Product management database, schema, connection handling

 - Introduce new configuration database schema, and database connection
   handling to this new database
 - Added basic API for product management
 - PostgreSQL server is not started automatically anymore
 - Products have a distinct endpoint
   (localhost:8001/ProductName/CodeCheckerService) in the request, and the
   requests are routed to the individual products based on this.
 - If server is started with SQLite configuration database and this is a
   brand new start, automatically configure a 'Default' product in the
   same folder.
 - Store a connection failure state for products that could not be
   connected to at server start.

As no other packages manage and use database connections anymore, put
the ORM and the run database handling into libcodechecker.server.

This commit breaks the tests. It will be fixed later on.
[ci skip]

Makefile

@@ -32,6 +32,9 @@ thrift: build_dir
 	thrift -r -o $(BUILD_DIR) -I api/ \
 		--gen py --gen js:jquery api/authentication.thrift
 
+	thrift -r -o $(BUILD_DIR) -I api/ \
+		--gen py --gen js:jquery api/products.thrift
+
 package: build_dir gen-docs thrift
 	if [ ! -d "$(BUILD_DIR)/CodeChecker" ]; then \
 		./scripts/build_package.py -r $(ROOT) -o $(BUILD_DIR) -b $(BUILD_DIR); \

alembic.ini

@@ -1,6 +1,35 @@
-# A generic, single database configuration.
+[product_db]
+# path to migration scripts
+script_location = product_db_migrate
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# max length of characters to apply to the
+# "slug" field
+#truncate_slug_length = 40
 
-[alembic]
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; this defaults
+# to database/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path
+# version_locations = %(here)s/bar %(here)s/bat database/versions
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+sqlalchemy.url = postgres://codechecker@localhost:5432/codechecker
+
+[run_db]
 # path to migration scripts
 script_location = db_migrate
 
@@ -29,7 +58,7 @@ script_location = db_migrate
 # are written from script.py.mako
 # output_encoding = utf-8
 
-sqlalchemy.url = postgres://codechecker@localhost:5432/codechecker
+sqlalchemy.url = postgres://codechecker@localhost:5432/default
 
 # Logging configuration
 [loggers]

api/products.thrift

@@ -0,0 +1,63 @@
+// -------------------------------------------------------------------------
+//                     The CodeChecker Infrastructure
+//   This file is distributed under the University of Illinois Open Source
+//   License. See LICENSE.TXT for details.
+// -------------------------------------------------------------------------
+
+include "shared.thrift"
+
+namespace py ProductManagement
+namespace js codeCheckerProductManagement
+
+
+/*
+struct PrivilegeRecord {
+  1: string   name,
+  2: bool     isGroup
+}
+typedef list<PrivilegeRecord> PrivilegeRecords
+*/
+
+struct DatabaseConnection {
+  1:          string engine,
+  2:          string host,
+  3:          i32    port,
+  4:          string username_b64,
+  5: optional string password_b64,   // Database password is NOT sent server->client!
+  6:          string database
+}
+
+/* ProductConfiguration carries administrative data regarding product settings */
+struct ProductConfiguration {
+  1:          i64                id,
+  2:          string             endpoint,
+  3:          string             displayedName,
+  4:          string             description,
+  5: optional DatabaseConnection connection
+}
+typedef list<ProductConfiguration> ProductConfigurations
+
+/* Product carries data to the end user's product list and tasks */
+struct Product {
+  1: i64    id,
+  2: string endpoint,
+  3: string displayedName,
+  4: string description,
+  5: bool   connected,      // Indicates that the server could set up the database connection properly.
+  6: bool   accessible      // Indicates whether the current user can access this product.
+}
+typedef list<Product> Products
+
+service codeCheckerProductService {
+
+  // *** Handling the add-modify-remove of products registered *** //
+  Products getProducts()
+                       throws (1: shared.RequestFailed requestError),
+
+  bool addProduct(1: Product product)
+                  throws (1: shared.RequestFailed requestError),
+
+  bool removeProduct(1: i64 productId)
+                     throws (1: shared.RequestFailed requestError)
+
+}

api/report_server.thrift

@@ -17,7 +17,6 @@ namespace js codeCheckerDBAccess
 namespace cpp cc.service.codechecker
 
 //=================================================
-const string API_VERSION = '6.0'
 const i64 MAX_QUERY_SIZE = 500
 //=================================================
 

api/shared.thrift

@@ -4,6 +4,8 @@
 //   License. See LICENSE.TXT for details.
 // -------------------------------------------------------------------------
 
+const string API_VERSION = '6.0'
+
 //-----------------------------------------------------------------------------
 struct BugPathEvent {
   1: i64    startLine,

config/package_layout.json

@@ -19,7 +19,8 @@
     "ld_logger": "ld_logger",
     "libcodechecker": "lib/python2.7/libcodechecker",
     "gencodechecker": "lib/python2.7/gencodechecker",
-    "codechecker_db_migrate": "lib/python2.7/db_migrate"
+    "run_db_migrate": "lib/python2.7/run_migrate",
+    "config_db_migrate": "lib/python2.7/config_migrate"
   },
   "runtime": {
     "analyzers": {

config/version.json

@@ -1,10 +1,14 @@
 {
-    "version":{
+    "version": {
         "major" : "6",
         "minor" : "0",
         "revision" : "0"
     },
-    "db_version":{
+    "product_db_version": {
+        "major" : "6",
+        "minor" : "0"
+    },
+    "run_db_version": {
         "major" : "6",
         "minor" : "0"
     }

db_migrate/env.py

@@ -9,15 +9,15 @@
 # Add your model's MetaData object here
 # for 'autogenerate' support.
 try:
-    from libcodechecker.orm_model import Base
+    from libcodechecker.server.run_db_model import Base
 except ImportError:
     # Assume we are in the source directory
     import sys
     import os
 
     sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__),
                                                  "..")))
-    from libcodechecker.orm_model import Base
+    from libcodechecker.server.run_db_model import Base
 
 target_metadata = Base.metadata
 

libcodechecker/analyze/store_handler.py

@@ -18,7 +18,8 @@
 
 from libcodechecker.analyze import skiplist_handler
 from libcodechecker.logger import LoggerFactory
-from libcodechecker.orm_model import *
+# TODO: This is a cross-subpackage import.
+from libcodechecker.server.run_db_model import *
 
 LOG = LoggerFactory.get_new_logger('STORE HANDLER')
 

libcodechecker/context_base.py

@@ -128,9 +128,14 @@ def www_root(self):
                             self.pckg_layout['www'])
 
     @property
-    def migration_root(self):
+    def run_migration_root(self):
         return os.path.join(self._package_root,
-                            self.pckg_layout['codechecker_db_migrate'])
+                            self.pckg_layout['run_db_migrate'])
+
+    @property
+    def config_migration_root(self):
+        return os.path.join(self._package_root,
+                            self.pckg_layout['config_db_migrate'])
 
     @property
     def db_username(self):

libcodechecker/generic_package_context.py

@@ -46,7 +46,8 @@ def __init__(self, package_root, pckg_layout, cfg_dict):
         self.__package_root = package_root
 
         self.__package_version = None
-        self.__db_version_info = None
+        self.__product_db_version_info = None
+        self.__run_db_version_info = None
         self.__package_build_date = None
         self.__package_git_hash = None
         self.__analyzers = {}
@@ -82,14 +83,18 @@ def __set_version(self):
             package_git_hash = vfile_data['git_hash']
             package_git_tag = vfile_data['git_describe']['tag']
             package_git_dirtytag = vfile_data['git_describe']['dirty']
-            database_version = vfile_data['db_version']
+            product_database_version = vfile_data['product_db_version']
+            run_database_version = vfile_data['run_db_version']
 
             self.__package_version = package_version['major'] + '.' + \
                 package_version['minor'] + '.' + \
                 package_version['revision']
-            self.__db_version_info = db_version.DBVersionInfo(
-                database_version['major'],
-                database_version['minor'])
+            self.__product_db_version_info = db_version.DBVersionInfo(
+                product_database_version['major'],
+                product_database_version['minor'])
+            self.__run_db_version_info = db_version.DBVersionInfo(
+                run_database_version['major'],
+                run_database_version['minor'])
 
             self.__package_build_date = package_build_date
             self.__package_git_hash = package_git_hash
@@ -154,8 +159,12 @@ def package_git_tag(self):
         return self.__package_git_tag
 
     @property
-    def db_version_info(self):
-        return self.__db_version_info
+    def product_db_version_info(self):
+        return self.__product_db_version_info
+
+    @property
+    def run_db_version_info(self):
+        return self.__run_db_version_info
 
     @property
     def version_file(self):

libcodechecker/libclient/client.py

@@ -7,20 +7,19 @@
 import getpass
 import sys
 
-from Authentication import ttypes as AuthTypes
-
 from thrift.Thrift import TApplicationException
 
-from . import thrift_helper
-from . import authentication_helper
+import shared
+from Authentication import ttypes as AuthTypes
 
 from libcodechecker import session_manager
 from libcodechecker.logger import LoggerFactory
 
-import shared
+from . import thrift_helper
+from . import authentication_helper
 
 LOG = LoggerFactory.get_new_logger('CLIENT')
-SUPPORTED_VERSION = '6.0'
+SUPPORTED_API_VERSION = '6.0'
 
 
 def check_api_version(client):
@@ -29,7 +28,7 @@ def check_api_version(client):
     """
 
     version = client.getAPIVersion()
-    supp_major_version = SUPPORTED_VERSION.split('.')[0]
+    supp_major_version = SUPPORTED_API_VERSION.split('.')[0]
     api_major_version = version.split('.')[0]
 
     # There is NO compatibility between major versions.

libcodechecker/libhandlers/server.py

@@ -22,11 +22,12 @@
 from libcodechecker import session_manager
 from libcodechecker import util
 from libcodechecker.analyze import analyzer_env
-from libcodechecker.database_handler import SQLServer
-from libcodechecker.logger import add_verbose_arguments
 from libcodechecker.logger import LoggerFactory
+from libcodechecker.logger import add_verbose_arguments
 from libcodechecker.server import client_db_access_server
+from libcodechecker.server import config_database
 from libcodechecker.server import instance_manager
+from libcodechecker.server import run_database
 
 LOG = LoggerFactory.get_new_logger('SERVER')
 
@@ -149,7 +150,7 @@ def add_arguments_to_parser(parser):
                          metavar='SQLITE_FILE',
                          default=os.path.join(
                              util.get_default_workspace(),
-                             "codechecker.sqlite"),
+                             "config.sqlite"),
                          required=False,
                          help="Path of the SQLite database file to use.")
 
@@ -299,12 +300,12 @@ def arg_match(options):
             parser.error("argument --config-directory: not allowed with "
                          "argument --workspace")
 
-        # If workspace is specified, sqlite is workspace/codechecker.sqlite
+        # If workspace is specified, sqlite is workspace/config.sqlite
         # and config_directory is the workspace directory.
         if len(arg_match(['--workspace', '-w'])) > 0:
             args.config_directory = args.workspace
             args.sqlite = os.path.join(args.workspace,
-                                       'codechecker.sqlite')
+                                       'config.sqlite')
             setattr(args, 'dbdatadir', os.path.join(args.workspace,
                                                     'pgsql_data'))
 
@@ -421,17 +422,41 @@ def main(args):
     check_env = analyzer_env.get_check_env(context.path_env_extra,
                                            context.ld_lib_path_extra)
 
-    sql_server = SQLServer.from_cmdline_args(args,
-                                             context.migration_root,
-                                             check_env)
+    # Create the main database link from the arguments passed over the
+    # command line.
+    default_product_path = os.path.join(args.config_directory,
+                                        'Default.sqlite')
+    create_default_product = 'sqlite' in args and \
+                             not os.path.exists(args.sqlite) and \
+                             not os.path.exists(default_product_path)
 
-    LOG.debug("Starting database server.")
-    sql_server.start(context.db_version_info, wait_for_start=True,
-                     init=True)
+    sql_server = config_database.SQLServer.from_cmdline_args(
+        args, context.config_migration_root, check_env)
+
+    LOG.debug("Connecting to product configuration database.")
+    sql_server.connect(context.product_db_version_info, init=True)
 
     # Start database viewer.
     db_connection_string = sql_server.get_connection_string()
 
+    if create_default_product:
+        # Create a default product and add it to the configuration database.
+
+        LOG.debug("Create default product...")
+        LOG.debug("Configuring schema and migration...")
+        prod_server = run_database.SQLiteDatabase(
+            default_product_path, context.run_migration_root, check_env)
+        prod_server.connect(context.run_db_version_info, init=True)
+        LOG.debug("Connecting database engine for default product")
+        product_conn_string = prod_server.get_connection_string()
+        LOG.debug("Default database created and connected.")
+
+        client_db_access_server.add_initial_run_database(
+            db_connection_string, product_conn_string)
+
+        LOG.info("Product 'Default' at '{0}' created and set up."
+                 .format(default_product_path))
+
     checker_md_docs = os.path.join(context.doc_root, 'checker_md_docs')
     checker_md_docs_map = os.path.join(checker_md_docs,
                                        'checker_doc_map.json')
@@ -450,7 +475,8 @@ def main(args):
                                              db_connection_string,
                                              suppress_handler,
                                              args.listen_address,
-                                             context)
+                                             context,
+                                             check_env)
     except socket.error as err:
         if err.errno == errno.EADDRINUSE:
             LOG.error("Server can't be started, maybe the given port number "