Rewrite the tests to use the Product system

 - Tests now start a single server at the beginning of testing
 - PostgreSQL must be started prior the tests being ran
 - Each test suite registers a new product with a unique endpoint, and
   uses that particular run database to execute the tests that were
   already implemented.
 - These databases are dropped when the test is over

.travis.yml

@@ -58,7 +58,7 @@ install:
     - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then pg_ctl -w start -l postgres.log --pgdata ${PG_DATA}; cat postgres.log; fi
     - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then cat postgres.log; fi
     - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then createuser -s postgres; fi
-    - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then psql -c 'create database travis_ci_test;' -U postgres; fi
+    - psql -c 'create database codechecker_config;' -U postgres
     - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then cat postgres.log; fi
 
 

libcodechecker/server/instance_manager.py

@@ -17,12 +17,15 @@
 import stat
 
 
-def __getInstanceDescriptorPath():
-    return os.path.join(os.path.expanduser("~"), ".codechecker.instances.json")
+def __getInstanceDescriptorPath(folder=None):
+    if not folder:
+        folder = os.path.expanduser("~")
 
+    return os.path.join(folder, ".codechecker.instances.json")
 
-def __makeInstanceDescriptorFile():
-    descriptor = __getInstanceDescriptorPath()
+
+def __makeInstanceDescriptorFile(folder=None):
+    descriptor = __getInstanceDescriptorPath(folder)
     if not os.path.exists(descriptor):
         with open(descriptor, 'w') as f:
             json.dump([], f)
@@ -49,12 +52,12 @@ def __checkInstance(hostname, pid):
         return False
 
 
-def __rewriteInstanceFile(append, remove):
+def __rewriteInstanceFile(append, remove, folder=None):
     """This helper method reads the user's instance descriptor and manages it
     eliminating dead records, appending new ones and reserialising the file."""
 
-    __makeInstanceDescriptorFile()
-    with open(__getInstanceDescriptorPath(), 'r+') as f:
+    __makeInstanceDescriptorFile(folder)
+    with open(__getInstanceDescriptorPath(folder), 'r+') as f:
         portalocker.lock(f, portalocker.LOCK_EX)
 
         # After reading, check every instance if they are still valid and
@@ -77,7 +80,7 @@ def __rewriteInstanceFile(append, remove):
         portalocker.unlock(f)
 
 
-def register(pid, workspace, port):
+def register(pid, workspace, port, folder=None):
     """
     Adds the specified CodeChecker server instance to the user's instance
     descriptor.
@@ -87,24 +90,25 @@ def register(pid, workspace, port):
                             "hostname": socket.gethostname(),
                             "workspace": workspace,
                             "port": port}],
-                          [])
+                          [],
+                          folder)
 
 
-def unregister(pid):
+def unregister(pid, folder=None):
     """
     Removes the specified CodeChecker server instance from the user's instance
     descriptor.
     """
 
-    __rewriteInstanceFile([], [socket.gethostname() + ":" + str(pid)])
+    __rewriteInstanceFile([], [socket.gethostname() + ":" + str(pid)], folder)
 
 
-def list():
+def list(folder=None):
     """Returns the list of running servers for the current user."""
 
     # This method does NOT write the descriptor file.
 
-    descriptor = __getInstanceDescriptorPath()
+    descriptor = __getInstanceDescriptorPath(folder)
     instances = []
     if os.path.exists(descriptor):
         with open(descriptor, 'r') as f:

tests/Makefile

@@ -25,13 +25,29 @@ pep8:
 	pep8 bin libcodechecker scripts tests
 
 UNIT_TEST_CMD = nosetests $(NOSECFG) tests/unit
+
+CODECHECKER_CMD = $(BUILD_DIR)/CodeChecker/bin/CodeChecker
+SHUTDOWN_SERVER_CMD = echo "Shutting down server..."; \
+    ${CODECHECKER_CMD} server -l; \
+    ${CODECHECKER_CMD} server --config-directory $(BUILD_DIR)/workspace \
+    --port `cat "$(BUILD_DIR)/workspace/serverport"` --stop; \
+    rm -f "$(BUILD_DIR)/workspace/serverport"; \
+    ${CODECHECKER_CMD} server -l
+
+# Preserve the error or no error status of the previous command but always
+# be able to shut down servers.
+EXIT_HAPPY = { ${SHUTDOWN_SERVER_CMD}; exit 0; }
+EXIT_ERROR = { ${SHUTDOWN_SERVER_CMD}; exit 1; }
+
 FUNCTIONAL_TEST_CMD = $(REPO_ROOT) $(CLANG_VERSION) $(TEST_PROJECT) \
-		nosetests $(NOSECFG) tests/functional
+		nosetests $(NOSECFG) tests/functional \
+		&& ${EXIT_HAPPY} || ${EXIT_ERROR}
 
 run_test: package venv_dev
 	$(ACTIVATE_DEV_VENV) && \
 		$(REPO_ROOT) $(CLANG_VERSION) $(TEST_PROJECT) \
-		nosetests $(NOSECFG) ${TEST}
+		nosetests $(NOSECFG) ${TEST} \
+		&& ${EXIT_HAPPY} || ${EXIT_ERROR}
 
 test_unit: venv_dev
 	$(ACTIVATE_DEV_VENV) && $(UNIT_TEST_CMD)

tests/functional/analyze/__init__.py

@@ -11,10 +11,7 @@
 import os
 import shutil
 
-from libtest import codechecker
 from libtest import env
-from libtest import get_free_port
-from libtest import project
 
 
 # Test workspace should be initialized in this module.

tests/functional/authentication/__init__.py

@@ -7,12 +7,10 @@
 
 """Setup for the package tests."""
 
-import json
 import multiprocessing
 import os
 import shutil
 import subprocess
-import sys
 import time
 
 from libtest import project
@@ -36,8 +34,6 @@ def setup_package():
 
     test_project = 'cpp'
 
-    pg_db_config = env.get_postgresql_cfg()
-
     test_config = {}
 
     project_info = project.get_info(test_project)
@@ -48,17 +44,19 @@ def setup_package():
 
     skip_list_file = None
 
-    # Setup environment varaibled for the test cases.
-    host_port_cfg = env.get_host_port_cfg()
+    # Setup environment variables for the test cases.
+    host_port_cfg = {'viewer_host': 'localhost',
+                     'viewer_port': env.get_free_port(),
+                     'viewer_product': 'authentication'}
 
     test_env = env.test_env()
+    test_env['HOME'] = TEST_WORKSPACE
 
     codechecker_cfg = {
         'suppress_file': suppress_file,
         'skip_list_file': skip_list_file,
         'check_env': test_env,
         'workspace': TEST_WORKSPACE,
-        'pg_db_config': pg_db_config,
         'checkers': []
     }
 
@@ -78,17 +76,31 @@ def setup_package():
 
 
 def teardown_package():
-    """Stop the CodeChecker server."""
-    __STOP_SERVER.set()
-
+    """Stop the CodeChecker server and clean up after the tests."""
     # TODO If environment variable is set keep the workspace
     # and print out the path.
     global TEST_WORKSPACE
 
+    # Removing the product through this server requires credentials.
+    codechecker_cfg = env.import_test_cfg(TEST_WORKSPACE)['codechecker_cfg']
+    codechecker.login(codechecker_cfg,
+                      TEST_WORKSPACE, "cc", "test")
+    codechecker.remove_test_package_product(TEST_WORKSPACE,
+                                            codechecker_cfg['check_env'])
+
+    __STOP_SERVER.set()
+
+    # The custom server stated in a separate home needs to be waited, so it
+    # can properly execute its finalizers.
+    time.sleep(5)
+
     print("Removing: " + TEST_WORKSPACE)
-    shutil.rmtree(TEST_WORKSPACE)
+    shutil.rmtree(TEST_WORKSPACE, ignore_errors=True)
 
 
+# This server uses custom server configuration, which is brought up here
+# and torn down by the package itself --- it does not connect to the
+# test run's "master" server.
 def _start_server(codechecker_cfg, test_config, auth=False):
     """Start the CodeChecker server."""
     def start_server_proc(event, server_cmd, checking_env):
@@ -102,13 +114,16 @@ def start_server_proc(event, server_cmd, checking_env):
         if proc.poll() is None:
             proc.terminate()
 
-    server_cmd = codechecker.serv_cmd(codechecker_cfg, test_config)
+    server_cmd = codechecker.serv_cmd(codechecker_cfg['workspace'],
+                                      str(codechecker_cfg['viewer_port']),
+                                      env.get_postgresql_cfg())
 
     server_proc = multiprocessing.Process(
         name='server',
         target=start_server_proc,
         args=(__STOP_SERVER, server_cmd, codechecker_cfg['check_env']))
 
     server_proc.start()
+
     # Wait for server to start and connect to database.
     time.sleep(20)

tests/functional/authentication/test_authentication.py

@@ -14,6 +14,7 @@
 
 from thrift.protocol.TProtocol import TProtocolException
 
+from libtest import codechecker
 from libtest import env
 
 
@@ -30,11 +31,7 @@ def setUp(self):
         test_class = self.__class__.__name__
         print('Running ' + test_class + ' tests in ' + self._test_workspace)
 
-        tcfg = os.path.join(self._test_workspace, 'test_config.json')
-        with open(tcfg, 'r') as cfg:
-            t = json.load(cfg)
-            self._host = t['codechecker_cfg']['viewer_host']
-            self._port = t['codechecker_cfg']['viewer_port']
+        self._test_cfg = env.import_test_cfg(self._test_workspace)
 
     def test_privileged_access(self):
         """
@@ -63,6 +60,19 @@ def test_privileged_access(self):
         self.assertIsNotNone(self.sessionToken,
                              "Valid credentials didn't give us a token!")
 
+        # We need to run an authentication on the command-line, so that the
+        # product-adding feature is accessible by us.
+        codechecker.login(self._test_cfg['codechecker_cfg'],
+                          self._test_workspace, "cc", "test")
+        # We still need to create a product on the new server, because
+        # in PostgreSQL mode, the same database is used for configuration
+        # by the newly started instance of this test suite too.
+        codechecker.add_test_package_product(
+            self._test_cfg['codechecker_cfg'],
+            self._test_workspace,
+            # Use the test's home directory to find the session token file.
+            self._test_cfg['codechecker_cfg']['check_env'])
+
         handshake = auth_client.getAuthParameters()
         self.assertTrue(handshake.requiresAuthentication,
                         "Privileged server " +
@@ -88,6 +98,10 @@ def test_privileged_access(self):
 
         self.assertTrue(result, "Server did not allow us to destroy session.")
 
+        # Kill the session token that was created by login() too.
+        codechecker.logout(self._test_cfg['codechecker_cfg'],
+                           self._test_workspace)
+
         try:
             client.getAPIVersion()
             success = False
@@ -114,8 +128,11 @@ def test_nonauth_storage(self):
         test_dir = os.path.dirname(os.path.realpath(__file__))
         report_file = os.path.join(test_dir, 'clang-5.0-trunk.plist')
 
+        codechecker_cfg = self._test_cfg['codechecker_cfg']
+
         store_cmd = [env.codechecker_cmd(), 'store', '--name', 'auth',
-                     '--host', str(self._host), '--port', str(self._port),
+                     # Use the 'Default' product.
+                     '--url', env.parts_to_url(codechecker_cfg),
                      report_file]
 
         with self.assertRaises(subprocess.CalledProcessError):