Setup infrastructure for running db scripts (squashed)

Squashed:
- pick 146ff8019a Setup infrastructure for running db scripts
- squash e0a26dcf3a Move is-one-db check into model.database_utils

lib/galaxy/config/__init__.py

@@ -41,8 +41,11 @@
 from galaxy.containers import parse_containers_config
 from galaxy.exceptions import ConfigurationError
 from galaxy.model import mapping
-from galaxy.model.database_utils import database_exists
 from galaxy.schema.fields import BaseDatabaseIdField
+from galaxy.model.database_utils import (
+    database_exists,
+    is_one_database,
+)
 from galaxy.model.orm.engine_factory import build_engine
 from galaxy.structured_app import BasicSharedApp
 from galaxy.util import (
@@ -1475,11 +1478,6 @@ def _configure_tool_shed_registry(self):
         else:
             self.tool_shed_registry = galaxy.tool_shed.tool_shed_registry.Registry()
 
-    def _is_one_database(self, db_url, install_db_url):
-        # TODO: Consider more aggressive check here that this is not the same
-        # database file under the hood.
-        return not(db_url and install_db_url and install_db_url != db_url)
-
     def _configure_engines(self, db_url, install_db_url, combined_install_database):
         trace_logger = getattr(self, "trace_logger", None)
         engine = build_engine(
@@ -1505,7 +1503,7 @@ def _configure_models(self, check_migrate_databases=False, config_file=None):
 
         db_url = get_database_url(self.config)
         install_db_url = self.config.install_database_connection
-        combined_install_database = self._is_one_database(db_url, install_db_url)
+        combined_install_database = is_one_database(db_url, install_db_url)
         engine, install_engine = self._configure_engines(db_url, install_db_url, combined_install_database)
 
         if self.config.database_wait:

lib/galaxy/model/database_utils.py

@@ -1,5 +1,6 @@
 import sqlite3
 from contextlib import contextmanager
+from typing import Optional
 
 from sqlalchemy import create_engine
 from sqlalchemy.engine.url import make_url
@@ -118,3 +119,14 @@ def create(self, encoding, *arg):
             stmt = f"CREATE DATABASE {database} CHARACTER SET = '{encoding}'"
             with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as conn:
                 conn.execute(stmt)
+
+
+def is_one_database(db1_url: str, db2_url: Optional[str]):
+    """
+    Check if the arguments refer to one database. This will be true
+    if only one argument is passed, or if the urls are the same.
+    URLs are strings, so sameness is determined via string comparison.
+    """
+    # TODO: Consider more aggressive check here that this is not the same
+    # database file under the hood.
+    return not(db1_url and db2_url and db1_url != db2_url)

lib/galaxy/model/migrations/__init__.py

@@ -6,11 +6,16 @@
 from alembic import command, script
 from alembic.config import Config
 from alembic.runtime import migration
-from sqlalchemy import MetaData
+from sqlalchemy import create_engine, MetaData
 
 from galaxy.model import Base as gxy_base
-from galaxy.model.database_utils import create_database, database_exists
+from galaxy.model.database_utils import (
+    create_database,
+    database_exists,
+    is_one_database,
+)
 from galaxy.model.mapping import create_additional_database_objects
+from galaxy.model.migrations.scripts import DatabaseConfig
 from galaxy.model.tool_shed_install import Base as tsi_base
 
 # These identifiers are used throughout the migrations system to distinquish
@@ -200,31 +205,72 @@ def _load_sqlalchemymigrate_version(self, conn):
             return conn.execute(sql).scalar()
 
 
-def verify_databases(engine, install_engine=None, config=None):
-    # Get config values for gxy model.
-    template, encoding, is_auto_migrate = None, None, False
-    if config:
-        is_auto_migrate = config.database_auto_migrate  # Applied for both gxy and tsi.
-        template = config.database_template
-        encoding = config.database_encoding
+def verify_databases_via_script(
+    gxy_config: DatabaseConfig,
+    tsi_config: DatabaseConfig,
+    is_auto_migrate: bool = False,
+):
+    # This function serves a use case when an engine has not been created yet
+    # (e.g. when called from a script).
+    gxy_engine = create_engine(gxy_config.url)
+    tsi_engine = None
+    if tsi_config.url and tsi_config.url != gxy_config.url:
+        tsi_engine = create_engine(tsi_config.url)
+
+    _verify(
+        gxy_engine, gxy_config.template, gxy_config.encoding,
+        tsi_engine, tsi_config.template, tsi_config.encoding,
+        is_auto_migrate
+    )
 
-    # Verify galaxy (gxy) model.
-    gxy_dsv = DatabaseStateVerifier(engine, GXY, template, encoding, is_auto_migrate)
-    gxy_dsv.run()
 
-    # Update database template and encoding for tsi model if needed, falling back to gxy values.
-    if install_engine != engine:
-        template = getattr(config, 'install_database_template', template)
-        encoding = getattr(config, 'install_database_encoding', encoding)
+def verify_databases(gxy_engine, tsi_engine=None, config=None):
+    gxy_template, gxy_encoding = None, None
+    tsi_template, tsi_encoding = None, None
+    is_auto_migrate = False
+
+    if config:
+        is_auto_migrate = config.database_auto_migrate
+        gxy_template = config.database_template
+        gxy_encoding = config.database_encoding
+
+        is_combined = gxy_engine and tsi_engine and \
+            is_one_database(str(gxy_engine.url), str(tsi_engine.url))
+        if not is_combined:  # Otherwise not used.
+            tsi_template = getattr(config, 'install_database_template', None)
+            tsi_encoding = getattr(config, 'install_database_encoding', None)
+
+    _verify(
+        gxy_engine, gxy_template, gxy_encoding,
+        tsi_engine, tsi_template, tsi_encoding,
+        is_auto_migrate
+    )
+
+
+def _verify(
+    gxy_engine,
+    gxy_template,
+    gxy_encoding,
+    tsi_engine,
+    tsi_template,
+    tsi_encoding,
+    is_auto_migrate,
+):
+    # Verify gxy model.
+    gxy_verifier = DatabaseStateVerifier(
+        gxy_engine, GXY, gxy_template, gxy_encoding, is_auto_migrate)
+    gxy_verifier.run()
 
-    # Determine install_engine.
-    install_engine = install_engine or engine
     # New database = same engine, and gxy model has just been initialized.
-    is_new_database = install_engine == engine and gxy_dsv.is_new_database
+    is_new_database = gxy_engine == tsi_engine and gxy_verifier.is_new_database
+
+    # Determine engine for tsi model.
+    tsi_engine = tsi_engine or gxy_engine
 
-    # Verify tool_shed_install model (tsi) model.
-    tsi_dsv = DatabaseStateVerifier(install_engine, TSI, template, encoding, is_auto_migrate, is_new_database)
-    tsi_dsv.run()
+    # Verify tsi model model.
+    tsi_verifier = DatabaseStateVerifier(
+        tsi_engine, TSI, tsi_template, tsi_encoding, is_auto_migrate, is_new_database)
+    tsi_verifier.run()
 
 
 class DatabaseStateVerifier:
@@ -329,7 +375,7 @@ def _get_upgrade_message(self, model, db_version, code_version):
         msg = f'Your {model} database has version {db_version}, but this code expects '
         msg += f'version {code_version}. '
         msg += 'This database can be upgraded automatically if database_auto_migrate is set. '
-        msg += 'To upgrade manually, run `migrate.sh` (see instructions in that file). '
+        msg += 'To upgrade manually, run `migrate_db.sh` (see instructions in that file). '
         msg += 'Please remember to backup your database before migrating.'
         return msg
 

lib/galaxy/model/migrations/scripts.py

@@ -0,0 +1,52 @@
+import os
+from collections import namedtuple
+
+from galaxy.util.properties import (
+    find_config_file,
+    get_data_dir,
+    load_app_properties,
+)
+
+DEFAULT_CONFIG_NAMES = ['galaxy', 'universe_wsgi']
+CONFIG_FILE_ARG = '--galaxy-config'
+CONFIG_DIR_NAME = 'config'
+GXY_CONFIG_PREFIX = 'GALALXY_CONFIG_'
+TSI_CONFIG_PREFIX = 'GALALXY_INSTALL_CONFIG_'
+
+DatabaseConfig = namedtuple('DatabaseConfig', ['url', 'template', 'encoding'])
+
+
+def _pop_config_file(argv):
+    if CONFIG_FILE_ARG in argv:
+        pos = argv.index(CONFIG_FILE_ARG)
+        argv.pop(pos)  # pop argument name
+        return argv.pop(pos)  # pop and return argument value
+
+
+def get_configuration(argv, cwd) -> tuple[DatabaseConfig, DatabaseConfig, bool]:
+    """
+    Return a 3-item-tuple with configuration values used for managing databases.
+    """
+    config_file = _pop_config_file(argv)
+    if config_file is None:
+        cwds = [cwd, os.path.join(cwd, CONFIG_DIR_NAME)]
+        config_file = find_config_file(DEFAULT_CONFIG_NAMES, dirs=cwds)
+
+    # load gxy properties and auto-migrate
+    properties = load_app_properties(config_file=config_file, config_prefix=GXY_CONFIG_PREFIX)
+    default_url = f"sqlite:///{os.path.join(get_data_dir(properties), 'universe.sqlite')}?isolation_level=IMMEDIATE"
+    url = properties.get('database_connection', default_url)
+    template = properties.get('database_template', None)
+    encoding = properties.get('database_encoding', None)
+    is_auto_migrate = properties.get('database_auto_migrate', False)
+    gxy_config = DatabaseConfig(url, template, encoding)
+
+    # load tsi properties
+    properties = load_app_properties(config_file=config_file, config_prefix=TSI_CONFIG_PREFIX)
+    default_url = gxy_config.url
+    url = properties.get('install_database_connection', default_url)
+    template = properties.get('database_template', None)
+    encoding = properties.get('database_encoding', None)
+    tsi_config = DatabaseConfig(url, template, encoding)
+
+    return (gxy_config, tsi_config, is_auto_migrate)