From a82c44ac12b14fffe45caedca549c5339cf86e7f Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Wed, 5 Jan 2022 19:09:01 -0500 Subject: [PATCH] Setup infrastructure for running db scripts (squashed) Squashed: - pick 146ff8019a Setup infrastructure for running db scripts - squash e0a26dcf3a Move is-one-db check into model.database_utils --- lib/galaxy/config/__init__.py | 12 ++-- lib/galaxy/model/database_utils.py | 12 ++++ lib/galaxy/model/migrations/__init__.py | 92 ++++++++++++++++++------- lib/galaxy/model/migrations/scripts.py | 52 ++++++++++++++ 4 files changed, 138 insertions(+), 30 deletions(-) create mode 100644 lib/galaxy/model/migrations/scripts.py diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index 4fd49554e38b..9ee72e636ab6 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -41,8 +41,11 @@ from galaxy.containers import parse_containers_config from galaxy.exceptions import ConfigurationError from galaxy.model import mapping -from galaxy.model.database_utils import database_exists from galaxy.schema.fields import BaseDatabaseIdField +from galaxy.model.database_utils import ( + database_exists, + is_one_database, +) from galaxy.model.orm.engine_factory import build_engine from galaxy.structured_app import BasicSharedApp from galaxy.util import ( @@ -1475,11 +1478,6 @@ def _configure_tool_shed_registry(self): else: self.tool_shed_registry = galaxy.tool_shed.tool_shed_registry.Registry() - def _is_one_database(self, db_url, install_db_url): - # TODO: Consider more aggressive check here that this is not the same - # database file under the hood. - return not(db_url and install_db_url and install_db_url != db_url) - def _configure_engines(self, db_url, install_db_url, combined_install_database): trace_logger = getattr(self, "trace_logger", None) engine = build_engine( @@ -1505,7 +1503,7 @@ def _configure_models(self, check_migrate_databases=False, config_file=None): db_url = get_database_url(self.config) install_db_url = self.config.install_database_connection - combined_install_database = self._is_one_database(db_url, install_db_url) + combined_install_database = is_one_database(db_url, install_db_url) engine, install_engine = self._configure_engines(db_url, install_db_url, combined_install_database) if self.config.database_wait: diff --git a/lib/galaxy/model/database_utils.py b/lib/galaxy/model/database_utils.py index ec0312bce658..d12fa72c9301 100644 --- a/lib/galaxy/model/database_utils.py +++ b/lib/galaxy/model/database_utils.py @@ -1,5 +1,6 @@ import sqlite3 from contextlib import contextmanager +from typing import Optional from sqlalchemy import create_engine from sqlalchemy.engine.url import make_url @@ -118,3 +119,14 @@ def create(self, encoding, *arg): stmt = f"CREATE DATABASE {database} CHARACTER SET = '{encoding}'" with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as conn: conn.execute(stmt) + + +def is_one_database(db1_url: str, db2_url: Optional[str]): + """ + Check if the arguments refer to one database. This will be true + if only one argument is passed, or if the urls are the same. + URLs are strings, so sameness is determined via string comparison. + """ + # TODO: Consider more aggressive check here that this is not the same + # database file under the hood. + return not(db1_url and db2_url and db1_url != db2_url) diff --git a/lib/galaxy/model/migrations/__init__.py b/lib/galaxy/model/migrations/__init__.py index 6f6456ef6efe..3ced7b6825c5 100644 --- a/lib/galaxy/model/migrations/__init__.py +++ b/lib/galaxy/model/migrations/__init__.py @@ -6,11 +6,16 @@ from alembic import command, script from alembic.config import Config from alembic.runtime import migration -from sqlalchemy import MetaData +from sqlalchemy import create_engine, MetaData from galaxy.model import Base as gxy_base -from galaxy.model.database_utils import create_database, database_exists +from galaxy.model.database_utils import ( + create_database, + database_exists, + is_one_database, +) from galaxy.model.mapping import create_additional_database_objects +from galaxy.model.migrations.scripts import DatabaseConfig from galaxy.model.tool_shed_install import Base as tsi_base # These identifiers are used throughout the migrations system to distinquish @@ -200,31 +205,72 @@ def _load_sqlalchemymigrate_version(self, conn): return conn.execute(sql).scalar() -def verify_databases(engine, install_engine=None, config=None): - # Get config values for gxy model. - template, encoding, is_auto_migrate = None, None, False - if config: - is_auto_migrate = config.database_auto_migrate # Applied for both gxy and tsi. - template = config.database_template - encoding = config.database_encoding +def verify_databases_via_script( + gxy_config: DatabaseConfig, + tsi_config: DatabaseConfig, + is_auto_migrate: bool = False, +): + # This function serves a use case when an engine has not been created yet + # (e.g. when called from a script). + gxy_engine = create_engine(gxy_config.url) + tsi_engine = None + if tsi_config.url and tsi_config.url != gxy_config.url: + tsi_engine = create_engine(tsi_config.url) + + _verify( + gxy_engine, gxy_config.template, gxy_config.encoding, + tsi_engine, tsi_config.template, tsi_config.encoding, + is_auto_migrate + ) - # Verify galaxy (gxy) model. - gxy_dsv = DatabaseStateVerifier(engine, GXY, template, encoding, is_auto_migrate) - gxy_dsv.run() - # Update database template and encoding for tsi model if needed, falling back to gxy values. - if install_engine != engine: - template = getattr(config, 'install_database_template', template) - encoding = getattr(config, 'install_database_encoding', encoding) +def verify_databases(gxy_engine, tsi_engine=None, config=None): + gxy_template, gxy_encoding = None, None + tsi_template, tsi_encoding = None, None + is_auto_migrate = False + + if config: + is_auto_migrate = config.database_auto_migrate + gxy_template = config.database_template + gxy_encoding = config.database_encoding + + is_combined = gxy_engine and tsi_engine and \ + is_one_database(str(gxy_engine.url), str(tsi_engine.url)) + if not is_combined: # Otherwise not used. + tsi_template = getattr(config, 'install_database_template', None) + tsi_encoding = getattr(config, 'install_database_encoding', None) + + _verify( + gxy_engine, gxy_template, gxy_encoding, + tsi_engine, tsi_template, tsi_encoding, + is_auto_migrate + ) + + +def _verify( + gxy_engine, + gxy_template, + gxy_encoding, + tsi_engine, + tsi_template, + tsi_encoding, + is_auto_migrate, +): + # Verify gxy model. + gxy_verifier = DatabaseStateVerifier( + gxy_engine, GXY, gxy_template, gxy_encoding, is_auto_migrate) + gxy_verifier.run() - # Determine install_engine. - install_engine = install_engine or engine # New database = same engine, and gxy model has just been initialized. - is_new_database = install_engine == engine and gxy_dsv.is_new_database + is_new_database = gxy_engine == tsi_engine and gxy_verifier.is_new_database + + # Determine engine for tsi model. + tsi_engine = tsi_engine or gxy_engine - # Verify tool_shed_install model (tsi) model. - tsi_dsv = DatabaseStateVerifier(install_engine, TSI, template, encoding, is_auto_migrate, is_new_database) - tsi_dsv.run() + # Verify tsi model model. + tsi_verifier = DatabaseStateVerifier( + tsi_engine, TSI, tsi_template, tsi_encoding, is_auto_migrate, is_new_database) + tsi_verifier.run() class DatabaseStateVerifier: @@ -329,7 +375,7 @@ def _get_upgrade_message(self, model, db_version, code_version): msg = f'Your {model} database has version {db_version}, but this code expects ' msg += f'version {code_version}. ' msg += 'This database can be upgraded automatically if database_auto_migrate is set. ' - msg += 'To upgrade manually, run `migrate.sh` (see instructions in that file). ' + msg += 'To upgrade manually, run `migrate_db.sh` (see instructions in that file). ' msg += 'Please remember to backup your database before migrating.' return msg diff --git a/lib/galaxy/model/migrations/scripts.py b/lib/galaxy/model/migrations/scripts.py new file mode 100644 index 000000000000..e37cff1844e2 --- /dev/null +++ b/lib/galaxy/model/migrations/scripts.py @@ -0,0 +1,52 @@ +import os +from collections import namedtuple + +from galaxy.util.properties import ( + find_config_file, + get_data_dir, + load_app_properties, +) + +DEFAULT_CONFIG_NAMES = ['galaxy', 'universe_wsgi'] +CONFIG_FILE_ARG = '--galaxy-config' +CONFIG_DIR_NAME = 'config' +GXY_CONFIG_PREFIX = 'GALALXY_CONFIG_' +TSI_CONFIG_PREFIX = 'GALALXY_INSTALL_CONFIG_' + +DatabaseConfig = namedtuple('DatabaseConfig', ['url', 'template', 'encoding']) + + +def _pop_config_file(argv): + if CONFIG_FILE_ARG in argv: + pos = argv.index(CONFIG_FILE_ARG) + argv.pop(pos) # pop argument name + return argv.pop(pos) # pop and return argument value + + +def get_configuration(argv, cwd) -> tuple[DatabaseConfig, DatabaseConfig, bool]: + """ + Return a 3-item-tuple with configuration values used for managing databases. + """ + config_file = _pop_config_file(argv) + if config_file is None: + cwds = [cwd, os.path.join(cwd, CONFIG_DIR_NAME)] + config_file = find_config_file(DEFAULT_CONFIG_NAMES, dirs=cwds) + + # load gxy properties and auto-migrate + properties = load_app_properties(config_file=config_file, config_prefix=GXY_CONFIG_PREFIX) + default_url = f"sqlite:///{os.path.join(get_data_dir(properties), 'universe.sqlite')}?isolation_level=IMMEDIATE" + url = properties.get('database_connection', default_url) + template = properties.get('database_template', None) + encoding = properties.get('database_encoding', None) + is_auto_migrate = properties.get('database_auto_migrate', False) + gxy_config = DatabaseConfig(url, template, encoding) + + # load tsi properties + properties = load_app_properties(config_file=config_file, config_prefix=TSI_CONFIG_PREFIX) + default_url = gxy_config.url + url = properties.get('install_database_connection', default_url) + template = properties.get('database_template', None) + encoding = properties.get('database_encoding', None) + tsi_config = DatabaseConfig(url, template, encoding) + + return (gxy_config, tsi_config, is_auto_migrate)