[22.01] Create config package, move app-related functionality to app

lib/galaxy/model/mapping.py

@@ -9,6 +9,7 @@
 from typing import Optional, Type
 
 from galaxy import model
+from galaxy.config import GalaxyAppConfiguration
 from galaxy.model import mapper_registry
 from galaxy.model.base import SharedModelMapping
 from galaxy.model.migrate.triggers.update_audit_table import install as install_timestamp_triggers
@@ -64,3 +65,20 @@ def init(file_path, url, engine_options=None, create_tables=False, map_install_m
     result.security_agent = GalaxyRBACAgent(result)
     result.thread_local_log = thread_local_log
     return result
+
+
+def init_models_from_config(config: GalaxyAppConfiguration, map_install_models=False, object_store=None, trace_logger=None):
+    model = init(
+        config.file_path,
+        config.database_connection,
+        config.database_engine_options,
+        map_install_models=map_install_models,
+        database_query_profiling_proxy=config.database_query_profiling_proxy,
+        object_store=object_store,
+        trace_logger=trace_logger,
+        use_pbkdf2=config.get_bool('use_pbkdf2', True),
+        slow_query_log_threshold=config.slow_query_log_threshold,
+        thread_local_log=config.thread_local_log,
+        log_query_counts=config.database_log_query_counts,
+    )
+    return model

lib/galaxy/web_stack/__init__.py

@@ -633,10 +633,6 @@ def get_app_kwds(config_section, app_name=None):
     return application_stack_class().get_app_kwds(config_section, app_name=app_name)
 
 
-def get_stack_facts(config=None):
-    return application_stack_instance(config=config).facts
-
-
 def _uwsgi_configured_mules():
     mules = uwsgi.opt.get('mule', [])
     return [mules] if isinstance(mules, str) or mules is True else mules

lib/tool_shed/webapp/config.py

@@ -12,11 +12,11 @@
     CommonConfigurationMixin,
     get_database_engine_options,
 )
+from galaxy.config import expand_pretty_datetime_format
 from galaxy.config.schema import AppSchema
 from galaxy.exceptions import ConfigurationError
 from galaxy.util import string_as_bool
 from galaxy.version import VERSION, VERSION_MAJOR, VERSION_MINOR
-from galaxy.web.formatting import expand_pretty_datetime_format
 
 try:
     from importlib.resources import files  # type: ignore[attr-defined]

packages/app/MANIFEST.in

@@ -1,5 +1,3 @@
 include *.rst *.txt LICENSE
-include galaxy/config/schemas/*.yml
-include galaxy/config/sample/*.sample*
 include galaxy/jobs/runners/util/job_script/*.sh
 include galaxy/tools/*tsv

packages/app/requirements.txt

@@ -1,7 +1,9 @@
 galaxy-auth
+galaxy-config
 galaxy-data
 galaxy-job-execution
 galaxy-job-metrics
+galaxy-objectstore
 galaxy-tool-util[cwl,edam]
 galaxy-web-framework
 galaxy-web-stack
@@ -18,7 +20,6 @@ Markdown
 paramiko!=2.9.0,!=2.9.1
 pulsar-galaxy-lib>=0.14.13
 pydantic
-pykwalify
 refgenconf>=0.12.0
 sqlitedict
 svgwrite

packages/app/setup.py

@@ -34,7 +34,6 @@ def get_var(var_name):
     'galaxy.actions',
     'galaxy.app_unittest_utils',
     'galaxy.authnz',
-    'galaxy.config',
     'galaxy.dependencies',
     'galaxy.forms',
     'galaxy.jobs',
@@ -95,14 +94,9 @@ def get_var(var_name):
 ENTRY_POINTS = '''
         [console_scripts]
         galaxy-main=galaxy.main:main
-        galaxy-config=galaxy.config.script:main
 '''
 PACKAGE_DATA = {
     # Be sure to update MANIFEST.in for source dist.
-    'galaxy': [
-        'config/schemas/*.yml',
-        'config/sample/*',
-    ],
     'tool_shed': [
         'scripts/bootstrap_tool_shed/user_info.xml',
     ],

packages/app/test-requirements.txt

@@ -1,3 +1,4 @@
 mock-ssh-server
+pykwalify
 pytest
 testfixtures

packages/config/HISTORY.rst

@@ -0,0 +1,12 @@
+.. :changelog:
+
+History
+-------
+
+.. to_doc
+
+---------------------
+22.1.0.dev0
+---------------------
+
+* Initial release

packages/config/LICENSE

@@ -0,0 +1 @@
+../../LICENSE.txt

packages/config/MANIFEST.in

@@ -0,0 +1,3 @@
+include *.rst *.txt LICENSE
+include galaxy/config/schemas/*.yml
+include galaxy/config/sample/*.sample*

packages/config/Makefile

@@ -0,0 +1 @@
+../package.Makefile

packages/config/README.rst

@@ -0,0 +1,14 @@
+
+.. image:: https://badge.fury.io/py/galaxy-config.svg
+   :target: https://pypi.org/project/galaxy-config/
+
+
+Overview
+--------
+
+The Galaxy_ config module.
+
+* Free software: Academic Free License version 3.0
+* Code: https://github.com/galaxyproject/galaxy
+
+.. _Galaxy: http://galaxyproject.org/

packages/config/dev-requirements.txt

@@ -0,0 +1 @@
+../package-dev-requirements.txt

packages/config/galaxy/__init__.py

@@ -0,0 +1 @@
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)  # type: ignore[has-type]

packages/config/galaxy/project_galaxy_config.py

@@ -0,0 +1,9 @@
+__version__ = "22.1.0rc1"
+
+PROJECT_NAME = "galaxy-config"
+PROJECT_OWNER = PROJECT_USERAME = "galaxyproject"
+PROJECT_URL = "https://github.com/galaxyproject/galaxy"
+PROJECT_AUTHOR = "Galaxy Project and Community"
+PROJECT_DESCRIPTION = "Galaxy Configuration"
+PROJECT_EMAIL = "galaxy-committers@lists.galaxyproject.org"
+RAW_CONTENT_URL = "https://raw.github.com/{}/{}/master/".format(PROJECT_USERAME, PROJECT_NAME)

packages/config/requirements.txt

@@ -0,0 +1,5 @@
+galaxy-containers
+boltons
+pykwalify
+PyYAML
+requests

packages/config/scripts

@@ -0,0 +1 @@
+../build_scripts

packages/config/setup.cfg

@@ -0,0 +1 @@
+../../setup.cfg

packages/config/setup.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python
+
+import ast
+import os
+import re
+
+try:
+    from setuptools import setup
+except ImportError:
+    from distutils.core import setup
+
+SOURCE_DIR = "galaxy"
+
+_version_re = re.compile(r"__version__\s+=\s+(.*)")
+
+project_short_name = os.path.basename(os.path.dirname(os.path.realpath(__file__)))
+with open(f"{SOURCE_DIR}/project_galaxy_{project_short_name}.py", "rb") as f:
+    init_contents = f.read().decode("utf-8")
+
+    def get_var(var_name):
+        pattern = re.compile(rf"{var_name}\s+=\s+(.*)")
+        match = pattern.search(init_contents).group(1)
+        return str(ast.literal_eval(match))
+
+    version = get_var("__version__")
+    PROJECT_NAME = get_var("PROJECT_NAME")
+    PROJECT_URL = get_var("PROJECT_URL")
+    PROJECT_AUTHOR = get_var("PROJECT_AUTHOR")
+    PROJECT_EMAIL = get_var("PROJECT_EMAIL")
+    PROJECT_DESCRIPTION = get_var("PROJECT_DESCRIPTION")
+
+TEST_DIR = "tests"
+PACKAGES = [
+    "galaxy",
+    "galaxy.config",
+]
+ENTRY_POINTS = """
+        [console_scripts]
+        galaxy-config=galaxy.config.script:main
+"""
+PACKAGE_DATA = {
+    # Be sure to update MANIFEST.in for source dist.
+    "galaxy": [
+        "config/schemas/*.yml",
+        "config/sample/*",
+    ],
+}
+PACKAGE_DIR = {
+    SOURCE_DIR: SOURCE_DIR,
+}
+
+readme = open("README.rst").read()
+history = open("HISTORY.rst").read().replace(".. :changelog:", "")
+
+if os.path.exists("requirements.txt"):
+    requirements = open("requirements.txt").read().split("\n")
+else:
+    # In tox, it will cover them anyway.
+    requirements = []
+
+
+test_requirements = open("test-requirements.txt").read().split("\n")
+
+
+setup(
+    name=PROJECT_NAME,
+    version=version,
+    description=PROJECT_DESCRIPTION,
+    long_description=readme + "\n\n" + history,
+    long_description_content_type="text/x-rst",
+    author=PROJECT_AUTHOR,
+    author_email=PROJECT_EMAIL,
+    url=PROJECT_URL,
+    packages=PACKAGES,
+    entry_points=ENTRY_POINTS,
+    package_data=PACKAGE_DATA,
+    package_dir=PACKAGE_DIR,
+    include_package_data=True,
+    install_requires=requirements,
+    license="AFL",
+    zip_safe=False,
+    keywords="galaxy",
+    classifiers=[
+        "Development Status :: 5 - Production/Stable",
+        "Intended Audience :: Developers",
+        "Environment :: Console",
+        "License :: OSI Approved :: Academic Free License (AFL)",
+        "Operating System :: POSIX",
+        "Topic :: Software Development",
+        "Topic :: Software Development :: Code Generators",
+        "Topic :: Software Development :: Testing",
+        "Natural Language :: English",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+    ],
+    test_suite=TEST_DIR,
+    tests_require=test_requirements,
+)

packages/config/test-data

@@ -0,0 +1 @@
+../../test-data

packages/config/test-requirements.txt

@@ -0,0 +1 @@
+pytest<6.1

packages/config/tests/config

@@ -0,0 +1 @@
+../../../test/unit/config

packages/test.sh

@@ -23,6 +23,7 @@ PACKAGE_DIRS=(
     objectstore
     job_metrics
     containers
+    config
     files
     tool_util
     data

scripts/cleanup_datasets/cleanup_datasets.py

@@ -18,6 +18,7 @@
 import galaxy.config
 from galaxy.datatypes.registry import Registry
 from galaxy.exceptions import ObjectNotFound
+from galaxy.model.mapping import init_models_from_config
 from galaxy.objectstore import build_object_store_from_config
 from galaxy.util import unicodify
 from galaxy.util.script import app_properties_from_args, populate_config_args
@@ -519,7 +520,7 @@ class CleanupDatasetsApplication:
     def __init__(self, config):
         self.object_store = build_object_store_from_config(config)
         # Setup the database engine and ORM
-        self.model = galaxy.config.init_models_from_config(config, object_store=self.object_store)
+        self.model = init_models_from_config(config, object_store=self.object_store)
         registry = Registry()
         registry.load_datatypes()
         galaxy.model.set_datatypes_registry(registry)

scripts/cleanup_datasets/pgcleanup.py

@@ -1112,7 +1112,7 @@ def actions(self):
     @property
     def conn(self):
         if self.__conn is None:
-            url = make_url(galaxy.config.get_database_url(self.config))
+            url = make_url(self.config.database_connection)
             log.info(f'Connecting to database with URL: {url}')
             args = url.translate_connect_args(username='user')
             args.update(url.query)

scripts/cleanup_datasets/populate_uuid.py

@@ -15,6 +15,7 @@
 sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, 'lib')))
 
 import galaxy.config
+from galaxy.model.mapping import init_models_from_config
 from galaxy.util.script import app_properties_from_args, populate_config_args
 
 DESCRIPTION = """
@@ -32,7 +33,7 @@ def main():
 
     app_properties = app_properties_from_args(args)
     config = galaxy.config.Configuration(**app_properties)
-    model = galaxy.config.init_models_from_config(config)
+    model = init_models_from_config(config)
 
     for row in model.context.query(model.Dataset):
         if row.uuid is None:

scripts/grt/export.py

@@ -18,6 +18,7 @@
 import galaxy
 import galaxy.app
 import galaxy.config
+from galaxy.model.mapping import init_models_from_config
 from galaxy.objectstore import build_object_store_from_config
 from galaxy.util import (
     hash_util,
@@ -36,7 +37,7 @@ def _init(args):
     if not config.database_connection:
         logging.warning("The database connection is empty. If you are using the default value, please uncomment that in your galaxy.yml")
 
-    model = galaxy.config.init_models_from_config(config, object_store=object_store)
+    model = init_models_from_config(config, object_store=object_store)
     return (
         model,
         object_store,

scripts/helper.py

@@ -12,6 +12,7 @@
 sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir, 'lib'))
 
 import galaxy.config
+from galaxy.model.mapping import init_models_from_config
 from galaxy.security import idencoding
 from galaxy.util.script import app_properties_from_args, populate_config_args
 
@@ -26,7 +27,7 @@
 app_properties = app_properties_from_args(args)
 config = galaxy.config.Configuration(**app_properties)
 helper = idencoding.IdEncodingHelper(id_secret=app_properties.get('id_secret'))
-model = galaxy.config.init_models_from_config(config)
+model = init_models_from_config(config)
 
 if args.encode_id:
     print(f'Encoded "{args.encode_id}": {helper.encode_id(args.encode_id)}')