WIP: Test user config objects and provide detailed graphical messages…

… before create.

client/src/api/schema/schema.ts

@@ -324,6 +324,10 @@ export interface paths {
         /** Create a user-bound object store. */
         post: operations["file_sources__create_instance"];
     };
+    "/api/file_source_instances/test": {
+        /** Test payload for creating user-bound object store. */
+        post: operations["file_sources__test_new_instance_configuration"];
+    };
     "/api/file_source_instances/{user_file_source_id}": {
         /** Get a list of persisted file source instances defined by the requesting user. */
         get: operations["file_sources__instances_get"];
@@ -10446,12 +10450,28 @@ export interface components {
          * @enum {string}
          */
         PersonalNotificationCategory: "message" | "new_shared_item";
+        /** PluginAspectStatus */
+        PluginAspectStatus: {
+            /** Message */
+            message: string;
+            /**
+             * State
+             * @enum {string}
+             */
+            state: "ok" | "not_ok" | "unknown";
+        };
         /**
          * PluginKind
          * @description Enum to distinguish between different kinds or categories of plugins.
          * @enum {string}
          */
         PluginKind: "rfs" | "drs" | "rdm" | "stock";
+        /** PluginStatus */
+        PluginStatus: {
+            connection?: components["schemas"]["PluginAspectStatus"] | null;
+            template_definition: components["schemas"]["PluginAspectStatus"];
+            template_settings?: components["schemas"]["PluginAspectStatus"] | null;
+        };
         /** Position */
         Position: {
             /** Left */
@@ -14994,6 +15014,34 @@ export interface operations {
             };
         };
     };
+    file_sources__test_new_instance_configuration: {
+        /** Test payload for creating user-bound object store. */
+        parameters?: {
+            /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */
+            header?: {
+                "run-as"?: string | null;
+            };
+        };
+        requestBody: {
+            content: {
+                "application/json": components["schemas"]["CreateInstancePayload"];
+            };
+        };
+        responses: {
+            /** @description Successful Response */
+            200: {
+                content: {
+                    "application/json": components["schemas"]["PluginStatus"];
+                };
+            };
+            /** @description Validation Error */
+            422: {
+                content: {
+                    "application/json": components["schemas"]["HTTPValidationError"];
+                };
+            };
+        };
+    };
     file_sources__instances_get: {
         /** Get a list of persisted file source instances defined by the requesting user. */
         parameters: {

lib/galaxy/files/templates/manager.py

@@ -12,6 +12,7 @@
     find_template,
     find_template_by,
     InstanceDefinition,
+    PluginAspectStatus,
     RawTemplateConfig,
     TemplateReference,
     validate_secrets_and_variables,
@@ -80,6 +81,13 @@ def validate(self, instance: InstanceDefinition):
         template = self.find_template(instance)
         validate_secrets_and_variables(instance, template)
 
+    def status_template_definition(self, instance: InstanceDefinition) -> PluginAspectStatus:
+        template = self.find_template(instance)
+        if template:
+            return PluginAspectStatus(state="ok", message="Template definition found and validates against schema.")
+        else:
+            return PluginAspectStatus(state="not_ok", message="Template not found or not loaded.")
+
 
 def raw_config_to_catalog(raw_config: List[RawTemplateConfig]) -> FileSourceTemplateCatalog:
     effective_root = apply_syntactic_sugar(raw_config)

lib/galaxy/managers/_config_templates.py

@@ -36,6 +36,7 @@
     secrets_as_dict,
     SecretsDict,
     Template,
+    TemplateEnvironmentEntry,
     TemplateEnvironmentSecret,
     TemplateEnvironmentVariable,
     TemplateVariableValueType,
@@ -102,8 +103,14 @@ def recover_secrets(
 def prepare_environment(
     configuration_template: HasConfigEnvironment, vault: Vault, app_config: UsesTemplatesAppConfig
 ) -> EnvironmentDict:
+    return prepare_environment_from_root(configuration_template.template_environment.root, vault, app_config)
+
+
+def prepare_environment_from_root(
+    root: Optional[List[TemplateEnvironmentEntry]], vault: Vault, app_config: UsesTemplatesAppConfig
+):
     environment: EnvironmentDict = {}
-    for environment_entry in configuration_template.template_environment.root:
+    for environment_entry in root or []:
         e_type = environment_entry.type
         e_name = environment_entry.name
         if e_type == "secret":

lib/galaxy/managers/file_source_instances.py

@@ -7,10 +7,12 @@
     Literal,
     Optional,
     Set,
+    Tuple,
     Union,
 )
 from uuid import uuid4
 
+from jinja2 import UndefinedError
 from pydantic import (
     BaseModel,
     ValidationError,
@@ -23,6 +25,7 @@
 from galaxy.files import (
     FileSourceScore,
     FileSourcesUserContext,
+    ProvidesFileSourcesUserContext,
     UserDefinedFileSources,
 )
 from galaxy.files.plugins import (
@@ -34,13 +37,15 @@
     file_source_type_is_browsable,
     FilesSourceProperties,
     PluginKind,
+    SupportsBrowsing,
 )
 from galaxy.files.templates import (
     ConfiguredFileSourceTemplates,
     FileSourceConfiguration,
     FileSourceTemplate,
     FileSourceTemplateSummaries,
     FileSourceTemplateType,
+    template_to_configuration,
 )
 from galaxy.managers.context import ProvidesUserContext
 from galaxy.model import (
@@ -50,6 +55,8 @@
 from galaxy.model.scoped_session import galaxy_scoped_session
 from galaxy.security.vault import Vault
 from galaxy.util.config_templates import (
+    PluginAspectStatus,
+    PluginStatus,
     TemplateVariableValueType,
     validate_no_extra_secrets_defined,
     validate_no_extra_variables_defined,
@@ -59,6 +66,7 @@
     CreateInstancePayload,
     ModifyInstancePayload,
     prepare_environment,
+    prepare_environment_from_root,
     purge_template_instance,
     recover_secrets,
     save_template_instance,
@@ -114,18 +122,21 @@ class FileSourceInstancesManager:
     _sa_session: galaxy_scoped_session
     _app_vault: Vault
     _app_config: UserDefinedFileSourcesConfig
+    _resolver: "UserDefinedFileSourcesImpl"
 
     def __init__(
         self,
         catalog: ConfiguredFileSourceTemplates,
         sa_session: galaxy_scoped_session,
         vault: Vault,
         app_config: UserDefinedFileSourcesConfig,
+        resolver: "UserDefinedFileSourcesImpl",
     ):
         self._catalog = catalog
         self._sa_session = sa_session
         self._app_vault = vault
         self._app_config = app_config
+        self._resolver = resolver
 
     @property
     def summaries(self) -> FileSourceTemplateSummaries:
@@ -222,6 +233,86 @@ def create_instance(self, trans: ProvidesUserContext, payload: CreateInstancePay
         self._save(persisted_file_source)
         return self._to_model(trans, persisted_file_source)
 
+    # do we need all of create here - would a subset work?
+    def plugin_status(self, trans: ProvidesUserContext, payload: CreateInstancePayload) -> PluginStatus:
+        template_definition_status = self._catalog.status_template_definition(payload)
+        status_kwds = {"template_definition": template_definition_status}
+        if template_definition_status.is_not_ok:
+            return PluginStatus(**status_kwds)
+        configuration, template_settings_status = self._template_settings_status(trans, payload)
+        print(configuration)
+        status_kwds["template_settings"] = template_settings_status
+        if template_settings_status.is_not_ok:
+            return PluginStatus(**status_kwds)
+        assert configuration
+        file_source, connection_status = self._connection_status(trans, payload, configuration)
+        status_kwds["connection"] = connection_status
+        if connection_status.is_not_ok:
+            return PluginStatus(**status_kwds)
+        assert file_source
+        # Lets circle back to this - we need to add an entry point to the file source plugins
+        # to test if things are writable. We could ping remote APIs or do something like os.access('/path/to/folder', os.W_OK)
+        # locally.
+        return PluginStatus(**status_kwds)
+
+    def _template_settings_status(
+        self, trans: ProvidesUserContext, payload: CreateInstancePayload
+    ) -> Tuple[Optional[FileSourceConfiguration], PluginAspectStatus]:
+        # we've already tested this is found and valid
+        template = self._catalog.find_template(payload)
+
+        secrets = payload.secrets
+        variables = payload.variables
+        environment = prepare_environment_from_root(template.environment, self._app_vault, self._app_config)
+        user_details = trans.user.config_template_details()
+
+        configuration = None
+        try:
+            configuration = template_to_configuration(
+                template,
+                variables=variables,
+                secrets=secrets,
+                user_details=user_details,
+                environment=environment,
+            )
+            status = PluginAspectStatus(state="ok", message="Valid configuration resulted from supplied settings")
+        except UndefinedError as e:
+            message = f"Problem with template definition causing invalid settings resolution, please contact admin to correct template: {e}"
+            status = PluginAspectStatus(state="not_ok", message=message)
+        except ValidationError as e:
+            message = f"Problem with template definition causing invalid configuration, template expanded without error but resulting configuration is invalid. please contact admin to correct template: {e}"
+            status = PluginAspectStatus(state="not_ok", message=message)
+        except Exception as e:
+            pass
+        return configuration, status
+
+    def _connection_status(
+        self, trans: ProvidesUserContext, payload: CreateInstancePayload, configuration: FileSourceConfiguration
+    ) -> Tuple[Optional[BaseFilesSource], PluginAspectStatus]:
+        file_source = None
+        try:
+            file_source_properties = configuration_to_file_source_properties(
+                configuration,
+                label=payload.name,
+                doc=payload.description,
+                id=uuid4().hex,
+            )
+            file_source = self._resolver._file_source(file_source_properties)
+            if hasattr(file_source, "list"):
+                # if we can list the root, do that and assume there is
+                # a connection problem if we cannot
+                file_source = cast(SupportsBrowsing, file_source)
+                user_context = ProvidesFileSourcesUserContext(trans)
+                file_source.list("/", recursive=False, user_context=user_context)
+
+            connection_status = PluginAspectStatus(
+                state="ok", message="Valid connection resulted from supplied settings"
+            )
+        except Exception as e:
+            message = f"Failed to connect to a file source with supplied setting: {e}"
+            connection_status = PluginAspectStatus(state="not_ok", message=message)
+        return file_source, connection_status
+
     def _index_filter(self, id: Union[str, int]):
         index_by = self._app_config.user_config_templates_index_by
         index_filter: Any
@@ -344,21 +435,12 @@ def _file_source_properties(self, user_file_source: UserFileSource) -> FilesSour
         file_source_configuration: FileSourceConfiguration = user_file_source.file_source_configuration(
             secrets=secrets, environment=environment, templates=templates
         )
-        file_source_properties = cast(FilesSourceProperties, file_source_configuration.model_dump())
-        file_source_properties["label"] = user_file_source.name
-        file_source_properties["doc"] = user_file_source.description
-        file_source_properties["id"] = f"{user_file_source.uuid}"
-        file_source_properties["scheme"] = USER_FILE_SOURCES_SCHEME
-        # Moved this into templates - plugins should just define this and decide what
-        # that looks like. aws public buckets are clearly not writable, private buckets
-        # maybe should give users the option, etc..
-        # file_source_properties["writable"] = True
-
-        # We did templating with Jinja - disable Galaxy's Cheetah templating for
-        # these plugins. I can't imagine a use case for that and I would hate to templating
-        # languages having odd interactions.
-        file_source_properties["disable_templating"] = True
-        return file_source_properties
+        return configuration_to_file_source_properties(
+            file_source_configuration,
+            label=user_file_source.name,
+            doc=user_file_source.description,
+            id=f"{user_file_source.uuid}",
+        )
 
     def validate_uri_root(self, uri: str, user_context: FileSourcesUserContext) -> None:
         user_object_store = self._user_file_source(uri)
@@ -424,6 +506,31 @@ def user_file_sources_to_dicts(
         return as_dicts
 
 
+# Turn the validated Pydantic thing describe what is possible to configure to the
+# raw TypedDict consumed by the actual galaxy.files plugins.
+def configuration_to_file_source_properties(
+    file_source_configuration: FileSourceConfiguration,
+    label: str,
+    doc: Optional[str],
+    id: str,
+) -> FilesSourceProperties:
+    file_source_properties = cast(FilesSourceProperties, file_source_configuration.model_dump())
+    file_source_properties["label"] = label
+    file_source_properties["doc"] = doc
+    file_source_properties["id"] = id
+    file_source_properties["scheme"] = USER_FILE_SOURCES_SCHEME
+    # Moved this into templates - plugins should just define this and decide what
+    # that looks like. aws public buckets are clearly not writable, private buckets
+    # maybe should give users the option, etc..
+    # file_source_properties["writable"] = True
+
+    # We did templating with Jinja - disable Galaxy's Cheetah templating for
+    # these plugins. I can't imagine a use case for that and I would hate to templating
+    # languages having odd interactions.
+    file_source_properties["disable_templating"] = True
+    return file_source_properties
+
+
 __all__ = (
     "CreateInstancePayload",
     "FileSourceInstancesManager",

lib/galaxy/model/__init__.py

@@ -1206,6 +1206,16 @@ def expand_user_properties(user, in_string):
         environment = User.user_template_environment(user)
         return Template(in_string).safe_substitute(environment)
 
+    # above templating is for Cheetah in tools where we discouraged user details from being exposed.
+    # the following templating if user details in Jinja for object stores and file sources where user
+    # details are critical and documented.
+    def config_template_details(self) -> Dict[str, Any]:
+        return {
+            "username": self.username,
+            "email": self.email,
+            "id": self.id,
+        }
+
     def is_active(self):
         return self.active
 
@@ -11060,12 +11070,7 @@ def object_store_configuration(
     ) -> ObjectStoreConfiguration:
         if templates is None:
             templates = [self.template]
-        user = self.user
-        user_details = {
-            "username": user.username,
-            "email": user.email,
-            "id": user.id,
-        }
+        user_details = self.user.config_template_details()
         variables: CONFIGURATION_TEMPLATE_CONFIGURATION_VARIABLES_TYPE = self.template_variables or {}
         first_exception = None
         for template in templates:
@@ -11133,12 +11138,7 @@ def file_source_configuration(
     ) -> FileSourceConfiguration:
         if templates is None:
             templates = [self.template]
-        user = self.user
-        user_details = {
-            "username": user.username,
-            "email": user.email,
-            "id": user.id,
-        }
+        user_details = self.user.config_template_details()
         variables: CONFIGURATION_TEMPLATE_CONFIGURATION_VARIABLES_TYPE = self.template_variables or {}
         first_exception = None
         for template in templates: