Skip to content

Commit

Permalink
Merge pull request #18598 from sanjaysrikakulam/webdav_as_user_file_s…
Browse files Browse the repository at this point in the history
…ource

Bring your own file sources: Add the WebDAV template and configuration
  • Loading branch information
mvdbeek authored Oct 14, 2024
2 parents 3867ae9 + fd69a85 commit fcb3bdf
Show file tree
Hide file tree
Showing 12 changed files with 97 additions and 5 deletions.
4 changes: 2 additions & 2 deletions client/src/api/schema/schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9035,7 +9035,7 @@ export interface components {
* Type
* @enum {string}
*/
type: "ftp" | "posix" | "s3fs" | "azure" | "onedata";
type: "ftp" | "posix" | "s3fs" | "azure" | "onedata" | "webdav";
/** Variables */
variables?:
| (
Expand Down Expand Up @@ -17366,7 +17366,7 @@ export interface components {
* Type
* @enum {string}
*/
type: "ftp" | "posix" | "s3fs" | "azure" | "onedata";
type: "ftp" | "posix" | "s3fs" | "azure" | "onedata" | "webdav";
/** Uri Root */
uri_root: string;
/**
Expand Down
1 change: 1 addition & 0 deletions client/src/components/FileSources/FileSourceTypeSpan.vue
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ const MESSAGES = {
azure: "This is an remote file source plugin based on the Azure service.",
onedata: "This is an remote file source plugin based on the Onedata service.",
ftp: "This is an remote file source plugin based on the FTP protocol.",
webdav: "This is an remote file source plugin based on the WebDAV protocol.",
};
interface Props {
Expand Down
21 changes: 21 additions & 0 deletions doc/source/admin/data.md
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,19 @@ configuration).

![](file_source_azure_configuration.png)

#### `webdav`

The syntax for the `configuration` section of `webdav` templates looks like this.

![](file_source_webdav_configuration_template.png)

At runtime, after the `configuration` template is expanded, the resulting dictionary
passed to Galaxy's file source plugin infrastructure looks like this and should match a subset
of what you'd be able to add directly to `file_sources_conf.yml` (Galaxy's global file source
configuration).

![](file_source_webdav_configuration.png)

### YAML Syntax

![galaxy.files.templates.models](file_source_templates.png)
Expand Down Expand Up @@ -398,6 +411,14 @@ and you are comfortable with it storing your user's secrets.
:language: yaml
```

#### Allow Users to Define WebDAV Servers as File Sources

```{literalinclude} ../../../lib/galaxy/files/templates/examples/production_webdav.yml
:language: yaml
```

![Screenshot](user_file_source_form_full_webdav.png)

## Playing Nicer with Ansible

Many large instances of Galaxy are configured with Ansible and much of the existing administrator
Expand Down
Binary file modified doc/source/admin/file_source_templates.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
4 changes: 4 additions & 0 deletions doc/source/admin/gen_diagrams.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
PosixFileSourceTemplateConfiguration,
S3FSFileSourceConfiguration,
S3FSFileSourceTemplateConfiguration,
WebdavFileSourceConfiguration,
WebdavFileSourceTemplateConfiguration,
)
from galaxy.objectstore.templates.models import (
AwsS3ObjectStoreConfiguration,
Expand Down Expand Up @@ -53,6 +55,8 @@
S3FSFileSourceConfiguration: "file_source_s3fs_configuration",
FtpFileSourceTemplateConfiguration: "file_source_ftp_configuration_template",
FtpFileSourceConfiguration: "file_source_ftp_configuration",
WebdavFileSourceTemplateConfiguration: "file_source_webdav_configuration_template",
WebdavFileSourceConfiguration: "file_source_webdav_configuration",
}

for clazz, diagram_name in class_to_diagram.items():
Expand Down
1 change: 1 addition & 0 deletions doc/source/admin/search_for_new_screenshots.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
"user_file_source_form_full_aws_public.png",
"user_file_source_form_full_azure.png",
"user_file_source_form_full_ftp.png",
"user_file_source_form_full_webdav.png",
]


Expand Down
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
4 changes: 2 additions & 2 deletions lib/galaxy/config/schemas/config_schema.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3746,7 +3746,7 @@ mapping:
Workflows launched with URI/URL inputs that are not marked as 'deferred'
are "materialized" (or undeferred) by the workflow scheduler. This might be
a lengthy process. Setting this to 'True' will place the invocation back in
the queue after materialization before scheduling the workflow so it is less
the queue after materialization before scheduling the workflow so it is less
likely to starve other workflow scheduling. Ideally, Galaxy would allow more
fine grain control of handlers but until then, this provides a way to tip the
balance between "doing more work" and "being more fair". The default here is
Expand Down Expand Up @@ -4110,7 +4110,7 @@ mapping:
required: false
desc: |
Directory to store temporary files for file sources. This defaults to new_file_path if not set.
file_source_webdav_use_temp_files:
type: bool
default: true
Expand Down
42 changes: 42 additions & 0 deletions lib/galaxy/files/templates/examples/production_webdav.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
- id: webdav
version: 0
name: WebDAV
description: |
The WebDAV protocol is a simple way to access files over the internet. This template
configuration allows you to connect to a WebDAV server.
variables:
url:
label: Server Domain (e.g. https://myowncloud.org)
type: string
help: |
The domain of the WebDAV server you are connecting to. This should be the full URL
including the protocol (http or https) and the domain name.
root:
label: WebDAV server Path (should end with /remote.php/webdav, e.g. /a/sub/path/remote.php/webdav)
type: string
help: |
The full server path to the WebDAV service. Ensure the path includes /remote.php/webdav.
login:
label: Username
type: string
help: |
The username to use to connect to the WebDAV server. This should be the username you use
to log in to the WebDAV server.
writable:
label: Writable?
type: boolean
default: false
help: Allow Galaxy to write data to this WebDAV server.
secrets:
password:
label: Password
help: |
The password to use to connect to the WebDAV server. This should be the password you use
to log in to the WebDAV server.
configuration:
type: webdav
url: '{{ variables.url }}'
root: '{{ variables.root }}'
login: '{{ variables.login }}'
writable: '{{ variables.writable }}'
password: '{{ secrets.password }}'
25 changes: 24 additions & 1 deletion lib/galaxy/files/templates/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
UserDetailsDict,
)

FileSourceTemplateType = Literal["ftp", "posix", "s3fs", "azure", "onedata"]
FileSourceTemplateType = Literal["ftp", "posix", "s3fs", "azure", "onedata", "webdav"]


class PosixFileSourceTemplateConfiguration(StrictModel):
Expand Down Expand Up @@ -122,19 +122,41 @@ class OnedataFileSourceConfiguration(StrictModel):
writable: bool = False


class WebdavFileSourceTemplateConfiguration(StrictModel):
type: Literal["webdav"]
url: Union[str, TemplateExpansion]
root: Union[str, TemplateExpansion]
login: Union[str, TemplateExpansion]
password: Union[str, TemplateExpansion]
writable: Union[bool, TemplateExpansion] = False
template_start: Optional[str] = None
template_end: Optional[str] = None


class WebdavFileSourceConfiguration(StrictModel):
type: Literal["webdav"]
url: str
root: str
login: str
password: str
writable: bool = False


FileSourceTemplateConfiguration = Union[
PosixFileSourceTemplateConfiguration,
S3FSFileSourceTemplateConfiguration,
FtpFileSourceTemplateConfiguration,
AzureFileSourceTemplateConfiguration,
OnedataFileSourceTemplateConfiguration,
WebdavFileSourceTemplateConfiguration,
]
FileSourceConfiguration = Union[
PosixFileSourceConfiguration,
S3FSFileSourceConfiguration,
FtpFileSourceConfiguration,
AzureFileSourceConfiguration,
OnedataFileSourceConfiguration,
WebdavFileSourceConfiguration,
]


Expand Down Expand Up @@ -196,6 +218,7 @@ def template_to_configuration(
"s3fs": S3FSFileSourceConfiguration,
"azure": AzureFileSourceConfiguration,
"onedata": OnedataFileSourceConfiguration,
"webdav": WebdavFileSourceConfiguration,
}


Expand Down

0 comments on commit fcb3bdf

Please sign in to comment.