Add support for virtual environments (#392)

* Initial virtual env support

* Make import ignoring part of model

* Add migrations

* Improve stdout/stderr behavior

* Fix stderr append

* Move venv setup to separate function

* Handle pip setup

* unit tests

* Add more settings and help text

* maybe windows

* windows stuff

* win

* dows

* more windows....

* Add test case for running script in venv

* test setup

* Remove debug print

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.7
+FROM python:3.11
 
 ARG HOST_USER=1000
 ENV HOST_USER=${HOST_USER}
@@ -14,7 +14,7 @@ ENV BUILD_DIR=${BUILD_DIR}
 WORKDIR ${BUILD_DIR}
 RUN chown wooey:wooey ${BUILD_DIR}
 
-RUN pip install docker psycopg2
+RUN pip install docker psycopg2 redis
 
 COPY --chown=wooey:wooey setup.py MANIFEST.in Makefile README.md ${BUILD_DIR}/
 COPY --chown=wooey:wooey scripts ${BUILD_DIR}/scripts

docker/docker-compose.yml

@@ -12,6 +12,7 @@ services:
       - 8081:8080
     depends_on:
       - rabbit
+      - redis
       - db
       - celery
     command: ./run-server
@@ -22,6 +23,7 @@ services:
       service: common
     depends_on:
       - rabbit
+      - redis
       - db
     command: watchmedo auto-restart --directory=$BUILD_DIR/wooey --recursive --ignore-patterns="*.pyc" -- celery -A $WOOEY_PROJECT worker -c 4 -B -l debug -s schedule
 
@@ -38,3 +40,6 @@ services:
       POSTGRES_USER: wooey
       POSTGRES_PASSWORD: wooey
       POSTGRES_DB: wooey
+
+  redis:
+    image: redis:7.2.3

docker/user_settings.py

@@ -6,15 +6,16 @@
 
 WOOEY_ENABLE_API_KEYS = True
 
-## Celery related options
+WOOEY_REALTIME_CACHE = "default"
+CACHES = {
+    "default": {
+        "BACKEND": "django.core.cache.backends.redis.RedisCache",
+        "LOCATION": "redis://redis:6379",
+    }
+}
 
+## Celery related options
 WOOEY_CELERY = True
-broker_url = "amqp://guest@rabbit"
-task_track_started = True
-worker_send_task_events = True
-imports = ("wooey.tasks",)
-task_serializer = "json"
-task_acks_late = True
 
 # the directory for uploads (physical directory)
 MEDIA_ROOT = os.path.join(BASE_DIR, "user_uploads")  # noqa: F405

setup.py

@@ -20,7 +20,7 @@
     python_requires=">3.5.0",
     install_requires=[
         "celery>=4,<6",
-        "clinto>=0.3.0",
+        "clinto>=0.5.1",
         "Django>=3,<5",
         "django-autoslug",
         "django-storages",

wooey/admin.py

@@ -1,8 +1,11 @@
 from __future__ import absolute_import
 import os
+import sys
 
 from django.contrib.admin import ModelAdmin, site, TabularInline
 
+from wooey import settings as wooey_settings
+
 from .models import (
     Script,
     ScriptVersion,
@@ -13,6 +16,7 @@
     UserFile,
     WooeyJob,
     WooeyWidget,
+    VirtualEnvironment,
 )
 
 
@@ -117,6 +121,14 @@ class FileAdmin(ModelAdmin):
     pass
 
 
+class VirtualEnvironmentAdmin(ModelAdmin):
+    def get_changeform_initial_data(self, request):
+        return {
+            "python_binary": sys.executable,
+            "venv_directory": wooey_settings.WOOEY_VIRTUAL_ENVIRONMENT_DIRECTORY,
+        }
+
+
 site.register(WooeyWidget)
 site.register(WooeyJob, JobAdmin)
 site.register(UserFile, FileAdmin)
@@ -126,3 +138,4 @@ class FileAdmin(ModelAdmin):
 site.register(ScriptParameterGroup, ParameterGroupAdmin)
 site.register(ScriptParser, ScriptParserAdmin)
 site.register(ScriptVersion, ScriptVersionAdmin)
+site.register(VirtualEnvironment, VirtualEnvironmentAdmin)

wooey/api/forms.py

@@ -1,4 +1,5 @@
 from django import forms
+from django.utils.translation import gettext_lazy as _
 
 
 class SubmitForm(forms.Form):
@@ -12,8 +13,19 @@ class SubmitForm(forms.Form):
 class AddScriptForm(forms.Form):
     group = forms.CharField(required=False)
     default = forms.NullBooleanField(required=False)
+    ignore_bad_imports = forms.BooleanField(
+        required=False,
+        help_text=_(
+            "Ignore bad imports when adding scripts. This is useful if a script is under a virtual environment."
+        ),
+    )
 
     def clean_default(self):
         if self.cleaned_data["default"] is None:
             return True
         return self.cleaned_data["default"]
+
+    def clean_ignore_bad_imports(self):
+        if self.cleaned_data["ignore_bad_imports"] is None:
+            return False
+        return self.cleaned_data["ignore_bad_imports"]

wooey/api/scripts.py

@@ -217,6 +217,7 @@ def add_or_update_script(request):
             "group": group,
             "script_name": script_name,
             "set_default_version": data["default"],
+            "ignore_bad_imports": data["ignore_bad_imports"],
         }
         results = utils.add_wooey_script(**add_kwargs)
         output = {

wooey/backend/utils.py

@@ -90,9 +90,13 @@ def purge_output(job=None):
             user_file.delete()
 
 
-def get_job_commands(job=None):
+def get_job_commands(job=None, executable=None):
     script_version = job.script_version
-    com = [sys.executable] if sys.executable else []
+    com = (
+        [executable]
+        if executable is not None
+        else ([sys.executable] if sys.executable else [])
+    )
     com.extend([script_version.get_script_path()])
 
     parameters = job.get_parameters()
@@ -330,7 +334,9 @@ def add_wooey_script(
     group=None,
     script_name=None,
     set_default_version=True,
+    ignore_bad_imports=False,
 ):
+
     # There is a class called 'Script' which contains the general information about a script. However, that is not where the file details
     # of the script lie. That is the ScriptVersion model. This allows the end user to tag a script as a favorite/etc. and set
     # information such as script descriptions/names that do not constantly need to be updated with every version change. Thus,
@@ -444,7 +450,11 @@ def add_wooey_script(
     basename, extension = os.path.splitext(script)
     filename = os.path.split(basename)[1]
 
-    parser = Parser(script_name=filename, script_path=local_storage.path(local_file))
+    parser = Parser(
+        script_name=filename,
+        script_path=local_storage.path(local_file),
+        ignore_bad_imports=ignore_bad_imports,
+    )
     if not parser.valid:
         return {
             "valid": False,
@@ -470,6 +480,7 @@ def add_wooey_script(
         script_kwargs = {
             "script_group": script_group,
             "script_name": script_name or script_schema["name"],
+            "ignore_bad_imports": ignore_bad_imports,
         }
         version_kwargs = {
             "script_version": version_string,

wooey/management/commands/addscript.py

@@ -28,6 +28,11 @@ def add_arguments(self, parser):
             default=None,
             help="The name of the script. Default: None (uses the filename)",
         )
+        parser.add_argument(
+            "--ignore-bad-imports",
+            action="store_true",
+            help="Ignore failed imports. Useful when importing into a VirtualEnv",
+        )
         parser.add_argument(
             "--update", dest="update", action="store_true", help=argparse.SUPPRESS
         )
@@ -48,6 +53,7 @@ def handle(self, *args, **options):
         if not os.path.exists(script):
             raise CommandError("{0} does not exist.".format(script))
         group = options.get("group", wooey_settings.WOOEY_DEFAULT_SCRIPT_GROUP)
+        ignore_bad_imports = options.get("ignore_bad_imports")
         scripts = (
             [os.path.join(script, i) for i in os.listdir(script)]
             if os.path.isdir(script)
@@ -84,6 +90,7 @@ def handle(self, *args, **options):
                     "script_path": script,
                     "group": group,
                     "script_name": base_name,
+                    "ignore_bad_imports": ignore_bad_imports,
                 }
                 res = add_wooey_script(**add_kwargs)
                 if res["valid"]:

wooey/migrations/0051_add_virtual_env.py

@@ -0,0 +1,45 @@
+# Generated by Django 3.2.23 on 2023-11-22 02:05
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("wooey", "0050_add_api_keys"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="VirtualEnvironment",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("name", models.CharField(max_length=25)),
+                ("python_binary", models.CharField(max_length=1024)),
+                ("requirements", models.TextField(null=True, blank=True)),
+                ("venv_directory", models.CharField(max_length=1024)),
+            ],
+            options={
+                "verbose_name": "virtual environment",
+                "verbose_name_plural": "virtual environments",
+            },
+        ),
+        migrations.AddField(
+            model_name="script",
+            name="virtual_environment",
+            field=models.ForeignKey(
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                to="wooey.virtualenvironment",
+            ),
+        ),
+    ]

wooey/migrations/0052_add_ignore_bad_imports_to_script_model.py

@@ -0,0 +1,21 @@
+# Generated by Django 3.2.23 on 2023-11-22 22:37
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("wooey", "0051_add_virtual_env"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="script",
+            name="ignore_bad_imports",
+            field=models.BooleanField(
+                default=False,
+                help_text="Ignore bad imports when adding scripts. This is useful if a script is under a virtual environment.",
+            ),
+        ),
+    ]

wooey/models/core.py

@@ -59,6 +59,12 @@ class Script(models.Model):
     script_order = models.PositiveSmallIntegerField(default=1)
     is_active = models.BooleanField(default=True)
     user_groups = models.ManyToManyField(Group, blank=True)
+    ignore_bad_imports = models.BooleanField(
+        default=False,
+        help_text=_(
+            "Ignore bad imports when adding scripts. This is useful if a script is under a virtual environment."
+        ),
+    )
 
     execute_full_path = models.BooleanField(
         default=True
@@ -70,6 +76,9 @@ class Script(models.Model):
         help_text="By default save to the script name,"
         " this will change the output folder.",
     )
+    virtual_environment = models.ForeignKey(
+        "VirtualEnvironment", on_delete=models.SET_NULL, null=True, blank=True
+    )
 
     created_date = models.DateTimeField(auto_now_add=True)
     modified_date = models.DateTimeField(auto_now=True)
@@ -259,10 +268,13 @@ def submit_to_celery(self, **kwargs):
                     param.recreate()
                     param.save()
         self.status = self.SUBMITTED
+        rerun = kwargs.pop("rerun", False)
+        if rerun:
+            self.command = ""
         self.save()
-        task_kwargs = {"wooey_job": self.pk, "rerun": kwargs.pop("rerun", False)}
+        task_kwargs = {"wooey_job": self.pk, "rerun": rerun}
 
-        if task_kwargs.get("rerun"):
+        if rerun:
             utils.purge_output(job=self)
         if wooey_settings.WOOEY_CELERY:
             transaction.on_commit(lambda: tasks.submit_script.delay(**task_kwargs))
@@ -717,3 +729,51 @@ class Meta:
 
     def __str__(self):
         return self.filepath.name
+
+
+class VirtualEnvironment(models.Model):
+    name = models.CharField(
+        max_length=25, help_text=_("The name of the virtual environment.")
+    )
+    python_binary = models.CharField(
+        max_length=1024,
+        help_text=_(
+            'The binary to use for creating the virtual environment. Should be in your path (e.g. "python3" or "/usr/bin/python3")'
+        ),
+    )
+    requirements = models.TextField(
+        null=True,
+        blank=True,
+        help_text=_(
+            'A list of requirements for the virtualenv. This gets passed directly to "pip install -r".'
+        ),
+    )
+    venv_directory = models.CharField(
+        max_length=1024,
+        help_text=_("The directory to place the virtual environment under."),
+    )
+
+    class Meta:
+        app_label = "wooey"
+        verbose_name = _("virtual environment")
+        verbose_name_plural = _("virtual environments")
+
+    def get_venv_python_binary(self):
+        return os.path.join(
+            self.get_install_path(),
+            "Scripts" if wooey_settings.IS_WINDOWS else "bin",
+            "python.exe" if wooey_settings.IS_WINDOWS else "python",
+        )
+
+    def get_install_path(self, ensure_exists=False):
+        path = os.path.join(
+            self.venv_directory,
+            "".join(x for x in self.python_binary if x.isalnum()),
+            self.name,
+        )
+        if ensure_exists:
+            os.makedirs(path, exist_ok=True)
+        return path
+
+    def __str__(self):
+        return self.name