Export and Import of Projects, update and refactor for cli and drf

rdmo/accounts/utils.py

@@ -1,10 +1,14 @@
+from __future__ import annotations
+
 import logging
 
 from django.conf import settings
 from django.contrib.auth import authenticate, get_user_model
 from django.contrib.auth.models import Group, Permission
 from django.contrib.contenttypes.models import ContentType
 from django.core.exceptions import ObjectDoesNotExist
+from django.utils.crypto import get_random_string
+from django.utils.text import slugify
 
 from .models import Role
 from .settings import GROUPS
@@ -93,3 +97,59 @@ def get_user_from_db_or_none(username: str, email: str):
     except ObjectDoesNotExist:
         log.error('Retrieval of user "%s" with email "%s" failed, user does not exist', username, email)
         return None
+
+
+def find_user(user_id=None, username="", email=""):
+    username = (username or "").strip()
+    email = (email or "").strip().lower()
+
+    if user_id:
+        user = get_user_model().objects.filter(pk=user_id).first()
+        if user:
+            return user
+
+    if username:
+        user = get_user_model().objects.filter(username=username).first()
+        if user:
+            return user
+
+    if email:
+        return get_user_model().objects.filter(email__iexact=email).first()
+
+    return None
+
+
+def make_unique_username(seed: str) -> str:
+    base = slugify(seed) or "user"
+    user_model = get_user_model()
+    for suffix in range(0, 8):
+        candidate = base if suffix == 0 else f"{base}_{suffix}"
+        if not user_model.objects.filter(username=candidate).exists():
+            return candidate
+    # fallback
+    return f"{base}_{get_random_string(8)}"
+
+
+def create_user_from_fields(username, email, first_name, last_name):
+    username = (username or "").strip()
+    email = (email or "").strip().lower()
+    first_name = (first_name or "").strip()
+    last_name = (last_name or "").strip()
+
+    base = username or (email.split("@")[0] if email else "") or "imported"
+    unique = make_unique_username(base)
+
+    user = get_user_model().objects.create_user(
+        username=unique,
+        email=email,
+        first_name=first_name,
+        last_name=last_name,
+        is_active=True,
+    )
+    user.set_unusable_password()
+    user.save(update_fields=["password"])
+
+    if unique != base:
+        log.info("Username '%s' taken, created unique name '%s'.", base, unique)
+
+    return user

rdmo/core/imports.py

@@ -1,13 +1,15 @@
 import logging
 import tempfile
-import time
 from collections import defaultdict
 from enum import Enum
-from os.path import join as pj
-from random import randint
+from pathlib import Path
 from typing import Optional, Union
 
-from django.core.exceptions import ObjectDoesNotExist, ValidationError
+from django.conf import settings
+from django.core.exceptions import ObjectDoesNotExist, SuspiciousOperation, ValidationError
+from django.core.files.base import ContentFile
+from django.core.files.storage import FileSystemStorage
+from django.core.files.uploadedfile import UploadedFile
 from django.db import models
 from django.utils.translation import gettext_lazy as _
 
@@ -20,6 +22,9 @@
 
 logger = logging.getLogger(__name__)
 
+_TEMP_DIR = Path(tempfile.gettempdir()) / "rdmo_uploads"
+_TEMP_DIR.mkdir(exist_ok=True)
+_temp_storage = FileSystemStorage(location=_TEMP_DIR)
 
 class ImportElementFields(str, Enum):
     DIFF = "updated_and_changed"
@@ -32,26 +37,26 @@ class ImportElementFields(str, Enum):
     CHANGED_FIELDS = "changedFields"  # for ignored_keys when ordering at save
 
 
-def handle_uploaded_file(filedata):
-    tempfilename = generate_tempfile_name()
-    with open(tempfilename, 'wb+') as destination:
-        for chunk in filedata.chunks():
-            destination.write(chunk)
-    return tempfilename
-
-
-def handle_fetched_file(filedata):
-    tempfilename = generate_tempfile_name()
-    with open(tempfilename, 'wb+') as destination:
-        destination.write(filedata)
-    return tempfilename
-
+def store_temp_file(filedata, *, suffix=".xml") -> str:
+    max_size = getattr(settings, "MAX_UPLOAD_SIZE", None)
+
+    if isinstance(filedata, bytes):
+        if max_size is not None and len(filedata) > max_size:
+            raise SuspiciousOperation("File too large.")
+        content = ContentFile(filedata)
+        filename = _temp_storage.get_available_name(f"upload{suffix}")
+    elif isinstance(filedata, UploadedFile):
+        if max_size is not None and filedata.size > max_size:
+            raise SuspiciousOperation("Uploaded file too large.")
+        content = filedata
+        filename = _temp_storage.get_available_name(
+            f"upload{Path(filedata.name).suffix or suffix}"
+        )
+    else:
+        raise TypeError(f"Unsupported filedata type: {type(filedata)}")
 
-def generate_tempfile_name():
-    t = round(time.time() * 1000)
-    r = randint(10000, 99999)
-    fn = pj(tempfile.gettempdir(), 'upload_' + str(t) + '_' + str(r) + '.xml')
-    return fn
+    saved_path = _temp_storage.save(filename, content)
+    return _temp_storage.path(saved_path)
 
 
 def get_or_return_instance(model: models.Model, uri: Optional[str] = None) -> tuple[models.Model, bool]:

rdmo/core/serializers.py

@@ -259,3 +259,16 @@ class Meta:
             'id',
             'name'
         )
+
+
+class FileUploadSerializer(serializers.Serializer):
+    file = serializers.FileField(
+        help_text="The file to upload."
+    )
+    format = serializers.CharField(
+        default="xml",
+        required=False,
+        allow_blank=False,
+        allow_null=False,
+        help_text="Format that can be mapped to an import plugin key."
+    )

rdmo/core/settings.py

@@ -206,6 +206,7 @@
     'SWAGGER_UI_DIST': 'SIDECAR',
     'SWAGGER_UI_FAVICON_HREF': 'SIDECAR',
     'REDOC_DIST': 'SIDECAR',
+    'COMPONENT_SPLIT_REQUEST': True,  # this makes file upload for FileField work in Swagger UI
 }
 
 SETTINGS_EXPORT = [

rdmo/core/tests/test_openapi.py

@@ -21,7 +21,7 @@ def test_openapi_schema(db, client, login, settings, username):
         assert response.status_code == 200
         schema = yaml.safe_load(response.content)
         assert schema['openapi'] == '3.0.3'
-        assert len(schema['paths']) == 124
+        assert len(schema['paths']) == 129
     else:
         assert response.status_code == 302
 

rdmo/management/viewsets.py

@@ -9,7 +9,7 @@
 from rest_framework.response import Response
 from rest_framework.serializers import ValidationError
 
-from rdmo.core.imports import handle_uploaded_file
+from rdmo.core.imports import store_temp_file
 from rdmo.core.permissions import CanToggleElementCurrentSite
 from rdmo.core.utils import get_model_field_meta, is_truthy
 from rdmo.core.xml import parse_xml_to_elements
@@ -40,7 +40,7 @@ def create(self, request, *args, **kwargs):
         except KeyError as e:
             raise ValidationError({'file': [_('This field may not be blank.')]}) from e
         else:
-            import_tmpfile_name = handle_uploaded_file(uploaded_file)
+            import_tmpfile_name = store_temp_file(uploaded_file)
         try:
             # step 1.1: initialize parse_xml_to_elements
             # step 2-6: parse xml, validate and convert to

rdmo/projects/exports.py

@@ -21,6 +21,7 @@ def __init__(self, *args, **kwargs):
 
         self.project = None
         self.snapshot = None
+        self.include_memberships = False
 
     def render(self):
         raise NotImplementedError
@@ -158,7 +159,9 @@ class RDMOXMLExport(Export):
     def render(self):
         if self.project:
             content_disposition = f'attachment; filename="{self.project.title}.xml"'
-            serializer = ProjectExportSerializer(self.project)
+            serializer = ProjectExportSerializer(self.project, context={
+                'include_memberships': self.include_memberships
+            })
 
         else:
             content_disposition = f'attachment; filename="{self.snapshot.title}.xml"'