✨(backend) add duplicate action to the document API endpoint

We took this opportunity to refactor the way access is controlled on
media attachments. We now add the media key to a list on the document
instance each time a media is uploaded to a document. This list is
passed along when a document is duplicated, allowing us to grant
access to readers on the new document, even if they don't have or
lost access to the original document.

We also propose an option to reproduce the same access rights on the
duplicate document as what was in place on the original document.
This can be requested by passing the "with_accesses=true" option in
the query string.

The tricky point is that we need to extract attachment keys from the
existing documents and set them on the new "attachments" field that is
now used to track access rights on media files.

Author: sampaccoud

CHANGELOG.md

@@ -128,6 +128,7 @@ and this project adheres to
 
 ## Added
 
+- ✨(backend) add duplicate action to the document API endpoint
 - ⚗️(backend) add util to extract text from base64 yjs document
 - ✨(backend) add soft delete and restore API endpoints to documents #516
 - ✨(backend) allow organizing documents in a tree structure #516

src/backend/core/admin.py

@@ -151,6 +151,8 @@ class DocumentAdmin(TreeAdmin):
                     "path",
                     "depth",
                     "numchild",
+                    "duplicated_from",
+                    "attachments",
                 )
             },
         ),
@@ -166,8 +168,10 @@ class DocumentAdmin(TreeAdmin):
         "updated_at",
     )
     readonly_fields = (
+        "attachments",
         "creator",
         "depth",
+        "duplicated_from",
         "id",
         "numchild",
         "path",

src/backend/core/api/serializers.py

@@ -381,6 +381,27 @@ class Meta:
         ]
 
 
+class DocumentDuplicationSerializer(serializers.Serializer):
+    """
+    Serializer for duplicating a document.
+    Allows specifying whether to keep access permissions.
+    """
+
+    with_accesses = serializers.BooleanField(default=False)
+
+    def create(self, validated_data):
+        """
+        This serializer is not intended to create objects.
+        """
+        raise NotImplementedError("This serializer does not support creation.")
+
+    def update(self, instance, validated_data):
+        """
+        This serializer is not intended to update objects.
+        """
+        raise NotImplementedError("This serializer does not support updating.")
+
+
 # Suppress the warning about not implementing `create` and `update` methods
 # since we don't use a model and only rely on the serializer for validation
 # pylint: disable=abstract-method

src/backend/core/api/viewsets.py

@@ -16,6 +16,8 @@
 from django.db.models.expressions import RawSQL
 from django.db.models.functions import Left, Length
 from django.http import Http404, StreamingHttpResponse
+from django.utils.text import capfirst
+from django.utils.translation import gettext_lazy as _
 
 import requests
 import rest_framework as drf
@@ -28,26 +30,13 @@
 from core import authentication, enums, models
 from core.services.ai_services import AIService
 from core.services.collaboration_services import CollaborationService
+from core.utils import extract_attachments
 
 from . import permissions, serializers, utils
 from .filters import DocumentFilter, ListDocumentFilter
 
 logger = logging.getLogger(__name__)
 
-<<<<<<< HEAD
-ATTACHMENTS_FOLDER = "attachments"
-UUID_REGEX = (
-    r"[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}"
-)
-FILE_EXT_REGEX = r"\.[a-zA-Z0-9]{1,10}"
-MEDIA_STORAGE_URL_PATTERN = re.compile(
-    f"{settings.MEDIA_URL:s}(?P<pk>{UUID_REGEX:s})/"
-    f"(?P<key>{ATTACHMENTS_FOLDER:s}/{UUID_REGEX:s}(?:-unsafe)?{FILE_EXT_REGEX:s})$"
-)
-COLLABORATION_WS_URL_PATTERN = re.compile(rf"(?:^|&)room=(?P<pk>{UUID_REGEX})(?:&|$)")
-=======
->>>>>>> 8076486a (✅(backend) add missing test on media-auth and collaboration-auth)
-
 # pylint: disable=too-many-ancestors
 
 
@@ -902,6 +891,78 @@ def tree(self, request, pk, *args, **kwargs):
         )
         return drf.response.Response(
             utils.nest_tree(serializer.data, self.queryset.model.steplen)
+        
+    @drf.decorators.action(
+        detail=True,
+        methods=["post"],
+        permission_classes=[permissions.IsAuthenticated, permissions.AccessPermission],
+        url_path="duplicate",
+    )
+    @transaction.atomic
+    def duplicate(self, request, *args, **kwargs):
+        """
+        Duplicate a document and store the links to attached files in the duplicated
+        document to allow cross-access.
+
+        Optionally duplicates accesses if `with_accesses` is set to true
+        in the payload.
+        """
+        serializer = serializers.DocumentDuplicationSerializer(data=request.GET)
+        serializer.is_valid(raise_exception=True)
+        with_accesses = serializer.validated_data["with_accesses"]
+
+        # Get document while checking permissions
+        document = self.get_object()
+        base64_yjs_content = document.content
+
+        # Duplicate the document instance
+        link_kwargs = (
+            {"link_reach": document.link_reach, "link_role": document.link_role}
+            if with_accesses
+            else {}
+        )
+        extracted_attachments = set(extract_attachments(document.content))
+        attachments = list(extracted_attachments & set(document.attachments))
+        duplicated_document = document.add_sibling(
+            "right",
+            title=capfirst(_("copy of {title}").format(title=document.title)),
+            content=base64_yjs_content,
+            attachments=attachments,
+            duplicated_from=document,
+            creator=request.user,
+            **link_kwargs,
+        )
+
+        # Always add the logged-in user as OWNER
+        accesses_to_create = [
+            models.DocumentAccess(
+                document=duplicated_document,
+                user=request.user,
+                role=models.RoleChoices.OWNER,
+            )
+        ]
+
+        # If accesses should be duplicated, add other users' accesses as per original document
+        if with_accesses:
+            original_accesses = models.DocumentAccess.objects.filter(
+                document=document
+            ).exclude(user=request.user)
+
+            accesses_to_create.extend(
+                models.DocumentAccess(
+                    document=duplicated_document,
+                    user_id=access.user_id,
+                    team=access.team,
+                    role=access.role,
+                )
+                for access in original_accesses
+            )
+
+        # Bulk create all the duplicated accesses
+        models.DocumentAccess.objects.bulk_create(accesses_to_create)
+
+        return drf_response.Response(
+            {"id": str(duplicated_document.id)}, status=status.HTTP_201_CREATED
         )
 
     @drf.decorators.action(detail=True, methods=["get"], url_path="versions")
@@ -1085,6 +1146,10 @@ def attachment_upload(self, request, *args, **kwargs):
             file, default_storage.bucket_name, key, ExtraArgs=extra_args
         )
 
+        # Make the attachment readable by document readers
+        document.attachments.append(key)
+        document.save()
+
         return drf.response.Response(
             {"file": f"{settings.MEDIA_URL:s}{key:s}"},
             status=drf.status.HTTP_201_CREATED,
@@ -1152,20 +1217,20 @@ def media_auth(self, request, *args, **kwargs):
         url_params = self._auth_get_url_params(
             enums.MEDIA_STORAGE_URL_PATTERN, parsed_url.path
         )
-        document = self._auth_get_document(url_params["pk"])
 
-        if not document.get_abilities(request.user).get(self.action, False):
-            logger.debug(
-                "User '%s' lacks permission for document '%s'",
-                request.user,
-                document.pk,
-            )
+        user = request.user
+        key = f"{url_params['pk']:s}/{url_params['attachment']:s}"
+
+        if (
+            not self.queryset.readable(user)
+            .filter(attachments__contains=[key])
+            .exists()
+        ):
+            logger.debug("User '%s' lacks permission for image", user)
             raise drf.exceptions.PermissionDenied()
 
         # Generate S3 authorization headers using the extracted URL parameters
-        request = utils.generate_s3_authorization_headers(
-            f"{url_params['pk']:s}/{url_params['key']:s}"
-        )
+        request = utils.generate_s3_authorization_headers(key)
 
         return drf.response.Response("authorized", headers=request.headers, status=200)
 

src/backend/core/enums.py

@@ -15,7 +15,10 @@
 FILE_EXT_REGEX = r"\.[a-zA-Z]{3,4}"
 MEDIA_STORAGE_URL_PATTERN = re.compile(
     f"{settings.MEDIA_URL:s}(?P<pk>{UUID_REGEX:s})/"
-    f"(?P<key>{ATTACHMENTS_FOLDER:s}/{UUID_REGEX:s}{FILE_EXT_REGEX:s})$"
+    f"(?P<key>{ATTACHMENTS_FOLDER:s}/{UUID_REGEX:s}(?:-unsafe)?{FILE_EXT_REGEX:s})$"
+)
+MEDIA_STORAGE_URL_EXTRACT = re.compile(
+    f"{settings.MEDIA_URL:s}({UUID_REGEX}/{ATTACHMENTS_FOLDER}/{UUID_REGEX}{FILE_EXT_REGEX})"
 )
 COLLABORATION_WS_URL_PATTERN = re.compile(rf"(?:^|&)room=(?P<pk>{UUID_REGEX})(?:&|$)")
 

src/backend/core/migrations/0018_remove_is_public_add_field_attachments_and_duplicated_from.py

@@ -0,0 +1,55 @@
+# Generated by Django 5.1.4 on 2025-01-18 11:53
+import re
+
+import core.models
+import django.contrib.postgres.fields
+import django.db.models.deletion
+from django.db import migrations, models
+
+from core.utils import extract_attachments
+
+
+def populate_attachments_on_all_documents(apps, schema_editor):
+    """Populate "attachments" field on all existing documents in the database."""
+    Document = apps.get_model("core", "Document")
+
+    for document in  Document.objects.all():
+        document.attachments = extract_attachments(document.content)
+        document.save(update_fields=['attachments'])
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0017_add_fields_for_soft_delete'),
+    ]
+
+    operations = [
+        # v2.0.0 was released so we can now remove BC field "is_public"
+        migrations.RemoveField(
+            model_name='document',
+            name='is_public',
+        ),
+        migrations.AlterModelManagers(
+            name='user',
+            managers=[
+                ('objects', core.models.UserManager()),
+            ],
+        ),
+        migrations.AddField(
+            model_name='document',
+            name='attachments',
+            field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, editable=False, null=True, size=None),
+        ),
+        migrations.AddField(
+            model_name='document',
+            name='duplicated_from',
+            field=models.ForeignKey(blank=True, editable=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='duplicates', to='core.document'),
+        ),
+        migrations.AlterField(
+            model_name='user',
+            name='language',
+            field=models.CharField(choices="(('en-us', 'English'), ('fr-fr', 'French'), ('de-de', 'German'))", default='en-us', help_text='The language in which the user wants to see the interface.', max_length=10, verbose_name='language'),
+        ),
+        migrations.RunPython(populate_attachments_on_all_documents),
+    ]

src/backend/core/models.py

@@ -13,6 +13,7 @@
 from django.conf import settings
 from django.contrib.auth import models as auth_models
 from django.contrib.auth.base_user import AbstractBaseUser
+from django.contrib.postgres.fields import ArrayField
 from django.contrib.sites.models import Site
 from django.core import mail, validators
 from django.core.cache import cache
@@ -486,6 +487,21 @@ class Document(MP_Node, BaseModel):
     )
     deleted_at = models.DateTimeField(null=True, blank=True)
     ancestors_deleted_at = models.DateTimeField(null=True, blank=True)
+    duplicated_from = models.ForeignKey(
+        "self",
+        on_delete=models.SET_NULL,
+        related_name="duplicates",
+        editable=False,
+        blank=True,
+        null=True,
+    )
+    attachments = ArrayField(
+        models.CharField(max_length=255),
+        default=list,
+        editable=False,
+        blank=True,
+        null=True,
+    )
 
     _content = None
 
@@ -800,6 +816,7 @@ def get_abilities(self, user, ancestors_links=None):
             "cors_proxy": can_get,
             "descendants": can_get,
             "destroy": is_owner,
+            "duplicate": can_get,
             "favorite": can_get and user.is_authenticated,
             "link_configuration": is_owner_or_admin,
             "invite_owner": is_owner,