[Bug]: Large file upload with an S3 backend always fails with 423 LOCKED

[JulienFS]

⚠️ This issue respects the following points: ⚠️

• This is a bug, not a question or a configuration/webserver/proxy issue.
• This issue is not already reported on Github OR Nextcloud Community Forum (I've searched it).
• Nextcloud Server is up to date. See Maintenance and Release Schedule for supported versions.
• I agree to follow Nextcloud's Code of Conduct.

Bug description

I'm trying to upload a 60GB file to a managed nextcloud instance running with an S3 backend. I'm using rclone as a client. The upload constantly fails for many reasons (judging by the stack traces in the logs) but the client-facing error is always "REDACTED/REDACTED/REDACTED.REDACTED" is locked, existing lock on file: exclusive: OCA\DAV\Connector\Sabre\Exception\FileLocked: 423 Locked.

Steps to reproduce

1. Upload a very large file on an instance with an S3 backend

Expected behavior

A successful upload with a 2XX status code or an actual 4XX/5XX error explaining what failed.

Nextcloud Server version

28

Operating system

None

PHP engine version

PHP 8.2

Web server

None

Database engine version

MySQL

Is this bug present after an update or on a fresh install?

None

Are you using the Nextcloud Server Encryption module?

None

What user-backends are you using?

• Default user-backend (database)
• LDAP/ Active Directory
• SSO - SAML
• Other

Configuration report

No response

List of activated Apps

No response

Nextcloud Signing status

No response

Nextcloud Logs

{
    "reqId": "Hprz482Q7i9wfzLu4WET",
    "level": 3,
    "time": "2024-10-17T08:57:45+00:00",
    "remoteAddr": "10.216.170.70",
    "user": "REDACTED",
    "app": "objectstore",
    "method": "MOVE",
    "url": "/remote.php/dav/uploads/REDACTED/rclone-chunked-upload-87ba1266bc39584e10931fa1a0532198/.file",
    "message": "Could not create object urn:oid:255190 for files/REDACTED/REDACTED/REDACTED/REDACTED.REDACTED",
    "userAgent": "rclone/v1.68.1",
    "version": "28.0.7.5",
    "exception": {
        "Exception": "RuntimeException",
        "Message": "Unable to read from stream",
        "Code": 0,
        "Trace": [{
            "file": "/var/www/html/3rdparty/guzzlehttp/psr7/src/AppendStream.php",
            "line": 203,
            "function": "read",
            "class": "GuzzleHttp\\Psr7\\Stream",
            "type": "->",
            "args": [8192]
        }, {
            "file": "/var/www/html/3rdparty/guzzlehttp/psr7/src/LimitStream.php",
            "line": 152,
            "function": "read",
            "class": "GuzzleHttp\\Psr7\\AppendStream",
            "type": "->",
            "args": [8192]
        }, {
            "file": "/var/www/html/3rdparty/aws/aws-sdk-php/src/HashingStream.php",
            "line": 38,
            "function": "read",
            "class": "GuzzleHttp\\Psr7\\LimitStream",
            "type": "->",
            "args": [8192]
        }, {
            "file": "/var/www/html/3rdparty/guzzlehttp/psr7/src/Utils.php",
            "line": 53,
            "function": "read",
            "class": "Aws\\HashingStream",
            "type": "->",
            "args": [8192]
        }, {
            "file": "/var/www/html/3rdparty/aws/aws-sdk-php/src/S3/MultipartUploader.php",
            "line": 117,
            "function": "copyToStream",
            "class": "GuzzleHttp\\Psr7\\Utils",
            "type": "::",
            "args": [
                ["Aws\\HashingStream", ["GuzzleHttp\\Psr7\\LimitStream"]],
                ["GuzzleHttp\\Psr7\\Stream"]
            ]
        }, {
            "file": "/var/www/html/3rdparty/aws/aws-sdk-php/src/Multipart/AbstractUploader.php",
            "line": 54,
            "function": "createPart",
            "class": "Aws\\S3\\MultipartUploader",
            "type": "->",
            "args": ["*** sensitive parameters replaced ***"]
        }, {
            "file": "/var/www/html/3rdparty/aws/aws-sdk-php/src/CommandPool.php",
            "line": 56,
            "function": "getUploadCommands",
            "class": "Aws\\Multipart\\AbstractUploader",
            "type": "->",
            "args": [
                ["Closure"]
            ]
        }, {
            "function": "Aws\\{closure}",
            "class": "Aws\\CommandPool",
            "type": "->",
            "args": ["*** sensitive parameters replaced ***"]
        }, {
            "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/EachPromise.php",
            "line": 205,
            "function": "next",
            "class": "Generator",
            "type": "->",
            "args": []
        }, {
            "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/EachPromise.php",
            "line": 231,
            "function": "advanceIterator",
            "class": "GuzzleHttp\\Promise\\EachPromise",
            "type": "->",
            "args": []
        }, {
            "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/EachPromise.php",
            "line": 176,
            "function": "step",
            "class": "GuzzleHttp\\Promise\\EachPromise",
            "type": "->",
            "args": [9]
        }, {
            "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/Promise.php",
            "line": 204,
            "function": "GuzzleHttp\\Promise\\{closure}",
            "class": "GuzzleHttp\\Promise\\EachPromise",
            "type": "->",
            "args": ["*** sensitive parameters replaced ***"]
        }, {
            "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/Promise.php",
            "line": 153,
            "function": "callHandler",
            "class": "GuzzleHttp\\Promise\\Promise",
            "type": "::",
            "args": [1, "*** sensitive parameters replaced ***", null]
        }, {
            "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/TaskQueue.php",
            "line": 48,
            "function": "GuzzleHttp\\Promise\\{closure}",
            "class": "GuzzleHttp\\Promise\\Promise",
            "type": "::",
            "args": ["*** sensitive parameters replaced ***"]
        }, {
            "file": "/var/www/html/3rdparty/guzzlehttp/guzzle/src/Handler/CurlMultiHandler.php",
            "line": 159,
            "function": "run",
            "class": "GuzzleHttp\\Promise\\TaskQueue",
            "type": "->",
            "args": []
        }, {
            "file": "/var/www/html/3rdparty/guzzlehttp/guzzle/src/Handler/CurlMultiHandler.php",
            "line": 184,
            "function": "tick",
            "class": "GuzzleHttp\\Handler\\CurlMultiHandler",
            "type": "->",
            "args": []
        }, {
            "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/Promise.php",
            "line": 248,
            "function": "execute",
            "class": "GuzzleHttp\\Handler\\CurlMultiHandler",
            "type": "->",
            "args": ["*** sensitive parameters replaced ***"]
        }, {
            "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/Promise.php",
            "line": 224,
            "function": "invokeWaitFn",
            "class": "GuzzleHttp\\Promise\\Promise",
            "type": "->",
            "args": []
        }, {
            "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/Promise.php",
            "line": 269,
            "function": "waitIfPending",
            "class": "GuzzleHttp\\Promise\\Promise",
            "type": "->",
            "args": []
        }, {
            "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/Promise.php",
            "line": 226,
            "function": "invokeWaitList",
            "class": "GuzzleHttp\\Promise\\Promise",
            "type": "->",
            "args": []
        }, {
            "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/Promise.php",
            "line": 269,
            "function": "waitIfPending",
            "class": "GuzzleHttp\\Promise\\Promise",
            "type": "->",
            "args": []
        }, {
            "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/Promise.php",
            "line": 226,
            "function": "invokeWaitList",
            "class": "GuzzleHttp\\Promise\\Promise",
            "type": "->",
            "args": []
        }, {
            "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/Promise.php",
            "line": 62,
            "function": "waitIfPending",
            "class": "GuzzleHttp\\Promise\\Promise",
            "type": "->",
            "args": []
        }, {
            "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/Coroutine.php",
            "line": 67,
            "function": "wait",
            "class": "GuzzleHttp\\Promise\\Promise",
            "type": "->",
            "args": []
        }, {
            "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/Promise.php",
            "line": 248,
            "function": "GuzzleHttp\\Promise\\{closure}",
            "class": "GuzzleHttp\\Promise\\Coroutine",
            "type": "->",
            "args": ["*** sensitive parameters replaced ***"]
        }, {
            "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/Promise.php",
            "line": 224,
            "function": "invokeWaitFn",
            "class": "GuzzleHttp\\Promise\\Promise",
            "type": "->",
            "args": []
        }, {
            "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/Promise.php",
            "line": 269,
            "function": "waitIfPending",
            "class": "GuzzleHttp\\Promise\\Promise",
            "type": "->",
            "args": []
        }, {
            "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/Promise.php",
            "line": 226,
            "function": "invokeWaitList",
            "class": "GuzzleHttp\\Promise\\Promise",
            "type": "->",
            "args": []
        }, {
            "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/Promise.php",
            "line": 62,
            "function": "waitIfPending",
            "class": "GuzzleHttp\\Promise\\Promise",
            "type": "->",
            "args": []
        }, {
            "file": "/var/www/html/3rdparty/aws/aws-sdk-php/src/Multipart/AbstractUploadManager.php",
            "line": 83,
            "function": "wait",
            "class": "GuzzleHttp\\Promise\\Promise",
            "type": "->",
            "args": []
        }, {
            "file": "/var/www/html/lib/private/Files/ObjectStore/S3ObjectTrait.php",
            "line": 139,
            "function": "upload",
            "class": "Aws\\Multipart\\AbstractUploadManager",
            "type": "->",
            "args": []
        }, {
            "file": "/var/www/html/lib/private/Files/ObjectStore/S3ObjectTrait.php",
            "line": 172,
            "function": "writeMultiPart",
            "class": "OC\\Files\\ObjectStore\\S3",
            "type": "->",
            "args": ["urn:oid:255190", ["GuzzleHttp\\Psr7\\AppendStream"], "video/x-matroska"]
        }, {
            "file": "/var/www/html/lib/private/Files/ObjectStore/ObjectStoreStorage.php",
            "line": 533,
            "function": "writeObject",
            "class": "OC\\Files\\ObjectStore\\S3",
            "type": "->",
            "args": ["urn:oid:255190", null, "video/x-matroska"]
        }, {
            "file": "/var/www/html/lib/private/Files/Storage/Wrapper/Wrapper.php",
            "line": 653,
            "function": "writeStream",
            "class": "OC\\Files\\ObjectStore\\ObjectStoreStorage",
            "type": "->",
            "args": ["files/REDACTED/REDACTED/REDACTED/REDACTED.REDACTED", null, null]
        }, {
            "file": "/var/www/html/lib/private/Files/Storage/Wrapper/Wrapper.php",
            "line": 653,
            "function": "writeStream",
            "class": "OC\\Files\\Storage\\Wrapper\\Wrapper",
            "type": "->",
            "args": ["files/REDACTED/REDACTED/REDACTED/REDACTED.REDACTED", null, null]
        }, {
            "file": "/var/www/html/lib/private/Files/Storage/Wrapper/Wrapper.php",
            "line": 653,
            "function": "writeStream",
            "class": "OC\\Files\\Storage\\Wrapper\\Wrapper",
            "type": "->",
            "args": ["files/REDACTED/REDACTED/REDACTED/REDACTED.REDACTED", null, null]
        }, {
            "file": "/var/www/html/apps/dav/lib/Connector/Sabre/File.php",
            "line": 250,
            "function": "writeStream",
            "class": "OC\\Files\\Storage\\Wrapper\\Wrapper",
            "type": "->",
            "args": ["files/REDACTED/REDACTED/REDACTED/REDACTED.REDACTED", null]
        }, {
            "file": "/var/www/html/apps/dav/lib/Connector/Sabre/Directory.php",
            "line": 148,
            "function": "put",
            "class": "OCA\\DAV\\Connector\\Sabre\\File",
            "type": "->",
            "args": ["*** sensitive parameters replaced ***"]
        }, {
            "file": "/var/www/html/3rdparty/sabre/dav/lib/DAV/Tree.php",
            "line": 325,
            "function": "createFile",
            "class": "OCA\\DAV\\Connector\\Sabre\\Directory",
            "type": "->",
            "args": ["*** sensitive parameters replaced ***"]
        }, {
            "file": "/var/www/html/3rdparty/sabre/dav/lib/DAV/Tree.php",
            "line": 151,
            "function": "copyNode",
            "class": "Sabre\\DAV\\Tree",
            "type": "->",
            "args": [
                ["OCA\\DAV\\Upload\\FutureFile"],
                ["OCA\\DAV\\Connector\\Sabre\\Directory"], "*** sensitive parameters replaced ***"
            ]
        }, {
            "file": "/var/www/html/3rdparty/sabre/dav/lib/DAV/Tree.php",
            "line": 181,
            "function": "copy",
            "class": "Sabre\\DAV\\Tree",
            "type": "->",
            "args": ["uploads/REDACTED/rclone-chunked-upload-87ba1266bc39584e10931fa1a0532198/.file", "files/REDACTED/REDACTED/REDACTED/REDACTED/REDACTED.REDACTED"]
        }, {
            "file": "/var/www/html/apps/dav/lib/Upload/ChunkingPlugin.php",
            "line": 94,
            "function": "move",
            "class": "Sabre\\DAV\\Tree",
            "type": "->",
            "args": ["uploads/REDACTED/rclone-chunked-upload-87ba1266bc39584e10931fa1a0532198/.file", "files/REDACTED/REDACTED/REDACTED/REDACTED/REDACTED.REDACTED"]
        }, {
            "file": "/var/www/html/apps/dav/lib/Upload/ChunkingPlugin.php",
            "line": 76,
            "function": "performMove",
            "class": "OCA\\DAV\\Upload\\ChunkingPlugin",
            "type": "->",
            "args": ["uploads/REDACTED/rclone-chunked-upload-87ba1266bc39584e10931fa1a0532198/.file", "files/REDACTED/REDACTED/REDACTED/REDACTED/REDACTED.REDACTED"]
        }, {
            "file": "/var/www/html/3rdparty/sabre/event/lib/WildcardEmitterTrait.php",
            "line": 89,
            "function": "beforeMove",
            "class": "OCA\\DAV\\Upload\\ChunkingPlugin",
            "type": "->",
            "args": ["uploads/REDACTED/rclone-chunked-upload-87ba1266bc39584e10931fa1a0532198/.file", "files/REDACTED/REDACTED/REDACTED/REDACTED/REDACTED.REDACTED"]
        }, {
            "file": "/var/www/html/3rdparty/sabre/dav/lib/DAV/CorePlugin.php",
            "line": 603,
            "function": "emit",
            "class": "Sabre\\DAV\\Server",
            "type": "->",
            "args": ["beforeMove", ["uploads/REDACTED/rclone-chunked-upload-87ba1266bc39584e10931fa1a0532198/.file", "files/REDACTED/REDACTED/REDACTED/REDACTED/REDACTED.REDACTED"]]
        }, {
            "file": "/var/www/html/3rdparty/sabre/event/lib/WildcardEmitterTrait.php",
            "line": 89,
            "function": "httpMove",
            "class": "Sabre\\DAV\\CorePlugin",
            "type": "->",
            "args": [
                ["Sabre\\HTTP\\Request"],
                ["Sabre\\HTTP\\Response"]
            ]
        }, {
            "file": "/var/www/html/3rdparty/sabre/dav/lib/DAV/Server.php",
            "line": 472,
            "function": "emit",
            "class": "Sabre\\DAV\\Server",
            "type": "->",
            "args": ["method:MOVE", [
                ["Sabre\\HTTP\\Request"],
                ["Sabre\\HTTP\\Response"]
            ]]
        }, {
            "file": "/var/www/html/3rdparty/sabre/dav/lib/DAV/Server.php",
            "line": 253,
            "function": "invokeMethod",
            "class": "Sabre\\DAV\\Server",
            "type": "->",
            "args": [
                ["Sabre\\HTTP\\Request"],
                ["Sabre\\HTTP\\Response"]
            ]
        }, {
            "file": "/var/www/html/3rdparty/sabre/dav/lib/DAV/Server.php",
            "line": 321,
            "function": "start",
            "class": "Sabre\\DAV\\Server",
            "type": "->",
            "args": []
        }, {
            "file": "/var/www/html/apps/dav/lib/Server.php",
            "line": 378,
            "function": "exec",
            "class": "Sabre\\DAV\\Server",
            "type": "->",
            "args": []
        }, {
            "file": "/var/www/html/apps/dav/appinfo/v2/remote.php",
            "line": 35,
            "function": "exec",
            "class": "OCA\\DAV\\Server",
            "type": "->",
            "args": []
        }, {
            "file": "/var/www/html/remote.php",
            "line": 172,
            "args": ["/var/www/html/apps/dav/appinfo/v2/remote.php"],
            "function": "require_once"
        }],
        "File": "/var/www/html/3rdparty/guzzlehttp/psr7/src/Stream.php",
        "Line": 233,
        "Previous": {
            "Exception": "Sabre\\DAV\\Exception\\ServiceUnavailable",
            "Message": "Impossible d'ouvrir le fichier",
            "Code": 0,
            "Trace": [{
                "file": "/var/www/html/apps/dav/lib/Upload/AssemblyStream.php",
                "line": 294,
                "function": "get",
                "class": "OCA\\DAV\\Connector\\Sabre\\File",
                "type": "->",
                "args": []
            }, {
                "file": "/var/www/html/apps/dav/lib/Upload/AssemblyStream.php",
                "line": 164,
                "function": "getStream",
                "class": "OCA\\DAV\\Upload\\AssemblyStream",
                "type": "->",
                "args": [
                    ["OCA\\DAV\\Connector\\Sabre\\File"]
                ]
            }, {
                "function": "stream_read",
                "class": "OCA\\DAV\\Upload\\AssemblyStream",
                "type": "->",
                "args": [8192]
            }, {
                "file": "/var/www/html/3rdparty/icewind/streams/src/Wrapper.php",
                "line": 55,
                "function": "fread",
                "args": ["*** sensitive parameters replaced ***", 8192]
            }, {
                "file": "/var/www/html/3rdparty/icewind/streams/src/CallbackWrapper.php",
                "line": 96,
                "function": "stream_read",
                "class": "Icewind\\Streams\\Wrapper",
                "type": "->",
                "args": [8192]
            }, {
                "function": "stream_read",
                "class": "Icewind\\Streams\\CallbackWrapper",
                "type": "->",
                "args": [8192]
            }, {
                "file": "/var/www/html/3rdparty/icewind/streams/src/Wrapper.php",
                "line": 55,
                "function": "fread",
                "args": [null, 8192]
            }, {
                "file": "/var/www/html/3rdparty/icewind/streams/src/CountWrapper.php",
                "line": 87,
                "function": "stream_read",
                "class": "Icewind\\Streams\\Wrapper",
                "type": "->",
                "args": [8192]
            }, {
                "function": "stream_read",
                "class": "Icewind\\Streams\\CountWrapper",
                "type": "->",
                "args": [8192]
            }, {
                "file": "/var/www/html/3rdparty/guzzlehttp/psr7/src/Stream.php",
                "line": 231,
                "function": "fread",
                "args": [null, 8192]
            }, {
                "file": "/var/www/html/3rdparty/guzzlehttp/psr7/src/AppendStream.php",
                "line": 203,
                "function": "read",
                "class": "GuzzleHttp\\Psr7\\Stream",
                "type": "->",
                "args": [8192]
            }, {
                "file": "/var/www/html/3rdparty/guzzlehttp/psr7/src/LimitStream.php",
                "line": 152,
                "function": "read",
                "class": "GuzzleHttp\\Psr7\\AppendStream",
                "type": "->",
                "args": [8192]
            }, {
                "file": "/var/www/html/3rdparty/aws/aws-sdk-php/src/HashingStream.php",
                "line": 38,
                "function": "read",
                "class": "GuzzleHttp\\Psr7\\LimitStream",
                "type": "->",
                "args": [8192]
            }, {
                "file": "/var/www/html/3rdparty/guzzlehttp/psr7/src/Utils.php",
                "line": 53,
                "function": "read",
                "class": "Aws\\HashingStream",
                "type": "->",
                "args": [8192]
            }, {
                "file": "/var/www/html/3rdparty/aws/aws-sdk-php/src/S3/MultipartUploader.php",
                "line": 117,
                "function": "copyToStream",
                "class": "GuzzleHttp\\Psr7\\Utils",
                "type": "::",
                "args": [
                    ["Aws\\HashingStream", ["GuzzleHttp\\Psr7\\LimitStream"]],
                    ["GuzzleHttp\\Psr7\\Stream"]
                ]
            }, {
                "file": "/var/www/html/3rdparty/aws/aws-sdk-php/src/Multipart/AbstractUploader.php",
                "line": 54,
                "function": "createPart",
                "class": "Aws\\S3\\MultipartUploader",
                "type": "->",
                "args": ["*** sensitive parameters replaced ***"]
            }, {
                "file": "/var/www/html/3rdparty/aws/aws-sdk-php/src/CommandPool.php",
                "line": 56,
                "function": "getUploadCommands",
                "class": "Aws\\Multipart\\AbstractUploader",
                "type": "->",
                "args": [
                    ["Closure"]
                ]
            }, {
                "function": "Aws\\{closure}",
                "class": "Aws\\CommandPool",
                "type": "->",
                "args": ["*** sensitive parameters replaced ***"]
            }, {
                "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/EachPromise.php",
                "line": 205,
                "function": "next",
                "class": "Generator",
                "type": "->",
                "args": []
            }, {
                "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/EachPromise.php",
                "line": 231,
                "function": "advanceIterator",
                "class": "GuzzleHttp\\Promise\\EachPromise",
                "type": "->",
                "args": []
            }, {
                "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/EachPromise.php",
                "line": 176,
                "function": "step",
                "class": "GuzzleHttp\\Promise\\EachPromise",
                "type": "->",
                "args": [9]
            }, {
                "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/Promise.php",
                "line": 204,
                "function": "GuzzleHttp\\Promise\\{closure}",
                "class": "GuzzleHttp\\Promise\\EachPromise",
                "type": "->",
                "args": ["*** sensitive parameters replaced ***"]
            }, {
                "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/Promise.php",
                "line": 153,
                "function": "callHandler",
                "class": "GuzzleHttp\\Promise\\Promise",
                "type": "::",
                "args": [1, "*** sensitive parameters replaced ***", null]
            }, {
                "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/TaskQueue.php",
                "line": 48,
                "function": "GuzzleHttp\\Promise\\{closure}",
                "class": "GuzzleHttp\\Promise\\Promise",
                "type": "::",
                "args": ["*** sensitive parameters replaced ***"]
            }, {
                "file": "/var/www/html/3rdparty/guzzlehttp/guzzle/src/Handler/CurlMultiHandler.php",
                "line": 159,
                "function": "run",
                "class": "GuzzleHttp\\Promise\\TaskQueue",
                "type": "->",
                "args": []
            }, {
                "file": "/var/www/html/3rdparty/guzzlehttp/guzzle/src/Handler/CurlMultiHandler.php",
                "line": 184,
                "function": "tick",
                "class": "GuzzleHttp\\Handler\\CurlMultiHandler",
                "type": "->",
                "args": []
            }, {
                "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/Promise.php",
                "line": 248,
                "function": "execute",
                "class": "GuzzleHttp\\Handler\\CurlMultiHandler",
                "type": "->",
                "args": ["*** sensitive parameters replaced ***"]
            }, {
                "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/Promise.php",
                "line": 224,
                "function": "invokeWaitFn",
                "class": "GuzzleHttp\\Promise\\Promise",
                "type": "->",
                "args": []
            }, {
                "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/Promise.php",
                "line": 269,
                "function": "waitIfPending",
                "class": "GuzzleHttp\\Promise\\Promise",
                "type": "->",
                "args": []
            }, {
                "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/Promise.php",
                "line": 226,
                "function": "invokeWaitList",
                "class": "GuzzleHttp\\Promise\\Promise",
                "type": "->",
                "args": []
            }, {
                "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/Promise.php",
                "line": 269,
                "function": "waitIfPending",
                "class": "GuzzleHttp\\Promise\\Promise",
                "type": "->",
                "args": []
            }, {
                "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/Promise.php",
                "line": 226,
                "function": "invokeWaitList",
                "class": "GuzzleHttp\\Promise\\Promise",
                "type": "->",
                "args": []
            }, {
                "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/Promise.php",
                "line": 62,
                "function": "waitIfPending",
                "class": "GuzzleHttp\\Promise\\Promise",
                "type": "->",
                "args": []
            }, {
                "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/Coroutine.php",
                "line": 67,
                "function": "wait",
                "class": "GuzzleHttp\\Promise\\Promise",
                "type": "->",
                "args": []
            }, {
                "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/Promise.php",
                "line": 248,
                "function": "GuzzleHttp\\Promise\\{closure}",
                "class": "GuzzleHttp\\Promise\\Coroutine",
                "type": "->",
                "args": ["*** sensitive parameters replaced ***"]
            }, {
                "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/Promise.php",
                "line": 224,
                "function": "invokeWaitFn",
                "class": "GuzzleHttp\\Promise\\Promise",
                "type": "->",
                "args": []
            }, {
                "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/Promise.php",
                "line": 269,
                "function": "waitIfPending",
                "class": "GuzzleHttp\\Promise\\Promise",
                "type": "->",
                "args": []
            }, {
                "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/Promise.php",
                "line": 226,
                "function": "invokeWaitList",
                "class": "GuzzleHttp\\Promise\\Promise",
                "type": "->",
                "args": []
            }, {
                "file": "/var/www/html/3rdparty/guzzlehttp/promises/src/Promise.php",
                "line": 62,
                "function": "waitIfPending",
                "class": "GuzzleHttp\\Promise\\Promise",
                "type": "->",
                "args": []
            }, {
                "file": "/var/www/html/3rdparty/aws/aws-sdk-php/src/Multipart/AbstractUploadManager.php",
                "line": 83,
                "function": "wait",
                "class": "GuzzleHttp\\Promise\\Promise",
                "type": "->",
                "args": []
            }, {
                "file": "/var/www/html/lib/private/Files/ObjectStore/S3ObjectTrait.php",
                "line": 139,
                "function": "upload",
                "class": "Aws\\Multipart\\AbstractUploadManager",
                "type": "->",
                "args": []
            }, {
                "file": "/var/www/html/lib/private/Files/ObjectStore/S3ObjectTrait.php",
                "line": 172,
                "function": "writeMultiPart",
                "class": "OC\\Files\\ObjectStore\\S3",
                "type": "->",
                "args": ["urn:oid:255190", ["GuzzleHttp\\Psr7\\AppendStream"], "video/x-matroska"]
            }, {
                "file": "/var/www/html/lib/private/Files/ObjectStore/ObjectStoreStorage.php",
                "line": 533,
                "function": "writeObject",
                "class": "OC\\Files\\ObjectStore\\S3",
                "type": "->",
                "args": ["urn:oid:255190", null, "video/x-matroska"]
            }, {
                "file": "/var/www/html/lib/private/Files/Storage/Wrapper/Wrapper.php",
                "line": 653,
                "function": "writeStream",
                "class": "OC\\Files\\ObjectStore\\ObjectStoreStorage",
                "type": "->",
                "args": ["files/REDACTED/REDACTED/REDACTED/REDACTED.REDACTED", null, null]
            }, {
                "file": "/var/www/html/lib/private/Files/Storage/Wrapper/Wrapper.php",
                "line": 653,
                "function": "writeStream",
                "class": "OC\\Files\\Storage\\Wrapper\\Wrapper",
                "type": "->",
                "args": ["files/REDACTED/REDACTED/REDACTED/REDACTED.REDACTED", null, null]
            }, {
                "file": "/var/www/html/lib/private/Files/Storage/Wrapper/Wrapper.php",
                "line": 653,
                "function": "writeStream",
                "class": "OC\\Files\\Storage\\Wrapper\\Wrapper",
                "type": "->",
                "args": ["files/REDACTED/REDACTED/REDACTED/REDACTED.REDACTED", null, null]
            }, {
                "file": "/var/www/html/apps/dav/lib/Connector/Sabre/File.php",
                "line": 250,
                "function": "writeStream",
                "class": "OC\\Files\\Storage\\Wrapper\\Wrapper",
                "type": "->",
                "args": ["files/REDACTED/REDACTED/REDACTED/REDACTED.REDACTED", null]
            }, {
                "file": "/var/www/html/apps/dav/lib/Connector/Sabre/Directory.php",
                "line": 148,
                "function": "put",
                "class": "OCA\\DAV\\Connector\\Sabre\\File",
                "type": "->",
                "args": ["*** sensitive parameters replaced ***"]
            }, {
                "file": "/var/www/html/3rdparty/sabre/dav/lib/DAV/Tree.php",
                "line": 325,
                "function": "createFile",
                "class": "OCA\\DAV\\Connector\\Sabre\\Directory",
                "type": "->",
                "args": ["*** sensitive parameters replaced ***"]
            }, {
                "file": "/var/www/html/3rdparty/sabre/dav/lib/DAV/Tree.php",
                "line": 151,
                "function": "copyNode",
                "class": "Sabre\\DAV\\Tree",
                "type": "->",
                "args": [
                    ["OCA\\DAV\\Upload\\FutureFile"],
                    ["OCA\\DAV\\Connector\\Sabre\\Directory"], "*** sensitive parameters replaced ***"
                ]
            }, {
                "file": "/var/www/html/3rdparty/sabre/dav/lib/DAV/Tree.php",
                "line": 181,
                "function": "copy",
                "class": "Sabre\\DAV\\Tree",
                "type": "->",
                "args": ["uploads/REDACTED/rclone-chunked-upload-87ba1266bc39584e10931fa1a0532198/.file", "files/REDACTED/REDACTED/REDACTED/REDACTED/REDACTED.REDACTED"]
            }, {
                "file": "/var/www/html/apps/dav/lib/Upload/ChunkingPlugin.php",
                "line": 94,
                "function": "move",
                "class": "Sabre\\DAV\\Tree",
                "type": "->",
                "args": ["uploads/REDACTED/rclone-chunked-upload-87ba1266bc39584e10931fa1a0532198/.file", "files/REDACTED/REDACTED/REDACTED/REDACTED/REDACTED.REDACTED"]
            }, {
                "file": "/var/www/html/apps/dav/lib/Upload/ChunkingPlugin.php",
                "line": 76,
                "function": "performMove",
                "class": "OCA\\DAV\\Upload\\ChunkingPlugin",
                "type": "->",
                "args": ["uploads/REDACTED/rclone-chunked-upload-87ba1266bc39584e10931fa1a0532198/.file", "files/REDACTED/REDACTED/REDACTED/REDACTED/REDACTED.REDACTED"]
            }, {
                "file": "/var/www/html/3rdparty/sabre/event/lib/WildcardEmitterTrait.php",
                "line": 89,
                "function": "beforeMove",
                "class": "OCA\\DAV\\Upload\\ChunkingPlugin",
                "type": "->",
                "args": ["uploads/REDACTED/rclone-chunked-upload-87ba1266bc39584e10931fa1a0532198/.file", "files/REDACTED/REDACTED/REDACTED/REDACTED/REDACTED.REDACTED"]
            }, {
                "file": "/var/www/html/3rdparty/sabre/dav/lib/DAV/CorePlugin.php",
                "line": 603,
                "function": "emit",
                "class": "Sabre\\DAV\\Server",
                "type": "->",
                "args": ["beforeMove", ["uploads/REDACTED/rclone-chunked-upload-87ba1266bc39584e10931fa1a0532198/.file", "files/REDACTED/REDACTED/REDACTED/REDACTED/REDACTED.REDACTED"]]
            }, {
                "file": "/var/www/html/3rdparty/sabre/event/lib/WildcardEmitterTrait.php",
                "line": 89,
                "function": "httpMove",
                "class": "Sabre\\DAV\\CorePlugin",
                "type": "->",
                "args": [
                    ["Sabre\\HTTP\\Request"],
                    ["Sabre\\HTTP\\Response"]
                ]
            }, {
                "file": "/var/www/html/3rdparty/sabre/dav/lib/DAV/Server.php",
                "line": 472,
                "function": "emit",
                "class": "Sabre\\DAV\\Server",
                "type": "->",
                "args": ["method:MOVE", [
                    ["Sabre\\HTTP\\Request"],
                    ["Sabre\\HTTP\\Response"]
                ]]
            }, {
                "file": "/var/www/html/3rdparty/sabre/dav/lib/DAV/Server.php",
                "line": 253,
                "function": "invokeMethod",
                "class": "Sabre\\DAV\\Server",
                "type": "->",
                "args": [
                    ["Sabre\\HTTP\\Request"],
                    ["Sabre\\HTTP\\Response"]
                ]
            }, {
                "file": "/var/www/html/3rdparty/sabre/dav/lib/DAV/Server.php",
                "line": 321,
                "function": "start",
                "class": "Sabre\\DAV\\Server",
                "type": "->",
                "args": []
            }, {
                "file": "/var/www/html/apps/dav/lib/Server.php",
                "line": 378,
                "function": "exec",
                "class": "Sabre\\DAV\\Server",
                "type": "->",
                "args": []
            }, {
                "file": "/var/www/html/apps/dav/appinfo/v2/remote.php",
                "line": 35,
                "function": "exec",
                "class": "OCA\\DAV\\Server",
                "type": "->",
                "args": []
            }, {
                "file": "/var/www/html/remote.php",
                "line": 172,
                "args": ["/var/www/html/apps/dav/appinfo/v2/remote.php"],
                "function": "require_once"
            }],
            "File": "/var/www/html/apps/dav/lib/Connector/Sabre/File.php",
            "Line": 505
        },
        "message": "Could not create object urn:oid:255190 for files/REDACTED/REDACTED/REDACTED/REDACTED.REDACTED",
        "CustomMessage": "Could not create object urn:oid:255190 for files/REDACTED/REDACTED/REDACTED/REDACTED.REDACTED"
    },
    "id": "6710d7cf2ef4b"
}

Additional info

More context :

• there's an issue on rclone and the "fix" is not pretty at all (423 + 404 considered as upload OK) webdav: nextcloud: rclone incorrectly retries long server-side chunks merge operations before they complete rclone/rclone#7109
• the problem happens on the move to assemble the file part. The upload toward NC is fine, it fails when pushing the parts from the virtually assembled stream to the S3 bucket (judging by the traces I've seen)
• smaller yet big files (~20GB) managed to be uploaded even if it took few attempts in some cases
• I'm using a paid provider (IONOS to name them). They are said to be Platinum NC partners but the only answer I got from the admin team was sorry can't fix that at this point
• the file eventually gets uploaded : at least I have a file with the same name and size. I didn't compare them though. Yet the upload process never "officially" succeeded (it's this paradox that lead to the previous "fix")

Some investigations :

• I'm NOT in a chunkv2 scenario : the chunk are uploaded to some tmp directory and then moved to the S3 bucket as they are virtually assembled
• The stack traces show multiple S3 upload failure scenarios : 408 on S3 operation, broken stream on S3 operation... It look like what's triggering the whole thing is the handling around failures of the AWS S3 PHP SDK used in writeMultiPart of the S3ObjectTrait
• At this point I don't know if its simply the failure of the AWS MultipartUploader that triggers the problem or if the timing also plays a role : the whole upload process takes up to an hour.

[solracsf]

This should be fixed with #27034

This requires memcache.distributed to be set to use Redis (or Memcached), otherwise we fall back on the prior behavior, probably what you're facing.

[JulienFS]

I'll check this out, both to see if the server is ready for chunking v2 and the client sends what's needed.

However unless chunking v1 is meant to disappear it would still be nice to have it fixed. Since there's no locking problem on files that manage to be properly uploaded, even the smaller big ones, I guess there's something messing up about locking that only happens when there's a NC to S3 error. Mayble a simple lock on a part that doesn't get released because of an uncaught exception.

I'm going to try to setup an S3 test env and see if I can trigger systematically the behavior by mocking few things here and there.

[joshtrichards]

If you get a chance, it'd be helpful if you included your config (occ config:list system) as requested in the template. :-)

Particularly since there are several elements at play (locking config, memcache config, object storage config, etc.)

Re: the MultipartUpload fallback behavior: https://docs.nextcloud.com/server/latest/admin_manual/configuration_files/big_file_upload_configuration.html#large-file-upload-on-object-storage

[joshtrichards]

Follow-up: I was just reminded of this Issue: nextcloud/docker#1928

[adippl]

I'm getting similar error and cannot reliably upload files larger than 1GiB.
I'm using official docker image configured by environment variables.

example of an error:

{
  "reqId": "0z7RKmCbBSfl1d5q6nES",
  "level": 3,
  "time": "2024-11-01T10:00:55+00:00",
  "remoteAddr": "removed",
  "user": "test",
  "app": "no app in context",
  "method": "MOVE",
  "url": "/remote.php/dav/uploads/test/rclone-chunked-upload-b0434fe74cedaee84bb8dacf3a1c951d/.file",
  "message": "Unable to read from stream",
  "userAgent": "rclone/1.67.0",
  "version": "28.0.11.1",
  "exception": {
    "Exception": "RuntimeException",
    "Message": "Unable to read from stream",
    "Code": 0,
    "Trace": [
      {
        "file": "/var/www/html/3rdparty/guzzlehttp/psr7/src/Utils.php",
        "line": 60,
        "function": "read",
        "class": "GuzzleHttp\\Psr7\\Stream",
        "type": "->",
        "args": [
          8192
        ]
      },

Nextcloud config from my test instance.

{
    "system": {
        "htaccess.RewriteBase": "\/",
        "memcache.local": "\\OC\\Memcache\\APCu",
        "apps_paths": [
            {
                "path": "\/var\/www\/html\/apps",
                "url": "\/apps",
                "writable": false
            },
            {
                "path": "\/var\/www\/html\/custom_apps",
                "url": "\/custom_apps",
                "writable": true
            }
        ],
        "memcache.distributed": "\\OC\\Memcache\\Redis",
        "memcache.locking": "\\OC\\Memcache\\Redis",
        "redis": {
            "host": "***REMOVED SENSITIVE VALUE***",
            "password": "***REMOVED SENSITIVE VALUE***",
            "port": 6379
        },
        "overwritehost": "<<<maybe config dump should remove this?>>>",
        "overwriteprotocol": "https",
        "trusted_proxies": "***REMOVED SENSITIVE VALUE***",
        "objectstore": {
            "class": "\\OC\\Files\\ObjectStore\\S3",
            "arguments": {
                "bucket": "nextcloud-2024-data",
                "region": "region1",
                "hostname": "<<<maybe config dump should remove this?>>>",
                "port": "443",
                "storageClass": "",
                "objectPrefix": "urn:oid:",
                "autocreate": true,
                "use_ssl": true,
                "use_path_style": true,
                "legacy_auth": false,
                "key": "***REMOVED SENSITIVE VALUE***",
                "secret": "***REMOVED SENSITIVE VALUE***"
            }
        },
        "upgrade.disable-web": true,
        "passwordsalt": "***REMOVED SENSITIVE VALUE***",
        "secret": "***REMOVED SENSITIVE VALUE***",
        "trusted_domains": [
            "localhost"
        ],
        "datadirectory": "***REMOVED SENSITIVE VALUE***",
        "dbtype": "mysql",
        "version": "28.0.11.1",
        "overwrite.cli.url": "https:\/\/localhost",
        "dbname": "***REMOVED SENSITIVE VALUE***",
        "dbhost": "***REMOVED SENSITIVE VALUE***",
        "dbport": "",
        "dbtableprefix": "oc_",
        "mysql.utf8mb4": true,
        "dbuser": "***REMOVED SENSITIVE VALUE***",
        "dbpassword": "***REMOVED SENSITIVE VALUE***",
        "installed": true,
        "instanceid": "***REMOVED SENSITIVE VALUE***",
        "skeletondirectory": "",
        "loglevel": 2,
        "maintenance": false
    }
}