Merge branch 'feature/parallel-downloads' into 'devel'

New upload and download implementation

Closes #931, #966, and #1137

See merge request sds-dev/sd-connect/swift-browser-ui!158

Author: Sampsa Penna

.github/config/.finnishwords.txt

@@ -441,6 +441,7 @@ pyyntö
 päivitä
 päänäkymään
 pääsy
+pääsyä
 päättyä
 rajan
 rajattu
@@ -585,6 +586,7 @@ toiminto
 toiseen
 toiselle
 toisen
+toisessa
 toisesta
 toista
 toteuttaa

CHANGELOG.md

@@ -71,6 +71,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - (GL #1047) Added redis and vault monitoring to `/health` endpoint
 - (GL #947) Added cypress tests for sharing folders
 - (GL #987) Added keyboard navigation for all modals in the UI
+- (GL #931) Added proper parallel upload and download support
+  - Move tar archiving to frontend
+  - Add frontend support for downloading arbitrary files as an archive
+  - Use msgpack for parsing websocket data
+  - Separate upload and download workers
+  - Use direct file system writes if available on system
+  - Move upload, download fetch calls to workers to reduce messaging overhead
 
 ### Changed
 
@@ -185,6 +192,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - (GL #1078) Fix overwriting existing files with upload
 - (GL #1108) Fix deleting large amount of objects (~100 items) with long names
 - (GL #1140) Fix Sharing permissions swapping between options after sharing
+- (GL #1125) Remove shared containers if the owner container is deleted
+- (GL #966) Fix upload cancel/finish breaking upload and download functions due to `ServiceWorker` issues
+- (GL #1137) Fix upload repeatability and worker lifecycle issues leading to upload unavailability
 
 ### Removed
 

dockerfiles/Dockerfile-build-crypt

@@ -1,7 +1,27 @@
-# Build wasm encryption
+# Build encryption post-script
+FROM node:18-alpine3.17 AS WEBPACK_WASM_PREPROCESS
+
+RUN --mount=type=cache,target=/var/cache/apk/ \
+    apk add git python3
+
+COPY swift_browser_ui_frontend/package.json /root/swift-browser-ui/swift_browser_ui_frontend/package.json
+COPY swift_browser_ui_frontend/pnpm-lock.yaml /root/swift-browser-ui/swift_browser_ui_frontend/pnpm-lock.yaml
+
+RUN --mount=type=cache,target=/root/.pnpm-store \
+    cd /root/swift-browser-ui/swift_browser_ui_frontend/ \
+    && npm install -g pnpm@8 \
+    && pnpm install --prod
+
+COPY . /root/swift-browser-ui
+
+RUN cd /root/swift-browser-ui/swift_browser_ui_frontend/wasm \
+    && mkdir build \
+    && npx webpack --config "./wasm-webpack.config.js"
+
+# Build upworker and downworker WebAssembly contents
 FROM ghcr.io/cscfi/docker-emscripten-crypt4gh:1.2.0 AS WASMCRYPT
 
-COPY swift_browser_ui_frontend/wasm/ /src/
+COPY --from=WEBPACK_WASM_PREPROCESS /root/swift-browser-ui/swift_browser_ui_frontend/wasm/ /src/
 
 RUN bash /bin/build_wasm.sh all
 
@@ -20,6 +40,14 @@ RUN --mount=type=cache,target=/root/.pnpm-store \
 
 COPY swift_browser_ui_frontend /root/swift_ui/swift_browser_ui_frontend
 
+# Copy wasm files for build
+COPY --from=WASMCRYPT "/src/build/upworker.js" "/root/swift_ui/swift_browser_ui_frontend/public/upworker.js"
+COPY --from=WASMCRYPT "/src/build/downworker.js" "/root/swift_ui/swift_browser_ui_frontend/public/downworker.js"
+COPY --from=WASMCRYPT "/src/build/upworker-post.js.map" "/root/swift_ui/swift_browser_ui_frontend/public/upworker-post.js.map"
+COPY --from=WASMCRYPT "/src/build/downworker-post.js.map" "/root/swift_ui/swift_browser_ui_frontend/public/downworker-post.js.map"
+COPY --from=WASMCRYPT "/src/build/upworker.wasm" "/root/swift_ui/swift_browser_ui_frontend/public/upworker.wasm"
+COPY --from=WASMCRYPT "/src/build/downworker.wasm" "/root/swift_ui/swift_browser_ui_frontend/public/downworker.wasm"
+
 RUN cd /root/swift_ui/swift_browser_ui_frontend \
     && OIDC_ENABLED=$OIDC_ENABLED pnpm run build
 
@@ -28,9 +56,13 @@ FROM python:3.11-alpine3.17 as BACKEND
 COPY pyproject.toml /root/swift_ui/pyproject.toml
 COPY README.md /root/swift_ui/README.md
 COPY swift_browser_ui /root/swift_ui/swift_browser_ui
-COPY --from=FRONTEND /root/swift_ui/swift_browser_ui_frontend/dist /root/swift_ui/swift_browser_ui_frontend/dist
-COPY --from=WASMCRYPT /src/src/libupload.js /root/swift_ui/swift_browser_ui_frontend/dist/libupload.js
-COPY --from=WASMCRYPT /src/src/libupload.wasm /root/swift_ui/swift_browser_ui_frontend/dist/libupload.wasm
+COPY --from=FRONTEND /root/swift_ui/swift_browser_ui_frontend/dist/ /root/swift_ui/swift_browser_ui_frontend/dist/
+COPY --from=WASMCRYPT "/src/build/downworker.wasm" "/usr/local/lib/python3.11/site-packages/swift_browser_ui/ui/static/downworker.wasm"
+COPY --from=WASMCRYPT "/src/build/downworker.js" "/usr/local/lib/python3.11/site-packages/swift_browser_ui/ui/static/downworker.js"
+COPY --from=WASMCRYPT "/src/build/upworker-post.js.map" "/usr/local/lib/python3.11/site-packages/swift_browser_ui/ui/static/upworker-post.js.map"
+COPY --from=WASMCRYPT "/src/build/downworker-post.js.map" "/usr/local/lib/python3.11/site-packages/swift_browser_ui/ui/static/downworker-post.js.map"
+COPY --from=WASMCRYPT "/src/build/upworker.wasm" "/usr/local/lib/python3.11/site-packages/swift_browser_ui/ui/static/upworker.wasm"
+COPY --from=WASMCRYPT "/src/build/upworker.js" "/usr/local/lib/python3.11/site-packages/swift_browser_ui/ui/static/upworker.js"
 
 RUN --mount=type=cache,target=/root/.cache/pip \
     pip install --upgrade pip \

dockerfiles/Dockerfile-build-crypt-devel

@@ -1,7 +1,27 @@
-# Build wasm encryption
+# Build encryption post-script
+FROM node:18-alpine3.17 AS WEBPACK_WASM_PREPROCESS
+
+RUN --mount=type=cache,target=/var/cache/apk/ \
+    apk add git python3
+
+COPY swift_browser_ui_frontend/package.json /root/swift-browser-ui/swift_browser_ui_frontend/package.json
+COPY swift_browser_ui_frontend/pnpm-lock.yaml /root/swift-browser-ui/swift_browser_ui_frontend/pnpm-lock.yaml
+
+RUN --mount=type=cache,target=/root/.pnpm-store \
+    cd /root/swift-browser-ui/swift_browser_ui_frontend/ \
+    && npm install -g pnpm@8 \
+    && pnpm install
+
+COPY . /root/swift-browser-ui
+
+RUN cd /root/swift-browser-ui/swift_browser_ui_frontend/wasm \
+    && mkdir build \
+    && npx webpack --config "./wasm-webpack.config.js"
+
+# Build upworker and downworker WebAssembly contents
 FROM ghcr.io/cscfi/docker-emscripten-crypt4gh:1.2.0 AS WASMCRYPT
 
-COPY swift_browser_ui_frontend/wasm/ /src/
+COPY --from=WEBPACK_WASM_PREPROCESS /root/swift-browser-ui/swift_browser_ui_frontend/wasm/ /src/
 
 RUN bash /bin/build_wasm.sh all
 
@@ -16,10 +36,18 @@ COPY swift_browser_ui_frontend/pnpm-lock.yaml /root/swift_ui/swift_browser_ui_fr
 RUN --mount=type=cache,target=/root/.pnpm-store \
     cd /root/swift_ui/swift_browser_ui_frontend \
     && npm install -g pnpm@8 \
-    && pnpm install --prod
+    && pnpm install
 
 COPY . /root/swift_ui/
 
+# Copy wasm files for build
+COPY --from=WASMCRYPT "/src/build/upworker.js" "/root/swift_ui/swift_browser_ui_frontend/public/upworker.js"
+COPY --from=WASMCRYPT "/src/build/downworker.js" "/root/swift_ui/swift_browser_ui_frontend/public/downworker.js"
+COPY --from=WASMCRYPT "/src/build/upworker-post.js.map" "/root/swift_ui/swift_browser_ui_frontend/public/upworker-post.js.map"
+COPY --from=WASMCRYPT "/src/build/downworker-post.js.map" "/root/swift_ui/swift_browser_ui_frontend/public/downworker-post.js.map"
+COPY --from=WASMCRYPT "/src/build/upworker.wasm" "/root/swift_ui/swift_browser_ui_frontend/public/upworker.wasm"
+COPY --from=WASMCRYPT "/src/build/downworker.wasm" "/root/swift_ui/swift_browser_ui_frontend/public/downworker.wasm"
+
 RUN cd /root/swift_ui/swift_browser_ui_frontend \
     && OIDC_ENABLED=$OIDC_ENABLED pnpm run build-devel
 
@@ -28,9 +56,13 @@ FROM python:3.11-alpine3.17 as BACKEND
 COPY pyproject.toml /root/swift_ui/pyproject.toml
 COPY README.md /root/swift_ui/README.md
 COPY swift_browser_ui /root/swift_ui/swift_browser_ui
-COPY --from=FRONTEND /root/swift_ui/swift_browser_ui_frontend/dist /root/swift_ui/swift_browser_ui_frontend/dist
-COPY --from=WASMCRYPT /src/src/libupload.js /root/swift_ui/swift_browser_ui_frontend/dist/libupload.js
-COPY --from=WASMCRYPT /src/src/libupload.wasm /root/swift_ui/swift_browser_ui_frontend/dist/libupload.wasm
+COPY --from=FRONTEND /root/swift_ui/swift_browser_ui_frontend/dist/ /root/swift_ui/swift_browser_ui_frontend/dist/
+COPY --from=WASMCRYPT "/src/build/downworker.wasm" "/usr/local/lib/python3.11/site-packages/swift_browser_ui/ui/static/downworker.wasm"
+COPY --from=WASMCRYPT "/src/build/downworker.js" "/usr/local/lib/python3.11/site-packages/swift_browser_ui/ui/static/downworker.js"
+COPY --from=WASMCRYPT "/src/build/upworker-post.js.map" "/usr/local/lib/python3.11/site-packages/swift_browser_ui/ui/static/upworker-post.js.map"
+COPY --from=WASMCRYPT "/src/build/downworker-post.js.map" "/usr/local/lib/python3.11/site-packages/swift_browser_ui/ui/static/downworker-post.js.map"
+COPY --from=WASMCRYPT "/src/build/upworker.wasm" "/usr/local/lib/python3.11/site-packages/swift_browser_ui/ui/static/upworker.wasm"
+COPY --from=WASMCRYPT "/src/build/upworker.js" "/usr/local/lib/python3.11/site-packages/swift_browser_ui/ui/static/upworker.js"
 
 RUN --mount=type=cache,target=/root/.cache/pip \
     pip install --upgrade pip \

pyproject.toml

@@ -34,6 +34,7 @@ dependencies = [
     "oidcrp==2.1.4",
     "python-swiftclient==4.4.0",
     "uvloop==0.18.0",
+    "msgpack==1.0.5",
 ]
 
 [project.optional-dependencies]

requirements.txt

@@ -10,3 +10,4 @@ keystoneauth1==5.3.0
 oidcrp==2.1.4
 python-swiftclient==4.4.0
 uvloop==0.18.0
+msgpack==1.0.5

scripts/vault.sh

@@ -26,6 +26,7 @@ SWIFT_BROWSER_UI_DIR=${SWIFT_BROWSER_UI_DIR:-$ROOT}
 function initVault {
     cd "$SWIFT_BROWSER_UI_DIR"
     export VAULT_ADDR='http://127.0.0.1:8200'
+    wget --retry-connrefused --waitretry=1 --timeout=60 --spider http://127.0.0.1:8200/v1/sys/health?standbyok=true
     vault login token=devroot
     vault auth enable approle
     vault secrets enable c4ghtransit
@@ -46,7 +47,7 @@ mkdir -p vault/plugins
 go build -v -o vault/plugins/c4ghtransit c4ghtransit/cmd/c4ghtransit/main.go
 
 # setup vault for swift-browser-ui in the background, after the server is up
-(sleep 1; initVault) 2>&1 &
+initVault 2>&1 &
 
 # start vault server in development mode
 VAULT_LOG_LEVEL=ERROR exec vault server -dev -dev-plugin-dir=vault/plugins -dev-root-token-id="devroot"

swift_browser_ui/common/vault_client.py

@@ -382,6 +382,8 @@ async def put_header(
         :param header: header as b64 encoded string
         :param owner: name of the project that owns the container
         """
+        LOGGER.info(f"Pushing header to vault: {header}")
+
         if owner:
             await self._request(
                 "POST",

swift_browser_ui/ui/api.py

@@ -891,6 +891,34 @@ async def get_crypted_upload_session(
     )
 
 
+async def get_crypted_upload_socket_info(
+    request: aiohttp.web.Request,
+) -> aiohttp.web.Response:
+    """Return a pre-signed upload socket for specified project."""
+    session = await aiohttp_session.get_session(request)
+    request.app["Log"].info(
+        "API call for upload socket signature from "
+        f"{request.remote}, sess: {session} :: {time.ctime()}"
+    )
+
+    project = ""
+    if "project" in request.query:
+        project = request.query["project"]
+
+    runner_id = await open_upload_runner_session(request, project=project)
+    path = f"/cryptic/{request.match_info['project']}"
+    signature = await sign(28800, path)
+
+    return aiohttp.web.json_response(
+        {
+            "id": runner_id,
+            "wsurl": f"{setd['upload_external_endpoint']}{path}".replace("https", "wss"),
+            "host": setd["upload_external_endpoint"],
+            "wssignature": signature,
+        }
+    )
+
+
 async def close_upload_session(
     request: aiohttp.web.Request,
     project: str = "",

swift_browser_ui/ui/front.py

@@ -9,10 +9,10 @@
 from swift_browser_ui.ui.settings import setd
 
 
-async def swjs(_: aiohttp.web.Request) -> aiohttp.web.FileResponse:
+async def up_swjs(_: aiohttp.web.Request) -> aiohttp.web.FileResponse:
     """Serve worker js in worker scope."""
     return aiohttp.web.FileResponse(
-        str(setd["static_directory"]) + "/libupload.js",
+        str(setd["static_directory"]) + "/upworker.js",
         headers={
             "Cache-Control": "no-cache, no-store, must-revalidate",
             "Pragma": "no-cache",
@@ -22,10 +22,75 @@ async def swjs(_: aiohttp.web.Request) -> aiohttp.web.FileResponse:
     )
 
 
-async def swasm(_: aiohttp.web.Request) -> aiohttp.web.FileResponse:
+async def up_swasm(_: aiohttp.web.Request) -> aiohttp.web.FileResponse:
     """Serve worker wasm in worker scope."""
     return aiohttp.web.FileResponse(
-        str(setd["static_directory"]) + "/libupload.wasm",
+        str(setd["static_directory"]) + "/upworker.wasm",
+        headers={
+            "Cache-Control": "no-cache, no-store, must-revalidate",
+            "Pragma": "no-cache",
+            "Expires": "0",
+            "Service-Worker-Allowed": "/",
+        },
+    )
+
+
+async def down_swjs(_: aiohttp.web.Request) -> aiohttp.web.FileResponse:
+    """Serve worker js in worker scope."""
+    return aiohttp.web.FileResponse(
+        str(setd["static_directory"]) + "/downworker.js",
+        headers={
+            "Cache-Control": "no-cache, no-store, must-revalidate",
+            "Pragma": "no-cache",
+            "Expires": "0",
+            "Service-Worker-Allowed": "/",
+        },
+    )
+
+
+async def down_swasm(_: aiohttp.web.Request) -> aiohttp.web.FileResponse:
+    """Serve worker wasm in worker scope."""
+    return aiohttp.web.FileResponse(
+        str(setd["static_directory"]) + "/downworker.wasm",
+        headers={
+            "Cache-Control": "no-cache, no-store, must-revalidate",
+            "Pragma": "no-cache",
+            "Expires": "0",
+            "Service-Worker-Allowed": "/",
+        },
+    )
+
+
+async def map_down_swjs(_: aiohttp.web.Request) -> aiohttp.web.FileResponse:
+    """Serve worker js in worker scope."""
+    return aiohttp.web.FileResponse(
+        str(setd["static_directory"]) + "/downworker-post.js.map",
+        headers={
+            "Cache-Control": "no-cache, no-store, must-revalidate",
+            "Pragma": "no-cache",
+            "Expires": "0",
+            "Service-Worker-Allowed": "/",
+        },
+    )
+
+
+async def map_up_swjs(_: aiohttp.web.Request) -> aiohttp.web.FileResponse:
+    """Serve worker js in worker scope."""
+    return aiohttp.web.FileResponse(
+        str(setd["static_directory"]) + "/upworker-post.js.map",
+        headers={
+            "Cache-Control": "no-cache, no-store, must-revalidate",
+            "Pragma": "no-cache",
+            "Expires": "0",
+            "Service-Worker-Allowed": "/",
+        },
+    )
+
+
+async def agg_swjs(_: aiohttp.web.Request) -> aiohttp.web.FileResponse:
+    """Serve worker js in worker scope."""
+    return aiohttp.web.FileResponse(
+        str(setd["static_directory"]) + "/aggregatorsw.js",
         headers={
             "Cache-Control": "no-cache, no-store, must-revalidate",
             "Pragma": "no-cache",