Allow image pasting in rich text mode in RTE (#11049)

* add comments to rough first solution

* allow eventRelation prop to pass to both composers

* use eventRelation in image paste

* add image pasting to rich text mode of rich text editor

* extract error handling to function

* type the error handler

* add tests

* make behaviour mimic SendMessage

* add sad path tests

* refactor to use catch throughout

* update comments

* tidy up tests

* add special case and change function signature

* add comment

* bump rte to 2.2.2

package.json

@@ -61,7 +61,7 @@
     "dependencies": {
         "@babel/runtime": "^7.12.5",
         "@matrix-org/analytics-events": "^0.5.0",
-        "@matrix-org/matrix-wysiwyg": "^2.0.0",
+        "@matrix-org/matrix-wysiwyg": "^2.2.2",
         "@matrix-org/react-sdk-module-api": "^0.0.5",
         "@sentry/browser": "^7.0.0",
         "@sentry/tracing": "^7.0.0",

src/components/views/rooms/wysiwyg_composer/SendWysiwygComposer.tsx

@@ -57,11 +57,10 @@ export default function SendWysiwygComposer({
     isRichTextEnabled,
     e2eStatus,
     menuPosition,
-    eventRelation,
     ...props
 }: SendWysiwygComposerProps): JSX.Element {
     const Composer = isRichTextEnabled ? WysiwygComposer : PlainTextComposer;
-    const defaultContextValue = useRef(getDefaultContextValue({ eventRelation }));
+    const defaultContextValue = useRef(getDefaultContextValue({ eventRelation: props.eventRelation }));
 
     return (
         <ComposerContext.Provider value={defaultContextValue.current}>

src/components/views/rooms/wysiwyg_composer/components/PlainTextComposer.tsx

@@ -15,6 +15,7 @@ limitations under the License.
 */
 
 import classNames from "classnames";
+import { IEventRelation } from "matrix-js-sdk/src/matrix";
 import React, { MutableRefObject, ReactNode } from "react";
 
 import { useComposerFunctions } from "../hooks/useComposerFunctions";
@@ -36,6 +37,7 @@ interface PlainTextComposerProps {
     leftComponent?: ReactNode;
     rightComponent?: ReactNode;
     children?: (ref: MutableRefObject<HTMLDivElement | null>, composerFunctions: ComposerFunctions) => ReactNode;
+    eventRelation?: IEventRelation;
 }
 
 export function PlainTextComposer({

src/components/views/rooms/wysiwyg_composer/components/WysiwygComposer.tsx

@@ -15,6 +15,7 @@ limitations under the License.
 */
 
 import React, { memo, MutableRefObject, ReactNode, useEffect, useRef } from "react";
+import { IEventRelation } from "matrix-js-sdk/src/matrix";
 import { useWysiwyg, FormattingFunctions } from "@matrix-org/matrix-wysiwyg";
 import classNames from "classnames";
 
@@ -40,6 +41,7 @@ interface WysiwygComposerProps {
     leftComponent?: ReactNode;
     rightComponent?: ReactNode;
     children?: (ref: MutableRefObject<HTMLDivElement | null>, wysiwyg: FormattingFunctions) => ReactNode;
+    eventRelation?: IEventRelation;
 }
 
 export const WysiwygComposer = memo(function WysiwygComposer({
@@ -52,11 +54,12 @@ export const WysiwygComposer = memo(function WysiwygComposer({
     leftComponent,
     rightComponent,
     children,
+    eventRelation,
 }: WysiwygComposerProps) {
     const { room } = useRoomContext();
     const autocompleteRef = useRef<Autocomplete | null>(null);
 
-    const inputEventProcessor = useInputEventProcessor(onSend, autocompleteRef, initialContent);
+    const inputEventProcessor = useInputEventProcessor(onSend, autocompleteRef, initialContent, eventRelation);
     const { ref, isWysiwygReady, content, actionStates, wysiwyg, suggestion } = useWysiwyg({
         initialContent,
         inputEventProcessor,

src/components/views/rooms/wysiwyg_composer/hooks/useInputEventProcessor.ts

@@ -16,7 +16,7 @@ limitations under the License.
 
 import { Wysiwyg, WysiwygEvent } from "@matrix-org/matrix-wysiwyg";
 import { useCallback } from "react";
-import { MatrixClient } from "matrix-js-sdk/src/matrix";
+import { IEventRelation, MatrixClient } from "matrix-js-sdk/src/matrix";
 
 import { useSettingValue } from "../../../../../hooks/useSettings";
 import { getKeyBindingsManager } from "../../../../../KeyBindingsManager";
@@ -34,11 +34,15 @@ import { getEventsFromEditorStateTransfer, getEventsFromRoom } from "../utils/ev
 import { endEditing } from "../utils/editing";
 import Autocomplete from "../../Autocomplete";
 import { handleEventWithAutocomplete } from "./utils";
+import ContentMessages from "../../../../../ContentMessages";
+import { getBlobSafeMimeType } from "../../../../../utils/blobs";
+import { isNotNull } from "../../../../../Typeguards";
 
 export function useInputEventProcessor(
     onSend: () => void,
     autocompleteRef: React.RefObject<Autocomplete>,
     initialContent?: string,
+    eventRelation?: IEventRelation,
 ): (event: WysiwygEvent, composer: Wysiwyg, editor: HTMLElement) => WysiwygEvent | null {
     const roomContext = useRoomContext();
     const composerContext = useComposerContext();
@@ -47,10 +51,6 @@ export function useInputEventProcessor(
 
     return useCallback(
         (event: WysiwygEvent, composer: Wysiwyg, editor: HTMLElement) => {
-            if (event instanceof ClipboardEvent) {
-                return event;
-            }
-
             const send = (): void => {
                 event.stopPropagation?.();
                 event.preventDefault?.();
@@ -61,6 +61,21 @@ export function useInputEventProcessor(
                 onSend();
             };
 
+            // this is required to handle edge case image pasting in Safari, see
+            // https://github.com/vector-im/element-web/issues/25327 and it is caught by the
+            // `beforeinput` listener attached to the composer
+            const isInputEventForClipboard =
+                event instanceof InputEvent && event.inputType === "insertFromPaste" && isNotNull(event.dataTransfer);
+            const isClipboardEvent = event instanceof ClipboardEvent;
+
+            const shouldHandleAsClipboardEvent = isClipboardEvent || isInputEventForClipboard;
+
+            if (shouldHandleAsClipboardEvent) {
+                const data = isClipboardEvent ? event.clipboardData : event.dataTransfer;
+                const handled = handleClipboardEvent(event, data, roomContext, mxClient, eventRelation);
+                return handled ? null : event;
+            }
+
             const isKeyboardEvent = event instanceof KeyboardEvent;
             if (isKeyboardEvent) {
                 return handleKeyboardEvent(
@@ -78,7 +93,16 @@ export function useInputEventProcessor(
                 return handleInputEvent(event, send, isCtrlEnterToSend);
             }
         },
-        [isCtrlEnterToSend, onSend, initialContent, roomContext, composerContext, mxClient, autocompleteRef],
+        [
+            isCtrlEnterToSend,
+            onSend,
+            initialContent,
+            roomContext,
+            composerContext,
+            mxClient,
+            autocompleteRef,
+            eventRelation,
+        ],
     );
 }
 
@@ -220,3 +244,88 @@ function handleInputEvent(event: InputEvent, send: Send, isCtrlEnterToSend: bool
 
     return event;
 }
+
+/**
+ * Takes an event and handles image pasting. Returns a boolean to indicate if it has handled
+ * the event or not. Must accept either clipboard or input events in order to prevent issue:
+ * https://github.com/vector-im/element-web/issues/25327
+ *
+ * @param event - event to process
+ * @param roomContext - room in which the event occurs
+ * @param mxClient - current matrix client
+ * @param eventRelation - used to send the event to the correct place eg timeline vs thread
+ * @returns - boolean to show if the event was handled or not
+ */
+export function handleClipboardEvent(
+    event: ClipboardEvent | InputEvent,
+    data: DataTransfer | null,
+    roomContext: IRoomState,
+    mxClient: MatrixClient,
+    eventRelation?: IEventRelation,
+): boolean {
+    // Logic in this function follows that of `SendMessageComposer.onPaste`
+    const { room, timelineRenderingType, replyToEvent } = roomContext;
+
+    function handleError(error: unknown): void {
+        if (error instanceof Error) {
+            console.log(error.message);
+        } else if (typeof error === "string") {
+            console.log(error);
+        }
+    }
+
+    if (event.type !== "paste" || data === null || room === undefined) {
+        return false;
+    }
+
+    // Prioritize text on the clipboard over files if RTF is present as Office on macOS puts a bitmap
+    // in the clipboard as well as the content being copied. Modern versions of Office seem to not do this anymore.
+    // We check text/rtf instead of text/plain as when copy+pasting a file from Finder or Gnome Image Viewer
+    // it puts the filename in as text/plain which we want to ignore.
+    if (data.files.length && !data.types.includes("text/rtf")) {
+        ContentMessages.sharedInstance()
+            .sendContentListToRoom(Array.from(data.files), room.roomId, eventRelation, mxClient, timelineRenderingType)
+            .catch(handleError);
+        return true;
+    }
+
+    // Safari `Insert from iPhone or iPad`
+    // data.getData("text/html") returns a string like: <img src="blob:https://...">
+    if (data.types.includes("text/html")) {
+        const imgElementStr = data.getData("text/html");
+        const parser = new DOMParser();
+        const imgDoc = parser.parseFromString(imgElementStr, "text/html");
+
+        if (
+            imgDoc.getElementsByTagName("img").length !== 1 ||
+            !imgDoc.querySelector("img")?.src.startsWith("blob:") ||
+            imgDoc.childNodes.length !== 1
+        ) {
+            handleError("Failed to handle pasted content as Safari inserted content");
+            return false;
+        }
+        const imgSrc = imgDoc.querySelector("img")!.src;
+
+        fetch(imgSrc)
+            .then((response) => {
+                response
+                    .blob()
+                    .then((imgBlob) => {
+                        const type = imgBlob.type;
+                        const safetype = getBlobSafeMimeType(type);
+                        const ext = type.split("/")[1];
+                        const parts = response.url.split("/");
+                        const filename = parts[parts.length - 1];
+                        const file = new File([imgBlob], filename + "." + ext, { type: safetype });
+                        ContentMessages.sharedInstance()
+                            .sendContentToRoom(file, room.roomId, eventRelation, mxClient, replyToEvent)
+                            .catch(handleError);
+                    })
+                    .catch(handleError);
+            })
+            .catch(handleError);
+        return true;
+    }
+
+    return false;
+}