[IMP] Clipboard: support images in the clipboard

demo/file_store.js

@@ -21,4 +21,9 @@ export class FileStore {
   async delete(path) {
     console.warn("cannot delete file. Not implemented");
   }
+
+  async getFile(path) {
+    const response = await fetch(path);
+    return await response.blob();
+  }
 }

src/actions/edit_actions.ts

@@ -29,7 +29,7 @@ export const copy: ActionSpec = {
   isReadonlyAllowed: true,
   execute: async (env) => {
     env.model.dispatch("COPY");
-    await env.clipboard.write(env.model.getters.getClipboardContent());
+    await env.clipboard.write(await env.model.getters.getOsClipboardContentAsync());
   },
   icon: "o-spreadsheet-Icon.COPY",
 };
@@ -39,7 +39,7 @@ export const cut: ActionSpec = {
   description: "Ctrl+X",
   execute: async (env) => {
     interactiveCut(env);
-    await env.clipboard.write(env.model.getters.getClipboardContent());
+    await env.clipboard.write(await env.model.getters.getOsClipboardContentAsync());
   },
   icon: "o-spreadsheet-Icon.CUT",
 };

src/actions/menu_items_actions.ts

@@ -1,5 +1,6 @@
 import { CellPopoverStore } from "../components/popover";
 import { DEFAULT_FIGURE_HEIGHT, DEFAULT_FIGURE_WIDTH } from "../constants";
+import { parseOSClipboardContent } from "../helpers/clipboard/clipboard_helpers";
 import {
   getChartPositionAtCenterOfViewport,
   getSmartChartDefinition,
@@ -54,20 +55,13 @@ async function paste(env: SpreadsheetChildEnv, pasteOption?: ClipboardPasteOptio
   const osClipboard = await env.clipboard.read();
   switch (osClipboard.status) {
     case "ok":
-      const htmlDocument = new DOMParser().parseFromString(
-        osClipboard.content[ClipboardMIMEType.Html] ?? "<div></div>",
-        "text/html"
-      );
-      const osClipboardSpreadsheetContent =
-        osClipboard.content[ClipboardMIMEType.OSpreadsheet] || "{}";
-      const clipboardId =
-        JSON.parse(osClipboardSpreadsheetContent).clipboardId ??
-        htmlDocument.querySelector("div")?.getAttribute("data-clipboard-id");
+      const clipboardContent = await parseOSClipboardContent(env, osClipboard.content);
+      const clipboardId = clipboardContent.data?.clipboardId;
 
       const target = env.model.getters.getSelectedZones();
 
       if (env.model.getters.getClipboardId() !== clipboardId) {
-        interactivePasteFromOS(env, target, osClipboard.content, pasteOption);
+        interactivePasteFromOS(env, target, clipboardContent, pasteOption);
       } else {
         interactivePaste(env, target, pasteOption);
       }

src/components/grid/grid.ts

@@ -21,6 +21,7 @@ import {
   HEADER_WIDTH,
   SCROLLBAR_WIDTH,
 } from "../../constants";
+import { parseOSClipboardContent } from "../../helpers/clipboard/clipboard_helpers";
 import { isInside } from "../../helpers/index";
 import { openLink } from "../../helpers/links";
 import { isStaticTable } from "../../helpers/table_helpers";
@@ -607,11 +608,8 @@ export class Grid extends Component<Props, SpreadsheetChildEnv> {
     } else {
       this.env.model.dispatch("COPY");
     }
-    const content = this.env.model.getters.getClipboardContent();
-    const clipboardData = ev.clipboardData;
-    for (const type in content) {
-      clipboardData?.setData(type, content[type]);
-    }
+    const osContent = await this.env.model.getters.getOsClipboardContentAsync();
+    await this.env.clipboard.write(osContent);
     ev.preventDefault();
   }
 
@@ -626,32 +624,23 @@ export class Grid extends Component<Props, SpreadsheetChildEnv> {
     if (!clipboardData) {
       return;
     }
-    const clipboardDataTextContent = clipboardData?.getData(ClipboardMIMEType.PlainText);
-    const clipboardDataHtmlContent = clipboardData?.getData(ClipboardMIMEType.Html);
-    const htmlDocument = new DOMParser().parseFromString(
-      clipboardDataHtmlContent ?? "<div></div>",
-      "text/html"
-    );
-    const osClipboardSpreadsheetContent =
-      clipboardData.getData(ClipboardMIMEType.OSpreadsheet) || "{}";
+    const osClipboard = {
+      content: {
+        [ClipboardMIMEType.PlainText]: clipboardData?.getData(ClipboardMIMEType.PlainText),
+        [ClipboardMIMEType.Html]: clipboardData?.getData(ClipboardMIMEType.Html),
+        // TODORAR add all image/* allowed types by getting their data file env.clipbardData.files//  with the correct mime type
+        [ClipboardMIMEType.Png]: clipboardData?.files?.[0],
+      },
+    };
 
     const target = this.env.model.getters.getSelectedZones();
     const isCutOperation = this.env.model.getters.isCutOperation();
 
-    const clipboardId =
-      JSON.parse(osClipboardSpreadsheetContent).clipboardId ??
-      htmlDocument.querySelector("div")?.getAttribute("data-clipboard-id");
-
+    const clipboardContent = await parseOSClipboardContent(this.env, osClipboard.content);
+    const clipboardId = clipboardContent.data?.clipboardId;
     if (this.env.model.getters.getClipboardId() === clipboardId) {
       interactivePaste(this.env, target);
     } else {
-      const clipboardContent = {
-        [ClipboardMIMEType.PlainText]: clipboardDataTextContent,
-        [ClipboardMIMEType.Html]: clipboardDataHtmlContent,
-      };
-      if (osClipboardSpreadsheetContent !== "{}") {
-        clipboardContent[ClipboardMIMEType.OSpreadsheet] = osClipboardSpreadsheetContent;
-      }
       interactivePasteFromOS(this.env, target, clipboardContent);
     }
     if (isCutOperation) {

src/helpers/clipboard/clipboard_helpers.ts

@@ -1,4 +1,12 @@
-import { ClipboardCellData, UID, Zone } from "../../types";
+import {
+  ClipboardCellData,
+  ClipboardMIMEType,
+  ImportClipboardContent,
+  OSClipboardContent,
+  SpreadsheetChildEnv,
+  UID,
+  Zone,
+} from "../../types";
 import { mergeOverlappingZones, positions } from "../zones";
 
 export function getClipboardDataPositions(sheetId: UID, zones: Zone[]): ClipboardCellData {
@@ -54,3 +62,27 @@ export function getPasteZones<T>(target: Zone[], content: T[][]): Zone[] {
     height = content.length;
   return target.map((t) => splitZoneForPaste(t, width, height)).flat();
 }
+
+export async function parseOSClipboardContent(
+  env: SpreadsheetChildEnv,
+  content: OSClipboardContent
+): Promise<ImportClipboardContent> {
+  // TODORAR should not  upload the imge, we first need to check if it's a "same tab" copy/paste
+  const htmlDocument = new DOMParser().parseFromString(
+    content[ClipboardMIMEType.Html] ?? "<div></div>",
+    "text/html"
+  );
+  const oSheetClipboardData = htmlDocument
+    .querySelector("div")
+    ?.getAttribute("data-osheet-clipboard");
+  const spreadsheetContent = oSheetClipboardData && JSON.parse(oSheetClipboardData);
+  const importContent: ImportClipboardContent = {
+    text: content[ClipboardMIMEType.PlainText],
+    data: spreadsheetContent,
+  };
+  if (content[ClipboardMIMEType.Png]) {
+    const imageData = await env.imageProvider?.upload(content[ClipboardMIMEType.Png]);
+    importContent.imageData = imageData;
+  }
+  return importContent;
+}

src/helpers/clipboard/navigator_clipboard_wrapper.ts

@@ -1,11 +1,11 @@
-import { ClipboardContent, ClipboardMIMEType } from "./../../types/clipboard";
+import { ClipboardMIMEType, OSClipboardContent } from "./../../types/clipboard";
 
 export type ClipboardReadResult =
-  | { status: "ok"; content: ClipboardContent }
+  | { status: "ok"; content: OSClipboardContent }
   | { status: "permissionDenied" | "notImplemented" };
 
 export interface ClipboardInterface {
-  write(clipboardContent: ClipboardContent): Promise<void>;
+  write(clipboardContent: OSClipboardContent): Promise<void>;
   writeText(text: string): Promise<void>;
   read(): Promise<ClipboardReadResult>;
 }
@@ -18,7 +18,7 @@ class WebClipboardWrapper implements ClipboardInterface {
   // Can be undefined because navigator.clipboard doesn't exist in old browsers
   constructor(private clipboard: Clipboard | undefined) {}
 
-  async write(clipboardContent: ClipboardContent): Promise<void> {
+  async write(clipboardContent: OSClipboardContent): Promise<void> {
     if (this.clipboard?.write) {
       try {
         await this.clipboard?.write(this.getClipboardItems(clipboardContent));
@@ -60,12 +60,17 @@ class WebClipboardWrapper implements ClipboardInterface {
     if (this.clipboard?.read) {
       try {
         const clipboardItems = await this.clipboard.read();
-        const clipboardContent: ClipboardContent = {};
+        const clipboardContent: OSClipboardContent = {};
         for (const item of clipboardItems) {
           for (const type of item.types) {
+            // TODORAR read should work based on the type (if starts with, text, etc)
             const blob = await item.getType(type);
-            const text = await blob.text();
-            clipboardContent[type as ClipboardMIMEType] = text;
+            if (type === ClipboardMIMEType.Png) {
+              clipboardContent[type] = blob;
+            } else {
+              const text = await blob.text();
+              clipboardContent[type] = text;
+            }
           }
         }
         return { status: "ok", content: clipboardContent };
@@ -83,22 +88,20 @@ class WebClipboardWrapper implements ClipboardInterface {
     }
   }
 
-  private getClipboardItems(content: ClipboardContent): ClipboardItems {
+  private getClipboardItems(content: OSClipboardContent): ClipboardItems {
     const clipboardItemData = {
       [ClipboardMIMEType.PlainText]: this.getBlob(content, ClipboardMIMEType.PlainText),
       [ClipboardMIMEType.Html]: this.getBlob(content, ClipboardMIMEType.Html),
+      [ClipboardMIMEType.Png]: this.getBlob(content, ClipboardMIMEType.Png),
     };
-    const spreadsheetData = content[ClipboardMIMEType.OSpreadsheet];
-    if (spreadsheetData) {
-      clipboardItemData[ClipboardMIMEType.OSpreadsheet] = this.getBlob(
-        content,
-        ClipboardMIMEType.OSpreadsheet
-      );
-    }
     return [new ClipboardItem(clipboardItemData)];
   }
 
-  private getBlob(clipboardContent: ClipboardContent, type: ClipboardMIMEType): Blob {
+  private getBlob(clipboardContent: OSClipboardContent, type: ClipboardMIMEType): Blob {
+    const content = clipboardContent[type];
+    if (content instanceof Blob) {
+      return content;
+    }
     return new Blob([clipboardContent[type] || ""], {
       type,
     });

src/helpers/figures/charts/chart_ui_common.ts

@@ -360,6 +360,61 @@ export function chartToImage(
   return undefined;
 }
 
+export async function chartToImageBlob(
+  runtime: ChartRuntime,
+  figure: Figure,
+  type: string
+): Promise<Blob | null> {
+  // wrap the canvas in a div with a fixed size because chart.js would
+  // fill the whole page otherwise
+  const div = document.createElement("div");
+  div.style.width = `${figure.width}px`;
+  div.style.height = `${figure.height}px`;
+  const canvas = document.createElement("canvas");
+  div.append(canvas);
+  canvas.setAttribute("width", figure.width.toString());
+  canvas.setAttribute("height", figure.height.toString());
+  let finalContent: Blob | null = null;
+  // we have to add the canvas to the DOM otherwise it won't be rendered
+  document.body.append(div);
+  if ("chartJsConfig" in runtime) {
+    const config = deepCopy(runtime.chartJsConfig);
+    config.plugins = [backgroundColorChartJSPlugin];
+    const chart = new window.Chart(canvas, config);
+    const imgContent = chart.toBase64Image() as string;
+    finalContent = base64ToBlob(imgContent, "image/png");
+    chart.destroy();
+    div.remove();
+  } else if (type === "scorecard") {
+    const design = getScorecardConfiguration(figure, runtime as ScorecardChartRuntime);
+    drawScoreChart(design, canvas);
+    finalContent = await new Promise((resolve) => canvas.toBlob(resolve, "image/png"));
+    //canvas.toBlob((blob) => (finalContent = blob), "image/png");
+    div.remove();
+  } else if (type === "gauge") {
+    drawGaugeChart(canvas, runtime as GaugeChartRuntime);
+    // canvas.toBlob((blob) => (finalContent = blob), "image/png");
+    finalContent = await new Promise((resolve) => canvas.toBlob(resolve, "image/png"));
+    div.remove();
+  }
+  return finalContent;
+}
+
+function base64ToBlob(base64: string, mimeType: string): Blob {
+  // Remove the data URL part if present
+  const byteCharacters = atob(base64.split(",")[1]);
+
+  const byteNumbers = new Array(byteCharacters.length);
+  for (let i = 0; i < byteCharacters.length; i++) {
+    byteNumbers[i] = byteCharacters.charCodeAt(i);
+  }
+
+  const byteArray = new Uint8Array(byteNumbers);
+
+  // Create a Blob object from the byteArray
+  return new Blob([byteArray], { type: mimeType });
+}
+
 /**
  * Custom chart.js plugin to set the background color of the canvas
  * https://github.com/chartjs/Chart.js/blob/8fdf76f8f02d31684d34704341a5d9217e977491/docs/configuration/canvas-background.md

src/helpers/figures/images/image_provider.ts

@@ -16,6 +16,12 @@ export class ImageProvider implements ImageProviderInterface {
     return { path, size, mimetype: file.type };
   }
 
+  async upload(file: File): Promise<Image> {
+    const path = await this.fileStore.upload(file);
+    const size = await this.getImageOriginalSize(path);
+    return { path, size, mimetype: file.type };
+  }
+
   private getImageFromUser(): Promise<File> {
     return new Promise((resolve, reject) => {
       const input = document.createElement("input");

src/helpers/ui/paste_interactive.ts

@@ -1,11 +1,10 @@
 import { CURRENT_VERSION } from "../../migrations/data";
 import { _t } from "../../translation";
 import {
-  ClipboardContent,
-  ClipboardMIMEType,
   ClipboardPasteOptions,
   CommandResult,
   DispatchResult,
+  ImportClipboardContent,
   SpreadsheetChildEnv,
   Zone,
 } from "../../types";
@@ -45,7 +44,7 @@ export function interactivePaste(
 export function interactivePasteFromOS(
   env: SpreadsheetChildEnv,
   target: Zone[],
-  clipboardContent: ClipboardContent,
+  clipboardContent: ImportClipboardContent,
   pasteOption?: ClipboardPasteOptions
 ) {
   let result: DispatchResult;
@@ -59,10 +58,9 @@ export function interactivePasteFromOS(
       pasteOption,
     });
   } catch (error) {
-    const parsedSpreadsheetContent = clipboardContent[ClipboardMIMEType.OSpreadsheet]
-      ? JSON.parse(clipboardContent[ClipboardMIMEType.OSpreadsheet])
-      : {};
-    if (parsedSpreadsheetContent.version && parsedSpreadsheetContent.version !== CURRENT_VERSION) {
+    const parsedSpreadsheetContent = clipboardContent.data;
+
+    if (parsedSpreadsheetContent?.version !== CURRENT_VERSION) {
       env.raiseError(
         _t(
           "An unexpected error occurred while pasting content.\
@@ -73,7 +71,7 @@ export function interactivePasteFromOS(
     result = env.model.dispatch("PASTE_FROM_OS_CLIPBOARD", {
       target,
       clipboardContent: {
-        [ClipboardMIMEType.PlainText]: clipboardContent[ClipboardMIMEType.PlainText],
+        text: clipboardContent.text,
       },
       pasteOption,
     });

src/model.ts

@@ -448,6 +448,7 @@ export class Model extends EventBus<any> implements CommandDispatcher {
       session: this.session,
       defaultCurrency: this.config.defaultCurrency,
       customColors: this.config.customColors || [],
+      external: this.config.external,
     };
   }
 

src/plugins/ui_plugin.ts

@@ -28,6 +28,7 @@ export interface UIPluginConfig {
   readonly session: Session;
   readonly defaultCurrency?: Partial<Currency>;
   readonly customColors: Color[];
+  readonly external: ModelConfig["external"];
 }
 
 export interface UIPluginConstructor {