Reupload files on whiteboard content update

package.json

@@ -100,6 +100,7 @@
     "jwt-decode": "^3.1.2",
     "lodash": "^4.17.21",
     "logform": "^2.6.1",
+    "mime-detect": "^1.2.0",
     "module-alias": "^2.2.3",
     "mysql2": "^3.10.3",
     "nest-winston": "^1.9.7",

src/common/enums/logging.context.ts

@@ -52,6 +52,7 @@ export enum LogContext {
   TAGSET = 'tagset',
   EXCALIDRAW_SERVER = 'excalidraw-server',
   WHITEBOARD_INTEGRATION = 'whiteboard-integration',
+  WHITEBOARD = 'whiteboard',
   RESOLVER_FIELD = 'resolver-field',
   RESOLVER_QUERY = 'resolver-query',
   MUTATION = 'mutation',

src/common/exceptions/exception.details.ts

@@ -7,5 +7,5 @@ export type ExceptionExtraDetails = {
    * A probable cause added manually by the developer
    */
   cause?: string;
-  originalException?: Error;
+  originalException?: any;
 };

src/common/utils/buffer.from.url.ts

@@ -0,0 +1,11 @@
+import fetch from 'node-fetch';
+
+export const bufferFromUrl = async (url: string) => {
+  const response = await fetch(url);
+
+  if (!response.ok) {
+    throw new Error(`Failed to fetch: ${response.statusText}`);
+  }
+
+  return response.buffer();
+};

src/common/utils/index.ts

@@ -13,3 +13,4 @@ export * from './get.session';
 export * from './calculate.buffer.hash';
 export * from './untildify';
 export * from './path.resolve';
+export * from './buffer.from.url';

src/domain/common/whiteboard/whiteboard.service.ts

@@ -1,4 +1,4 @@
-import { Injectable } from '@nestjs/common';
+import { Inject, Injectable, LoggerService } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
 import { FindOneOptions, FindOptionsRelations, Repository } from 'typeorm';
 import {
@@ -25,10 +25,13 @@ import { UpdateWhiteboardInput } from './dto/whiteboard.dto.update';
 import { LicenseEngineService } from '@core/license-engine/license.engine.service';
 import { LicensePrivilege } from '@common/enums/license.privilege';
 import { AuthorizationPolicyType } from '@common/enums/authorization.policy.type';
+import { WINSTON_MODULE_NEST_PROVIDER } from 'nest-winston';
 
 @Injectable()
 export class WhiteboardService {
   constructor(
+    @Inject(WINSTON_MODULE_NEST_PROVIDER)
+    private readonly logger: LoggerService,
     @InjectRepository(Whiteboard)
     private whiteboardRepository: Repository<Whiteboard>,
     private authorizationPolicyService: AuthorizationPolicyService,
@@ -167,12 +170,16 @@ export class WhiteboardService {
 
     // TODO: is this still needed? It is a lot of work to be doing on every
     // whiteboard content save. Plus I think it is an inherent risk.
-    const newContentWithFiles = await this.reuploadDocumentsIfNotInBucket(
-      newWhiteboardContent,
-      whiteboard?.profile.id
-    );
+    try {
+      const newContentWithFiles = await this.reuploadDocumentsIfNotInBucket(
+        newWhiteboardContent,
+        whiteboard?.profile.id
+      );
 
-    whiteboard.content = JSON.stringify(newContentWithFiles);
+      whiteboard.content = JSON.stringify(newContentWithFiles);
+    } catch (e: any) {
+      this.logger.error(e?.message, e?.stack, LogContext.WHITEBOARD);
+    }
 
     return this.save(whiteboard);
   }
@@ -251,16 +258,31 @@ export class WhiteboardService {
         continue;
       }
 
-      const newDocUrl =
-        await this.profileDocumentsService.reuploadDocumentToProfile(
-          file.url,
-          profile
+      let newDocUrl: string | undefined;
+      try {
+        newDocUrl =
+          await this.profileDocumentsService.reuploadDocumentToProfileOrFail(
+            file.url,
+            profile
+          );
+      } catch (e: any) {
+        this.logger.error(
+          {
+            message: `Skipping failed to upload document to profile: ${e?.message}`,
+            fileUrl: file.url,
+            profileId: profile.id,
+            originalException: e,
+          },
+          e?.stack,
+          LogContext.WHITEBOARD
         );
-
+        // skip this file
+        continue;
+      }
+      // skip; file already in the bucket
       if (!newDocUrl) {
         continue;
       }
-
       // change the url to the new document
       whiteboardContent.files[file.id] = {
         ...file,

src/domain/profile-documents/profile.documents.service.ts

@@ -7,6 +7,10 @@ import { DocumentAuthorizationService } from '@domain/storage/document/document.
 import { EntityNotInitializedException } from '@common/exceptions';
 import { IProfile } from '@domain/common/profile';
 import { AuthorizationPolicyService } from '@domain/common/authorization-policy/authorization.policy.service';
+import { bufferFromUrl } from '@common/utils';
+import { detectBufferMime } from 'mime-detect';
+import { randomUUID } from 'crypto';
+import { IStorageBucket } from '@domain/storage/storage-bucket/storage.bucket.interface';
 
 @Injectable()
 export class ProfileDocumentsService {
@@ -17,22 +21,88 @@ export class ProfileDocumentsService {
     private authorizationPolicyService: AuthorizationPolicyService
   ) {}
 
-  /***
-   * Checks if a document is living under the storage bucket
-   * of a profile and adds it if not there
-   */
-  public async reuploadDocumentToProfile(
-    fileUrl: string,
-    profile: IProfile
-  ): Promise<string | undefined> {
-    if (!this.documentService.isAlkemioDocumentURL(fileUrl)) {
+  private async uploadDocumentFromAlkemioOrFail(
+    url: string,
+    storageBucket: IStorageBucket
+  ) {
+    // find the document by URL
+    const docInAlkemio = await this.documentService.getDocumentFromURL(url);
+
+    if (!docInAlkemio) {
+      throw new BaseException(
+        `File with URL '${url}' not found in Alkemio`,
+        LogContext.COLLABORATION,
+        AlkemioErrorStatus.NOT_FOUND,
+        { url }
+      );
+    }
+    // is the document in this bucket?
+    const docInThisBucket = storageBucket.documents.find(
+      doc => doc.id === docInAlkemio.id
+    );
+    // if in this bucket - skip
+    if (docInThisBucket) {
+      return undefined;
+    }
+    // if NOT in this bucket - create it inside it
+    const newDocument = await this.documentService.createDocument({
+      createdBy: '',
+      displayName: docInAlkemio.displayName,
+      externalID: docInAlkemio.externalID,
+      mimeType: docInAlkemio.mimeType,
+      size: docInAlkemio.size,
+      temporaryLocation: false,
+    });
+
+    await this.storageBucketService.addDocumentToBucketOrFail(
+      storageBucket.id,
+      newDocument
+    );
+    return this.documentService.saveDocument(newDocument);
+  }
+
+  private async uploadFileFromUrlOrFail(url: string, storageBucketId: string) {
+    let imageBuffer: Buffer | undefined;
+    try {
+      imageBuffer = await bufferFromUrl(url);
+    } catch (e) {
+      throw new BaseException(
+        'Unable to download image from URL',
+        LogContext.COLLABORATION,
+        AlkemioErrorStatus.UNSPECIFIED,
+        { url, originalException: e }
+      );
+    }
+
+    let mimeType: string | undefined;
+    try {
+      mimeType = await detectBufferMime(imageBuffer);
+    } catch (e) {
       throw new BaseException(
-        'File URL not inside Alkemio',
+        'Unable to detect file mime type',
         LogContext.COLLABORATION,
-        AlkemioErrorStatus.UNSPECIFIED
+        AlkemioErrorStatus.UNSPECIFIED,
+        { url, originalException: e }
       );
     }
 
+    return this.storageBucketService.uploadFileAsDocumentFromBuffer(
+      storageBucketId,
+      imageBuffer,
+      randomUUID(),
+      mimeType,
+      '',
+      false
+    );
+  }
+
+  /***
+   * Checks if a document is living under the storage bucket of a profile and adds it if not there
+   */
+  public async reuploadDocumentToProfileOrFail(
+    fileUrl: string,
+    profile: IProfile
+  ): Promise<string | undefined | never> {
     const storageBucketToCheck = profile.storageBucket;
 
     if (!storageBucketToCheck) {
@@ -48,50 +118,23 @@ export class ProfileDocumentsService {
         LogContext.PROFILE
       );
     }
+    const isAlkemioUrl = this.documentService.isAlkemioDocumentURL(fileUrl);
 
-    const docInContent = await this.documentService.getDocumentFromURL(fileUrl);
-
-    if (!docInContent) {
-      throw new BaseException(
-        `File with URL '${fileUrl}' not found`,
-        LogContext.COLLABORATION,
-        AlkemioErrorStatus.NOT_FOUND
-      );
-    }
-
-    const docInThisBucket = storageBucketToCheck.documents.find(
-      doc => doc.id === docInContent.id
-    );
-
-    if (docInThisBucket) {
+    const newDocument = await (isAlkemioUrl
+      ? this.uploadDocumentFromAlkemioOrFail(fileUrl, storageBucketToCheck)
+      : this.uploadFileFromUrlOrFail(fileUrl, storageBucketToCheck.id));
+    // no new document was generated
+    if (!newDocument) {
       return undefined;
     }
 
-    if (!docInThisBucket) {
-      // if not in this bucket - create it inside it
-      const newDoc = await this.documentService.createDocument({
-        createdBy: docInContent.createdBy,
-        displayName: docInContent.displayName,
-        externalID: docInContent.externalID,
-        mimeType: docInContent.mimeType,
-        size: docInContent.size,
-        temporaryLocation: false,
-      });
-      await this.storageBucketService.addDocumentToBucketOrFail(
-        storageBucketToCheck.id,
-        newDoc
+    const authorizations =
+      this.documentAuthorizationService.applyAuthorizationPolicy(
+        newDocument,
+        storageBucketToCheck.authorization
       );
-      await this.documentService.saveDocument(newDoc);
-
-      const authorizations =
-        this.documentAuthorizationService.applyAuthorizationPolicy(
-          newDoc,
-          storageBucketToCheck.authorization
-        );
-      await this.authorizationPolicyService.saveAll(authorizations);
-      return this.documentService.getPubliclyAccessibleURL(newDoc);
-    }
+    await this.authorizationPolicyService.saveAll(authorizations);
 
-    return undefined;
+    return this.documentService.getPubliclyAccessibleURL(newDocument);
   }
 }