fix: enhacement of content import

Author: Emnaghz

api/src/cms/controllers/content.controller.spec.ts

@@ -1,24 +1,16 @@
 /*
- * Copyright © 2024 Hexastack. All rights reserved.
+ * Copyright © 2025 Hexastack. All rights reserved.
  *
  * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms:
  * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission.
  * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
  */
 
-import fs from 'fs';
-
 import { NotFoundException } from '@nestjs/common/exceptions';
 import { EventEmitter2 } from '@nestjs/event-emitter';
 import { MongooseModule } from '@nestjs/mongoose';
 import { Test, TestingModule } from '@nestjs/testing';
 
-import { AttachmentRepository } from '@/attachment/repositories/attachment.repository';
-import {
-  Attachment,
-  AttachmentModel,
-} from '@/attachment/schemas/attachment.schema';
-import { AttachmentService } from '@/attachment/services/attachment.service';
 import { LoggerService } from '@/logger/logger.service';
 import { NOT_FOUND_ID } from '@/utils/constants/mock';
 import { PageQueryDto } from '@/utils/pagination/pagination-query.dto';
@@ -48,10 +40,8 @@ describe('ContentController', () => {
   let contentController: ContentController;
   let contentService: ContentService;
   let contentTypeService: ContentTypeService;
-  let attachmentService: AttachmentService;
   let contentType: ContentType | null;
   let content: Content | null;
-  let attachment: Attachment | null;
   let updatedContent;
   let pageQuery: PageQueryDto<Content>;
 
@@ -60,34 +50,24 @@ describe('ContentController', () => {
       controllers: [ContentController],
       imports: [
         rootMongooseTestModule(installContentFixtures),
-        MongooseModule.forFeature([
-          ContentTypeModel,
-          ContentModel,
-          AttachmentModel,
-        ]),
+        MongooseModule.forFeature([ContentTypeModel, ContentModel]),
       ],
       providers: [
         LoggerService,
         ContentTypeService,
         ContentService,
         ContentRepository,
-        AttachmentService,
         ContentTypeRepository,
-        AttachmentRepository,
         EventEmitter2,
       ],
     }).compile();
     contentController = module.get<ContentController>(ContentController);
     contentService = module.get<ContentService>(ContentService);
-    attachmentService = module.get<AttachmentService>(AttachmentService);
     contentTypeService = module.get<ContentTypeService>(ContentTypeService);
     contentType = await contentTypeService.findOne({ name: 'Product' });
     content = await contentService.findOne({
       title: 'Jean',
     });
-    attachment = await attachmentService.findOne({
-      name: 'store1.jpg',
-    });
 
     pageQuery = getPageQuery<Content>({
       limit: 1,
@@ -243,91 +223,76 @@ describe('ContentController', () => {
   });
 
   describe('import', () => {
-    it('should import content from a CSV file', async () => {
-      const mockCsvData: string = `other,title,status,image
-should not appear,store 3,true,image.jpg`;
-
-      const mockCsvContentDto: ContentCreateDto = {
-        entity: '0',
-        title: 'store 3',
-        status: true,
-        dynamicFields: {
-          image: 'image.jpg',
-        },
-      };
-      jest.spyOn(contentService, 'createMany');
-      jest.spyOn(fs, 'existsSync').mockReturnValueOnce(true);
-      jest.spyOn(fs, 'readFileSync').mockReturnValueOnce(mockCsvData);
+    const mockCsvData: string = `other,title,status,image
+      should not appear,store 3,true,image.jpg`;
+
+    const file: Express.Multer.File = {
+      buffer: Buffer.from(mockCsvData, 'utf-8'),
+      originalname: 'test.csv',
+      mimetype: 'text/csv',
+      size: mockCsvData.length,
+      fieldname: 'file',
+      encoding: '7bit',
+      stream: null,
+      destination: '',
+      filename: '',
+      path: '',
+    } as unknown as Express.Multer.File;
 
-      const contentType = await contentTypeService.findOne({
+    it('should import content from a CSV file', async () => {
+      const mockContentType = {
+        id: '0',
         name: 'Store',
-      });
-
-      const result = await contentController.import({
-        idFileToImport: attachment!.id,
-        idTargetContentType: contentType!.id,
-      });
-      expect(contentService.createMany).toHaveBeenCalledWith([
-        { ...mockCsvContentDto, entity: contentType!.id },
+      } as unknown as ContentType;
+      jest
+        .spyOn(contentTypeService, 'findOne')
+        .mockResolvedValueOnce(mockContentType);
+      jest.spyOn(contentService, 'parseAndSaveDataset').mockResolvedValueOnce([
+        {
+          entity: mockContentType.id,
+          title: 'store 3',
+          status: true,
+          dynamicFields: {
+            image: 'image.jpg',
+          },
+          id: '',
+          createdAt: null as unknown as Date,
+          updatedAt: null as unknown as Date,
+        },
       ]);
 
-      expect(result).toEqualPayload(
-        [
-          {
-            ...mockCsvContentDto,
-            entity: contentType!.id,
-          },
-        ],
-        [...IGNORED_TEST_FIELDS, 'rag'],
+      const result = await contentController.import(file, mockContentType.id);
+      expect(contentService.parseAndSaveDataset).toHaveBeenCalledWith(
+        mockCsvData,
+        mockContentType.id,
+        mockContentType,
       );
+      expect(result).toEqual([
+        {
+          entity: mockContentType.id,
+          title: 'store 3',
+          status: true,
+          dynamicFields: {
+            image: 'image.jpg',
+          },
+          id: '',
+          createdAt: null as unknown as Date,
+          updatedAt: null as unknown as Date,
+        },
+      ]);
     });
 
     it('should throw NotFoundException if content type is not found', async () => {
+      jest.spyOn(contentTypeService, 'findOne').mockResolvedValueOnce(null);
       await expect(
-        contentController.import({
-          idFileToImport: attachment!.id,
-          idTargetContentType: NOT_FOUND_ID,
-        }),
+        contentController.import(file, 'INVALID_ID'),
       ).rejects.toThrow(new NotFoundException('Content type is not found'));
     });
 
-    it('should throw NotFoundException if file is not found in attachment database', async () => {
-      const contentType = await contentTypeService.findOne({
-        name: 'Product',
-      });
-      jest.spyOn(contentTypeService, 'findOne');
-      await expect(
-        contentController.import({
-          idFileToImport: NOT_FOUND_ID,
-          idTargetContentType: contentType!.id.toString(),
-        }),
-      ).rejects.toThrow(new NotFoundException('File does not exist'));
-    });
-
-    it('should throw NotFoundException if file does not exist in the given path ', async () => {
-      jest.spyOn(fs, 'existsSync').mockReturnValue(false);
-      await expect(
-        contentController.import({
-          idFileToImport: attachment!.id,
-          idTargetContentType: contentType!.id,
-        }),
-      ).rejects.toThrow(new NotFoundException('File does not exist'));
+    it('should throw NotFoundException if idTargetContentType is missing', async () => {
+      await expect(contentController.import(file, '')).rejects.toThrow(
+        new NotFoundException('Missing parameter'),
+      );
     });
-
-    it.each([
-      ['file param and content type params are missing', '', ''],
-      ['content type param is missing', '', NOT_FOUND_ID],
-      ['file param is missing', NOT_FOUND_ID, ''],
-    ])(
-      'should throw NotFoundException if %s',
-      async (_message, fileToImport, targetContentType) => {
-        await expect(
-          contentController.import({
-            idFileToImport: fileToImport,
-            idTargetContentType: targetContentType,
-          }),
-        ).rejects.toThrow(new NotFoundException('Missing params'));
-      },
-    );
   });
 });

api/src/cms/controllers/content.controller.ts

@@ -1,14 +1,11 @@
 /*
- * Copyright © 2024 Hexastack. All rights reserved.
+ * Copyright © 2025 Hexastack. All rights reserved.
  *
  * Licensed under the GNU Affero General Public License v3.0 (AGPLv3) with the following additional terms:
  * 1. The name "Hexabot" is a trademark of Hexastack. You may not use this name in derivative works without express written permission.
  * 2. All derivative works must include clear attribution to the original creator and software, Hexastack and Hexabot, in a prominent location (e.g., in the software's "About" section, documentation, and README file).
  */
 
-import fs from 'fs';
-import path from 'path';
-
 import {
   Body,
   Controller,
@@ -20,14 +17,12 @@ import {
   Patch,
   Post,
   Query,
+  UploadedFile,
   UseInterceptors,
 } from '@nestjs/common';
-import { BadRequestException } from '@nestjs/common/exceptions';
+import { FileInterceptor } from '@nestjs/platform-express';
 import { CsrfCheck } from '@tekuconcept/nestjs-csrf';
-import Papa from 'papaparse';
 
-import { AttachmentService } from '@/attachment/services/attachment.service';
-import { config } from '@/config';
 import { CsrfInterceptor } from '@/interceptors/csrf.interceptor';
 import { LoggerService } from '@/logger/logger.service';
 import { BaseController } from '@/utils/generics/base-controller';
@@ -59,7 +54,6 @@ export class ContentController extends BaseController<
   constructor(
     private readonly contentService: ContentService,
     private readonly contentTypeService: ContentTypeService,
-    private readonly attachmentService: AttachmentService,
     private readonly logger: LoggerService,
   ) {
     super(contentService);
@@ -92,29 +86,22 @@ export class ContentController extends BaseController<
   /**
    * Imports content from a CSV file based on the provided content type and file ID.
    *
-   * @param idTargetContentType - The content type to match the CSV data against.
-   * @param idFileToImport - The ID of the file to be imported.
-   *
+   * @param idTargetContentType - The content type to match the CSV data against.   *
    * @returns A promise that resolves to the newly created content documents.
    */
-  @Get('import/:idTargetContentType/:idFileToImport')
+  @CsrfCheck(true)
+  @Post('import')
+  @UseInterceptors(FileInterceptor('file'))
   async import(
-    @Param()
-    {
-      idTargetContentType: targetContentType,
-      idFileToImport: fileToImport,
-    }: {
-      idTargetContentType: string;
-      idFileToImport: string;
-    },
+    @UploadedFile() file: Express.Multer.File,
+    @Query('idTargetContentType')
+    targetContentType: string,
   ) {
-    // Check params
-    if (!fileToImport || !targetContentType) {
-      this.logger.warn(`Parameters are missing`);
-      throw new NotFoundException(`Missing params`);
+    const datasetContent = file.buffer.toString('utf-8');
+    if (!targetContentType) {
+      this.logger.warn(`Parameter is missing`);
+      throw new NotFoundException(`Missing parameter`);
     }
-
-    // Find the content type that corresponds to the given content
     const contentType =
       await this.contentTypeService.findOne(targetContentType);
     if (!contentType) {
@@ -124,56 +111,11 @@ export class ContentController extends BaseController<
       throw new NotFoundException(`Content type is not found`);
     }
 
-    // Get file location
-    const file = await this.attachmentService.findOne(fileToImport);
-    // Check if file is present
-    const filePath = file
-      ? path.join(config.parameters.uploadDir, file.location)
-      : undefined;
-
-    if (!file || !filePath || !fs.existsSync(filePath)) {
-      this.logger.warn(`Failed to find file type with id ${fileToImport}.`);
-      throw new NotFoundException(`File does not exist`);
-    }
-    //read file sync
-    const data = fs.readFileSync(filePath, 'utf8');
-
-    const result = Papa.parse<Record<string, string | boolean | number>>(data, {
-      header: true,
-      skipEmptyLines: true,
-      dynamicTyping: true,
-    });
-
-    if (result.errors.length > 0) {
-      this.logger.warn(
-        `Errors parsing the file: ${JSON.stringify(result.errors)}`,
-      );
-
-      throw new BadRequestException(result.errors, {
-        cause: result.errors,
-        description: 'Error while parsing CSV',
-      });
-    }
-
-    const contentsDto = result.data.reduce(
-      (acc, { title, status, ...rest }) => [
-        ...acc,
-        {
-          title: String(title),
-          status: Boolean(status),
-          entity: targetContentType,
-          dynamicFields: Object.keys(rest)
-            .filter((key) =>
-              contentType.fields?.map((field) => field.name).includes(key),
-            )
-            .reduce((filtered, key) => ({ ...filtered, [key]: rest[key] }), {}),
-        },
-      ],
-      [],
+    return await this.contentService.parseAndSaveDataset(
+      datasetContent,
+      targetContentType,
+      contentType,
     );
-
-    // Create content
-    return await this.contentService.createMany(contentsDto);
   }
 
   /**

api/src/cms/services/content.service.spec.ts

@@ -10,9 +10,6 @@ import { EventEmitter2 } from '@nestjs/event-emitter';
 import { MongooseModule } from '@nestjs/mongoose';
 import { Test, TestingModule } from '@nestjs/testing';
 
-import { AttachmentRepository } from '@/attachment/repositories/attachment.repository';
-import { AttachmentModel } from '@/attachment/schemas/attachment.schema';
-import { AttachmentService } from '@/attachment/services/attachment.service';
 import { OutgoingMessageFormat } from '@/chat/schemas/types/message';
 import { ContentOptions } from '@/chat/schemas/types/options';
 import { LoggerService } from '@/logger/logger.service';
@@ -44,19 +41,13 @@ describe('ContentService', () => {
     const module: TestingModule = await Test.createTestingModule({
       imports: [
         rootMongooseTestModule(installContentFixtures),
-        MongooseModule.forFeature([
-          ContentTypeModel,
-          ContentModel,
-          AttachmentModel,
-        ]),
+        MongooseModule.forFeature([ContentTypeModel, ContentModel]),
       ],
       providers: [
         ContentTypeRepository,
         ContentRepository,
-        AttachmentRepository,
         ContentTypeService,
         ContentService,
-        AttachmentService,
         LoggerService,
         EventEmitter2,
       ],