Merge pull request #192 from import-ai/chore/tasks

LucienShui · web-flow · commit 06698a8b389e · 2025-09-23T18:39:47.000+08:00
Chore/tasks
diff --git a/src/api-key/api-key.service.spec.ts b/src/api-key/api-key.service.spec.ts
@@ -22,7 +22,6 @@ describe('APIKeyService', () => {
   let applicationsRepository: jest.Mocked<Repository<Applications>>;
   let permissionsService: jest.Mocked<PermissionsService>;
   let namespacesService: jest.Mocked<NamespacesService>;
-  let userService: jest.Mocked<UserService>;
 
   const mockApiKey = {
     id: 'test-api-key-id',
@@ -100,7 +99,6 @@ describe('APIKeyService', () => {
     applicationsRepository = module.get(getRepositoryToken(Applications));
     permissionsService = module.get(PermissionsService);
     namespacesService = module.get(NamespacesService);
-    userService = module.get(UserService);
   });
 
   it('should be defined', () => {
diff --git a/src/groups/groups.service.ts b/src/groups/groups.service.ts
@@ -1,4 +1,4 @@
-import { Injectable, NotFoundException } from '@nestjs/common';
+import { Injectable } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
 import {
   DataSource,
diff --git a/src/namespace-resources/open.resource.controller.ts b/src/namespace-resources/open.resource.controller.ts
@@ -49,7 +49,7 @@ export class OpenResourcesController {
       content: data.content,
       tagIds: data.tag_ids || [],
       attrs: data.attrs || {},
-      resourceType: ResourceType.FILE,
+      resourceType: ResourceType.DOC,
       namespaceId: apiKey.namespaceId,
       parentId: apiKey.attrs.root_resource_id,
     };
diff --git a/src/tasks/task-pipeline.e2e-spec.ts b/src/tasks/task-pipeline.e2e-spec.ts
@@ -17,8 +17,11 @@ class MockWizardWorker {
   private isPolling = false;
   private pollingInterval: NodeJS.Timeout | null = null;
   private readonly pollIntervalMs = 500; // Fast polling for tests
+  private readonly namespaceId: string;
 
-  constructor(private readonly client: TestClient) {}
+  constructor(private readonly client: TestClient) {
+    this.namespaceId = client.namespace.id;
+  }
 
   /**
    * Starts polling for tasks and processing them
@@ -87,7 +90,7 @@ class MockWizardWorker {
   private async fetchTask(): Promise<TaskDto | null> {
     try {
       const response = await this.makeRequest()
-        .get('/internal/api/v1/wizard/task')
+        .get(`/internal/api/v1/wizard/task?namespace_id=${this.namespaceId}`)
         .timeout(5000); // 5 second timeout
 
       if (response.status === 204) {
@@ -282,20 +285,16 @@ describe('Task Pipeline (e2e)', () => {
   let client: TestClient;
   let mockWorker: MockWizardWorker;
 
-  beforeAll(async () => {
+  beforeEach(async () => {
     client = await TestClient.create();
     mockWorker = new MockWizardWorker(client);
   });
 
-  afterAll(async () => {
-    await mockWorker.deleteAllTasks();
+  afterEach(async () => {
+    mockWorker.stopPolling();
     await client.close();
   });
 
-  beforeEach(async () => {
-    await mockWorker.deleteAllTasks();
-  });
-
   describe('Basic Task Processing', () => {
     it('create resource trigger upsertIndex', async () => {
       mockWorker.startPolling();
diff --git a/src/user/user.e2e-spec.ts b/src/user/user.e2e-spec.ts
@@ -93,7 +93,7 @@ describe('UserController (e2e)', () => {
     });
 
     it('should return null for non-existent user', async () => {
-      const response = await client
+      await client
         .get('/api/v1/user/00000000-0000-0000-0000-000000000000')
         .expect(HttpStatus.NOT_FOUND);
     });
diff --git a/src/wizard/dto/collect-request.dto.ts b/src/wizard/dto/collect-request.dto.ts
@@ -1,10 +1,6 @@
 import { IsNotEmpty, IsString } from 'class-validator';
 
-export class CollectRequestDto {
-  @IsString()
-  @IsNotEmpty()
-  html: string;
-
+export class CollectZRequestDto {
   @IsString()
   @IsNotEmpty()
   url: string;
@@ -21,3 +17,9 @@ export class CollectRequestDto {
   @IsNotEmpty()
   parentId: string;
 }
+
+export class CollectRequestDto extends CollectZRequestDto {
+  @IsString()
+  @IsNotEmpty()
+  html: string;
+}
diff --git a/src/wizard/internal.wizard.e2e-spec.ts b/src/wizard/internal.wizard.e2e-spec.ts
@@ -1,18 +1,72 @@
 import { TestClient } from 'test/test-client';
 import { HttpStatus } from '@nestjs/common';
 import { Image } from 'omniboxd/wizard/types/wizard.types';
+import { gzipSync } from 'zlib';
 
 describe('InternalWizardController (e2e)', () => {
   let client: TestClient;
 
-  beforeAll(async () => {
+  beforeEach(async () => {
     client = await TestClient.create();
   });
 
-  afterAll(async () => {
+  afterEach(async () => {
     await client.close();
   });
 
+  it('gzip_collect_callback', async () => {
+    const html: string =
+      '<html><body><h1>Test Page</h1><p>This is test content.</p></body></html>';
+    const collectData = {
+      url: 'https://example.com/test-page',
+      title: 'Test Page Title',
+      namespace_id: client.namespace.id,
+      parentId: client.namespace.root_resource_id,
+    };
+
+    const compressedHtml = gzipSync(html);
+
+    const taskCreateResponse = await client
+      .post('/api/v1/wizard/collect/gzip')
+      .field('url', collectData.url)
+      .field('title', collectData.title)
+      .field('namespace_id', collectData.namespace_id)
+      .field('parentId', collectData.parentId)
+      .attach('html', compressedHtml, {
+        filename: 'html.gz',
+        contentType: 'application/gzip',
+      })
+      .expect(HttpStatus.CREATED);
+
+    const taskId = taskCreateResponse.body.task_id;
+
+    const task = await client
+      .get(`/internal/api/v1/wizard/task?namespace_id=${client.namespace.id}`)
+      .expect(HttpStatus.OK);
+
+    expect(task.body.id).toBe(taskId);
+    expect(task.body.input.html).toBe(html);
+
+    const response = await client
+      .post('/internal/api/v1/wizard/callback')
+      .send({
+        id: taskId,
+        output: {
+          title: 'Test Page Title',
+          markdown: '![图片](http://example.com/image.png)',
+          images: [
+            {
+              name: '图片',
+              link: 'http://example.com/image.png',
+              data: 'iVBORw0KGgoAAAANSUhEUgAAAAUA',
+              mimetype: 'image/png',
+            },
+          ] as Image[],
+        },
+      });
+    expect(response.status).toBe(HttpStatus.CREATED);
+  });
+
   it('collect_callback', async () => {
     const collectData = {
       html: '<html><body><h1>Test Page</h1><p>This is test content.</p></body></html>',
@@ -29,10 +83,13 @@ describe('InternalWizardController (e2e)', () => {
 
     const taskId = taskCreateResponse.body.task_id;
 
-    await client
+    const task = await client
       .get(`/internal/api/v1/wizard/task?namespace_id=${client.namespace.id}`)
       .expect(HttpStatus.OK);
 
+    expect(task.body.id).toBe(taskId);
+    expect(task.body.input.html).toBe(collectData.html);
+
     const response = await client
       .post('/internal/api/v1/wizard/callback')
       .send({
diff --git a/src/wizard/wizard.controller.ts b/src/wizard/wizard.controller.ts
@@ -1,10 +1,22 @@
-import { Body, Controller, Post, Req, Sse } from '@nestjs/common';
+import {
+  Body,
+  Controller,
+  Post,
+  Req,
+  Sse,
+  UploadedFile,
+  UseInterceptors,
+} from '@nestjs/common';
 import { WizardService } from 'omniboxd/wizard/wizard.service';
-import { CollectRequestDto } from 'omniboxd/wizard/dto/collect-request.dto';
+import {
+  CollectRequestDto,
+  CollectZRequestDto,
+} from 'omniboxd/wizard/dto/collect-request.dto';
 import { CollectResponseDto } from 'omniboxd/wizard/dto/collect-response.dto';
 import { AgentRequestDto } from 'omniboxd/wizard/dto/agent-request.dto';
 import { RequestId } from 'omniboxd/decorators/request-id.decorators';
 import { UserId } from 'omniboxd/decorators/user-id.decorator';
+import { FileInterceptor } from '@nestjs/platform-express';
 
 @Controller('api/v1/wizard')
 export class WizardController {
@@ -18,6 +30,16 @@ export class WizardController {
     return await this.wizardService.collect(userId, data);
   }
 
+  @Post('collect/gzip')
+  @UseInterceptors(FileInterceptor('html'))
+  async collectGzip(
+    @UserId() userId: string,
+    @Body() data: CollectZRequestDto,
+    @UploadedFile() zHtml: Express.Multer.File,
+  ): Promise<CollectResponseDto> {
+    return await this.wizardService.collectZ(userId, data, zHtml);
+  }
+
   @Post('ask')
   @Sse()
   async ask(
diff --git a/src/wizard/wizard.service.ts b/src/wizard/wizard.service.ts
@@ -3,7 +3,10 @@ import { Task } from 'omniboxd/tasks/tasks.entity';
 import { NamespaceResourcesService } from 'omniboxd/namespace-resources/namespace-resources.service';
 import { TagService } from 'omniboxd/tag/tag.service';
 import { CreateResourceDto } from 'omniboxd/namespace-resources/dto/create-resource.dto';
-import { CollectRequestDto } from 'omniboxd/wizard/dto/collect-request.dto';
+import {
+  CollectRequestDto,
+  CollectZRequestDto,
+} from 'omniboxd/wizard/dto/collect-request.dto';
 import { CollectResponseDto } from 'omniboxd/wizard/dto/collect-response.dto';
 import { TaskCallbackDto } from 'omniboxd/wizard/dto/task-callback.dto';
 import { ConfigService } from '@nestjs/config';
@@ -22,6 +25,8 @@ import { Image, ProcessedImage } from 'omniboxd/wizard/types/wizard.types';
 import { InternalTaskDto } from 'omniboxd/tasks/dto/task.dto';
 import { isEmpty } from 'omniboxd/utils/is-empty';
 import { FetchTaskRequest } from 'omniboxd/wizard/dto/fetch-task-request.dto';
+import { MinioService } from 'omniboxd/minio/minio.service';
+import { createGunzip } from 'zlib';
 
 @Injectable()
 export class WizardService {
@@ -30,13 +35,16 @@ export class WizardService {
   readonly streamService: StreamService;
   readonly wizardApiService: WizardAPIService;
 
+  private readonly gzipHtmlFolder: string = 'collect/html/gzip';
+
   constructor(
     private readonly wizardTaskService: WizardTaskService,
     private readonly namespaceResourcesService: NamespaceResourcesService,
     private readonly tagService: TagService,
     private readonly messagesService: MessagesService,
     private readonly configService: ConfigService,
     private readonly attachmentsService: AttachmentsService,
+    private readonly minioService: MinioService,
   ) {
     this.processors = {
       collect: new CollectProcessor(
@@ -69,6 +77,51 @@ export class WizardService {
     return await this.wizardTaskService.create(partialTask);
   }
 
+  async collectZ(
+    userId: string,
+    data: CollectZRequestDto,
+    file: Express.Multer.File,
+  ) {
+    if (!file) {
+      throw new BadRequestException('Missing file');
+    }
+    const { url, title, namespace_id, parentId } = data;
+    if (!namespace_id || !parentId || !url) {
+      throw new BadRequestException('Missing required fields');
+    }
+
+    const resourceDto: CreateResourceDto = {
+      name: title || url,
+      namespaceId: namespace_id,
+      resourceType: ResourceType.LINK,
+      parentId: parentId,
+      attrs: { url },
+    };
+    const resource = await this.namespaceResourcesService.create(
+      userId,
+      resourceDto,
+    );
+
+    const filename = 'html.gz';
+    const { id } = await this.minioService.put(
+      filename,
+      file.buffer,
+      file.mimetype,
+      {
+        folder: this.gzipHtmlFolder,
+        metadata: { resourceId: resource.id, url },
+      },
+    );
+
+    const task = await this.wizardTaskService.createCollectTask(
+      userId,
+      namespace_id,
+      resource.id,
+      { html: [this.gzipHtmlFolder, id].join('/'), url, title },
+    );
+    return { task_id: task.id, resource_id: resource.id };
+  }
+
   async collect(
     userId: string,
     data: CollectRequestDto,
@@ -305,9 +358,35 @@ export class WizardService {
         namespaceId: record.namespace_id,
       });
       const newTask = await this.wizardTaskService.taskRepository.save(task);
+      // Fetch HTML content from S3 for collect tasks
+      if (
+        newTask.function === 'collect' &&
+        newTask.input.html?.startsWith(this.gzipHtmlFolder) &&
+        newTask.input.html?.length === this.gzipHtmlFolder.length + 36 // 1 + 32 + 3
+      ) {
+        const htmlContent = await this.getHtmlFromMinioGzipFile(
+          newTask.input.html,
+        );
+        newTask.input = { ...newTask.input, html: htmlContent };
+      }
       return InternalTaskDto.fromEntity(newTask);
     }
 
     return null;
   }
+
+  async getHtmlFromMinioGzipFile(path: string) {
+    const stream = await this.minioService.getObject(path);
+    const gunzip = createGunzip();
+    return new Promise<string>((resolve, reject) => {
+      const chunks: Buffer[] = [];
+      stream
+        .pipe(gunzip)
+        .on('data', (chunk: Buffer) => chunks.push(chunk))
+        .on('end', () => {
+          resolve(Buffer.concat(chunks).toString('utf-8'));
+        })
+        .on('error', reject);
+    });
+  }
 }

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-import { Injectable, NotFoundException } from '@nestjs/common';`
	`1`	`+import { Injectable } from '@nestjs/common';`
`2`	`2`	`import { InjectRepository } from '@nestjs/typeorm';`
`3`	`3`	`import {`
`4`	`4`	`DataSource,`