Merge pull request #233 from import-ai/refactor/upload

refactor(files): update upload method

Author: LucienShui

package.json

@@ -53,6 +53,9 @@
     "typescript-eslint": "^8.46.3"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.926.0",
+    "@aws-sdk/s3-presigned-post": "^3.926.0",
+    "@aws-sdk/s3-request-presigner": "^3.926.0",
     "@keyv/redis": "^5.1.3",
     "@nestjs-modules/mailer": "^2.0.2",
     "@nestjs/cache-manager": "^3.0.1",

pnpm-lock.yaml

@@ -1,17 +1,24 @@
 import { HttpStatus, Injectable } from '@nestjs/common';
-import { AwsClient } from 'aws4fetch';
 import { ConfigService } from '@nestjs/config';
 import { Repository } from 'typeorm';
 import { File } from './entities/file.entity';
 import { InjectRepository } from '@nestjs/typeorm';
 import { AppException } from 'omniboxd/common/exceptions/app.exception';
 import { I18nService } from 'nestjs-i18n';
+import { S3Client } from '@aws-sdk/client-s3';
+import { createPresignedPost, PresignedPost } from '@aws-sdk/s3-presigned-post';
+import { AwsClient } from 'aws4fetch';
+import { formatFileSize } from '../utils/format-file-size';
 
 @Injectable()
 export class FilesService {
   private readonly awsClient: AwsClient;
   private readonly s3Url: URL;
   private readonly s3InternalUrl: URL;
+  private readonly s3Client: S3Client;
+  private readonly s3Bucket: string;
+  private readonly s3Prefix: string;
+  private readonly s3MaxFileSize: number;
 
   constructor(
     configService: ConfigService,
@@ -43,9 +50,39 @@ export class FilesService {
       s3InternalUrl += '/';
     }
 
+    const s3Endpoint = configService.get<string>('OBB_S3_ENDPOINT');
+    if (!s3Endpoint) {
+      throw new Error('S3 endpoint not set');
+    }
+
+    const s3Bucket = configService.get<string>('OBB_S3_BUCKET');
+    if (!s3Bucket) {
+      throw new Error('S3 bucket not set');
+    }
+
+    const s3Prefix = configService.get<string>('OBB_S3_PREFIX');
+    if (!s3Prefix) {
+      throw new Error('S3 prefix not set');
+    }
+
     this.awsClient = new AwsClient({ accessKeyId, secretAccessKey });
     this.s3Url = new URL(s3Url);
     this.s3InternalUrl = new URL(s3InternalUrl);
+    this.s3MaxFileSize = configService.get<number>(
+      'OBB_S3_MAX_FILE_SIZE',
+      20 * 1024 * 1024,
+    );
+    const s3Region = configService.get<string>('OBB_S3_REGION', 'us-east-1');
+    this.s3Client = new S3Client({
+      region: s3Region,
+      credentials: {
+        accessKeyId,
+        secretAccessKey,
+      },
+      endpoint: s3Endpoint,
+    });
+    this.s3Bucket = s3Bucket;
+    this.s3Prefix = s3Prefix;
   }
 
   async createFile(
@@ -81,6 +118,38 @@ export class FilesService {
     return signedReq.url;
   }
 
+  async generatePostForm(
+    fileId: string,
+    fileSize: number | undefined,
+    filename: string,
+    mimetype: string,
+  ): Promise<PresignedPost> {
+    if (fileSize && fileSize > this.s3MaxFileSize) {
+      const message = this.i18n.t('resource.errors.fileTooLarge', {
+        args: {
+          userSize: formatFileSize(fileSize),
+          limitSize: formatFileSize(this.s3MaxFileSize),
+        },
+      });
+      throw new AppException(message, 'FILE_TOO_LARGE', HttpStatus.BAD_REQUEST);
+    }
+    const disposition = `attachment; filename*=UTF-8''${encodeURIComponent(filename)}`;
+    return await createPresignedPost(this.s3Client, {
+      Bucket: this.s3Bucket,
+      Key: `${this.s3Prefix}/${fileId}`,
+      Conditions: [
+        ['content-length-range', 0, this.s3MaxFileSize],
+        { 'content-type': mimetype },
+        { 'content-disposition': disposition },
+      ],
+      Fields: {
+        'content-type': mimetype,
+        'content-disposition': disposition,
+      },
+      Expires: 900, // 900 seconds
+    });
+  }
+
   private async generateDownloadUrl(
     namespaceId: string,
     fileId: string,

src/files/files.service.ts

@@ -4,6 +4,7 @@
     "parentOrResourceIdRequired": "parent_id or resource_id is required",
     "resourceNotFound": "Resource not found",
     "fileNotFound": "File not found",
+    "fileTooLarge": "Your file ({userSize}) exceeds the maximum limit of {limitSize}",
     "cannotDeleteRoot": "Cannot delete root resource",
     "cannotDuplicateRoot": "Cannot duplicate root resource",
     "cannotRestoreRoot": "Cannot restore root resource",

src/i18n/en/resource.json

@@ -4,6 +4,7 @@
     "parentOrResourceIdRequired": "需要 parent_id 或 resource_id",
     "resourceNotFound": "资源未找到",
     "fileNotFound": "文件未找到",
+    "fileTooLarge": "您的文件（{userSize}）超过最大限制 {limitSize}",
     "cannotDeleteRoot": "无法删除根资源",
     "cannotDuplicateRoot": "无法复制根资源",
     "cannotRestoreRoot": "无法恢复根资源",

src/i18n/zh/resource.json

@@ -1,5 +1,11 @@
 import { Expose } from 'class-transformer';
-import { IsNotEmpty, IsString } from 'class-validator';
+import {
+  IsNotEmpty,
+  IsNumber,
+  IsOptional,
+  IsString,
+  Min,
+} from 'class-validator';
 
 export class CreateFileReqDto {
   @Expose()
@@ -11,4 +17,10 @@ export class CreateFileReqDto {
   @IsString()
   @IsNotEmpty()
   mimetype: string;
+
+  @Expose()
+  @IsNumber()
+  @Min(1)
+  @IsOptional()
+  size?: number;
 }

src/namespace-resources/dto/create-file-req.dto.ts

@@ -7,10 +7,25 @@ export class FileInfoDto {
   @Expose()
   url: string;
 
-  static new(id: string, url: string) {
+  @Expose({ name: 'post_url' })
+  postUrl?: string;
+
+  @Expose({ name: 'post_fields' })
+  postFields?: [string, string][];
+
+  static new(
+    id: string,
+    url: string,
+    postUrl?: string,
+    postFields?: Record<string, string>,
+  ) {
     const dto = new FileInfoDto();
     dto.id = id;
     dto.url = url;
+    dto.postUrl = postUrl;
+    if (postFields) {
+      dto.postFields = Object.entries(postFields);
+    }
     return dto;
   }
 }

src/namespace-resources/dto/file-info.dto.ts

@@ -26,12 +26,13 @@ describe('FileResourcesController (e2e)', () => {
 
   test.each(uploadLanguageDatasets)(
     'upload and download file: $filename',
-    async ({ filename }) => {
+    async ({ filename, content }) => {
       const uploadRes = await client
         .post(`/api/v1/namespaces/${client.namespace.id}/resources/files`)
         .send({
           name: filename,
           mimetype: 'text/plain',
+          size: content.length,
         });
       expect(uploadRes.status).toBe(201);
     },

src/namespace-resources/file-resources.e2e-spec.ts

@@ -715,7 +715,13 @@ export class NamespaceResourcesService {
       createReq.mimetype,
     );
     const url = await this.filesService.generateUploadUrl(file.id);
-    return FileInfoDto.new(file.id, url);
+    const postReq = await this.filesService.generatePostForm(
+      file.id,
+      createReq.size,
+      file.name,
+      file.mimetype,
+    );
+    return FileInfoDto.new(file.id, url, postReq.url, postReq.fields);
   }
 
   async update(userId: string, resourceId: string, data: UpdateResourceDto) {

src/namespace-resources/namespace-resources.service.ts

@@ -0,0 +1,30 @@
+import { formatFileSize } from './format-file-size';
+
+describe('formatFileSize', () => {
+  it('should format bytes correctly', () => {
+    expect(formatFileSize(0)).toBe('0 B');
+    expect(formatFileSize(100)).toBe('100 B');
+    expect(formatFileSize(1023)).toBe('1023 B');
+  });
+
+  it('should format kilobytes correctly', () => {
+    expect(formatFileSize(1024)).toBe('1.0 KB');
+    expect(formatFileSize(1536)).toBe('1.5 KB');
+    expect(formatFileSize(10240)).toBe('10.0 KB');
+  });
+
+  it('should format megabytes correctly', () => {
+    expect(formatFileSize(1024 * 1024)).toBe('1.0 MB');
+    expect(formatFileSize(20 * 1024 * 1024)).toBe('20.0 MB');
+    expect(formatFileSize(1.5 * 1024 * 1024)).toBe('1.5 MB');
+  });
+
+  it('should format gigabytes correctly', () => {
+    expect(formatFileSize(1024 * 1024 * 1024)).toBe('1.0 GB');
+    expect(formatFileSize(2.5 * 1024 * 1024 * 1024)).toBe('2.5 GB');
+  });
+
+  it('should format terabytes correctly', () => {
+    expect(formatFileSize(1024 * 1024 * 1024 * 1024)).toBe('1.0 TB');
+  });
+});