From 0d9b44c0e3b6347e4a9aee05fde97199f020fcfc Mon Sep 17 00:00:00 2001 From: Giteabot Date: Sun, 12 Mar 2023 07:45:39 -0400 Subject: [PATCH] Preserve file size when creating attachments (#23406) (#23426) Backport #23406 by @baez90 When creating attachments (issue, release, repo) the file size (being part of the multipart file header) is passed through the chain of creating an attachment to ensure the MinIO client can stream the file directly instead of having to read it to memory completely at first. Fixes #23393 Co-authored-by: Peter Co-authored-by: KN4CK3R Co-authored-by: techknowlogick --- routers/api/v1/repo/issue_attachment.go | 2 +- routers/api/v1/repo/issue_comment_attachment.go | 2 +- routers/api/v1/repo/release_attachment.go | 2 +- routers/web/repo/attachment.go | 2 +- services/attachment/attachment.go | 8 ++++---- services/attachment/attachment_test.go | 2 +- services/mailer/incoming/incoming_handler.go | 2 +- services/release/release_test.go | 7 +++++-- 8 files changed, 15 insertions(+), 12 deletions(-) diff --git a/routers/api/v1/repo/issue_attachment.go b/routers/api/v1/repo/issue_attachment.go index 8cbd2e11b699..92e11386882c 100644 --- a/routers/api/v1/repo/issue_attachment.go +++ b/routers/api/v1/repo/issue_attachment.go @@ -176,7 +176,7 @@ func CreateIssueAttachment(ctx *context.APIContext) { filename = query } - attachment, err := attachment.UploadAttachment(file, setting.Attachment.AllowedTypes, &repo_model.Attachment{ + attachment, err := attachment.UploadAttachment(file, setting.Attachment.AllowedTypes, header.Size, &repo_model.Attachment{ Name: filename, UploaderID: ctx.Doer.ID, RepoID: ctx.Repo.Repository.ID, diff --git a/routers/api/v1/repo/issue_comment_attachment.go b/routers/api/v1/repo/issue_comment_attachment.go index 4c8452380f03..6fe4dbc97715 100644 --- a/routers/api/v1/repo/issue_comment_attachment.go +++ b/routers/api/v1/repo/issue_comment_attachment.go @@ -180,7 +180,7 @@ func CreateIssueCommentAttachment(ctx *context.APIContext) { filename = query } - attachment, err := attachment.UploadAttachment(file, setting.Attachment.AllowedTypes, &repo_model.Attachment{ + attachment, err := attachment.UploadAttachment(file, setting.Attachment.AllowedTypes, header.Size, &repo_model.Attachment{ Name: filename, UploaderID: ctx.Doer.ID, RepoID: ctx.Repo.Repository.ID, diff --git a/routers/api/v1/repo/release_attachment.go b/routers/api/v1/repo/release_attachment.go index 597578aac578..305b2808df54 100644 --- a/routers/api/v1/repo/release_attachment.go +++ b/routers/api/v1/repo/release_attachment.go @@ -194,7 +194,7 @@ func CreateReleaseAttachment(ctx *context.APIContext) { } // Create a new attachment and save the file - attach, err := attachment.UploadAttachment(file, setting.Repository.Release.AllowedTypes, &repo_model.Attachment{ + attach, err := attachment.UploadAttachment(file, setting.Repository.Release.AllowedTypes, header.Size, &repo_model.Attachment{ Name: filename, UploaderID: ctx.Doer.ID, RepoID: release.RepoID, diff --git a/routers/web/repo/attachment.go b/routers/web/repo/attachment.go index 589632ad6e10..c6d8828fac60 100644 --- a/routers/web/repo/attachment.go +++ b/routers/web/repo/attachment.go @@ -44,7 +44,7 @@ func uploadAttachment(ctx *context.Context, repoID int64, allowedTypes string) { } defer file.Close() - attach, err := attachment.UploadAttachment(file, allowedTypes, &repo_model.Attachment{ + attach, err := attachment.UploadAttachment(file, allowedTypes, header.Size, &repo_model.Attachment{ Name: header.Filename, UploaderID: ctx.Doer.ID, RepoID: repoID, diff --git a/services/attachment/attachment.go b/services/attachment/attachment.go index 7fdacc6aae50..3e7df0cee0c0 100644 --- a/services/attachment/attachment.go +++ b/services/attachment/attachment.go @@ -19,14 +19,14 @@ import ( ) // NewAttachment creates a new attachment object, but do not verify. -func NewAttachment(attach *repo_model.Attachment, file io.Reader) (*repo_model.Attachment, error) { +func NewAttachment(attach *repo_model.Attachment, file io.Reader, size int64) (*repo_model.Attachment, error) { if attach.RepoID == 0 { return nil, fmt.Errorf("attachment %s should belong to a repository", attach.Name) } err := db.WithTx(db.DefaultContext, func(ctx context.Context) error { attach.UUID = uuid.New().String() - size, err := storage.Attachments.Save(attach.RelativePath(), file, -1) + size, err := storage.Attachments.Save(attach.RelativePath(), file, size) if err != nil { return fmt.Errorf("Create: %w", err) } @@ -39,7 +39,7 @@ func NewAttachment(attach *repo_model.Attachment, file io.Reader) (*repo_model.A } // UploadAttachment upload new attachment into storage and update database -func UploadAttachment(file io.Reader, allowedTypes string, opts *repo_model.Attachment) (*repo_model.Attachment, error) { +func UploadAttachment(file io.Reader, allowedTypes string, fileSize int64, opts *repo_model.Attachment) (*repo_model.Attachment, error) { buf := make([]byte, 1024) n, _ := util.ReadAtMost(file, buf) buf = buf[:n] @@ -48,5 +48,5 @@ func UploadAttachment(file io.Reader, allowedTypes string, opts *repo_model.Atta return nil, err } - return NewAttachment(opts, io.MultiReader(bytes.NewReader(buf), file)) + return NewAttachment(opts, io.MultiReader(bytes.NewReader(buf), file), fileSize) } diff --git a/services/attachment/attachment_test.go b/services/attachment/attachment_test.go index 72d1b2ab3a44..1b9af34427e5 100644 --- a/services/attachment/attachment_test.go +++ b/services/attachment/attachment_test.go @@ -36,7 +36,7 @@ func TestUploadAttachment(t *testing.T) { RepoID: 1, UploaderID: user.ID, Name: filepath.Base(fPath), - }, f) + }, f, -1) assert.NoError(t, err) attachment, err := repo_model.GetAttachmentByUUID(db.DefaultContext, attach.UUID) diff --git a/services/mailer/incoming/incoming_handler.go b/services/mailer/incoming/incoming_handler.go index d89a5eab3d1b..3872288f923c 100644 --- a/services/mailer/incoming/incoming_handler.go +++ b/services/mailer/incoming/incoming_handler.go @@ -87,7 +87,7 @@ func (h *ReplyHandler) Handle(ctx context.Context, content *MailContent, doer *u attachmentIDs := make([]string, 0, len(content.Attachments)) if setting.Attachment.Enabled { for _, attachment := range content.Attachments { - a, err := attachment_service.UploadAttachment(bytes.NewReader(attachment.Content), setting.Attachment.AllowedTypes, &repo_model.Attachment{ + a, err := attachment_service.UploadAttachment(bytes.NewReader(attachment.Content), setting.Attachment.AllowedTypes, int64(len(attachment.Content)), &repo_model.Attachment{ Name: attachment.Name, UploaderID: doer.ID, RepoID: issue.Repo.ID, diff --git a/services/release/release_test.go b/services/release/release_test.go index 9b8aaa364983..805269413d72 100644 --- a/services/release/release_test.go +++ b/services/release/release_test.go @@ -106,11 +106,13 @@ func TestRelease_Create(t *testing.T) { IsTag: false, }, nil, "")) + testPlayload := "testtest" + attach, err := attachment.NewAttachment(&repo_model.Attachment{ RepoID: repo.ID, UploaderID: user.ID, Name: "test.txt", - }, strings.NewReader("testtest")) + }, strings.NewReader(testPlayload), int64(len([]byte(testPlayload)))) assert.NoError(t, err) release := repo_model.Release{ @@ -239,11 +241,12 @@ func TestRelease_Update(t *testing.T) { assert.Equal(t, tagName, release.TagName) // Add new attachments + samplePayload := "testtest" attach, err := attachment.NewAttachment(&repo_model.Attachment{ RepoID: repo.ID, UploaderID: user.ID, Name: "test.txt", - }, strings.NewReader("testtest")) + }, strings.NewReader(samplePayload), int64(len([]byte(samplePayload)))) assert.NoError(t, err) assert.NoError(t, UpdateRelease(user, gitRepo, release, []string{attach.UUID}, nil, nil))