From 06dc10ddeb105a924dd37cdcfec45e9cd4b68ffa Mon Sep 17 00:00:00 2001 From: vvbbnn00 Date: Tue, 12 Mar 2024 01:33:05 +0800 Subject: [PATCH] feat: Support small file uploads and cancellation of upload sessions; modify previous irrational route design --- config/goflet.json | 3 +- .../{download.go => operation_download.go} | 0 route/file/operation_other.go | 143 ++++++++++++++++ route/file/partical.go | 130 ++++++++++++++ route/file/route.go | 160 ++---------------- route/route.go | 3 + storage/upload/upload.go | 17 +- 7 files changed, 307 insertions(+), 149 deletions(-) rename route/file/{download.go => operation_download.go} (100%) create mode 100644 route/file/operation_other.go create mode 100644 route/file/partical.go diff --git a/config/goflet.json b/config/goflet.json index 2b41483..7381779 100644 --- a/config/goflet.json +++ b/config/goflet.json @@ -41,7 +41,8 @@ "allowFolderCreation": true, "uploadPath": "upload", "uploadLimit": 1073741824, - "uploadTimeout": 7200 + "uploadTimeout": 7200, + "maxPostSize": 20971520 }, "cacheConfig": { "cacheType": "MemoryCache", diff --git a/route/file/download.go b/route/file/operation_download.go similarity index 100% rename from route/file/download.go rename to route/file/operation_download.go diff --git a/route/file/operation_other.go b/route/file/operation_other.go new file mode 100644 index 0000000..5c9dcd3 --- /dev/null +++ b/route/file/operation_other.go @@ -0,0 +1,143 @@ +package file + +import ( + "io" + "mime/multipart" + "net/http" + + "github.com/gin-gonic/gin" + + "github.com/vvbbnn00/goflet/config" + "github.com/vvbbnn00/goflet/storage" + "github.com/vvbbnn00/goflet/storage/upload" + "github.com/vvbbnn00/goflet/util/log" +) + +// routePostFile handler for POST /file/*path +// @Summary Upload Small File +// @Description Upload a small file using a POST request, {path} should be the relative path of the file, starting from the root directory, e.g. /file/path/to/file.txt +// @Tags File, Upload +// @Param path path string true "File path" +// @Accept multipart/form-data +// @Param file formData file true "File" +// @Success 201 {object} string "Created" +// @Failure 400 {object} string "Bad request" +// @Failure 404 {object} string "File not found or upload not started" +// @Failure 409 {object} string "File completion in progress" +// @Failure 413 {object} string "File too large, please use PUT method to upload large files" +// @Failure 500 {object} string "Internal server error" +// @Router /file/{path} [post] +// @Security Authorization +func routePostFile(c *gin.Context) { + // Set the request body limit + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, config.GofletCfg.FileConfig.MaxPostSize) + file, err := c.FormFile("file") + + // If error is not nil and the error is "http: request body too large", return a 413 status code + if err != nil && err.Error() == "http: request body too large" { + c.JSON(http.StatusRequestEntityTooLarge, gin.H{"error": "File too large, please use PUT method to upload large files"}) + return + } + if err != nil { + log.Warnf("Error getting file: %s", err.Error()) + c.JSON(http.StatusBadRequest, gin.H{"error": "Bad request"}) + return + } + // If the file is not nil, handle the single file upload + handleSingleFileUpload(file, c) + + // Complete the file upload + relativePath := c.GetString("relativePath") + handleCompleteFileUpload(relativePath, c) +} + +// routeDeleteFile handler for DELETE /file/*path +// @Summary Delete File +// @Description Delete a file by path, {path} should be the relative path of the file, starting from the root directory, e.g. /file/path/to/file.txt +// @Tags File +// @Param path path string true "File path" +// @Success 204 {object} string "Deleted" +// @Failure 400 {object} string "Bad request" +// @Failure 404 {object} string "File not found or upload not started" +// @Failure 500 {object} string "Internal server error" +// @Router /file/{path} [delete] +// @Security Authorization +func routeDeleteFile(c *gin.Context) { + fsPath := c.GetString("fsPath") + + err := storage.DeleteFile(fsPath) + if err != nil { + errStr := err.Error() + if errStr == "file_not_found" { + c.JSON(http.StatusNotFound, gin.H{"error": "File not found"}) + return + } + log.Warnf("Error deleting file: %s", errStr) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Error deleting file"}) + return + } + + c.Status(http.StatusNoContent) +} + +// handleSingleFileUpload handles the single file upload +func handleSingleFileUpload(file *multipart.FileHeader, c *gin.Context) { + // Get temp file write stream + relativePath := c.GetString("relativePath") + writeStream, err := upload.GetTempFileWriteStream(relativePath) + if err != nil { + errStr := err.Error() + if errStr == "directory_creation" { + c.JSON(http.StatusForbidden, gin.H{"error": "Directory creation not allowed"}) + return + } + log.Warnf("Error getting write stream: %s", errStr) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Error writing file"}) + return + } + + // Open the file + fileReader, err := file.Open() + if err != nil { + log.Warnf("Error opening file: %s", err.Error()) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Error reading file"}) + return + } + + // Copy the file to the write stream + _, err = io.Copy(writeStream, fileReader) + if err != nil { + log.Warnf("Error copying file: %s", err.Error()) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Error writing file"}) + return + } + + // Close the file + _ = fileReader.Close() + // Close the write stream + _ = writeStream.Close() + + // Complete the file upload + handleCompleteFileUpload(relativePath, c) +} + +// handleCompleteFileUpload handles the completion of the file upload +func handleCompleteFileUpload(relativePath string, c *gin.Context) { + err := upload.CompleteFileUpload(relativePath) + if err != nil { + errStr := err.Error() + if errStr == "file_uploading" { + c.JSON(http.StatusConflict, gin.H{"error": "The file completion is in progress"}) + return + } + if errStr == "file_not_found" { + c.JSON(http.StatusNotFound, gin.H{"error": "File not found or upload not started"}) + return + } + log.Warnf("Error completing file upload: %s", errStr) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Error completing file upload"}) + return + } + + c.Status(http.StatusCreated) +} diff --git a/route/file/partical.go b/route/file/partical.go new file mode 100644 index 0000000..de59416 --- /dev/null +++ b/route/file/partical.go @@ -0,0 +1,130 @@ +package file + +import ( + "io" + "net/http" + "os" + + "github.com/gin-gonic/gin" + "github.com/pkg/errors" + + "github.com/vvbbnn00/goflet/storage/upload" + "github.com/vvbbnn00/goflet/util" + "github.com/vvbbnn00/goflet/util/log" +) + +// routePutUpload handler for PUT /file/*path +// @Summary Partial File Upload +// @Description Create an upload session with a partial file upload, supports range requests, {path} should be the relative path of the file, starting from the root directory, e.g. /file/path/to/file.txt +// @Tags Upload +// @Accept */* +// @Param path path string true "File path" +// @Success 202 {object} string "Accepted" +// @Failure 400 {object} string "Bad request" +// @Failure 403 {object} string "Directory creation not allowed" +// @Failure 500 {object} string "Internal server error" +// @Router /upload/{path} [put] +// @Security Authorization +func routePutUpload(c *gin.Context) { + relativePath := c.GetString("relativePath") + + // Parse the range + byteStart, byteEnd, _, err := util.HeaderParseRangeUpload(c.GetHeader("Content-Range"), c.GetHeader("Content-Length")) + if err != nil { + c.JSON(http.StatusRequestedRangeNotSatisfiable, gin.H{"error": err.Error()}) + return + } + + // Get the write stream + writeStream, err := upload.GetTempFileWriteStream(relativePath) + if err != nil { + errStr := err.Error() + if errStr == "directory_creation" { + c.JSON(http.StatusForbidden, gin.H{"error": "Directory creation not allowed"}) + return + } + log.Warnf("Error getting write stream: %s", errStr) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Error writing file"}) + return + } + defer func() { + if closeErr := writeStream.Close(); closeErr != nil { + log.Warnf("Error closing write stream: %s", closeErr.Error()) + } + }() + + // Write the file + body := c.Request.Body + defer func(body io.ReadCloser) { + _ = body.Close() + }(body) + + // Seek to the start of the range + _, err = writeStream.Seek(byteStart, io.SeekStart) + if err != nil { + log.Warnf("Error seeking write stream: %s", err.Error()) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Error writing file"}) + return + } + + // Write the range to the file + written, err := io.CopyN(writeStream, body, byteEnd-byteStart+1) + if err != nil { + log.Warnf("Error writing to file: %s", err.Error()) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Error writing file"}) + return + } + if written != byteEnd-byteStart+1 { + log.Warnf("Incomplete write: expected %d bytes, wrote %d bytes", byteEnd-byteStart+1, written) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Incomplete write"}) + return + } + + log.Debugf("Successfully written %d bytes to %s", written, relativePath) + c.Status(http.StatusAccepted) +} + +// routePostUpload handler for POST /upload/*path +// @Summary Complete Partial File Upload +// @Description Complete an upload session with a partial file upload. You should first upload the file with a PUT request, then complete the upload with a POST request, {path} should be the relative path of the file, starting from the root directory, e.g. /file/path/to/file.txt +// @Tags Upload +// @Param path path string true "File path" +// @Success 201 {object} string "Created" +// @Failure 400 {object} string "Bad request" +// @Failure 404 {object} string "File not found or upload not started" +// @Failure 409 {object} string "File completion in progress" +// @Failure 500 {object} string "Internal server error" +// @Router /upload/{path} [post] +// @Security Authorization +func routePostUpload(c *gin.Context) { + // Complete the file upload + relativePath := c.GetString("relativePath") + handleCompleteFileUpload(relativePath, c) +} + +// routeDeleteUpload handler for DELETE /upload/*path +// @Summary Cancel Upload +// @Description Cancel an upload session, {path} should be the relative path of the file, starting from the root directory, e.g. /upload/path/to/file.txt +// @Tags Upload +// @Param path path string true "File path" +// @Success 204 {object} string "Deleted" +// @Failure 400 {object} string "Bad request" +// @Failure 404 {object} string "Upload session not found" +// @Failure 500 {object} string "Internal server error" +// @Router /upload/{path} [delete] +func routeDeleteUpload(c *gin.Context) { + relativePath := c.GetString("relativePath") + err := upload.RemoveTempFile(relativePath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + c.JSON(http.StatusNotFound, gin.H{"error": "Upload session not found"}) + return + } + errStr := err.Error() + log.Debugf("Error deleting file: %s", errStr) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Error deleting file"}) + return + } + + c.Status(http.StatusNoContent) +} diff --git a/route/file/route.go b/route/file/route.go index 66cd410..0339521 100644 --- a/route/file/route.go +++ b/route/file/route.go @@ -2,163 +2,31 @@ package file import ( - "io" - "net/http" - "github.com/gin-gonic/gin" "github.com/vvbbnn00/goflet/middleware" - "github.com/vvbbnn00/goflet/storage" - "github.com/vvbbnn00/goflet/storage/upload" - "github.com/vvbbnn00/goflet/util" - "github.com/vvbbnn00/goflet/util/log" ) // RegisterRoutes load all the enabled routes for the application func RegisterRoutes(router *gin.Engine) { - file := router.Group("/file", + f := router.Group("/file", middleware.AuthChecker(), middleware.FilePathChecker()) { - // Register the routes - file.HEAD("/*rpath", routeGetFile) - file.GET("/*rpath", routeGetFile) - file.PUT("/*rpath", routePutFile) - file.POST("/*rpath", routePostFile) - file.DELETE("/*rpath", routeDeleteFile) - } -} - -// routePutFile handler for PUT /file/*path -// @Summary Partial File Upload -// @Description Create an upload session with a partial file upload, supports range requests, {path} should be the relative path of the file, starting from the root directory, e.g. /file/path/to/file.txt -// @Tags File -// @Accept */* -// @Param path path string true "File path" -// @Success 202 {object} string "Accepted" -// @Failure 400 {object} string "Bad request" -// @Failure 403 {object} string "Directory creation not allowed" -// @Failure 500 {object} string "Internal server error" -// @Router /file/{path} [put] -// @Security Authorization -func routePutFile(c *gin.Context) { - relativePath := c.GetString("relativePath") - - // Parse the range - byteStart, byteEnd, _, err := util.HeaderParseRangeUpload(c.GetHeader("Content-Range"), c.GetHeader("Content-Length")) - if err != nil { - c.JSON(http.StatusRequestedRangeNotSatisfiable, gin.H{"error": err.Error()}) - return - } - - // Get the write stream - writeStream, err := upload.GetTempFileWriteStream(relativePath) - if err != nil { - errStr := err.Error() - if errStr == "directory_creation" { - c.JSON(http.StatusForbidden, gin.H{"error": "Directory creation not allowed"}) - return - } - log.Warnf("Error getting write stream: %s", errStr) - c.JSON(http.StatusInternalServerError, gin.H{"error": "Error writing file"}) - return - } - defer func() { - if closeErr := writeStream.Close(); closeErr != nil { - log.Warnf("Error closing write stream: %s", closeErr.Error()) - } - }() - - // Write the file - body := c.Request.Body - defer func(body io.ReadCloser) { - _ = body.Close() - }(body) - - // Seek to the start of the range - _, err = writeStream.Seek(byteStart, io.SeekStart) - if err != nil { - log.Warnf("Error seeking write stream: %s", err.Error()) - c.JSON(http.StatusInternalServerError, gin.H{"error": "Error writing file"}) - return - } - - // Write the range to the file - written, err := io.CopyN(writeStream, body, byteEnd-byteStart+1) - if err != nil { - log.Warnf("Error writing to file: %s", err.Error()) - c.JSON(http.StatusInternalServerError, gin.H{"error": "Error writing file"}) - return - } - if written != byteEnd-byteStart+1 { - log.Warnf("Incomplete write: expected %d bytes, wrote %d bytes", byteEnd-byteStart+1, written) - c.JSON(http.StatusInternalServerError, gin.H{"error": "Incomplete write"}) - return + // Register the routes for file operations + f.HEAD("/*rpath", routeGetFile) + f.GET("/*rpath", routeGetFile) + f.POST("/*rpath", routePostFile) + f.DELETE("/*rpath", routeDeleteFile) } - log.Debugf("Successfully written %d bytes to %s", written, relativePath) - c.Status(http.StatusAccepted) -} - -// routePostFile handler for POST /file/*path -// @Summary Complete File Upload -// @Description Complete an upload session with a partial file upload. You should first upload the file with a PUT request, then complete the upload with a POST request, {path} should be the relative path of the file, starting from the root directory, e.g. /file/path/to/file.txt -// @Tags File -// @Param path path string true "File path" -// @Success 201 {object} string "Created" -// @Failure 400 {object} string "Bad request" -// @Failure 404 {object} string "File not found or upload not started" -// @Failure 409 {object} string "File completion in progress" -// @Failure 500 {object} string "Internal server error" -// @Router /file/{path} [post] -// @Security Authorization -func routePostFile(c *gin.Context) { - relativePath := c.GetString("relativePath") - - err := upload.CompleteFileUpload(relativePath) - if err != nil { - errStr := err.Error() - if errStr == "file_uploading" { - c.JSON(http.StatusConflict, gin.H{"error": "The file completion is in progress"}) - return - } - if errStr == "file_not_found" { - c.JSON(http.StatusNotFound, gin.H{"error": "File not found"}) - return - } - log.Warnf("Error completing file upload: %s", errStr) - c.JSON(http.StatusInternalServerError, gin.H{"error": "Error completing file upload"}) - return - } - - c.Status(http.StatusCreated) -} - -// routeDeleteFile handler for DELETE /file/*path -// @Summary Delete File -// @Description Delete a file by path, {path} should be the relative path of the file, starting from the root directory, e.g. /file/path/to/file.txt -// @Tags File -// @Param path path string true "File path" -// @Success 204 {object} string "Deleted" -// @Failure 400 {object} string "Bad request" -// @Failure 404 {object} string "File not found or upload not started" -// @Failure 500 {object} string "Internal server error" -// @Router /file/{path} [delete] -// @Security Authorization -func routeDeleteFile(c *gin.Context) { - fsPath := c.GetString("fsPath") - - err := storage.DeleteFile(fsPath) - if err != nil { - errStr := err.Error() - if errStr == "file_not_found" { - c.JSON(http.StatusNotFound, gin.H{"error": "File not found"}) - return - } - log.Warnf("Error deleting file: %s", errStr) - c.JSON(http.StatusInternalServerError, gin.H{"error": "Error deleting file"}) - return + u := router.Group("/upload", + middleware.AuthChecker(), + middleware.FilePathChecker()) + { + // Register the routes for partial file upload + u.PUT("/*rpath", routePutUpload) + u.POST("/*rpath", routePostUpload) + u.DELETE("/*rpath", routeDeleteUpload) } - - c.Status(http.StatusNoContent) } diff --git a/route/route.go b/route/route.go index fee828a..299fa1c 100644 --- a/route/route.go +++ b/route/route.go @@ -35,6 +35,9 @@ func RegisterRoutes() *gin.Engine { // Router should be created after setting the mode router := gin.Default() + // Set the max POST data size + router.MaxMultipartMemory = config.GofletCfg.FileConfig.MaxPostSize + // Log the requests router.Use(middleware.SafeLogger()) diff --git a/storage/upload/upload.go b/storage/upload/upload.go index 8e27bc4..f7401cc 100644 --- a/storage/upload/upload.go +++ b/storage/upload/upload.go @@ -39,6 +39,20 @@ func init() { } } +// GetTempFilePath Get the temporary file path +func GetTempFilePath(relativePath string) string { + fileName := hash.StringSha3New256(relativePath) // Get the hash of the path + tmpPath := filepath.Join(uploadPath, fileName) + return tmpPath +} + +// RemoveTempFile Remove the temporary file +func RemoveTempFile(relativePath string) error { + tmpPath := GetTempFilePath(relativePath) + err := os.Remove(tmpPath) + return err +} + // GetTempFileWriteStream Get a write stream for the temporary file func GetTempFileWriteStream(relativePath string) (*os.File, error) { // If it has subdirectory, check whether the directory can be created @@ -47,8 +61,7 @@ func GetTempFileWriteStream(relativePath string) (*os.File, error) { return nil, errors.New("directory_creation") } - fileName := hash.StringSha3New256(relativePath) // Get the hash of the path - tmpPath := filepath.Join(uploadPath, fileName) + tmpPath := GetTempFilePath(relativePath) // Ensure the directory exists dir = filepath.Dir(tmpPath)