From a746b7ea1c137f5bc4b438250ca6db5d0dc74795 Mon Sep 17 00:00:00 2001 From: Frank Somers Date: Tue, 28 Nov 2017 15:38:00 +0000 Subject: [PATCH 1/2] internal/fs: Don't Sync() destination file after copy When copying files, directly check for errors when closing the destination file rather than using out.Sync() with a deferred Close(). The out.Sync() call can be incredibly expensive on systems which are I/O-heavy, yet may have more than ample memory and CPU resources. As part of a file copy, we only need to ensure that errors writing and closing the file handle are checked and dealt with, rather than forcing the entire copied file to physical storage. Environments such as Continuous Integration servers and container hosts are often I/O-heavy. In such environments, this change can make a significant difference - milliseconds vs minutes. --- internal/fs/fs.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/fs/fs.go b/internal/fs/fs.go index bfacfd95af..1ed5d31eb5 100644 --- a/internal/fs/fs.go +++ b/internal/fs/fs.go @@ -407,8 +407,7 @@ func CopyDir(src, dst string) error { // copyFile copies the contents of the file named src to the file named // by dst. The file will be created if it does not already exist. If the // destination file exists, all its contents will be replaced by the contents -// of the source file. The file mode will be copied from the source and -// the copied data is synced/flushed to stable storage. +// of the source file. The file mode will be copied from the source. func copyFile(src, dst string) (err error) { if sym, err := IsSymlink(src); err != nil { return errors.Wrap(err, "symlink check failed") @@ -442,13 +441,14 @@ func copyFile(src, dst string) (err error) { if err != nil { return } - defer out.Close() if _, err = io.Copy(out, in); err != nil { + out.Close() return } - if err = out.Sync(); err != nil { + // Check for write errors on Close + if err = out.Close(); err != nil { return } From ebef7c178af385a94253f38efb6752c8c0bdfe35 Mon Sep 17 00:00:00 2001 From: Frank Somers Date: Tue, 28 Nov 2017 16:36:59 +0000 Subject: [PATCH 2/2] CHANGELOG.md: Add reference to PR 1408 internal/fs: Don't Sync() destination file after copy https://github.com/golang/dep/pull/1408 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c327a9cd6f..810d2381e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ IMPROVEMENTS: * Log as dependencies are pre-fetched during dep init ([#1176](https://github.com/golang/dep/pull/1176)). * Make the gps package importable ([#1349](https://github.com/golang/dep/pull/1349)). +* Improve file copy performance by not forcing a file sync (PR #1408). # v0.3.2