Skip to content

Commit a38e3dc

Browse files
juancarlospacoAraq
andauthored
copyFile with bufferSize instead of hardcoded value (nim-lang#22769)
- `copyFile` allows to specify `bufferSize` instead of hardcoded wrong value. Tiny diff. # Performance - 1200% Performance improvement. # Check it yourself Execute: ```bash for i in $(seq 0 10); do bs=$((1024*2**$i)) printf "%7s Kb\t" $bs timeout --foreground -sINT 2 dd bs=$bs if=/dev/zero of=/dev/null 2>&1 | sed -n 's/.* \([0-9.,]* [GM]B\/s\)/\1/p' done ``` (This script can be ported to PowerShell for Windows I guess, it works in Windows MinGW Bash anyways). # Stats - Hardcoded `8192` or `8000` Kb bufferSize gives `5` GB/s. - Setting `262144` Kb bufferSize gives `65` GB/s (script suggestion). --------- Co-authored-by: Andreas Rumpf <rumpf_a@web.de>
1 parent 5eeafbf commit a38e3dc

File tree

2 files changed

+11
-9
lines changed

2 files changed

+11
-9
lines changed

changelog.md

+1
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
[//]: # "Changes:"
1010

11+
- Changed `std/osfiles.copyFile` to allow to specify `bufferSize` instead of a hardcoded one.
1112

1213
[//]: # "Additions:"
1314

lib/std/private/osfiles.nim

+10-9
Original file line numberDiff line numberDiff line change
@@ -173,7 +173,7 @@ type
173173

174174
const copyFlagSymlink = {cfSymlinkAsIs, cfSymlinkFollow, cfSymlinkIgnore}
175175

176-
proc copyFile*(source, dest: string, options = {cfSymlinkFollow}) {.rtl,
176+
proc copyFile*(source, dest: string, options = {cfSymlinkFollow}; bufferSize = 16_384) {.rtl,
177177
extern: "nos$1", tags: [ReadDirEffect, ReadIOEffect, WriteIOEffect],
178178
noWeirdTarget.} =
179179
## Copies a file from `source` to `dest`, where `dest.parentDir` must exist.
@@ -202,6 +202,7 @@ proc copyFile*(source, dest: string, options = {cfSymlinkFollow}) {.rtl,
202202
## On OSX, `copyfile` C api will be used (available since OSX 10.5) unless
203203
## `-d:nimLegacyCopyFile` is used.
204204
##
205+
## `copyFile` allows to specify `bufferSize` to improve I/O performance.
205206
## See also:
206207
## * `CopyFlag enum`_
207208
## * `copyDir proc`_
@@ -210,8 +211,7 @@ proc copyFile*(source, dest: string, options = {cfSymlinkFollow}) {.rtl,
210211
## * `removeFile proc`_
211212
## * `moveFile proc`_
212213

213-
doAssert card(copyFlagSymlink * options) == 1, "There should be exactly " &
214-
"one cfSymlink* in options"
214+
doAssert card(copyFlagSymlink * options) == 1, "There should be exactly one cfSymlink* in options"
215215
let isSymlink = source.symlinkExists
216216
if isSymlink and (cfSymlinkIgnore in options or defined(windows)):
217217
return
@@ -238,42 +238,43 @@ proc copyFile*(source, dest: string, options = {cfSymlinkFollow}) {.rtl,
238238
if status2 != 0: raiseOSError(osLastError(), $(source, dest))
239239
else:
240240
# generic version of copyFile which works for any platform:
241-
const bufSize = 8000 # better for memory manager
242241
var d, s: File
243242
if not open(s, source):raiseOSError(osLastError(), source)
244243
if not open(d, dest, fmWrite):
245244
close(s)
246245
raiseOSError(osLastError(), dest)
247-
var buf = alloc(bufSize)
246+
var buf = alloc(bufferSize)
248247
while true:
249-
var bytesread = readBuffer(s, buf, bufSize)
248+
var bytesread = readBuffer(s, buf, bufferSize)
250249
if bytesread > 0:
251250
var byteswritten = writeBuffer(d, buf, bytesread)
252251
if bytesread != byteswritten:
253252
dealloc(buf)
254253
close(s)
255254
close(d)
256255
raiseOSError(osLastError(), dest)
257-
if bytesread != bufSize: break
256+
if bytesread != bufferSize: break
258257
dealloc(buf)
259258
close(s)
260259
flushFile(d)
261260
close(d)
262261

263-
proc copyFileToDir*(source, dir: string, options = {cfSymlinkFollow})
262+
proc copyFileToDir*(source, dir: string, options = {cfSymlinkFollow}; bufferSize = 16_384)
264263
{.noWeirdTarget, since: (1,3,7).} =
265264
## Copies a file `source` into directory `dir`, which must exist.
266265
##
267266
## On non-Windows OSes, `options` specify the way file is copied; by default,
268267
## if `source` is a symlink, copies the file symlink points to. `options` is
269268
## ignored on Windows: symlinks are skipped.
270269
##
270+
## `copyFileToDir` allows to specify `bufferSize` to improve I/O performance.
271+
##
271272
## See also:
272273
## * `CopyFlag enum`_
273274
## * `copyFile proc`_
274275
if dir.len == 0: # treating "" as "." is error prone
275276
raise newException(ValueError, "dest is empty")
276-
copyFile(source, dir / source.lastPathPart, options)
277+
copyFile(source, dir / source.lastPathPart, options, bufferSize)
277278

278279

279280
proc copyFileWithPermissions*(source, dest: string,

0 commit comments

Comments
 (0)