Skip to content

Commit

Permalink
fs: fix async iterator partial writes
Browse files Browse the repository at this point in the history
fix an issue where chunks might be partially written when
using writeFile with an async iterator.

PR-URL: #38615
Reviewed-By: Antoine du Hamel <duhamelantoine1995@gmail.com>
Reviewed-By: Rich Trott <rtrott@gmail.com>
Reviewed-By: James M Snell <jasnell@gmail.com>
  • Loading branch information
Linkgoron authored and jasnell committed May 17, 2021
1 parent 61a67c1 commit 3612229
Show file tree
Hide file tree
Showing 2 changed files with 25 additions and 4 deletions.
14 changes: 10 additions & 4 deletions lib/internal/fs/promises.js
Original file line number Diff line number Diff line change
Expand Up @@ -296,10 +296,16 @@ async function writeFileHandle(filehandle, data, signal, encoding) {
if (isCustomIterable(data)) {
for await (const buf of data) {
checkAborted(signal);
await write(
filehandle, buf, undefined,
isArrayBufferView(buf) ? buf.byteLength : encoding);
checkAborted(signal);
const toWrite =
isArrayBufferView(buf) ? buf : Buffer.from(buf, encoding || 'utf8');
let remaining = toWrite.byteLength;
while (remaining > 0) {
const writeSize = MathMin(kWriteFileMaxChunkSize, remaining);
const { bytesWritten } = await write(
filehandle, toWrite, toWrite.byteLength - remaining, writeSize);
remaining -= bytesWritten;
checkAborted(signal);
}
}
return;
}
Expand Down
15 changes: 15 additions & 0 deletions test/parallel/test-fs-promises-writefile.js
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,14 @@ const iterable = {
yield 'c';
}
};

const veryLargeBuffer = {
expected: 'dogs running'.repeat(512 * 1024),
*[Symbol.iterator]() {
yield Buffer.from('dogs running'.repeat(512 * 1024), 'utf8');
}
};

function iterableWith(value) {
return {
*[Symbol.iterator]() {
Expand Down Expand Up @@ -106,6 +114,12 @@ async function doWriteAsyncIterable() {
assert.deepStrictEqual(data, asyncIterable.expected);
}

async function doWriteAsyncLargeIterable() {
await fsPromises.writeFile(dest, veryLargeBuffer);
const data = fs.readFileSync(dest, 'utf-8');
assert.deepStrictEqual(data, veryLargeBuffer.expected);
}

async function doWriteInvalidValues() {
await Promise.all(
[42, 42n, {}, Symbol('42'), true, undefined, null, NaN].map((value) =>
Expand Down Expand Up @@ -158,5 +172,6 @@ async function doReadWithEncoding() {
await doWriteIterableWithEncoding();
await doWriteBufferIterable();
await doWriteAsyncIterable();
await doWriteAsyncLargeIterable();
await doWriteInvalidValues();
})().then(common.mustCall());

0 comments on commit 3612229

Please sign in to comment.