Skip to content

Commit 3313df8

Browse files
author
HiroyukiYagihashi
committed
fs: add support for async iterators to fs.writeFile
Fixes: #38075
1 parent 52e4fb5 commit 3313df8

File tree

5 files changed

+157
-21
lines changed

5 files changed

+157
-21
lines changed

‎doc/api/fs.md‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3879,6 +3879,9 @@ details.
38793879
<!-- YAML
38803880
added: v0.1.29
38813881
changes:
3882+
- version: REPLACEME
3883+
pr-url: https://github.com/nodejs/node/pull/38525
3884+
description: add support for async iterators to `fs.writeFile`.
38823885
- version: v16.0.0
38833886
pr-url: https://github.com/nodejs/node/pull/37460
38843887
description: The error returned may be an `AggregateError` if more than one
@@ -3916,7 +3919,8 @@ changes:
39163919
-->
39173920
39183921
* `file` {string|Buffer|URL|integer} filename or file descriptor
3919-
* `data` {string|Buffer|TypedArray|DataView|Object}
3922+
* `data` {string|Buffer|TypedArray|DataView|Object
3923+
|AsyncIterable|Iterable|Stream}
39203924
* `options` {Object|string}
39213925
* `encoding` {string|null} **Default:** `'utf8'`
39223926
* `mode` {integer} **Default:** `0o666`

‎lib/fs.js‎

Lines changed: 57 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ const {
4444
StringPrototypeCharCodeAt,
4545
StringPrototypeIndexOf,
4646
StringPrototypeSlice,
47+
SymbolAsyncIterator,
48+
SymbolIterator,
4749
} = primordials;
4850

4951
const { fs: constants } = internalBinding('constants');
@@ -85,6 +87,7 @@ const {
8587
const { FSReqCallback } = binding;
8688
const { toPathIfFileURL } = require('internal/url');
8789
const internalUtil = require('internal/util');
90+
const { isCustomIterable } = require('internal/streams/utils');
8891
const {
8992
constants: {
9093
kIoMaxLength,
@@ -828,12 +831,12 @@ function write(fd, buffer, offset, length, position, callback) {
828831
} else {
829832
position = length;
830833
}
831-
length = 'utf8';
834+
length = length || 'utf8';
832835
}
833836

834837
const str = String(buffer);
835838
validateEncoding(str, length);
836-
callback = maybeCallback(position);
839+
callback = maybeCallback(callback || position);
837840

838841
const req = new FSReqCallback();
839842
req.oncomplete = wrapper;
@@ -2039,7 +2042,8 @@ function lutimesSync(path, atime, mtime) {
20392042
handleErrorFromBinding(ctx);
20402043
}
20412044

2042-
function writeAll(fd, isUserFd, buffer, offset, length, signal, callback) {
2045+
function writeAll(
2046+
fd, isUserFd, buffer, offset, length, signal, encoding, callback) {
20432047
if (signal?.aborted) {
20442048
const abortError = new AbortError();
20452049
if (isUserFd) {
@@ -2051,16 +2055,16 @@ function writeAll(fd, isUserFd, buffer, offset, length, signal, callback) {
20512055
}
20522056
return;
20532057
}
2054-
// write(fd, buffer, offset, length, position, callback)
2058+
2059+
if (isCustomIterable(buffer)) {
2060+
writeAllCustomIterable(
2061+
fd, isUserFd, buffer, offset, length, signal, encoding, callback)
2062+
.catch(reason => { throw reason });
2063+
return;
2064+
}
20552065
fs.write(fd, buffer, offset, length, null, (writeErr, written) => {
20562066
if (writeErr) {
2057-
if (isUserFd) {
2058-
callback(writeErr);
2059-
} else {
2060-
fs.close(fd, (err) => {
2061-
callback(aggregateTwoErrors(err, writeErr));
2062-
});
2063-
}
2067+
handleWriteAllErrorCallback(fd, isUserFd, writeErr, callback);
20642068
} else if (written === length) {
20652069
if (isUserFd) {
20662070
callback(null);
@@ -2070,11 +2074,43 @@ function writeAll(fd, isUserFd, buffer, offset, length, signal, callback) {
20702074
} else {
20712075
offset += written;
20722076
length -= written;
2073-
writeAll(fd, isUserFd, buffer, offset, length, signal, callback);
2077+
writeAll(
2078+
fd, isUserFd, buffer, offset, length, signal, encoding, callback);
20742079
}
20752080
});
20762081
}
20772082

2083+
async function writeAllCustomIterable(
2084+
fd, isUserFd, buffer, offset, length, signal, encoding, callback) {
2085+
const result = await buffer.next();
2086+
if (result.done) {
2087+
fs.close(fd, callback);
2088+
return;
2089+
}
2090+
const resultValue = result.value.toString();
2091+
fs.write(fd, resultValue, undefined,
2092+
isArrayBufferView(buffer) ? resultValue.byteLength : encoding,
2093+
null, (writeErr, _) => {
2094+
if (writeErr) {
2095+
handleWriteAllErrorCallback(fd, isUserFd, writeErr, callback);
2096+
} else {
2097+
writeAll(fd, isUserFd, buffer, offset,
2098+
length, signal, encoding, callback);
2099+
}
2100+
}
2101+
);
2102+
}
2103+
2104+
function handleWriteAllErrorCallback(fd, isUserFd, writeErr, callback) {
2105+
if (isUserFd) {
2106+
callback(writeErr);
2107+
} else {
2108+
fs.close(fd, (err) => {
2109+
callback(aggregateTwoErrors(err, writeErr));
2110+
});
2111+
}
2112+
}
2113+
20782114
/**
20792115
* Asynchronously writes data to the file.
20802116
* @param {string | Buffer | URL | number} path
@@ -2093,15 +2129,20 @@ function writeFile(path, data, options, callback) {
20932129
options = getOptions(options, { encoding: 'utf8', mode: 0o666, flag: 'w' });
20942130
const flag = options.flag || 'w';
20952131

2096-
if (!isArrayBufferView(data)) {
2132+
if (!isArrayBufferView(data) && !isCustomIterable(data)) {
20972133
validateStringAfterArrayBufferView(data, 'data');
20982134
data = Buffer.from(String(data), options.encoding || 'utf8');
20992135
}
21002136

2137+
if (isCustomIterable(data)) {
2138+
data = data[SymbolIterator]?.() ?? data[SymbolAsyncIterator]?.();
2139+
}
2140+
21012141
if (isFd(path)) {
21022142
const isUserFd = true;
21032143
const signal = options.signal;
2104-
writeAll(path, isUserFd, data, 0, data.byteLength, signal, callback);
2144+
writeAll(path, isUserFd, data,
2145+
0, data.byteLength, signal, options.encoding, callback);
21052146
return;
21062147
}
21072148

@@ -2114,7 +2155,8 @@ function writeFile(path, data, options, callback) {
21142155
} else {
21152156
const isUserFd = false;
21162157
const signal = options.signal;
2117-
writeAll(fd, isUserFd, data, 0, data.byteLength, signal, callback);
2158+
writeAll(fd, isUserFd, data,
2159+
0, data.byteLength, signal, options.encoding, callback);
21182160
}
21192161
});
21202162
}

‎lib/internal/fs/promises.js‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ const pathModule = require('path');
7979
const { promisify } = require('internal/util');
8080
const { EventEmitterMixin } = require('internal/event_target');
8181
const { watch } = require('internal/fs/watchers');
82-
const { isIterable } = require('internal/streams/utils');
82+
const { isCustomIterable } = require('internal/streams/utils');
8383
const assert = require('internal/assert');
8484

8585
const kHandle = Symbol('kHandle');
@@ -730,10 +730,6 @@ async function writeFile(path, data, options) {
730730
writeFileHandle(fd, data, options.signal, options.encoding), fd.close);
731731
}
732732

733-
function isCustomIterable(obj) {
734-
return isIterable(obj) && !isArrayBufferView(obj) && typeof obj !== 'string';
735-
}
736-
737733
async function appendFile(path, data, options) {
738734
options = getOptions(options, { encoding: 'utf8', mode: 0o666, flag: 'a' });
739735
options = copyObject(options);

‎lib/internal/streams/utils.js‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ const {
44
SymbolAsyncIterator,
55
SymbolIterator,
66
} = primordials;
7+
const { isArrayBufferView } = require('internal/util/types');
78

89
function isReadable(obj) {
910
return !!(obj && typeof obj.pipe === 'function' &&
@@ -27,7 +28,12 @@ function isIterable(obj, isAsync) {
2728
typeof obj[SymbolIterator] === 'function';
2829
}
2930

31+
function isCustomIterable(obj) {
32+
return isIterable(obj) && !isArrayBufferView(obj) && typeof obj !== 'string';
33+
}
34+
3035
module.exports = {
36+
isCustomIterable,
3137
isIterable,
3238
isReadable,
3339
isStream,

‎test/parallel/test-fs-write-file-async-iterators.js‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
'use strict';
2+
const common = require('../common');
3+
const assert = require('assert');
4+
const fs = require('fs');
5+
const join = require('path').join;
6+
const { Readable } = require('stream');
7+
8+
const tmpdir = require('../common/tmpdir');
9+
tmpdir.refresh();
10+
11+
{
12+
const filenameIterable = join(tmpdir.path, 'testIterable.txt');
13+
const iterable = {
14+
expected: 'abc',
15+
*[Symbol.iterator]() {
16+
yield 'a';
17+
yield 'b';
18+
yield 'c';
19+
}
20+
};
21+
22+
fs.writeFile(filenameIterable, iterable, common.mustSucceed(() => {
23+
const data = fs.readFileSync(filenameIterable, 'utf-8');
24+
assert.strictEqual(iterable.expected, data);
25+
}));
26+
}
27+
28+
{
29+
const filenameBufferIterable = join(tmpdir.path, 'testBufferIterable.txt');
30+
const bufferIterable = {
31+
expected: 'abc',
32+
*[Symbol.iterator]() {
33+
yield Buffer.from('a');
34+
yield Buffer.from('b');
35+
yield Buffer.from('c');
36+
}
37+
};
38+
39+
fs.writeFile(
40+
filenameBufferIterable, bufferIterable, common.mustSucceed(() => {
41+
const data = fs.readFileSync(filenameBufferIterable, 'utf-8');
42+
assert.strictEqual(bufferIterable.expected, data);
43+
})
44+
);
45+
}
46+
47+
48+
{
49+
const filenameAsyncIterable = join(tmpdir.path, 'testAsyncIterable.txt');
50+
const asyncIterable = {
51+
expected: 'abc',
52+
*[Symbol.asyncIterator]() {
53+
yield 'a';
54+
yield 'b';
55+
yield 'c';
56+
}
57+
};
58+
59+
fs.writeFile(filenameAsyncIterable, asyncIterable, common.mustSucceed(() => {
60+
const data = fs.readFileSync(filenameAsyncIterable, 'utf-8');
61+
assert.strictEqual(asyncIterable.expected, data);
62+
}));
63+
}
64+
65+
{
66+
const filenameStream = join(tmpdir.path, 'testStream.txt');
67+
const stream = Readable.from(['a', 'b', 'c']);
68+
const expected = 'abc';
69+
70+
fs.writeFile(filenameStream, stream, common.mustSucceed(() => {
71+
const data = fs.readFileSync(filenameStream, 'utf-8');
72+
assert.strictEqual(expected, data);
73+
}));
74+
}
75+
76+
{
77+
const filenameStreamWithEncoding =
78+
join(tmpdir.path, 'testStreamWithEncoding.txt');
79+
const stream = Readable.from(['ümlaut', ' ', 'sechzig']);
80+
const expected = 'ümlaut sechzig';
81+
82+
fs.writeFile(
83+
filenameStreamWithEncoding, stream, 'latin1', common.mustSucceed(() => {
84+
const data = fs.readFileSync(filenameStreamWithEncoding, 'latin1');
85+
assert.strictEqual(expected, data);
86+
})
87+
);
88+
}

0 commit comments

Comments
 (0)