stream: add Transform.by utility function

doc/api/errors.md

@@ -630,6 +630,12 @@ display if `block` does not throw.
 An iterable argument (i.e. a value that works with `for...of` loops) was
 required, but not provided to a Node.js API.
 
+<a id="ERR_ARG_RETURN_VALUE_NOT_ASYNC_ITERABLE"></a>
+### ERR_ARG_RETURN_VALUE_NOT_ASYNC_ITERABLE
+
+A function argument that returns an async iterable (i.e. a value that works
+with `for await...of` loops) was required, but not provided to a Node.js API.
+
 <a id="ERR_ASSERTION"></a>
 ### ERR_ASSERTION
 

doc/api/stream.md

@@ -43,8 +43,8 @@ There are four fundamental stream types within Node.js:
   is written and read (for example, [`zlib.createDeflate()`][]).
 
 Additionally, this module includes the utility functions
-[`stream.pipeline()`][], [`stream.finished()`][] and
-[`stream.Readable.from()`][].
+[`stream.pipeline()`][], [`stream.finished()`][],
+[`stream.Readable.from()`][] and [`stream.Transform.by()`][].
 
 ### Object Mode
 
@@ -1646,6 +1646,49 @@ Calling `Readable.from(string)` or `Readable.from(buffer)` will not have
 the strings or buffers be iterated to match the other streams semantics
 for performance reasons.
 
+### stream.Transform.by(asyncGeneratorFunction[, options])
+<!-- YAML
+added: REPLACEME
+-->
+
+* `asyncGeneratorFunction` {AsyncGeneratorFunction} A mapping function which
+accepts a `source` async iterable which can be used to read incoming data, while
+transformed data is pushed to the stream with the `yield` keyword.
+* `options` {Object} Options provided to `new stream.Transform([options])`.
+By default, `Transform.by()` will set `options.objectMode` to `true`,
+unless this is explicitly opted out by setting `options.objectMode` to `false`.
+* Returns: {stream.Transform}
+
+A utility method for creating Transform Streams with async generator functions.
+The async generator is supplied a single argument, `source`, which is used to
+read incoming chunks.
+
+Yielded values become the data chunks emitted from the stream.
+
+```js
+const { Readable, Transform } = require('stream');
+
+const readable = Readable.from(['hello', 'streams']);
+async function * mapper(source) {
+  for await (const chunk of source) {
+    // If objectMode was set to false, the buffer would have to be converted
+    // to a string here but since it is true by default for both Readable.from()
+    // and Transform.by() each chunk is already a string.
+    yield chunk.toUpperCase();
+  }
+}
+const transform = Transform.by(mapper);
+readable.pipe(transform);
+transform.on('data', (chunk) => {
+  console.log(chunk);
+});
+```
+
+The `source` parameter has an `encoding` property which represents the encoding
+of the `WriteableStream` side of the transform. This is the same `encoding`
+value that would be passed as the second parameter to the `transform()` function
+option (or `_transform()` method) supplied to `stream.Transform`.
+
 ## API for Stream Implementers
 
 <!--type=misc-->
@@ -1689,7 +1732,7 @@ on the type of stream being created, as detailed in the chart below:
 | Reading only | [`Readable`][] | [`_read()`][stream-_read] |
 | Writing only | [`Writable`][] | [`_write()`][stream-_write], [`_writev()`][stream-_writev], [`_final()`][stream-_final] |
 | Reading and writing | [`Duplex`][] | [`_read()`][stream-_read], [`_write()`][stream-_write], [`_writev()`][stream-_writev], [`_final()`][stream-_final] |
-| Operate on written data, then read the result | [`Transform`][] | [`_transform()`][stream-_transform], [`_flush()`][stream-_flush], [`_final()`][stream-_final] |
+| Operate on written data, then read the result | [`Transform`][] | [`_transform()`][], [`_flush()`][stream-_flush], [`_final()`][stream-_final] |
 
 The implementation code for a stream should *never* call the "public" methods
 of a stream that are intended for use by consumers (as described in the
@@ -2430,7 +2473,7 @@ The `stream.Transform` class is extended to implement a [`Transform`][] stream.
 The `stream.Transform` class prototypically inherits from `stream.Duplex` and
 implements its own versions of the `writable._write()` and `readable._read()`
 methods. Custom `Transform` implementations *must* implement the
-[`transform._transform()`][stream-_transform] method and *may* also implement
+[`transform._transform()`][] method and *may* also implement
 the [`transform._flush()`][stream-_flush] method.
 
 Care must be taken when using `Transform` streams in that data written to the
@@ -2442,7 +2485,7 @@ output on the `Readable` side is not consumed.
 * `options` {Object} Passed to both `Writable` and `Readable`
   constructors. Also has the following fields:
   * `transform` {Function} Implementation for the
-    [`stream._transform()`][stream-_transform] method.
+    [`stream._transform()`][] method.
   * `flush` {Function} Implementation for the [`stream._flush()`][stream-_flush]
     method.
 
@@ -2489,7 +2532,7 @@ const myTransform = new Transform({
 The [`'finish'`][] and [`'end'`][] events are from the `stream.Writable`
 and `stream.Readable` classes, respectively. The `'finish'` event is emitted
 after [`stream.end()`][stream-end] is called and all chunks have been processed
-by [`stream._transform()`][stream-_transform]. The `'end'` event is emitted
+by [`stream._transform()`][]. The `'end'` event is emitted
 after all data has been output, which occurs after the callback in
 [`transform._flush()`][stream-_flush] has been called. In the case of an error,
 neither `'finish'` nor `'end'` should be emitted.
@@ -2630,6 +2673,35 @@ readable.on('data', (chunk) => {
 });
 ```
 
+#### Creating Transform Streams with Async Generator Functions
+
+We can construct a Node.js Transform stream with an asynchronous
+generator function using the `Transform.by()` utility method.
+
+```js
+const { Readable, Transform } = require('stream');
+
+async function * toUpperCase(source) {
+  for await (const chunk of source) {
+    yield chunk.toUpperCase();
+  }
+}
+const transform = Transform.by(toUpperCase);
+
+async function * generate() {
+  yield 'a';
+  yield 'b';
+  yield 'c';
+}
+
+const readable = Readable.from(generate());
+
+readable.pipe(transform);
+transform.on('data', (chunk) => {
+  console.log(chunk);
+});
+```
+
 #### Piping to Writable Streams from Async Iterators
 
 In the scenario of writing to a writable stream from an async iterator, ensure
@@ -2819,6 +2891,7 @@ contain multi-byte characters.
 [`readable.push('')`]: #stream_readable_push
 [`readable.setEncoding()`]: #stream_readable_setencoding_encoding
 [`stream.Readable.from()`]: #stream_stream_readable_from_iterable_options
+[`stream.Transform.by()`]: #stream_stream_transform_by_asyncgeneratorfunction_options
 [`stream.cork()`]: #stream_writable_cork
 [`stream.finished()`]: #stream_stream_finished_stream_options_callback
 [`stream.pipe()`]: #stream_readable_pipe_destination_options
@@ -2853,7 +2926,7 @@ contain multi-byte characters.
 [stream-_final]: #stream_writable_final_callback
 [stream-_flush]: #stream_transform_flush_callback
 [stream-_read]: #stream_readable_read_size_1
-[stream-_transform]: #stream_transform_transform_chunk_encoding_callback
+[]: #stream_transform_transform_chunk_encoding_callback
 [stream-_write]: #stream_writable_write_chunk_encoding_callback_1
 [stream-_writev]: #stream_writable_writev_chunks_callback
 [stream-end]: #stream_writable_end_chunk_encoding_callback

lib/_stream_transform.js

@@ -65,16 +65,29 @@
 
 const {
   ObjectSetPrototypeOf,
+  ObjectGetPrototypeOf,
+  Symbol
 } = primordials;
 
 module.exports = Transform;
 const {
+  ERR_ARG_RETURN_VALUE_NOT_ASYNC_ITERABLE,
   ERR_METHOD_NOT_IMPLEMENTED,
   ERR_MULTIPLE_CALLBACK,
   ERR_TRANSFORM_ALREADY_TRANSFORMING,
   ERR_TRANSFORM_WITH_LENGTH_0
 } = require('internal/errors').codes;
 const Duplex = require('_stream_duplex');
+const AsyncIteratorPrototype = ObjectGetPrototypeOf(
+  ObjectGetPrototypeOf(async function* () {}).prototype);
+
+const kSourceIteratorPull = Symbol('kSourceIteratorPull');
+const kSourceIteratorResolve = Symbol('kSourceIteratorResolve');
+const kSourceIteratorChunk = Symbol('kSourceIteratorChunk');
+const kSourceIteratorStream = Symbol('kSourceIteratorStream');
+const kSourceIteratorPump = Symbol('kSourceIteratorPump');
+const kSourceIteratorGrabResolve = Symbol('kSourceIteratorGrabResolve');
+
 ObjectSetPrototypeOf(Transform.prototype, Duplex.prototype);
 ObjectSetPrototypeOf(Transform, Duplex);
 
@@ -203,7 +216,6 @@ Transform.prototype._destroy = function(err, cb) {
   });
 };
 
-
 function done(stream, er, data) {
   if (er)
     return stream.emit('error', er);
@@ -219,3 +231,111 @@ function done(stream, er, data) {
     throw new ERR_TRANSFORM_ALREADY_TRANSFORMING();
   return stream.push(null);
 }
+
+function SourceIterator(asyncGeneratorFn, opts) {
+  const source = this;
+  const result = asyncGeneratorFn(this);
+  if (typeof result[Symbol.asyncIterator] !== 'function') {
+    throw new ERR_ARG_RETURN_VALUE_NOT_ASYNC_ITERABLE('asyncGeneratorFn');
+  }
+  const iter = result[Symbol.asyncIterator]();
+  if (typeof iter.next !== 'function') {
+    throw new ERR_ARG_RETURN_VALUE_NOT_ASYNC_ITERABLE('asyncGeneratorFn');
+  }
+
+  this[kSourceIteratorPull] = null;
+  this[kSourceIteratorChunk] = null;
+  this[kSourceIteratorResolve] = null;
+  this[kSourceIteratorStream] = new Transform({
+    objectMode: true,
+    ...opts,
+    transform(chunk, encoding, cb) {
+      source.encoding = encoding;
+      if (source[kSourceIteratorResolve] === null) {
+        source[kSourceIteratorChunk] = chunk;
+        source[kSourceIteratorPull] = cb;
+        return;
+      }
+      source[kSourceIteratorResolve]({ value: chunk, done: false });
+      source[kSourceIteratorResolve] = null;
+      cb(null);
+    }
+  });
+  this.encoding = this[kSourceIteratorStream]._transformState.writeencoding;
+  this[kSourceIteratorGrabResolve] = (resolve) => {
+    this[kSourceIteratorResolve] = resolve;
+  };
+  const first = iter.next();
+  this[kSourceIteratorPump](iter, first);
+}
+
+SourceIterator.prototype[Symbol.asyncIterator] = function() {
+  return this;
+};
+
+ObjectSetPrototypeOf(SourceIterator.prototype, AsyncIteratorPrototype);
+
+SourceIterator.prototype.next = function next() {
+  if (this[kSourceIteratorPull] === null || this[kSourceIteratorChunk] === null)
+    return new Promise(this[kSourceIteratorGrabResolve]);
+
+  this[kSourceIteratorPull](null);
+  const result = Promise.resolve({
+    value: this[kSourceIteratorChunk],
+    done: false
+  });
+  this[kSourceIteratorChunk] = null;
+  this[kSourceIteratorPull] = null;
+  return result;
+};
+
+SourceIterator.prototype[kSourceIteratorPump] = async function pump(iter, p) {
+  const stream = this[kSourceIteratorStream];
+  try {
+    stream.removeListener('prefinish', prefinish);
+    stream.on('prefinish', () => {
+      if (this[kSourceIteratorResolve] !== null) {
+        this[kSourceIteratorResolve]({ value: undefined, done: true });
+      }
+    });
+    let next = await p;
+    while (true) {
+      const { done, value } = next;
+      if (done) {
+        if (value !== undefined) stream.push(value);
+
+        // In the event of an early return we explicitly
+        // discard any buffered state
+        if (stream._writableState.length > 0) {
+          const { length } = stream._writableState;
+          const { transforming } = stream._transformState;
+          stream._writableState.length = 0;
+          stream._transformState.transforming = false;
+          prefinish.call(stream);
+          stream._writableState.length = length;
+          stream._transformState.transforming = transforming;
+        } else {
+          prefinish.call(stream);
+        }
+        break;
+      }
+      stream.push(value);
+      next = await iter.next();
+    }
+  } catch (err) {
+    process.nextTick(() => stream.destroy(err));
+  } finally {
+    this[kSourceIteratorPull] = null;
+    this[kSourceIteratorChunk] = null;
+    this[kSourceIteratorResolve] = null;
+    this[kSourceIteratorStream] = null;
+  }
+};
+
+
+Transform.by = function by(asyncGeneratorFn, opts) {
+  const source = new SourceIterator(asyncGeneratorFn, opts);
+  const stream = source[kSourceIteratorStream];
+
+  return stream;
+};

lib/internal/errors.js

@@ -720,6 +720,8 @@ module.exports = {
 // Note: Node.js specific errors must begin with the prefix ERR_
 E('ERR_AMBIGUOUS_ARGUMENT', 'The "%s" argument is ambiguous. %s', TypeError);
 E('ERR_ARG_NOT_ITERABLE', '%s must be iterable', TypeError);
+E('ERR_ARG_RETURN_VALUE_NOT_ASYNC_ITERABLE', '%s must return an async iterable',
+  TypeError);
 E('ERR_ASSERTION', '%s', Error);
 E('ERR_ASYNC_CALLBACK', '%s must be a function', TypeError);
 E('ERR_ASYNC_TYPE', 'Invalid name for async "type": %s', TypeError);

test/parallel/test-stream-readable-async-iterators.js

@@ -420,6 +420,30 @@ async function tests() {
     mustReach[1]();
   }
 
+  {
+    console.log('readable side of a transform stream pushes null');
+    const transform = new Transform({
+      objectMode: true,
+      transform: (chunk, enc, cb) => { cb(null, chunk); }
+    });
+    transform.push(0);
+    transform.push(1);
+    process.nextTick(() => {
+      transform.push(null);
+    });
+
+    const mustReach = [ common.mustCall(), common.mustCall() ];
+
+    const iter = transform[Symbol.asyncIterator]();
+    assert.strictEqual((await iter.next()).value, 0);
+
+    for await (const d of iter) {
+      assert.strictEqual(d, 1);
+      mustReach[0]();
+    }
+    mustReach[1]();
+  }
+
   {
     console.log('all next promises must be resolved on end');
     const r = new Readable({