From 8f4a3080813868256ca8ee54216d44a31708ee88 Mon Sep 17 00:00:00 2001 From: Robert Nagy Date: Fri, 10 Feb 2023 16:10:42 +0100 Subject: [PATCH] stream: bump default highWaterMark This should give a performance boost accross the board. Given that the old limit is a decod old and memory capacity has doubled many times since I think it is appropriate to slightly bump the default limit. PR-URL: https://github.com/nodejs/node/pull/52037 Refs: https://github.com/nodejs/node/pull/46608 Refs: https://github.com/nodejs/node/pull/50120 --- benchmark/net/net-c2s.js | 2 +- benchmark/net/net-pipe.js | 2 +- benchmark/net/net-s2c.js | 2 +- benchmark/net/net-wrap-js-stream-passthrough.js | 2 +- benchmark/net/tcp-raw-c2s.js | 2 +- benchmark/net/tcp-raw-pipe.js | 2 +- benchmark/net/tcp-raw-s2c.js | 2 +- doc/api/stream.md | 4 ++-- lib/internal/streams/state.js | 2 +- test/parallel/test-http-no-read-no-dump.js | 2 +- test/parallel/test-https-hwm.js | 2 +- test/parallel/test-stream-duplex-readable-end.js | 2 ++ .../test-stream-pipe-await-drain-push-while-write.js | 2 ++ test/parallel/test-stream-readable-infinite-read.js | 1 + .../parallel/test-stream-transform-split-highwatermark.js | 4 ++-- test/parallel/test-stream-transform-split-objectmode.js | 8 +++++--- test/parallel/test-tls-connect-hwm-option.js | 2 +- test/parallel/test-zlib-brotli-16GB.js | 3 ++- 18 files changed, 27 insertions(+), 19 deletions(-) diff --git a/benchmark/net/net-c2s.js b/benchmark/net/net-c2s.js index fdb0bb4968c025..48e454c5815ba8 100644 --- a/benchmark/net/net-c2s.js +++ b/benchmark/net/net-c2s.js @@ -6,7 +6,7 @@ const net = require('net'); const PORT = common.PORT; const bench = common.createBenchmark(main, { - len: [64, 102400, 1024 * 1024 * 16], + len: [64, 102400, 1024 * 64 * 16], type: ['utf', 'asc', 'buf'], dur: [5], }, { diff --git a/benchmark/net/net-pipe.js b/benchmark/net/net-pipe.js index 0b9107357cabb3..dd9f2de497f5b6 100644 --- a/benchmark/net/net-pipe.js +++ b/benchmark/net/net-pipe.js @@ -6,7 +6,7 @@ const net = require('net'); const PORT = common.PORT; const bench = common.createBenchmark(main, { - len: [2, 64, 102400, 1024 * 1024 * 16], + len: [2, 64, 102400, 1024 * 64 * 16], type: ['utf', 'asc', 'buf'], dur: [5], }, { diff --git a/benchmark/net/net-s2c.js b/benchmark/net/net-s2c.js index 07c5a8806920b0..1b9c91b8e7802c 100644 --- a/benchmark/net/net-s2c.js +++ b/benchmark/net/net-s2c.js @@ -5,7 +5,7 @@ const common = require('../common.js'); const PORT = common.PORT; const bench = common.createBenchmark(main, { - sendchunklen: [256, 32 * 1024, 128 * 1024, 16 * 1024 * 1024], + sendchunklen: [256, 32 * 1024, 128 * 1024, 16 * 64 * 1024], type: ['utf', 'asc', 'buf'], recvbuflen: [0, 64 * 1024, 1024 * 1024], recvbufgenfn: ['true', 'false'], diff --git a/benchmark/net/net-wrap-js-stream-passthrough.js b/benchmark/net/net-wrap-js-stream-passthrough.js index bae976ad40b0ba..27cda28343ccaa 100644 --- a/benchmark/net/net-wrap-js-stream-passthrough.js +++ b/benchmark/net/net-wrap-js-stream-passthrough.js @@ -5,7 +5,7 @@ const common = require('../common.js'); const { PassThrough } = require('stream'); const bench = common.createBenchmark(main, { - len: [64, 102400, 1024 * 1024 * 16], + len: [64, 102400, 1024 * 64 * 16], type: ['utf', 'asc', 'buf'], dur: [5], }, { diff --git a/benchmark/net/tcp-raw-c2s.js b/benchmark/net/tcp-raw-c2s.js index edfab797d09967..5b174f50e81c2f 100644 --- a/benchmark/net/tcp-raw-c2s.js +++ b/benchmark/net/tcp-raw-c2s.js @@ -9,7 +9,7 @@ const util = require('util'); // run the function with those settings. // if not, then queue up a bunch of child processes. const bench = common.createBenchmark(main, { - len: [102400, 1024 * 1024 * 16], + len: [102400, 1024 * 64 * 16], type: ['utf', 'asc', 'buf'], dur: [5], }, { diff --git a/benchmark/net/tcp-raw-pipe.js b/benchmark/net/tcp-raw-pipe.js index 15279b32784d8d..fbaa21b5a3097e 100644 --- a/benchmark/net/tcp-raw-pipe.js +++ b/benchmark/net/tcp-raw-pipe.js @@ -9,7 +9,7 @@ const util = require('util'); // run the function with those settings. // if not, then queue up a bunch of child processes. const bench = common.createBenchmark(main, { - len: [102400, 1024 * 1024 * 16], + len: [102400, 1024 * 64 * 16], type: ['utf', 'asc', 'buf'], dur: [5], }, { diff --git a/benchmark/net/tcp-raw-s2c.js b/benchmark/net/tcp-raw-s2c.js index 7818719b7285a8..3ca03529fcea9f 100644 --- a/benchmark/net/tcp-raw-s2c.js +++ b/benchmark/net/tcp-raw-s2c.js @@ -9,7 +9,7 @@ const util = require('util'); // run the function with those settings. // If not, then queue up a bunch of child processes. const bench = common.createBenchmark(main, { - len: [102400, 1024 * 1024 * 16], + len: [102400, 1024 * 64 * 16], type: ['utf', 'asc', 'buf'], dur: [5], }, { diff --git a/doc/api/stream.md b/doc/api/stream.md index d5b0044d65c2fd..67e72faace8574 100644 --- a/doc/api/stream.md +++ b/doc/api/stream.md @@ -3500,7 +3500,7 @@ changes: * `options` {Object} * `highWaterMark` {number} Buffer level when [`stream.write()`][stream-write] starts returning `false`. **Default:** - `16384` (16 KiB), or `16` for `objectMode` streams. + `65536` (64 KiB), or `16` for `objectMode` streams. * `decodeStrings` {boolean} Whether to encode `string`s passed to [`stream.write()`][stream-write] to `Buffer`s (with the encoding specified in the [`stream.write()`][stream-write] call) before passing @@ -3873,7 +3873,7 @@ changes: * `options` {Object} * `highWaterMark` {number} The maximum [number of bytes][hwm-gotcha] to store in the internal buffer before ceasing to read from the underlying resource. - **Default:** `16384` (16 KiB), or `16` for `objectMode` streams. + **Default:** `65536` (64 KiB), or `16` for `objectMode` streams. * `encoding` {string} If specified, then buffers will be decoded to strings using the specified encoding. **Default:** `null`. * `objectMode` {boolean} Whether this stream should behave diff --git a/lib/internal/streams/state.js b/lib/internal/streams/state.js index 98f8d8a6cc33d5..97affc98bc6a8b 100644 --- a/lib/internal/streams/state.js +++ b/lib/internal/streams/state.js @@ -8,7 +8,7 @@ const { validateInteger } = require('internal/validators'); const { ERR_INVALID_ARG_VALUE } = require('internal/errors').codes; -let defaultHighWaterMarkBytes = 16 * 1024; +let defaultHighWaterMarkBytes = 64 * 1024; let defaultHighWaterMarkObjectMode = 16; function highWaterMarkFrom(options, isDuplex, duplexKey) { diff --git a/test/parallel/test-http-no-read-no-dump.js b/test/parallel/test-http-no-read-no-dump.js index 3f976bb087290c..4bf9d9fe1a4b8b 100644 --- a/test/parallel/test-http-no-read-no-dump.js +++ b/test/parallel/test-http-no-read-no-dump.js @@ -34,7 +34,7 @@ const server = http.createServer((req, res) => { }, common.mustCall((res) => { res.resume(); - post.write(Buffer.alloc(16 * 1024).fill('X')); + post.write(Buffer.alloc(64 * 1024).fill('X')); onPause = () => { post.end('something'); }; diff --git a/test/parallel/test-https-hwm.js b/test/parallel/test-https-hwm.js index 71ee33b7086861..b3f8960195e03d 100644 --- a/test/parallel/test-https-hwm.js +++ b/test/parallel/test-https-hwm.js @@ -62,5 +62,5 @@ const httpsServer = https.createServer({ port: this.address().port, rejectUnauthorized: false, highWaterMark: undefined, - }, loadCallback(16 * 1024)).on('error', common.mustNotCall()).end(); + }, loadCallback(64 * 1024)).on('error', common.mustNotCall()).end(); })); diff --git a/test/parallel/test-stream-duplex-readable-end.js b/test/parallel/test-stream-duplex-readable-end.js index 0e3e62aacb14bc..3b1d4d21ce0a39 100644 --- a/test/parallel/test-stream-duplex-readable-end.js +++ b/test/parallel/test-stream-duplex-readable-end.js @@ -7,6 +7,7 @@ const stream = require('stream'); let loops = 5; const src = new stream.Readable({ + highWaterMark: 16 * 1024, read() { if (loops--) this.push(Buffer.alloc(20000)); @@ -14,6 +15,7 @@ const src = new stream.Readable({ }); const dst = new stream.Transform({ + highWaterMark: 16 * 1024, transform(chunk, output, fn) { this.push(null); fn(); diff --git a/test/parallel/test-stream-pipe-await-drain-push-while-write.js b/test/parallel/test-stream-pipe-await-drain-push-while-write.js index a717291cda2b03..089767166c99b5 100644 --- a/test/parallel/test-stream-pipe-await-drain-push-while-write.js +++ b/test/parallel/test-stream-pipe-await-drain-push-while-write.js @@ -4,6 +4,7 @@ const stream = require('stream'); const assert = require('assert'); const writable = new stream.Writable({ + highWaterMark: 16 * 1024, write: common.mustCall(function(chunk, encoding, cb) { assert.strictEqual( readable._readableState.awaitDrainWriters, @@ -26,6 +27,7 @@ const writable = new stream.Writable({ // A readable stream which produces two buffers. const bufs = [Buffer.alloc(32 * 1024), Buffer.alloc(33 * 1024)]; // above hwm const readable = new stream.Readable({ + highWaterMark: 16 * 1024, read: function() { while (bufs.length > 0) { this.push(bufs.shift()); diff --git a/test/parallel/test-stream-readable-infinite-read.js b/test/parallel/test-stream-readable-infinite-read.js index 3df3e39a73ec36..df88d78b74c36f 100644 --- a/test/parallel/test-stream-readable-infinite-read.js +++ b/test/parallel/test-stream-readable-infinite-read.js @@ -7,6 +7,7 @@ const { Readable } = require('stream'); const buf = Buffer.alloc(8192); const readable = new Readable({ + highWaterMark: 16 * 1024, read: common.mustCall(function() { this.push(buf); }, 31) diff --git a/test/parallel/test-stream-transform-split-highwatermark.js b/test/parallel/test-stream-transform-split-highwatermark.js index b6255c704710ac..290c7d957cc2d7 100644 --- a/test/parallel/test-stream-transform-split-highwatermark.js +++ b/test/parallel/test-stream-transform-split-highwatermark.js @@ -2,9 +2,9 @@ require('../common'); const assert = require('assert'); -const { Transform, Readable, Writable } = require('stream'); +const { Transform, Readable, Writable, getDefaultHighWaterMark } = require('stream'); -const DEFAULT = 16 * 1024; +const DEFAULT = getDefaultHighWaterMark(); function testTransform(expectedReadableHwm, expectedWritableHwm, options) { const t = new Transform(options); diff --git a/test/parallel/test-stream-transform-split-objectmode.js b/test/parallel/test-stream-transform-split-objectmode.js index 55d613573493b9..c637056534ab19 100644 --- a/test/parallel/test-stream-transform-split-objectmode.js +++ b/test/parallel/test-stream-transform-split-objectmode.js @@ -25,12 +25,14 @@ const assert = require('assert'); const Transform = require('stream').Transform; -const parser = new Transform({ readableObjectMode: true }); +const parser = new Transform({ + readableObjectMode: true +}); assert(parser._readableState.objectMode); assert(!parser._writableState.objectMode); assert.strictEqual(parser.readableHighWaterMark, 16); -assert.strictEqual(parser.writableHighWaterMark, 16 * 1024); +assert.strictEqual(parser.writableHighWaterMark, 64 * 1024); assert.strictEqual(parser.readableHighWaterMark, parser._readableState.highWaterMark); assert.strictEqual(parser.writableHighWaterMark, @@ -57,7 +59,7 @@ const serializer = new Transform({ writableObjectMode: true }); assert(!serializer._readableState.objectMode); assert(serializer._writableState.objectMode); -assert.strictEqual(serializer.readableHighWaterMark, 16 * 1024); +assert.strictEqual(serializer.readableHighWaterMark, 64 * 1024); assert.strictEqual(serializer.writableHighWaterMark, 16); assert.strictEqual(parser.readableHighWaterMark, parser._readableState.highWaterMark); diff --git a/test/parallel/test-tls-connect-hwm-option.js b/test/parallel/test-tls-connect-hwm-option.js index e016ccc6cba0dc..e1fb6c187bc783 100644 --- a/test/parallel/test-tls-connect-hwm-option.js +++ b/test/parallel/test-tls-connect-hwm-option.js @@ -37,7 +37,7 @@ server.listen(0, common.mustCall(() => { rejectUnauthorized: false, highWaterMark: undefined, }, common.mustCall(() => { - assert.strictEqual(defaultHighBob.readableHighWaterMark, 16 * 1024); + assert.strictEqual(defaultHighBob.readableHighWaterMark, 64 * 1024); defaultHighBob.end(); })); diff --git a/test/parallel/test-zlib-brotli-16GB.js b/test/parallel/test-zlib-brotli-16GB.js index 6b7cd22abb74ec..1c2712c32c91ee 100644 --- a/test/parallel/test-zlib-brotli-16GB.js +++ b/test/parallel/test-zlib-brotli-16GB.js @@ -3,6 +3,7 @@ const common = require('../common'); const { createBrotliDecompress } = require('node:zlib'); const strictEqual = require('node:assert').strictEqual; +const { getDefaultHighWaterMark } = require('stream'); // This tiny HEX string is a 16GB file. // This test verifies that the stream actually stops. @@ -18,5 +19,5 @@ decoder.end(buf); // to process the data and the buffer is not empty. setTimeout(common.mustCall(() => { // There is only one chunk in the buffer - strictEqual(decoder._readableState.buffer.length, 1); + strictEqual(decoder._readableState.buffer.length, getDefaultHighWaterMark() / (16 * 1024)); }), common.platformTimeout(500));