Skip to content

Commit

Permalink
stream: bump default highWaterMark
Browse files Browse the repository at this point in the history
This should give a performance boost accross the board.

Given that the old limit is a decod old and memory capacity has
doubled many times since I think it is appropriate to slightly bump
the default limit.

PR-URL: nodejs#52037
Refs: nodejs#46608
Refs: nodejs#50120
  • Loading branch information
ronag committed Mar 10, 2024
1 parent 1f19316 commit 8f4a308
Show file tree
Hide file tree
Showing 18 changed files with 27 additions and 19 deletions.
2 changes: 1 addition & 1 deletion benchmark/net/net-c2s.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ const net = require('net');
const PORT = common.PORT;

const bench = common.createBenchmark(main, {
len: [64, 102400, 1024 * 1024 * 16],
len: [64, 102400, 1024 * 64 * 16],
type: ['utf', 'asc', 'buf'],
dur: [5],
}, {
Expand Down
2 changes: 1 addition & 1 deletion benchmark/net/net-pipe.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ const net = require('net');
const PORT = common.PORT;

const bench = common.createBenchmark(main, {
len: [2, 64, 102400, 1024 * 1024 * 16],
len: [2, 64, 102400, 1024 * 64 * 16],
type: ['utf', 'asc', 'buf'],
dur: [5],
}, {
Expand Down
2 changes: 1 addition & 1 deletion benchmark/net/net-s2c.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ const common = require('../common.js');
const PORT = common.PORT;

const bench = common.createBenchmark(main, {
sendchunklen: [256, 32 * 1024, 128 * 1024, 16 * 1024 * 1024],
sendchunklen: [256, 32 * 1024, 128 * 1024, 16 * 64 * 1024],
type: ['utf', 'asc', 'buf'],
recvbuflen: [0, 64 * 1024, 1024 * 1024],
recvbufgenfn: ['true', 'false'],
Expand Down
2 changes: 1 addition & 1 deletion benchmark/net/net-wrap-js-stream-passthrough.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ const common = require('../common.js');
const { PassThrough } = require('stream');

const bench = common.createBenchmark(main, {
len: [64, 102400, 1024 * 1024 * 16],
len: [64, 102400, 1024 * 64 * 16],
type: ['utf', 'asc', 'buf'],
dur: [5],
}, {
Expand Down
2 changes: 1 addition & 1 deletion benchmark/net/tcp-raw-c2s.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ const util = require('util');
// run the function with those settings.
// if not, then queue up a bunch of child processes.
const bench = common.createBenchmark(main, {
len: [102400, 1024 * 1024 * 16],
len: [102400, 1024 * 64 * 16],
type: ['utf', 'asc', 'buf'],
dur: [5],
}, {
Expand Down
2 changes: 1 addition & 1 deletion benchmark/net/tcp-raw-pipe.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ const util = require('util');
// run the function with those settings.
// if not, then queue up a bunch of child processes.
const bench = common.createBenchmark(main, {
len: [102400, 1024 * 1024 * 16],
len: [102400, 1024 * 64 * 16],
type: ['utf', 'asc', 'buf'],
dur: [5],
}, {
Expand Down
2 changes: 1 addition & 1 deletion benchmark/net/tcp-raw-s2c.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ const util = require('util');
// run the function with those settings.
// If not, then queue up a bunch of child processes.
const bench = common.createBenchmark(main, {
len: [102400, 1024 * 1024 * 16],
len: [102400, 1024 * 64 * 16],
type: ['utf', 'asc', 'buf'],
dur: [5],
}, {
Expand Down
4 changes: 2 additions & 2 deletions doc/api/stream.md
Original file line number Diff line number Diff line change
Expand Up @@ -3500,7 +3500,7 @@ changes:
* `options` {Object}
* `highWaterMark` {number} Buffer level when
[`stream.write()`][stream-write] starts returning `false`. **Default:**
`16384` (16 KiB), or `16` for `objectMode` streams.
`65536` (64 KiB), or `16` for `objectMode` streams.
* `decodeStrings` {boolean} Whether to encode `string`s passed to
[`stream.write()`][stream-write] to `Buffer`s (with the encoding
specified in the [`stream.write()`][stream-write] call) before passing
Expand Down Expand Up @@ -3873,7 +3873,7 @@ changes:
* `options` {Object}
* `highWaterMark` {number} The maximum [number of bytes][hwm-gotcha] to store
in the internal buffer before ceasing to read from the underlying resource.
**Default:** `16384` (16 KiB), or `16` for `objectMode` streams.
**Default:** `65536` (64 KiB), or `16` for `objectMode` streams.
* `encoding` {string} If specified, then buffers will be decoded to
strings using the specified encoding. **Default:** `null`.
* `objectMode` {boolean} Whether this stream should behave
Expand Down
2 changes: 1 addition & 1 deletion lib/internal/streams/state.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ const { validateInteger } = require('internal/validators');

const { ERR_INVALID_ARG_VALUE } = require('internal/errors').codes;

let defaultHighWaterMarkBytes = 16 * 1024;
let defaultHighWaterMarkBytes = 64 * 1024;
let defaultHighWaterMarkObjectMode = 16;

function highWaterMarkFrom(options, isDuplex, duplexKey) {
Expand Down
2 changes: 1 addition & 1 deletion test/parallel/test-http-no-read-no-dump.js
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ const server = http.createServer((req, res) => {
}, common.mustCall((res) => {
res.resume();

post.write(Buffer.alloc(16 * 1024).fill('X'));
post.write(Buffer.alloc(64 * 1024).fill('X'));
onPause = () => {
post.end('something');
};
Expand Down
2 changes: 1 addition & 1 deletion test/parallel/test-https-hwm.js
Original file line number Diff line number Diff line change
Expand Up @@ -62,5 +62,5 @@ const httpsServer = https.createServer({
port: this.address().port,
rejectUnauthorized: false,
highWaterMark: undefined,
}, loadCallback(16 * 1024)).on('error', common.mustNotCall()).end();
}, loadCallback(64 * 1024)).on('error', common.mustNotCall()).end();
}));
2 changes: 2 additions & 0 deletions test/parallel/test-stream-duplex-readable-end.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,15 @@ const stream = require('stream');
let loops = 5;

const src = new stream.Readable({
highWaterMark: 16 * 1024,
read() {
if (loops--)
this.push(Buffer.alloc(20000));
}
});

const dst = new stream.Transform({
highWaterMark: 16 * 1024,
transform(chunk, output, fn) {
this.push(null);
fn();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ const stream = require('stream');
const assert = require('assert');

const writable = new stream.Writable({
highWaterMark: 16 * 1024,
write: common.mustCall(function(chunk, encoding, cb) {
assert.strictEqual(
readable._readableState.awaitDrainWriters,
Expand All @@ -26,6 +27,7 @@ const writable = new stream.Writable({
// A readable stream which produces two buffers.
const bufs = [Buffer.alloc(32 * 1024), Buffer.alloc(33 * 1024)]; // above hwm
const readable = new stream.Readable({
highWaterMark: 16 * 1024,
read: function() {
while (bufs.length > 0) {
this.push(bufs.shift());
Expand Down
1 change: 1 addition & 0 deletions test/parallel/test-stream-readable-infinite-read.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ const { Readable } = require('stream');
const buf = Buffer.alloc(8192);

const readable = new Readable({
highWaterMark: 16 * 1024,
read: common.mustCall(function() {
this.push(buf);
}, 31)
Expand Down
4 changes: 2 additions & 2 deletions test/parallel/test-stream-transform-split-highwatermark.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@
require('../common');
const assert = require('assert');

const { Transform, Readable, Writable } = require('stream');
const { Transform, Readable, Writable, getDefaultHighWaterMark } = require('stream');

const DEFAULT = 16 * 1024;
const DEFAULT = getDefaultHighWaterMark();

function testTransform(expectedReadableHwm, expectedWritableHwm, options) {
const t = new Transform(options);
Expand Down
8 changes: 5 additions & 3 deletions test/parallel/test-stream-transform-split-objectmode.js
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,14 @@ const assert = require('assert');

const Transform = require('stream').Transform;

const parser = new Transform({ readableObjectMode: true });
const parser = new Transform({
readableObjectMode: true
});

assert(parser._readableState.objectMode);
assert(!parser._writableState.objectMode);
assert.strictEqual(parser.readableHighWaterMark, 16);
assert.strictEqual(parser.writableHighWaterMark, 16 * 1024);
assert.strictEqual(parser.writableHighWaterMark, 64 * 1024);
assert.strictEqual(parser.readableHighWaterMark,
parser._readableState.highWaterMark);
assert.strictEqual(parser.writableHighWaterMark,
Expand All @@ -57,7 +59,7 @@ const serializer = new Transform({ writableObjectMode: true });

assert(!serializer._readableState.objectMode);
assert(serializer._writableState.objectMode);
assert.strictEqual(serializer.readableHighWaterMark, 16 * 1024);
assert.strictEqual(serializer.readableHighWaterMark, 64 * 1024);
assert.strictEqual(serializer.writableHighWaterMark, 16);
assert.strictEqual(parser.readableHighWaterMark,
parser._readableState.highWaterMark);
Expand Down
2 changes: 1 addition & 1 deletion test/parallel/test-tls-connect-hwm-option.js
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ server.listen(0, common.mustCall(() => {
rejectUnauthorized: false,
highWaterMark: undefined,
}, common.mustCall(() => {
assert.strictEqual(defaultHighBob.readableHighWaterMark, 16 * 1024);
assert.strictEqual(defaultHighBob.readableHighWaterMark, 64 * 1024);
defaultHighBob.end();
}));

Expand Down
3 changes: 2 additions & 1 deletion test/parallel/test-zlib-brotli-16GB.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 8f4a308

Please sign in to comment.