Commit c7b1b6207b2 for nodejs
commit c7b1b6207b2ebb41300db3f5295735e3b5d9f18c
Author: Yagiz Nizipli <yagiz@nizipli.com>
Date: Mon Oct 19 13:07:51 2026 -0400
zlib: grow the output buffer of large async one-shot calls
The asynchronous convenience methods hand the whole input to zlib in
one write, but still take one threadpool round trip per `chunkSize`
(16 KiB by default) of output. Decompressing 4 MiB took 257 round
trips.
Only the concatenated result of these calls is observable, so double
the output buffer whenever it fills up, up to 1 MiB. 4 MiB of output
now takes 10 round trips. The buffer never grows past what is needed
to tell that `maxOutputLength` was exceeded, and results that fit in
the first buffer are unchanged.
Signed-off-by: Yagiz Nizipli <yagiz@nizipli.com>
Assisted-by: Grok
PR-URL: https://github.com/nodejs/node/pull/66863
Reviewed-By: Robert Nagy <ronagy@icloud.com>
Reviewed-By: Matteo Collina <matteo.collina@gmail.com>
diff --git a/lib/zlib.js b/lib/zlib.js
index e9458573577..f1d93665967 100644
--- a/lib/zlib.js
+++ b/lib/zlib.js
@@ -24,6 +24,7 @@
const {
ArrayBuffer,
MathMax,
+ MathMin,
NumberIsNaN,
ObjectDefineProperties,
ObjectDefineProperty,
@@ -182,6 +183,18 @@ function zlibBufferOnData(chunk) {
zlibBufferAddChunk(this, chunk);
}
+// Only the concatenated output of a one-shot call is observable, so its output
+// buffer can double in size whenever it fills up, and large results take far
+// fewer threadpool round trips. It never grows past what is needed to detect
+// that maxOutputLength was exceeded.
+const kMaxBufferResultChunkSize = 1024 * 1024;
+function zlibBufferGrowChunkSize(engine) {
+ const size = MathMin(engine._chunkSize * 2, kMaxBufferResultChunkSize,
+ engine._maxOutputLength - engine.nread + 1);
+ if (size > engine._chunkSize)
+ engine._chunkSize = size;
+}
+
function zlibBufferOnError(err) {
this.removeAllListeners('end');
this.cb(err);
@@ -691,6 +704,8 @@ function processCallback() {
// Exhausted the output buffer, or used all the input create a new one.
if (availOutAfter === 0 || self._outOffset >= self._chunkSize) {
+ if (availOutAfter === 0 && self[kBufferResult] === true)
+ zlibBufferGrowChunkSize(self);
handle.availOutBefore = self._chunkSize;
self._outOffset = 0;
self._outBuffer = Buffer.allocUnsafe(self._chunkSize);
diff --git a/test/parallel/test-zlib-convenience-async-large.js b/test/parallel/test-zlib-convenience-async-large.js
new file mode 100644
index 00000000000..03a7dab8684
--- /dev/null
+++ b/test/parallel/test-zlib-convenience-async-large.js
@@ -0,0 +1,82 @@
+'use strict';
+// The asynchronous convenience methods grow their output buffer while
+// processing large inputs. Check that large results, custom chunkSize values
+// and maxOutputLength behave exactly like the synchronous methods.
+
+const common = require('../common');
+const assert = require('assert');
+const fs = require('fs');
+const zlib = require('zlib');
+
+const { Z_MIN_CHUNK, BROTLI_PARAM_QUALITY } = zlib.constants;
+
+// Text plus pseudo-random bytes, so that both the compressed and the
+// decompressed output span many output buffers.
+const random = Buffer.alloc(256 * 1024);
+{
+ let seed = 0x2545f491;
+ for (let i = 0; i < random.length; i++) {
+ seed ^= seed << 13;
+ seed ^= seed >>> 17;
+ seed ^= seed << 5;
+ random[i] = seed & 0xff;
+ }
+}
+const data = Buffer.concat([
+ Buffer.alloc(2 * 1024 * 1024, fs.readFileSync(__filename)),
+ random,
+]);
+
+// Keep brotli fast enough for a test.
+const brotliOptions = { params: { [BROTLI_PARAM_QUALITY]: 4 } };
+const pairs = [
+ ['deflate', 'inflate'],
+ ['gzip', 'gunzip'],
+ ['gzip', 'unzip'],
+ ['deflateRaw', 'inflateRaw'],
+ ['brotliCompress', 'brotliDecompress', brotliOptions],
+ ['zstdCompress', 'zstdDecompress'],
+];
+
+function callAsync(method, input, options) {
+ return new Promise((resolve, reject) => {
+ zlib[method](input, options, common.mustCall((err, result) => {
+ if (err) reject(err);
+ else resolve(result);
+ }));
+ });
+}
+
+(async () => {
+ for (const [compress, decompress, options = {}] of pairs) {
+ const label = `${compress}/${decompress}`;
+ const compressed = await callAsync(compress, data, options);
+ const expected = zlib[`${compress}Sync`](data, options);
+ // zstdCompress() pledges the input size and ends the frame separately,
+ // which yields different (not larger) frames than zstdCompressSync().
+ if (compress !== 'zstdCompress') {
+ assert.deepStrictEqual(compressed, expected, label);
+ }
+ assert.deepStrictEqual(await callAsync(decompress, compressed, {}), data,
+ label);
+
+ for (const chunkSize of [Z_MIN_CHUNK, data.length + 1]) {
+ assert.deepStrictEqual(
+ await callAsync(decompress, compressed, { chunkSize }), data,
+ `${label} chunkSize=${chunkSize}`);
+ }
+
+ for (const maxOutputLength of [data.length - 1, 100000, 1]) {
+ await assert.rejects(
+ callAsync(decompress, compressed, { maxOutputLength }), {
+ name: 'RangeError',
+ code: 'ERR_BUFFER_TOO_LARGE',
+ message: `Cannot create a Buffer larger than ${maxOutputLength} bytes`,
+ }, `${label} maxOutputLength=${maxOutputLength}`);
+ }
+ assert.deepStrictEqual(
+ await callAsync(decompress, compressed,
+ { maxOutputLength: data.length }),
+ data, label);
+ }
+})().then(common.mustCall());
diff --git a/test/parallel/test-zlib-convenience-async-round-trip.js b/test/parallel/test-zlib-convenience-async-round-trip.js
index ab48f885272..e7b9fb217b0 100644
--- a/test/parallel/test-zlib-convenience-async-round-trip.js
+++ b/test/parallel/test-zlib-convenience-async-round-trip.js
@@ -1,7 +1,8 @@
'use strict';
// The asynchronous convenience methods know the whole input up front, so they
// process it with the finish flush in a single write. Check that a call whose
-// output fits in one output buffer takes exactly one threadpool round trip.
+// output fits in one output buffer takes exactly one threadpool round trip,
+// and that larger results need few round trips.
const common = require('../common');
const assert = require('assert');
@@ -71,4 +72,17 @@ function callAsync(method, buffer, options = {}) {
});
assert.deepStrictEqual(roundTrips, [1]);
assert.strictEqual(error.code, 'Z_DATA_ERROR');
+
+ // The output buffer doubles from the 16 KiB default up to 1 MiB, so 4 MiB of
+ // output takes 6 + 4 round trips instead of 256.
+ {
+ const large = Buffer.alloc(4 * 1024 * 1024, input);
+ const compressed = zlib.deflateSync(large);
+ let result;
+ const roundTrips = await countRoundTrips(async () => {
+ result = await callAsync('inflate', compressed);
+ });
+ assert.deepStrictEqual(roundTrips, [10]);
+ assert.deepStrictEqual(result, large);
+ }
})().then(common.mustCall());