Add support for gzip de/encode streaming

Author: pimterry

README.md

@@ -59,10 +59,17 @@ This library also exports consistent async methods to compress and decompress ea
 
 Each method accepts a buffer and returns a promise for a buffer.
 
+## Streaming methods
+
+This library also supports streaming decoding, using web de(compression) streams wherever possible.
+
+* `createGzipStream`
+* `createGunzipStream`
+
 ## Browser usage
 
 To use this in a browser, you'll need to use a bundler (e.g. Webpack) that can include standard Node.js polyfill packages, you may need to install those polyfill packages, and your bundler needs to support bundling WebAssembly (e.g. Webpack v4+).
 
-In Webpack v4 this should all work automatically. In Webpack v5 this will require explicit dependencies and configuration. See this package's own [test webpack config](./karma.conf.js#L14-L44) and [dev dependencies](./package.json) for a working example.
+In Webpack v4 this should all work automatically. In Webpack v5 and most other builders this will require explicit dependencies and configuration. See this package's own [test webpack config](./karma.conf.js#L14-L44) and [dev dependencies](./package.json) for a working example.
 
 Brotli and Zstandard are only supported in runtime environments that support WebAssembly. All WebAssembly packages are loaded on-demand and only when native methods (e.g. Node's `zlib.brotli*`) are not available.

src/index.ts

@@ -115,6 +115,39 @@ export { encodeBase64Promisified as encodeBase64 };
 const decodeBase64Promisified = promisify<Uint8Array, Uint8Array>(decodeBase64);
 export { decodeBase64Promisified as decodeBase64 };
 
+// --- Streaming APIs ---
+
+// Lazily loaded to avoid bundling Node's stream polyfill for browsers
+let Duplex: typeof import('stream').Duplex | undefined;
+const getDuplex = () => {
+    if (!Duplex) {
+        Duplex = require('stream').Duplex;
+    }
+    return Duplex!;
+};
+
+type BufferSource = ArrayBufferView | ArrayBuffer;
+
+export function createGzipStream(): TransformStream<BufferSource, Uint8Array> {
+    // Use native CompressionStream where available:
+    if (typeof CompressionStream !== 'undefined') {
+        return new CompressionStream('gzip');
+    }
+    // Turn zlib node built-in into a web stream if not:
+    return getDuplex().toWeb(zlib.createGzip()) as TransformStream<BufferSource, Uint8Array>;
+}
+
+export function createGunzipStream(): TransformStream<BufferSource, Uint8Array> {
+    // Use native DecompressionStream where available:
+    if (typeof DecompressionStream !== 'undefined') {
+        return new DecompressionStream('gzip');
+    }
+    // Turn zlib node built-in into a web stream if not:
+    return getDuplex().toWeb(zlib.createGunzip()) as TransformStream<BufferSource, Uint8Array>;
+}
+
+// --- Buffer helpers ---
+
 const asBuffer = (input: Buffer | Uint8Array | ArrayBuffer): Buffer => {
     if (Buffer.isBuffer(input)) {
         return input;

test/stream.spec.ts

@@ -0,0 +1,124 @@
+import * as zlib from 'zlib';
+
+import chai = require("chai");
+const expect = chai.expect;
+
+import {
+    createGzipStream,
+    createGunzipStream
+} from '../src/index';
+
+// Helper to collect all chunks from a ReadableStream into a single Uint8Array
+async function collectStream(stream: ReadableStream<Uint8Array>): Promise<Uint8Array> {
+    const reader = stream.getReader();
+    const chunks: Uint8Array[] = [];
+    let totalLength = 0;
+
+    while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        chunks.push(value);
+        totalLength += value.length;
+    }
+
+    const result = new Uint8Array(totalLength);
+    let offset = 0;
+    for (const chunk of chunks) {
+        result.set(chunk, offset);
+        offset += chunk.length;
+    }
+    return result;
+}
+
+// Helper to create a ReadableStream from a Uint8Array
+function createReadableStream(data: Uint8Array): ReadableStream<Uint8Array> {
+    return new ReadableStream({
+        start(controller) {
+            controller.enqueue(data);
+            controller.close();
+        }
+    });
+}
+
+describe("Streaming", () => {
+    describe("Gzip", () => {
+        it('should compress data with gzip stream', async () => {
+            const input = Buffer.from('Hello streaming gzip world!');
+            const inputStream = createReadableStream(input);
+
+            const compressedStream = inputStream.pipeThrough(createGzipStream());
+            const compressed = await collectStream(compressedStream);
+
+            // Verify the compressed data can be decompressed with zlib
+            const decompressed = zlib.gunzipSync(Buffer.from(compressed));
+            expect(decompressed.toString()).to.equal('Hello streaming gzip world!');
+        });
+
+        it('should decompress gzip data with stream', async () => {
+            const original = 'Hello streaming gunzip world!';
+            const compressed = zlib.gzipSync(original);
+            const inputStream = createReadableStream(compressed);
+
+            const decompressedStream = inputStream.pipeThrough(createGunzipStream());
+            const decompressed = await collectStream(decompressedStream);
+
+            expect(Buffer.from(decompressed).toString()).to.equal(original);
+        });
+
+        it('should handle round-trip compression and decompression', async () => {
+            const original = 'Round-trip streaming test with some repeated data data data data';
+            const inputStream = createReadableStream(Buffer.from(original));
+
+            const compressedStream = inputStream.pipeThrough(createGzipStream());
+            const decompressedStream = compressedStream.pipeThrough(createGunzipStream());
+            const result = await collectStream(decompressedStream);
+
+            expect(Buffer.from(result).toString()).to.equal(original);
+        });
+
+        it('should handle multiple chunks', async () => {
+            const chunks = ['chunk1', 'chunk2', 'chunk3', 'chunk4'];
+            const inputStream = new ReadableStream<Uint8Array>({
+                start(controller) {
+                    for (const chunk of chunks) {
+                        controller.enqueue(Buffer.from(chunk));
+                    }
+                    controller.close();
+                }
+            });
+
+            const compressedStream = inputStream.pipeThrough(createGzipStream());
+            const decompressedStream = compressedStream.pipeThrough(createGunzipStream());
+            const result = await collectStream(decompressedStream);
+
+            expect(Buffer.from(result).toString()).to.equal(chunks.join(''));
+        });
+
+        it('should handle empty input', async () => {
+            const inputStream = createReadableStream(new Uint8Array(0));
+
+            const compressedStream = inputStream.pipeThrough(createGzipStream());
+            const decompressedStream = compressedStream.pipeThrough(createGunzipStream());
+            const result = await collectStream(decompressedStream);
+
+            expect(result.length).to.equal(0);
+        });
+
+        it('should handle large data', async () => {
+            // Create a 1MB buffer of repeated data
+            const pattern = 'This is a test pattern that will be repeated many times. ';
+            const repeated = pattern.repeat(20000);
+            const inputStream = createReadableStream(Buffer.from(repeated));
+
+            const compressedStream = inputStream.pipeThrough(createGzipStream());
+            const compressed = await collectStream(compressedStream);
+
+            // Compressed should be smaller than original due to repetition
+            expect(compressed.length).to.be.lessThan(repeated.length);
+
+            // Verify decompression
+            const decompressed = zlib.gunzipSync(Buffer.from(compressed));
+            expect(decompressed.toString()).to.equal(repeated);
+        });
+    });
+});

tsconfig.json

@@ -1,6 +1,6 @@
 {
     "compilerOptions": {
-        "target": "es6",
+        "target": "es2020",
         "module": "commonjs",
         "outDir": "./dist",
         "declaration": true,