crossbind
GitHub

Zstandard for WebAssembly

v1.5.7WebAssembly

Zstandard 1.5.7 for browsers, Node.js and edge runtimes, precompiled for wasm32, single-threaded and multi-threaded as @crossbind/port-zstd-wasm.

npm install @crossbind/port-zstd-wasm@beta

Install

shell
npm install @crossbind/port-zstd-wasm@beta
crossbind.config.js
import zstdWasm from '@crossbind/port-zstd-wasm/crossbind.config.js';
 
export default {
dependencies: [zstdWasm],
paths: { config: import.meta.url },
};

crossbind itself arrives with your bundler plugin, or with a new project from npm create crossbind@beta; Bundlers has Vite, Webpack, Rspack and Rollup.

Usage

Each example runs here, in this tab, and prints what the site build checked.

Each example also has a JavaScript only tab: the same task with no C++ file, calling Zstandard's own headers from @crossbind/port-zstd directly. 3 of 4 work that way; the other says what stops it.

Compress and decompress a buffer

The two calls most zstd code makes: ZSTD_compress, and ZSTD_decompress with the original size read back from the frame.

src/native/zstd_codec.h
#pragma once
 
#include <zstd.h>
 
#include <stdexcept>
#include <string>
 
// One-shot Zstandard. Bytes cross the binding as a byte string: one UTF-16 code unit (0-255) per byte.
class Zstd {
public:
static std::string version() { return ZSTD_versionString(); }
 
static std::u16string compress(const std::string& text, int level) {
std::string out(ZSTD_compressBound(text.size()), '\0');
const size_t size = ZSTD_compress(&out[0], out.size(), text.data(), text.size(), level);
if (ZSTD_isError(size)) throw std::runtime_error(ZSTD_getErrorName(size));
std::u16string bytes(size, u'\0');
for (size_t i = 0; i < size; ++i) bytes[i] = static_cast<unsigned char>(out[i]);
return bytes;
}
 
static std::string decompress(const std::u16string& bytes) {
std::string in(bytes.size(), '\0');
for (size_t i = 0; i < bytes.size(); ++i) {
if (bytes[i] > 0xFF) throw std::invalid_argument("not a byte string");
in[i] = static_cast<char>(bytes[i]);
}
const unsigned long long size = ZSTD_getFrameContentSize(in.data(), in.size());
if (size == ZSTD_CONTENTSIZE_ERROR) throw std::runtime_error("not a zstd frame");
if (size == ZSTD_CONTENTSIZE_UNKNOWN) throw std::runtime_error("size not stored in the frame; use streaming");
if (size > (256u << 20)) throw std::runtime_error("refusing to allocate more than 256 MiB");
std::string out(static_cast<size_t>(size), '\0');
const size_t got = ZSTD_decompress(&out[0], out.size(), in.data(), in.size());
if (ZSTD_isError(got)) throw std::runtime_error(ZSTD_getErrorName(got));
out.resize(got);
return out;
}
};
main.js
import { initNative, Zstd } from './native/zstd_codec.h';
 
await initNative();
const text = 'crossbind '.repeat(100);
const frame = await Zstd.compress(text, 19);
const bytes = Uint8Array.from(frame, (c) => c.charCodeAt(0));
console.log(await Zstd.version(), bytes.length, [...bytes.slice(0, 4)].map((b) => b.toString(16)).join(' '));
console.log((await Zstd.decompress(frame)) === text);
PRINTSfirst run downloads 0.9 MB
1.5.7 27 28 b5 2f fd
true

Stream a file through zstd

For data you should not hold in one buffer: ZSTD_compressStream2 and ZSTD_decompressStream work file to file in 128 KB steps.

src/native/zstd_stream.h
#pragma once
 
#include <zstd.h>
 
#include <cstdio>
#include <memory>
#include <stdexcept>
#include <string>
#include <vector>
 
// Streaming Zstandard, file to file. Memory stays at two small buffers whatever the file size,
// the pattern of zstd's own examples/streaming_compression.c.
class ZstdStream {
public:
// Compresses `input` into `output` with a content checksum; returns the compressed size.
static double compressFile(const std::string& input, const std::string& output, int level) {
File in = open(input, "rb");
File out = open(output, "wb");
std::unique_ptr<ZSTD_CCtx, size_t (*)(ZSTD_CCtx*)> cctx(ZSTD_createCCtx(), ZSTD_freeCCtx);
check(ZSTD_CCtx_setParameter(cctx.get(), ZSTD_c_compressionLevel, level));
check(ZSTD_CCtx_setParameter(cctx.get(), ZSTD_c_checksumFlag, 1));
std::vector<char> inBuffer(ZSTD_CStreamInSize());
std::vector<char> outBuffer(ZSTD_CStreamOutSize());
double written = 0;
for (;;) {
const size_t read = std::fread(inBuffer.data(), 1, inBuffer.size(), in.get());
const bool last = read < inBuffer.size();
ZSTD_inBuffer source = {inBuffer.data(), read, 0};
bool finished = false;
while (!finished) {
ZSTD_outBuffer target = {outBuffer.data(), outBuffer.size(), 0};
const size_t remaining = check(ZSTD_compressStream2(cctx.get(), &target, &source, last ? ZSTD_e_end : ZSTD_e_continue));
write(out.get(), outBuffer.data(), target.pos);
written += static_cast<double>(target.pos);
finished = last ? remaining == 0 : source.pos == source.size;
}
if (last) return written;
}
}
 
// Decompresses `input` into `output`; returns the decompressed size.
static double decompressFile(const std::string& input, const std::string& output) {
File in = open(input, "rb");
File out = open(output, "wb");
std::unique_ptr<ZSTD_DCtx, size_t (*)(ZSTD_DCtx*)> dctx(ZSTD_createDCtx(), ZSTD_freeDCtx);
std::vector<char> inBuffer(ZSTD_DStreamInSize());
std::vector<char> outBuffer(ZSTD_DStreamOutSize());
double written = 0;
size_t pending = 0;
size_t read = 0;
while ((read = std::fread(inBuffer.data(), 1, inBuffer.size(), in.get())) > 0) {
ZSTD_inBuffer source = {inBuffer.data(), read, 0};
while (source.pos < source.size) {
ZSTD_outBuffer target = {outBuffer.data(), outBuffer.size(), 0};
pending = check(ZSTD_decompressStream(dctx.get(), &target, &source));
write(out.get(), outBuffer.data(), target.pos);
written += static_cast<double>(target.pos);
}
}
if (pending != 0) throw std::runtime_error("the input ends in the middle of a zstd frame");
return written;
}
 
private:
using File = std::unique_ptr<FILE, int (*)(FILE*)>;
 
static File open(const std::string& path, const char* mode) {
File file(std::fopen(path.c_str(), mode), std::fclose);
if (!file) throw std::runtime_error("cannot open " + path);
return file;
}
 
static void write(FILE* file, const char* data, size_t size) {
if (size && std::fwrite(data, 1, size, file) != size) throw std::runtime_error("write failed");
}
 
static size_t check(size_t code) {
if (ZSTD_isError(code)) throw std::runtime_error(ZSTD_getErrorName(code));
return code;
}
};
main.js
import { initNative } from './native/zstd_stream.h';
 
const m = await initNative();
const { ZstdStream } = m;
let seed = 42;
const random = (n) => (seed = (seed * 48271) % 2147483647) % n;
const lines = Array.from({ length: 50000 }, (_, i) => `2026-09-24T12:00:${String(i % 60).padStart(2, '0')}Z GET /api/items/${random(9000)} ${random(10) ? 200 : 404} ${random(900)}ms`);
// m.FS.writeFile adds to a file that already exists, so every run gets a fresh directory.
const dir = await m.getRandomPath('/memfs');
await m.FS.writeFile(`${dir}/access.log`, lines.join('\n'));
 
const packed = await ZstdStream.compressFile(`${dir}/access.log`, `${dir}/access.log.zst`, 3);
const unpacked = await ZstdStream.decompressFile(`${dir}/access.log.zst`, `${dir}/access.copy.log`);
const original = await m.getFileBytes(`${dir}/access.log`);
const copy = await m.getFileBytes(`${dir}/access.copy.log`);
console.log(`${original.length} B -> ${packed} B -> ${unpacked} B`);
console.log(copy.length === original.length && copy.every((byte, i) => byte === original[i]));
PRINTSfirst run downloads 0.9 MB
2537578 B -> 378699 B -> 2537578 B
true

Choose a level, a checksum and a window

A context set up with ZSTD_CCtx_setParameter decides what every frame it writes looks like; ZSTD_getFrameHeader reads the choices back.

src/native/zstd_frame.h
#pragma once
 
// ZSTD_getFrameHeader is in zstd's static API; safe with this statically linked, pinned library.
#define ZSTD_STATIC_LINKING_ONLY
#include <zstd.h>
 
#include <memory>
#include <stdexcept>
#include <string>
 
// A compression context with explicit parameters, and a reader for what they put in the frame header.
class ZstdFrame {
public:
// A windowLog of 0 keeps the level's default window.
static std::u16string compress(const std::string& text, int level, bool checksum, int windowLog) {
std::unique_ptr<ZSTD_CCtx, size_t (*)(ZSTD_CCtx*)> cctx(ZSTD_createCCtx(), ZSTD_freeCCtx);
check(ZSTD_CCtx_setParameter(cctx.get(), ZSTD_c_compressionLevel, level));
check(ZSTD_CCtx_setParameter(cctx.get(), ZSTD_c_checksumFlag, checksum ? 1 : 0));
if (windowLog) check(ZSTD_CCtx_setParameter(cctx.get(), ZSTD_c_windowLog, windowLog));
std::string out(ZSTD_compressBound(text.size()), '\0');
out.resize(check(ZSTD_compress2(cctx.get(), &out[0], out.size(), text.data(), text.size())));
std::u16string bytes(out.size(), u'\0');
for (size_t i = 0; i < out.size(); ++i) bytes[i] = static_cast<unsigned char>(out[i]);
return bytes;
}
 
static std::string header(const std::u16string& frame) {
std::string head;
for (size_t i = 0; i < frame.size() && i < ZSTD_FRAMEHEADERSIZE_MAX; ++i) head += static_cast<char>(frame[i]);
ZSTD_FrameHeader info;
const size_t status = ZSTD_getFrameHeader(&info, head.data(), head.size());
if (ZSTD_isError(status) || status > 0) throw std::runtime_error("not a zstd frame header");
const std::string content = info.frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN ? "not stored" : std::to_string(info.frameContentSize) + " B";
return "content " + content + ", window " + std::to_string(info.windowSize) + " B, checksum " + (info.checksumFlag ? "yes" : "no");
}
 
private:
static size_t check(size_t code) {
if (ZSTD_isError(code)) throw std::runtime_error(ZSTD_getErrorName(code));
return code;
}
};
main.js
import { initNative, ZstdFrame } from './native/zstd_frame.h';
 
await initNative();
let seed = 7;
const random = (n) => (seed = (seed * 48271) % 2147483647) % n;
const text = Array.from({ length: 400 }, (_, i) => `{"id":${i},"user":"user${random(5000)}","score":${random(1000)}}`).join('\n');
for (const level of [3, 9, 19]) {
const frame = await ZstdFrame.compress(text, level, false, 0);
console.log(`level ${level}: ${text.length} B -> ${frame.length} B`);
}
const small = await ZstdFrame.compress(text, 19, true, 10);
console.log(`level 19, 1 KiB window, checksum: ${small.length} B`);
console.log(await ZstdFrame.header(small));
PRINTSfirst run downloads 0.9 MB
level 3: 16160 B -> 3065 B
level 9: 16160 B -> 2749 B
level 19: 16160 B -> 2219 B
level 19, 1 KiB window, checksum: 2469 B
content 16160 B, window 1024 B, checksum yes

Compress small messages with a dictionary

Train once with ZDICT_trainFromBuffer, digest it once with ZSTD_createCDict and ZSTD_createDDict, then compress every message against it.

src/native/zstd_dictionary.h
#pragma once
 
#include <zdict.h>
#include <zstd.h>
 
#include <memory>
#include <stdexcept>
#include <string>
#include <vector>
 
// Dictionary compression for small messages: train once on samples, prepare the dictionary once,
// then compress and decompress each message against it.
class ZstdDictionary {
public:
// Trains on newline-separated samples; returns at most `capacity` bytes of dictionary.
static std::u16string train(const std::string& samples, int capacity) {
std::string joined;
std::vector<size_t> sizes;
for (size_t start = 0; start < samples.size();) {
size_t end = samples.find('\n', start);
if (end == std::string::npos) end = samples.size();
joined.append(samples, start, end - start);
sizes.push_back(end - start);
start = end + 1;
}
std::string dictionary(static_cast<size_t>(capacity), '\0');
const size_t size = ZDICT_trainFromBuffer(&dictionary[0], dictionary.size(), joined.data(), sizes.data(), static_cast<unsigned>(sizes.size()));
if (ZDICT_isError(size)) throw std::runtime_error(ZDICT_getErrorName(size));
dictionary.resize(size);
return toUnits(dictionary);
}
 
// Digests the dictionary once for compression and once for decompression.
ZstdDictionary(const std::u16string& dictionary, int level)
: compressDictionary(nullptr, ZSTD_freeCDict), decompressDictionary(nullptr, ZSTD_freeDDict) {
const std::string bytes = fromUnits(dictionary);
compressDictionary.reset(ZSTD_createCDict(bytes.data(), bytes.size(), level));
decompressDictionary.reset(ZSTD_createDDict(bytes.data(), bytes.size()));
if (!compressDictionary || !decompressDictionary) throw std::runtime_error("not a usable dictionary");
}
 
std::u16string compress(const std::string& message) const {
std::unique_ptr<ZSTD_CCtx, size_t (*)(ZSTD_CCtx*)> cctx(ZSTD_createCCtx(), ZSTD_freeCCtx);
std::string out(ZSTD_compressBound(message.size()), '\0');
out.resize(check(ZSTD_compress_usingCDict(cctx.get(), &out[0], out.size(), message.data(), message.size(), compressDictionary.get())));
return toUnits(out);
}
 
std::string decompress(const std::u16string& frame) const {
const std::string in = fromUnits(frame);
const unsigned long long size = ZSTD_getFrameContentSize(in.data(), in.size());
if (size == ZSTD_CONTENTSIZE_ERROR || size == ZSTD_CONTENTSIZE_UNKNOWN) throw std::runtime_error("not a zstd frame with its size");
if (size > (1u << 20)) throw std::runtime_error("a message above 1 MiB is not a small message");
std::unique_ptr<ZSTD_DCtx, size_t (*)(ZSTD_DCtx*)> dctx(ZSTD_createDCtx(), ZSTD_freeDCtx);
std::string out(static_cast<size_t>(size), '\0');
out.resize(check(ZSTD_decompress_usingDDict(dctx.get(), &out[0], out.size(), in.data(), in.size(), decompressDictionary.get())));
return out;
}
 
private:
static size_t check(size_t code) {
if (ZSTD_isError(code)) throw std::runtime_error(ZSTD_getErrorName(code));
return code;
}
 
static std::u16string toUnits(const std::string& data) {
std::u16string units(data.size(), u'\0');
for (size_t i = 0; i < data.size(); ++i) units[i] = static_cast<unsigned char>(data[i]);
return units;
}
 
static std::string fromUnits(const std::u16string& units) {
std::string data(units.size(), '\0');
for (size_t i = 0; i < units.size(); ++i) {
if (units[i] > 0xFF) throw std::invalid_argument("not a byte string");
data[i] = static_cast<char>(units[i]);
}
return data;
}
 
std::unique_ptr<ZSTD_CDict, size_t (*)(ZSTD_CDict*)> compressDictionary;
std::unique_ptr<ZSTD_DDict, size_t (*)(ZSTD_DDict*)> decompressDictionary;
};
main.js
import { initNative, ZstdDictionary } from './native/zstd_dictionary.h';
import { Zstd } from './native/zstd_codec.h';
 
await initNative();
const event = (i) => `{"event":"click","user":${1000 + ((i * 37) % 900)},"page":"/products/${i % 12}","ms":${(i * 7919) % 400}}`;
const samples = Array.from({ length: 4000 }, (_, i) => event(i)).join('\n');
const dictionary = await ZstdDictionary.train(samples, 2048);
const codec = await new ZstdDictionary(dictionary, 3);
 
const message = event(4321);
const alone = await Zstd.compress(message, 3); // the one-shot wrapper from the first example
const frame = await codec.compress(message);
console.log(`dictionary: ${dictionary.length} B`);
console.log(`${message.length} B message: ${alone.length} B alone, ${frame.length} B with the dictionary`);
console.log((await codec.decompress(frame)) === message);
PRINTSfirst run downloads 0.9 MB
dictionary: 2048 B
59 B message: 68 B alone, 29 B with the dictionary
true

What is different on WebAssembly

  • In a browser the module runs in a Worker by default (useWorker), so every call returns a promise: await calls and constructors alike.
  • The module has its own filesystem: m.FS writes files, m.getFileBytes reads them back and m.autoMountFiles mounts File objects from an <input type=file>. /memfs lives in memory; /opfs persists across reloads and needs the Worker. See Filesystem.
  • In Node.js, m.FS is the real disk, so use real paths there.
  • Multi-threaded builds (runtime: 'mt') need COOP and COEP headers in production. See Threading.

Other platforms

Facts on this page come from the port manifests in the repository and from what npm served on beta when the site was built. See the Libraries guide for the full consumer flow.

MORE LIBRARIES
cURLExpatGDALGEOSGeoTIFFiconvLERClibjpeg-turbolibTIFFOpenSSLPROJSpatiaLiteSQLiteWebPzlib
Type to search every guide page and section.
↑↓ navigate↵ openesc close