Use zflate instead of libdeflate for BGZF, sort runs and indexes

CHANGELOG.md

@@ -4,6 +4,20 @@ Each release's binaries and full notes are on the
 [releases page](https://github.com/DylanCosto/bonefish/releases). "Output" means the alignment
 records. Each entry quotes the MD5 of the genome-wide HG001 sample's SAM output, `@PG` lines removed.
 
+## Unreleased
+
+**zflate replaces libdeflate.** BGZF input and output, the sorter's temporary runs and the BAI/CSI
+indexes go through [zflate](https://github.com/DylanCosto/zflate), a pure-Rust port of libdeflate,
+instead of libdeflate's C library. A build without jemalloc or mimalloc no longer needs a C
+compiler. zflate needs Rust 1.91 or newer.
+
+- **Output:** unchanged (`9bc7ff3e…`). Sorted BAMs, from FASTQ and from dorado BAM input, hold the
+  same records, and their BAI and CSI indexes answer region queries as samtools' own do. At the
+  default level the genome-wide sample's BAM is 465 bytes smaller.
+- **Speed:** CPU time is unchanged.
+- **`--sort-level`** outside 0-12 is rejected with an error, instead of a panic in a compression
+  thread.
+
 ## v0.4.0 (2026-10-28)
 
 **New: `-x hifi`** (alias `map-hifi`), for PacBio HiFi genomic DNA. It is the `dna` preset, plus

bonefish-core/Cargo.toml

@@ -35,7 +35,7 @@ thiserror = "2"
 anyhow = "1"
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
-libdeflater = "1.26"
+zflate = "1.0.0"
 memmap2 = "0.9"
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]

bonefish-core/src/fasta/bam.rs

@@ -1,6 +1,6 @@
 //! BAM input (e.g. unaligned ONT basecaller output) as a FASTQ text stream.
 //!
-//! `BamFastq` decodes BGZF blocks with libdeflate and turns each primary or unmapped record
+//! `BamFastq` decodes BGZF blocks with zflate and turns each primary or unmapped record
 //! (flags 0x100 and 0x800 skipped, like `samtools fastq -F 0x900`) into a FASTQ record. Reverse-
 //! strand records are reverse-complemented back to the read's original orientation. With `-y`
 //! the record's tags are written as the FASTQ comment in SAM text form, except alignment tags
@@ -65,7 +65,7 @@ pub fn filter_comment_tags(comment: &str) -> Option<String> {
 /// list if the input is not BAM.
 pub fn header_lines<R: Read>(input: R) -> io::Result<Vec<String>> {
     let mut f = io::BufReader::new(input);
-    let mut dec = libdeflater::Decompressor::new();
+    let mut dec = zflate::Decompressor::new();
     let mut block = Vec::with_capacity(1 << 16);
     let mut data = Vec::new();
     let mut need = 8usize;
@@ -102,7 +102,7 @@ pub fn looks_like_bam(head: &[u8]) -> bool {
         return false;
     }
     let mut out = vec![0u8; isize];
-    let mut dec = libdeflater::Decompressor::new();
+    let mut dec = zflate::Decompressor::new();
     matches!(dec.gzip_decompress(&head[..bsize], &mut out), Ok(n) if n >= 4 && &out[..4] == b"BAM\x01")
 }
 
@@ -151,7 +151,7 @@ impl BamFastq {
         let (tx, rx) = std::sync::mpsc::sync_channel::<io::Result<Vec<u8>>>(8);
         std::thread::spawn(move || {
             let mut inner = inner;
-            let mut dec = libdeflater::Decompressor::new();
+            let mut dec = zflate::Decompressor::new();
             let mut block = Vec::with_capacity(1 << 16);
             loop {
                 // ~1 MB of decompressed data per message
@@ -296,7 +296,7 @@ impl Read for BamFastq {
 }
 
 /// Decompress the next BGZF block from `inner`, appending to `out`; false at end of input.
-fn next_block<R: Read>(inner: &mut R, dec: &mut libdeflater::Decompressor, block: &mut Vec<u8>, out: &mut Vec<u8>) -> io::Result<bool> {
+fn next_block<R: Read>(inner: &mut R, dec: &mut zflate::Decompressor, block: &mut Vec<u8>, out: &mut Vec<u8>) -> io::Result<bool> {
     let mut head = [0u8; 18];
     match read_full(inner, &mut head)? {
         0 => return Ok(false),
@@ -490,7 +490,7 @@ fn append_tags(mut aux: &[u8], out: &mut Vec<u8>) -> Result<(), ()> {
 
 /// BGZF decompression on several threads, as a `BufRead` of the decompressed stream (for
 /// bgzipped FASTQ/FASTA). A splitter thread cuts the file into batches of whole blocks (~1 MiB
-/// compressed), `workers` threads decompress them with libdeflate, and the reader puts them
+/// compressed), `workers` threads decompress them with zflate, and the reader puts them
 /// back in file order.
 pub struct BgzfParallel {
     rx: std::sync::mpsc::Receiver<(u64, io::Result<Vec<u8>>)>,
@@ -556,7 +556,7 @@ impl BgzfParallel {
             let job_rx = std::sync::Arc::clone(&job_rx);
             let res_tx = res_tx.clone();
             std::thread::spawn(move || {
-                let mut dec = libdeflater::Decompressor::new();
+                let mut dec = zflate::Decompressor::new();
                 loop {
                     let job = { let rx = job_rx.lock().unwrap(); rx.recv() };
                     let Ok((seq, job)) = job else { return; };

bonefish-core/src/sorted_bam.rs

@@ -4,7 +4,7 @@
 //! The mapper's SAM text is parsed into BAM records as it is written (one writer thread; parsing
 //! is cheap next to compression). Records collect in a memory batch. A full batch is sorted and
 //! handed to a background thread that writes it as a temporary run, in chunks compressed with
-//! libdeflate level 1 (a private format, much cheaper than the BGZF level 6 that samtools uses for
+//! DEFLATE level 1 (a private format, much cheaper than the BGZF level 6 that samtools uses for
 //! its temporary files), while the next batch fills. After the last read, the runs and the last
 //! batch are merged and written as BGZF blocks compressed in parallel, and the BAI index is built
 //! from the block offsets as they are written.
@@ -81,6 +81,9 @@ pub struct SortedBamWriter {
 
 impl SortedBamWriter {
     pub fn new(out_path: &str, cfg: SortConfig) -> io::Result<Self> {
+        if !(0..=12).contains(&cfg.level) {
+            return Err(io::Error::new(io::ErrorKind::InvalidInput, format!("BGZF compression level {} is not 0-12", cfg.level)));
+        }
         // Fail early if the output can't be created.
         File::create(out_path)?;
         // Rendezvous: at most one batch filling and one being spilled.
@@ -498,16 +501,21 @@ fn write_run(b: &Batch, path: &PathBuf, threads: usize, abort: &std::sync::atomi
     Ok(true)
 }
 
+/// A DEFLATE level 0-12 (`SortedBamWriter::new` checks the output level).
+fn zlevel(lvl: i32) -> zflate::CompressionLevel {
+    u8::try_from(lvl).ok().and_then(zflate::CompressionLevel::new).expect("DEFLATE level 0-12")
+}
+
 /// Deflate every buffer (level `lvl`) on up to `threads` threads, in order.
 fn compress_all(bufs: &[Vec<u8>], lvl: i32, threads: usize) -> Vec<Vec<u8>> {
     let mut out: Vec<Vec<u8>> = vec![Vec::new(); bufs.len()];
     let per = bufs.len().div_ceil(threads.max(1)).max(1);
     std::thread::scope(|s| {
         for (bs, os) in bufs.chunks(per).zip(out.chunks_mut(per)) {
             s.spawn(move || {
-                let mut c = libdeflater::Compressor::new(libdeflater::CompressionLvl::new(lvl).unwrap());
+                let mut c = zflate::Compressor::new(zlevel(lvl));
                 for (b, o) in bs.iter().zip(os.iter_mut()) {
-                    let mut v = vec![0u8; c.deflate_compress_bound(b.len())];
+                    let mut v = vec![0u8; zflate::deflate_compress_bound(b.len())];
                     let n = c.deflate_compress(b, &mut v).expect("deflate");
                     v.truncate(n);
                     *o = v;
@@ -543,7 +551,7 @@ impl RunReader {
         for _ in 0..dec {
             let jr = job_rx.clone();
             std::thread::Builder::new().name("sort-dec".into()).spawn(move || {
-                let mut d = libdeflater::Decompressor::new();
+                let mut d = zflate::Decompressor::new();
                 loop {
                     let job = jr.lock().unwrap().recv();
                     let Ok((c, raw, done)) = job else { break };
@@ -641,9 +649,7 @@ fn bgzf_block(raw: &[u8], deflated: &[u8]) -> Vec<u8> {
     b.extend_from_slice(&[0x1f, 0x8b, 8, 4, 0, 0, 0, 0, 0, 0xff, 6, 0, b'B', b'C', 2, 0]);
     b.extend_from_slice(&((total - 1) as u16).to_le_bytes());
     b.extend_from_slice(deflated);
-    let mut crc = libdeflater::Crc::new();
-    crc.update(raw);
-    b.extend_from_slice(&crc.sum().to_le_bytes());
+    b.extend_from_slice(&zflate::crc32(0, raw).to_le_bytes());
     b.extend_from_slice(&(raw.len() as u32).to_le_bytes());
     b
 }
@@ -797,10 +803,10 @@ impl Indexer {
         }
         raw.extend_from_slice(&self.n_no_coor.to_le_bytes());
         let mut w = BufWriter::new(File::create(path)?);
-        let mut c = libdeflater::Compressor::new(libdeflater::CompressionLvl::default());
+        let mut c = zflate::Compressor::new(zflate::CompressionLevel::DEFAULT);
         let mut tmp = Vec::new();
         for chunk in raw.chunks(0xff00) {
-            tmp.resize(c.deflate_compress_bound(chunk.len()), 0);
+            tmp.resize(zflate::deflate_compress_bound(chunk.len()), 0);
             let n = c.deflate_compress(chunk, &mut tmp).map_err(|_| bad("deflate failed"))?;
             w.write_all(&bgzf_block(chunk, &tmp[..n]))?;
         }
@@ -834,12 +840,12 @@ fn merge_and_write(out_path: &str, header: &str, refs: &[(String, u32)], runs: &
         let rx = job_rx.clone();
         let tx = done_tx.clone();
         workers.push(std::thread::spawn(move || {
-            let mut c = libdeflater::Compressor::new(libdeflater::CompressionLvl::new(level).unwrap());
+            let mut c = zflate::Compressor::new(zlevel(level));
             let mut tmp = Vec::new();
             loop {
                 let job = match rx.lock().unwrap().recv() { Ok(j) => j, Err(_) => break };
                 let out: Vec<Vec<u8>> = job.blocks.iter().map(|raw| {
-                    tmp.resize(c.deflate_compress_bound(raw.len()), 0);
+                    tmp.resize(zflate::deflate_compress_bound(raw.len()), 0);
                     let n = c.deflate_compress(raw, &mut tmp).expect("deflate");
                     bgzf_block(raw, &tmp[..n])
                 }).collect();

bonefish-core/tests/parser_stress.rs

@@ -214,19 +214,19 @@ fn gen_bam(rng: &mut Rng) -> Vec<u8> {
 }
 
 /// One BGZF block holding `data`.
-fn bgzf_block(c: &mut libdeflater::Compressor, data: &[u8], o: &mut Vec<u8>) {
-    let mut def = vec![0u8; c.deflate_compress_bound(data.len())];
+fn bgzf_block(c: &mut zflate::Compressor, data: &[u8], o: &mut Vec<u8>) {
+    let mut def = vec![0u8; zflate::deflate_compress_bound(data.len())];
     let n = c.deflate_compress(data, &mut def).unwrap();
     let bsize = 18 + n + 8;
     o.extend_from_slice(&[0x1f, 0x8b, 8, 4, 0, 0, 0, 0, 0, 0xff, 6, 0, b'B', b'C', 2, 0]);
     o.extend_from_slice(&((bsize - 1) as u16).to_le_bytes());
     o.extend_from_slice(&def[..n]);
-    o.extend_from_slice(&libdeflater::crc32(data).to_le_bytes());
+    o.extend_from_slice(&zflate::crc32(0, data).to_le_bytes());
     o.extend_from_slice(&(data.len() as u32).to_le_bytes());
 }
 
 fn bgzf(rng: &mut Rng, raw: &[u8]) -> Vec<u8> {
-    let mut c = libdeflater::Compressor::default();
+    let mut c = zflate::Compressor::new(zflate::CompressionLevel::DEFAULT);
     let max_block = if rng.chance(1, 2) { 65280 } else { 1 + rng.below(512) };
     let mut o = Vec::new();
     for chunk in raw.chunks(max_block) { bgzf_block(&mut c, chunk, &mut o); }
@@ -236,8 +236,8 @@ fn bgzf(rng: &mut Rng, raw: &[u8]) -> Vec<u8> {
 
 /// Plain (non-BGZF) gzip, which `open` hands to the threaded MultiGzDecoder.
 fn gzip(raw: &[u8]) -> Vec<u8> {
-    let mut c = libdeflater::Compressor::default();
-    let mut o = vec![0u8; c.gzip_compress_bound(raw.len())];
+    let mut c = zflate::Compressor::new(zflate::CompressionLevel::DEFAULT);
+    let mut o = vec![0u8; zflate::gzip_compress_bound(raw.len())];
     let n = c.gzip_compress(raw, &mut o).unwrap();
     o.truncate(n);
     o
@@ -564,7 +564,7 @@ fn bam_bgzf(l_text: i32, records: &[(&[u8], Option<i32>)]) -> Vec<u8> {
         put_i32(&mut raw, size.unwrap_or(rec.len() as i32));
         raw.extend_from_slice(rec);
     }
-    let mut c = libdeflater::Compressor::default();
+    let mut c = zflate::Compressor::new(zflate::CompressionLevel::DEFAULT);
     let mut o = Vec::new();
     bgzf_block(&mut c, &raw, &mut o);
     bgzf_block(&mut c, b"", &mut o);

bonefish/src/main.rs

@@ -249,7 +249,7 @@ struct AlignArgs {
     pub sort_tmp: Option<String>,
 
     /// --sort: BGZF compression level of the BAM
-    #[arg(long, value_name = "INT", default_value_t = 6)]
+    #[arg(long, value_name = "INT", default_value_t = 6, value_parser = clap::value_parser!(i32).range(0..=12))]
     pub sort_level: i32,
 
     /// Write CIGAR with >65535 ops at CG tag