diff --git a/CHANGELOG.md b/CHANGELOG.md index 73a27a6..534adaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,20 @@ Each release's binaries and full notes are on the [releases page](https://github.com/DylanCosto/bonefish/releases). "Output" means the alignment records. Each entry quotes the MD5 of the genome-wide HG001 sample's SAM output, `@PG` lines removed. +## Unreleased + +**zflate replaces libdeflate.** BGZF input and output, the sorter's temporary runs and the BAI/CSI +indexes go through [zflate](https://github.com/DylanCosto/zflate), a pure-Rust port of libdeflate, +instead of libdeflate's C library. A build without jemalloc or mimalloc no longer needs a C +compiler. zflate needs Rust 1.91 or newer. + +- **Output:** unchanged (`9bc7ff3e…`). Sorted BAMs, from FASTQ and from dorado BAM input, hold the + same records, and their BAI and CSI indexes answer region queries as samtools' own do. At the + default level the genome-wide sample's BAM is 465 bytes smaller. +- **Speed:** CPU time is unchanged. +- **`--sort-level`** outside 0-12 is rejected with an error, instead of a panic in a compression + thread. + ## v0.4.0 (2026-10-28) **New: `-x hifi`** (alias `map-hifi`), for PacBio HiFi genomic DNA. It is the `dna` preset, plus diff --git a/Cargo.lock b/Cargo.lock index 584ae17..5aac279 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -119,7 +119,6 @@ dependencies = [ "byteorder", "flate2", "hashbrown", - "libdeflater", "log", "memchr", "memmap2", @@ -128,6 +127,7 @@ dependencies = [ "serde", "thiserror", "web-time", + "zflate", ] [[package]] @@ -375,24 +375,6 @@ version = "0.2.182" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" -[[package]] -name = "libdeflate-sys" -version = "1.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7870e5fbd2766179a937c725fb11f4ca0ef025d982beb61bd3ce755425bd19c" -dependencies = [ - "cc", -] - -[[package]] -name = "libdeflater" -version = "1.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cbb8285d2e2bd4b8d4155840ae81929354b4ba0d7b3c272c84761ffa2141e1f" -dependencies = [ - "libdeflate-sys", -] - [[package]] name = "libmimalloc-sys" version = "0.1.49" @@ -775,6 +757,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "zflate" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f39b7883ef8cfe2e491f09b5f8db624cb3b55d829da220d89753155ab12e6bdf" +dependencies = [ + "simd-adler32", +] + [[package]] name = "zlib-rs" version = "0.6.8" diff --git a/bonefish-core/Cargo.toml b/bonefish-core/Cargo.toml index 4354fbd..0c0168d 100644 --- a/bonefish-core/Cargo.toml +++ b/bonefish-core/Cargo.toml @@ -35,7 +35,7 @@ thiserror = "2" anyhow = "1" [target.'cfg(not(target_arch = "wasm32"))'.dependencies] -libdeflater = "1.26" +zflate = "1.0.0" memmap2 = "0.9" [target.'cfg(target_arch = "wasm32")'.dependencies] diff --git a/bonefish-core/src/fasta/bam.rs b/bonefish-core/src/fasta/bam.rs index 1a5dd0e..a26b160 100644 --- a/bonefish-core/src/fasta/bam.rs +++ b/bonefish-core/src/fasta/bam.rs @@ -1,6 +1,6 @@ //! BAM input (e.g. unaligned ONT basecaller output) as a FASTQ text stream. //! -//! `BamFastq` decodes BGZF blocks with libdeflate and turns each primary or unmapped record +//! `BamFastq` decodes BGZF blocks with zflate and turns each primary or unmapped record //! (flags 0x100 and 0x800 skipped, like `samtools fastq -F 0x900`) into a FASTQ record. Reverse- //! strand records are reverse-complemented back to the read's original orientation. With `-y` //! the record's tags are written as the FASTQ comment in SAM text form, except alignment tags @@ -65,7 +65,7 @@ pub fn filter_comment_tags(comment: &str) -> Option { /// list if the input is not BAM. pub fn header_lines(input: R) -> io::Result> { let mut f = io::BufReader::new(input); - let mut dec = libdeflater::Decompressor::new(); + let mut dec = zflate::Decompressor::new(); let mut block = Vec::with_capacity(1 << 16); let mut data = Vec::new(); let mut need = 8usize; @@ -102,7 +102,7 @@ pub fn looks_like_bam(head: &[u8]) -> bool { return false; } let mut out = vec![0u8; isize]; - let mut dec = libdeflater::Decompressor::new(); + let mut dec = zflate::Decompressor::new(); matches!(dec.gzip_decompress(&head[..bsize], &mut out), Ok(n) if n >= 4 && &out[..4] == b"BAM\x01") } @@ -151,7 +151,7 @@ impl BamFastq { let (tx, rx) = std::sync::mpsc::sync_channel::>>(8); std::thread::spawn(move || { let mut inner = inner; - let mut dec = libdeflater::Decompressor::new(); + let mut dec = zflate::Decompressor::new(); let mut block = Vec::with_capacity(1 << 16); loop { // ~1 MB of decompressed data per message @@ -296,7 +296,7 @@ impl Read for BamFastq { } /// Decompress the next BGZF block from `inner`, appending to `out`; false at end of input. -fn next_block(inner: &mut R, dec: &mut libdeflater::Decompressor, block: &mut Vec, out: &mut Vec) -> io::Result { +fn next_block(inner: &mut R, dec: &mut zflate::Decompressor, block: &mut Vec, out: &mut Vec) -> io::Result { let mut head = [0u8; 18]; match read_full(inner, &mut head)? { 0 => return Ok(false), @@ -490,7 +490,7 @@ fn append_tags(mut aux: &[u8], out: &mut Vec) -> Result<(), ()> { /// BGZF decompression on several threads, as a `BufRead` of the decompressed stream (for /// bgzipped FASTQ/FASTA). A splitter thread cuts the file into batches of whole blocks (~1 MiB -/// compressed), `workers` threads decompress them with libdeflate, and the reader puts them +/// compressed), `workers` threads decompress them with zflate, and the reader puts them /// back in file order. pub struct BgzfParallel { rx: std::sync::mpsc::Receiver<(u64, io::Result>)>, @@ -556,7 +556,7 @@ impl BgzfParallel { let job_rx = std::sync::Arc::clone(&job_rx); let res_tx = res_tx.clone(); std::thread::spawn(move || { - let mut dec = libdeflater::Decompressor::new(); + let mut dec = zflate::Decompressor::new(); loop { let job = { let rx = job_rx.lock().unwrap(); rx.recv() }; let Ok((seq, job)) = job else { return; }; diff --git a/bonefish-core/src/sorted_bam.rs b/bonefish-core/src/sorted_bam.rs index ab27a2f..eb7c415 100644 --- a/bonefish-core/src/sorted_bam.rs +++ b/bonefish-core/src/sorted_bam.rs @@ -4,7 +4,7 @@ //! The mapper's SAM text is parsed into BAM records as it is written (one writer thread; parsing //! is cheap next to compression). Records collect in a memory batch. A full batch is sorted and //! handed to a background thread that writes it as a temporary run, in chunks compressed with -//! libdeflate level 1 (a private format, much cheaper than the BGZF level 6 that samtools uses for +//! DEFLATE level 1 (a private format, much cheaper than the BGZF level 6 that samtools uses for //! its temporary files), while the next batch fills. After the last read, the runs and the last //! batch are merged and written as BGZF blocks compressed in parallel, and the BAI index is built //! from the block offsets as they are written. @@ -81,6 +81,9 @@ pub struct SortedBamWriter { impl SortedBamWriter { pub fn new(out_path: &str, cfg: SortConfig) -> io::Result { + if !(0..=12).contains(&cfg.level) { + return Err(io::Error::new(io::ErrorKind::InvalidInput, format!("BGZF compression level {} is not 0-12", cfg.level))); + } // Fail early if the output can't be created. File::create(out_path)?; // Rendezvous: at most one batch filling and one being spilled. @@ -498,6 +501,11 @@ fn write_run(b: &Batch, path: &PathBuf, threads: usize, abort: &std::sync::atomi Ok(true) } +/// A DEFLATE level 0-12 (`SortedBamWriter::new` checks the output level). +fn zlevel(lvl: i32) -> zflate::CompressionLevel { + u8::try_from(lvl).ok().and_then(zflate::CompressionLevel::new).expect("DEFLATE level 0-12") +} + /// Deflate every buffer (level `lvl`) on up to `threads` threads, in order. fn compress_all(bufs: &[Vec], lvl: i32, threads: usize) -> Vec> { let mut out: Vec> = vec![Vec::new(); bufs.len()]; @@ -505,9 +513,9 @@ fn compress_all(bufs: &[Vec], lvl: i32, threads: usize) -> Vec> { std::thread::scope(|s| { for (bs, os) in bufs.chunks(per).zip(out.chunks_mut(per)) { s.spawn(move || { - let mut c = libdeflater::Compressor::new(libdeflater::CompressionLvl::new(lvl).unwrap()); + let mut c = zflate::Compressor::new(zlevel(lvl)); for (b, o) in bs.iter().zip(os.iter_mut()) { - let mut v = vec![0u8; c.deflate_compress_bound(b.len())]; + let mut v = vec![0u8; zflate::deflate_compress_bound(b.len())]; let n = c.deflate_compress(b, &mut v).expect("deflate"); v.truncate(n); *o = v; @@ -543,7 +551,7 @@ impl RunReader { for _ in 0..dec { let jr = job_rx.clone(); std::thread::Builder::new().name("sort-dec".into()).spawn(move || { - let mut d = libdeflater::Decompressor::new(); + let mut d = zflate::Decompressor::new(); loop { let job = jr.lock().unwrap().recv(); let Ok((c, raw, done)) = job else { break }; @@ -641,9 +649,7 @@ fn bgzf_block(raw: &[u8], deflated: &[u8]) -> Vec { b.extend_from_slice(&[0x1f, 0x8b, 8, 4, 0, 0, 0, 0, 0, 0xff, 6, 0, b'B', b'C', 2, 0]); b.extend_from_slice(&((total - 1) as u16).to_le_bytes()); b.extend_from_slice(deflated); - let mut crc = libdeflater::Crc::new(); - crc.update(raw); - b.extend_from_slice(&crc.sum().to_le_bytes()); + b.extend_from_slice(&zflate::crc32(0, raw).to_le_bytes()); b.extend_from_slice(&(raw.len() as u32).to_le_bytes()); b } @@ -797,10 +803,10 @@ impl Indexer { } raw.extend_from_slice(&self.n_no_coor.to_le_bytes()); let mut w = BufWriter::new(File::create(path)?); - let mut c = libdeflater::Compressor::new(libdeflater::CompressionLvl::default()); + let mut c = zflate::Compressor::new(zflate::CompressionLevel::DEFAULT); let mut tmp = Vec::new(); for chunk in raw.chunks(0xff00) { - tmp.resize(c.deflate_compress_bound(chunk.len()), 0); + tmp.resize(zflate::deflate_compress_bound(chunk.len()), 0); let n = c.deflate_compress(chunk, &mut tmp).map_err(|_| bad("deflate failed"))?; w.write_all(&bgzf_block(chunk, &tmp[..n]))?; } @@ -834,12 +840,12 @@ fn merge_and_write(out_path: &str, header: &str, refs: &[(String, u32)], runs: & let rx = job_rx.clone(); let tx = done_tx.clone(); workers.push(std::thread::spawn(move || { - let mut c = libdeflater::Compressor::new(libdeflater::CompressionLvl::new(level).unwrap()); + let mut c = zflate::Compressor::new(zlevel(level)); let mut tmp = Vec::new(); loop { let job = match rx.lock().unwrap().recv() { Ok(j) => j, Err(_) => break }; let out: Vec> = job.blocks.iter().map(|raw| { - tmp.resize(c.deflate_compress_bound(raw.len()), 0); + tmp.resize(zflate::deflate_compress_bound(raw.len()), 0); let n = c.deflate_compress(raw, &mut tmp).expect("deflate"); bgzf_block(raw, &tmp[..n]) }).collect(); diff --git a/bonefish-core/tests/parser_stress.rs b/bonefish-core/tests/parser_stress.rs index 9af642c..32a4475 100644 --- a/bonefish-core/tests/parser_stress.rs +++ b/bonefish-core/tests/parser_stress.rs @@ -214,19 +214,19 @@ fn gen_bam(rng: &mut Rng) -> Vec { } /// One BGZF block holding `data`. -fn bgzf_block(c: &mut libdeflater::Compressor, data: &[u8], o: &mut Vec) { - let mut def = vec![0u8; c.deflate_compress_bound(data.len())]; +fn bgzf_block(c: &mut zflate::Compressor, data: &[u8], o: &mut Vec) { + let mut def = vec![0u8; zflate::deflate_compress_bound(data.len())]; let n = c.deflate_compress(data, &mut def).unwrap(); let bsize = 18 + n + 8; o.extend_from_slice(&[0x1f, 0x8b, 8, 4, 0, 0, 0, 0, 0, 0xff, 6, 0, b'B', b'C', 2, 0]); o.extend_from_slice(&((bsize - 1) as u16).to_le_bytes()); o.extend_from_slice(&def[..n]); - o.extend_from_slice(&libdeflater::crc32(data).to_le_bytes()); + o.extend_from_slice(&zflate::crc32(0, data).to_le_bytes()); o.extend_from_slice(&(data.len() as u32).to_le_bytes()); } fn bgzf(rng: &mut Rng, raw: &[u8]) -> Vec { - let mut c = libdeflater::Compressor::default(); + let mut c = zflate::Compressor::new(zflate::CompressionLevel::DEFAULT); let max_block = if rng.chance(1, 2) { 65280 } else { 1 + rng.below(512) }; let mut o = Vec::new(); for chunk in raw.chunks(max_block) { bgzf_block(&mut c, chunk, &mut o); } @@ -236,8 +236,8 @@ fn bgzf(rng: &mut Rng, raw: &[u8]) -> Vec { /// Plain (non-BGZF) gzip, which `open` hands to the threaded MultiGzDecoder. fn gzip(raw: &[u8]) -> Vec { - let mut c = libdeflater::Compressor::default(); - let mut o = vec![0u8; c.gzip_compress_bound(raw.len())]; + let mut c = zflate::Compressor::new(zflate::CompressionLevel::DEFAULT); + let mut o = vec![0u8; zflate::gzip_compress_bound(raw.len())]; let n = c.gzip_compress(raw, &mut o).unwrap(); o.truncate(n); o @@ -564,7 +564,7 @@ fn bam_bgzf(l_text: i32, records: &[(&[u8], Option)]) -> Vec { put_i32(&mut raw, size.unwrap_or(rec.len() as i32)); raw.extend_from_slice(rec); } - let mut c = libdeflater::Compressor::default(); + let mut c = zflate::Compressor::new(zflate::CompressionLevel::DEFAULT); let mut o = Vec::new(); bgzf_block(&mut c, &raw, &mut o); bgzf_block(&mut c, b"", &mut o); diff --git a/bonefish/src/main.rs b/bonefish/src/main.rs index fc7b0d5..60e5704 100644 --- a/bonefish/src/main.rs +++ b/bonefish/src/main.rs @@ -249,7 +249,7 @@ struct AlignArgs { pub sort_tmp: Option, /// --sort: BGZF compression level of the BAM - #[arg(long, value_name = "INT", default_value_t = 6)] + #[arg(long, value_name = "INT", default_value_t = 6, value_parser = clap::value_parser!(i32).range(0..=12))] pub sort_level: i32, /// Write CIGAR with >65535 ops at CG tag