diff --git a/CHANGELOG.md b/CHANGELOG.md index 96a3427..29220e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,22 @@ Each release's binaries and full notes are on the [releases page](https://github.com/DylanCosto/bonefish/releases). "Output" means the alignment records. Each entry quotes the MD5 of the genome-wide HG001 sample's SAM output, `@PG` lines removed. +## v0.3.0 (unreleased) + +**New: `-x tx`.** The preset maps direct RNA or cDNA to a transcriptome for quantifiers such as +oarfish and NanoCount (minimap2 `-ax map-ont -N 100`). + +- **Speed:** on HG002 it is 2.3x (cDNA) and 2.8x (direct RNA) faster than minimap2. +- **Accuracy:** it maps the same reads as minimap2. oarfish gene counts agree at r = 0.9995 or + higher. Report: `docs/validation/2026-10-21-transcriptome/`. + +**Faster with many reference sequences:** the ALT-contig count is computed once per index, not +by scanning every target for every read. This matters for transcriptomes and metagenome +assemblies. + +**Faster chaining:** isolated anchors are dropped before chaining. DNA uses 1.2% less CPU, with +output unchanged (`9bc7ff3e…`). + ## v0.2.0 (2026-10-21) **New: direct RNA and cDNA.** Spliced alignment to a genome is available through two presets: diff --git a/README.md b/README.md index 64fb8d2..18f5107 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,8 @@ RNA and cDNA reads to a reference genome. It writes SAM, PAF or a sorted, indexe - **Accurate.** On whole human genomes (GIAB HG002 and HG005), SNP, indel and structural-variant calls match minimap2's. Methylation calls agree at r = 0.999. - **Fast.** - - Aligner alone: 2.6-8x faster than minimap2 on DNA, 2.6x on direct RNA and 5.4x on cDNA. + - Aligner alone: 2.6-8x faster than minimap2 on DNA, 2.6x on direct RNA, 5.4x on cDNA and + 2.3-2.8x for transcriptome quantification. - Reads to a sorted BAM: about 3x faster than `minimap2 | samtools sort`. - **Drop-in.** It reads dorado's BAM output, keeping methylation and other tags, and accepts minimap2's options and preset names. @@ -53,6 +54,7 @@ in a few minutes. | `dna` (default) | genomic DNA: R10.4.1 or R9.4.1, sup or hac, ultra-long (minimap2 `lr:hq`) | | `rna` | direct RNA (minimap2 `splice -uf -k14`) | | `cdna` | cDNA (minimap2 `splice`) | +| `tx` | RNA or cDNA to a transcriptome, for quantification with oarfish or NanoCount (minimap2 `map-ont -N 100`) | Input formats, output options, memory use and CPU support are covered in [docs/usage.md](docs/usage.md). To replace minimap2 in an existing pipeline, including ONT's @@ -81,9 +83,11 @@ Sniffles2 against GIAB v5.0q. | DNA, HG002 R9.4.1 chr20 (4.6 Gbp) | 412 s | 103 s | 4.0x | | cDNA, HG002 PCS114 (809k reads) | 1,223 s | 227 s | 5.4x | | direct RNA, HG002 RNA004 (902k reads) | 456 s | 177 s | 2.6x | +| direct RNA to the transcriptome (`-x tx`, 200k reads) | 106 s | 37 s | 2.8x | On RNA, 99.5-99.9% of primary alignments are identical to minimap2's, and junction accuracy -against GENCODE is the same. +against GENCODE is the same. Transcript quantification from `-x tx` alignments agrees with +minimap2's at gene level at r = 0.9995-0.99997. bonefish is also faster than dorado's aligner (4.3x), mm2plus (5.1x), mm2-fast (14x) and Winnowmap2 (38x). diff --git a/bonefish-core/src/align/index.rs b/bonefish-core/src/align/index.rs index 8dc61ba..9053ab4 100644 --- a/bonefish-core/src/align/index.rs +++ b/bonefish-core/src/align/index.rs @@ -135,6 +135,10 @@ pub struct Index { /// computed on first use for the ava self-skip fast path. See [`name_rank`]. #[serde(skip)] name_rank: std::sync::OnceLock, + /// Derived (not serialized): number of ALT contigs, counted on first use (after `--alt` has + /// marked them), so per-read code does not scan every target (500k+ for a transcriptome). + #[serde(skip)] + n_alt: std::sync::OnceLock, } /// Dense lexicographic ranking of target-sequence names, used to turn the @@ -211,6 +215,7 @@ impl IndexBuilder { backend: LookupBackend::BucketHash(super::index_bucket::BucketHashLookup::empty()), packed_seqs: Default::default(), name_rank: Default::default(), + n_alt: Default::default(), }, w, k, max_occ, bucket_bits, @@ -304,6 +309,11 @@ impl IndexBuilder { } impl Index { + /// Number of ALT contigs (computed once). + pub fn n_alt(&self) -> usize { + *self.n_alt.get_or_init(|| self.seqs.iter().filter(|s| s.is_alt).count()) + } + /// Dense lexicographic ranking of the target names, computed once on first use and cached pub fn name_rank(&self) -> &NameRankData { self.name_rank.get_or_init(|| { @@ -538,6 +548,7 @@ impl Index { backend: LookupBackend::BucketHash(lookup), packed_seqs: Storage::mapped(map.clone(), qo, qn), name_rank: Default::default(), + n_alt: Default::default(), }); pos += hdr.part_len; pos += pad_to(pos, RAW_ALIGN); @@ -707,6 +718,7 @@ impl Index { backend, packed_seqs: packed_seqs.into(), name_rank: Default::default(), + n_alt: Default::default(), }; eprintln!("[*] Index loaded: {} seqs, {}M bases in {:.1}s", @@ -727,6 +739,7 @@ impl Index { backend: LookupBackend::BucketHash(super::index_bucket::BucketHashLookup::empty()), packed_seqs: Default::default(), name_rank: Default::default(), + n_alt: Default::default(), } } @@ -740,6 +753,7 @@ impl Index { backend: LookupBackend::BucketHash(super::index_bucket::BucketHashLookup::empty()), packed_seqs: Default::default(), name_rank: Default::default(), + n_alt: Default::default(), } } @@ -769,6 +783,7 @@ impl Index { backend: LookupBackend::BucketHash(super::index_bucket::BucketHashLookup::empty()), packed_seqs: Default::default(), name_rank: Default::default(), + n_alt: Default::default(), }; let mut offset = 0usize; diff --git a/bonefish-core/src/align/map.rs b/bonefish-core/src/align/map.rs index b861294..7aff37b 100644 --- a/bonefish-core/src/align/map.rs +++ b/bonefish-core/src/align/map.rs @@ -652,6 +652,36 @@ fn get_minimizer_index(qlen: usize, a: &Minimizer, mini_pos: &[u64]) -> Option 1) always drops: the chains are +/// the same without it. A transcriptome yields many such scattered hits (a k-mer of the read +/// on hundreds of short transcripts). +fn drop_isolated_anchors(a: &mut Vec, max_dist: i32, min_cnt: i32) { + let n = a.len(); + if min_cnt <= 1 || n < 2 { + return; + } + let md = max_dist.max(0) as u64; + let near = |p: &Minimizer, q: &Minimizer| p.x >> 32 == q.x >> 32 && q.x.wrapping_sub(p.x) <= md; + let mut w = 0; + for i in 0..n { + if (i > 0 && near(&a[i - 1], &a[i])) || (i + 1 < n && near(&a[i], &a[i + 1])) { + a[w] = a[i]; + w += 1; + } + } + a.truncate(w); +} + +/// Copy each region's chain anchors (`anchor_count` from offset `original_as` of `chains`). +fn fill_anchors(regs: &mut [Mapping], chains: &[Minimizer]) { + for r in regs.iter_mut() { + let (a, n) = (r.original_as, r.anchor_count as usize); + r.anchors = chains[a..a + n].to_vec(); + } +} + fn estimate_divergence(regs: &mut [Mapping], _anchors: &[Minimizer], qlen: usize, mini_pos: &[u64], mi: &Index) { if mini_pos.is_empty() { return; } @@ -839,6 +869,13 @@ pub fn map_query( // presets), otherwise DP let is_splice = opt.flags.contains(AlignFlags::SPLICE); let mut anchors = std::mem::take(&mut ctx.anchors); + // The rescue's density test below counts every anchor, isolated ones included. + let n_anchors = anchors.len(); + if !is_splice && !opt.flags.intersects(AlignFlags::RMQ_CHAIN | AlignFlags::PAR_CHAIN) { + // Widest reference distance any chaining pass of this read uses (main, the -g re-chain). + let reach = opt.chaining.max_gap.max(max_chain_gap_ref).max(opt.chaining.bandwidth); + drop_isolated_anchors(&mut anchors, reach, opt.chaining.min_cnt); + } let (mut u, mut chains) = if is_splice { chain_anchors_cdna(&opt.chaining, max_chain_gap_ref, max_chain_gap_qry, &mut anchors, &mut ctx.chain_bufs) } else if opt.flags.contains(AlignFlags::RMQ_CHAIN) { @@ -889,7 +926,7 @@ pub fn map_query( let needs_rescue = ((qlen.saturating_sub(coverage) > opt.chaining.rmq_rescue_size as usize) || (coverage > (qlen as f32 * opt.chaining.rmq_rescue_ratio) as usize)) && crate::align::env_flags::EXP_RESCUE_COVER.is_none_or(|f| (coverage as f32) < f * qlen as f32) - && (opt.chaining.rescue_max_density == 0 || ctx.anchors.len() < opt.chaining.rescue_max_density as usize * qlen); + && (opt.chaining.rescue_max_density == 0 || n_anchors < opt.chaining.rescue_max_density as usize * qlen); let needs_rescue = needs_rescue && (!(opt.chaining.rescue_join || *crate::align::env_flags::EXP_RESCUE_JOIN) || { let max_dist = opt.chaining.max_gap.max(opt.chaining.bandwidth_long); let j = chains_joinable(&u, &chains, max_dist, opt.chaining.bandwidth_long); @@ -952,12 +989,12 @@ pub fn map_query( if cnt == 0 { continue; } - // Collect chain anchors - let chain_anchors_vec = chains[k_offset..k_offset + cnt as usize].to_vec(); - + // The chain's anchors are copied into the region only if it survives the parent / + // secondary filtering below (`fill_anchors`): with many overlapping chains per read + // (repeats, transcript isoforms) most regions are dropped there. let is_qstrand = opt.flags.contains(AlignFlags::QSTRAND); let ChainCoords { rid, rev, rs, re, qs, qe, hash: chain_hash } = - chain_coords(&chain_anchors_vec, qseq.len(), is_qstrand, read_hash); + chain_coords(&chains[k_offset..k_offset + cnt as usize], qseq.len(), is_qstrand, read_hash); regs.push(Mapping { ref_id: rid, @@ -968,7 +1005,7 @@ pub fn map_query( score: sc, initial_chain_score: sc, // initially same as score anchor_count: cnt, - anchors: chain_anchors_vec, + anchors: Vec::new(), is_reverse: rev, s2_score: None, is_secondary: false, @@ -1004,7 +1041,7 @@ pub fn map_query( } // Mark ALT-contig chains; then sort chains by descending score. - let n_alt = mi.seqs.iter().filter(|s| s.is_alt).count(); + let n_alt = mi.n_alt(); if n_alt > 0 { for r in regs.iter_mut() { if mi.seqs[r.ref_id].is_alt { r.is_alt = true; } @@ -1027,6 +1064,7 @@ pub fn map_query( // Skip parent-based filtering when ALL_CHAINS is set (e.g. ava-ont, ava-pb) if opt.flags.contains(AlignFlags::ALL_CHAINS) { + fill_anchors(&mut regs, &chains); // Nearby-seed computation on post-filter chains // Build squeezed anchor array from surviving chains, sorted by position let mut pos_order: Vec = (0..regs.len()).collect(); @@ -1219,6 +1257,7 @@ pub fn map_query( compacted.push(r); } regs = compacted; + fill_anchors(&mut regs, &chains); let mut filtered_regs = regs; let pc3 = Instant::now(); PC_PARENT_NS.fetch_add((pc3 - pc2).as_nanos() as u64, Relaxed); diff --git a/bonefish-core/src/align/pipeline.rs b/bonefish-core/src/align/pipeline.rs index 53a4e99..4053ad5 100644 --- a/bonefish-core/src/align/pipeline.rs +++ b/bonefish-core/src/align/pipeline.rs @@ -1769,7 +1769,7 @@ fn process_query_core( let squeezed = map_result.squeezed; let split_mode = out.split_mode; // Mark ALT contigs on chains before sorting - let n_alt = mi.seqs.iter().filter(|s| s.is_alt).count(); + let n_alt = mi.n_alt(); if n_alt > 0 { for r in regs.iter_mut() { if mi.seqs[r.ref_id].is_alt { r.is_alt = true; } diff --git a/bonefish-core/src/api.rs b/bonefish-core/src/api.rs index 51c2788..36e287e 100644 --- a/bonefish-core/src/api.rs +++ b/bonefish-core/src/api.rs @@ -43,6 +43,8 @@ pub enum Preset { Cdna, /// Accurate cDNA, spliced (minimap2 `-x splice:hq`). SpliceHq, + /// RNA or cDNA reads to a transcriptome, for quantification (minimap2 `-x map-ont -N 100`). + Tx, } impl Preset { @@ -53,6 +55,7 @@ impl Preset { Preset::Rna => "rna", Preset::Cdna => "cdna", Preset::SpliceHq => "splice:hq", + Preset::Tx => "tx", } } } @@ -568,7 +571,7 @@ fn parse_cigar_string(s: &str) -> Vec { // --------------------------------------------------------------------------- /// Apply a preset by string name ("dna" or one of its minimap2-name aliases, "rna", "cdna", -/// "splice" (= cdna) or "splice:hq"). +/// "splice" (= cdna), "splice:hq" or "tx"). /// /// This is the same function used by the CLI's `-x` flag. Modifies `opt`, /// `k` and `w` in place. Returns an error if the preset name @@ -613,6 +616,18 @@ pub fn apply_preset_str(opt: &mut MapOptions, k: &mut usize, w: &mut usize, pres opt.alignment.gap_subanchor = true; opt.alignment.gap_subanchor_tr = 150; }, + // Reads (direct RNA or cDNA) against a transcriptome, for quantifiers that take all + // alignments (oarfish, NanoCount, salmon): minimap2's map-ont seeding (k15 w10, no seed + // occurrence range, 5 kb query gap limit) for short and noisy RNA reads, up to 100 + // secondary alignments per read (isoforms share most of their sequence), and otherwise + // the dna preset's cheaper chaining and DP (there are no introns to align). + "tx" => { + apply_preset_str(opt, k, w, "dna")?; + *k = 15; *w = 10; + opt.chaining.max_gap = 5000; + opt.seeding.min_mid_occ = 10; opt.seeding.max_mid_occ = 1_000_000; + opt.filtering.best_n = 100; + }, // Spliced alignment of transcripts, as minimap2 2.30/2.31 `-x splice` (options.c): k15 w5, // both transcript strands tried (-ub) with splice-site flank scoring, introns up to 200 kb // (-G: chaining reference gap and bandwidth), cDNA-mode chaining, 2 kb query gap limit, @@ -1088,7 +1103,7 @@ mod tests { #[test] fn test_preset_as_str_roundtrip() { - let presets = [Preset::Dna, Preset::Rna, Preset::Cdna, Preset::SpliceHq]; + let presets = [Preset::Dna, Preset::Rna, Preset::Cdna, Preset::SpliceHq, Preset::Tx]; for p in &presets { let s = p.as_str(); assert!(!s.is_empty(), "preset {:?} should have a string form", p); diff --git a/bonefish/src/main.rs b/bonefish/src/main.rs index c70abb6..2685f99 100644 --- a/bonefish/src/main.rs +++ b/bonefish/src/main.rs @@ -52,11 +52,13 @@ struct AlignArgs { /// rna — direct RNA vs a reference genome (spliced; minimap2 -x splice -uf -k14) /// cdna — cDNA vs a reference genome (spliced; minimap2 -x splice); alias: splice /// splice:hq — accurate cDNA (minimap2 -x splice:hq) + /// tx — RNA/cDNA vs a transcriptome, for quantification (minimap2 -x map-ont -N 100) /// dna = minimap2 lr:hq seeding and scoring plus --ext-bw 200 --ext-adaptive 64 --ext-rescue /// --ext-probe 2000,1000 --gap-bw-margin 500 --gap-bw-narrow 32 --ungapped-fill --max-gap-ref 5000 /// --seg-rmq --clean-dp --rescue-join --rescue-max-density 2 --gap-subanchor-tr 150 /// cdna = -k15 -w5 -g2k -G200k -A1 -B2 -O2,32 -E1,0 -C9 -z200,100 -ub --junc-bonus 9 /// --junc-pen 5 --splice-flank yes; rna = cdna with -k14 -uf + /// tx = dna with -k15 -w10 -g5000 -U10,1000000 -N100 #[arg(short = 'x', long, verbatim_doc_comment)] pub preset: Option, diff --git a/bonefish/tests/data/expected.tsv b/bonefish/tests/data/expected.tsv index ee570ea..0cd2ec6 100644 --- a/bonefish/tests/data/expected.tsv +++ b/bonefish/tests/data/expected.tsv @@ -14,3 +14,4 @@ cdna_paf_cs bonefish -x cdna -c --cs 21afca13cdff0b35 cdna_junc_bed bonefish -x cdna -a --junc-bed {junc} 6292d532a2fcabc3 cdna_splice_hq bonefish -x splice:hq -a 1988a08a9ab7353a cdna_alias_splice bonefish -x splice -a 60d6d8c7c31cd299 +tx_sam bonefish -x tx -a d53bfcfb58fa2138 diff --git a/bonefish/tests/regression.rs b/bonefish/tests/regression.rs index 7220c5a..379d002 100644 --- a/bonefish/tests/regression.rs +++ b/bonefish/tests/regression.rs @@ -55,7 +55,8 @@ fn expected() -> Vec<(String, String, String, String)> { } /// Arguments for an expected.tsv case: its options (`{junc}` stands for the annotation BED), then -/// the reference and reads. Cases named `rna_*` / `cdna_*` use the spliced dataset (made by +/// the reference and reads (`tx_*` runs the transcriptome preset on the DNA dataset, for its +/// seeding and secondary settings). Cases named `rna_*` / `cdna_*` use the spliced dataset (made by /// `scripts/make_regression_rna_data.py`): direct-RNA-like or cDNA-like reads on a genome of /// multi-exon genes. fn map_args(name: &str, case_args: &str) -> Vec { diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 221b269..6046b2f 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -96,6 +96,20 @@ the reference near the chain's anchors with a 10-mer search for unanchored exons aligned once for both transcript strands. Details: [validation/2026-10-21-rna-cdna](validation/2026-10-21-rna-cdna/README.md). +## Transcriptome quantification + +`-x tx` against minimap2 `-ax map-ont -N 100` on GENCODE v49 transcripts (HG002, 200k reads each): + +| | minimap2 | bonefish `-x tx` | +|---|---:|---:| +| cDNA, wall | 185 s | 81 s | +| direct RNA, wall | 106 s | 37 s | +| reads mapped | | the same as minimap2 | +| oarfish gene counts vs minimap2's: Pearson r | | 0.9995 (cDNA), 0.99997 (direct RNA) | + +On simulated reads, the primary lands on the true transcript at least as often as with minimap2. +Details: [validation/2026-10-21-transcriptome](validation/2026-10-21-transcriptome/README.md). + ## Metagenomes, duplex reads and rDNA Against minimap2 `lr:hq` on the same reads (v0.1.7): ONT's Zymo fecal reference (1.70 M R10.4.1 sup diff --git a/docs/pipelines.md b/docs/pipelines.md index 1b1d221..6538170 100644 --- a/docs/pipelines.md +++ b/docs/pipelines.md @@ -16,6 +16,7 @@ minimap2 in a pipeline in two ways: | `minimap2 -ax splice -uf -k14 ref.fa drna.fq` | `bonefish -x rna -a ref.fa drna.fq` | | `minimap2 -ax splice ref.fa cdna.fq` | `bonefish -x cdna -a ref.fa cdna.fq` | | `minimap2 -ax splice --junc-bed anno.bed ...` | `bonefish -x cdna -a --junc-bed anno.bed ...` | +| `minimap2 -ax map-ont -N 100 transcripts.fa reads.fq` (for quantification) | `bonefish -x tx -a transcripts.fa reads.fq` | Notes: @@ -75,6 +76,22 @@ minimap2 does. reads ([report](validation/2026-10-21-rna-cdna/README.md)), but not yet against these downstream tools. +## Transcript quantification + +oarfish, NanoCount and similar quantifiers take reads aligned to the transcriptome with many +secondary alignments. `-x tx` writes up to 100 secondaries per read, as `minimap2 -N 100` does: + +```bash +bonefish -t 16 -x tx -a --eqx transcripts.fa reads.fq | samtools view -b -o tx.bam - +``` + +- **Agreement with minimap2:** on HG002 direct RNA and cDNA, oarfish gene counts from these + alignments agree with counts from minimap2's at r = 0.9995-0.99997. Simulated reads land on + their true transcript at least as often as with minimap2 + ([report](validation/2026-10-21-transcriptome/README.md)). +- **oarfish needs a workaround for now:** it only accepts BAMs whose `@PG` header names an + aligner it has validated, and bonefish is not yet on its list. + ## Containers ```bash diff --git a/docs/usage.md b/docs/usage.md index 16920f6..b295f09 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -8,6 +8,7 @@ | `rna` | direct RNA (RNA004), spliced to a genome | `-x splice -uf -k14` | | `cdna` | cDNA, spliced to a genome, transcript strand from splice signals | `-x splice` | | `splice:hq` | accurate cDNA | `-x splice:hq` | +| `tx` | direct RNA or cDNA to a transcriptome, for quantification (oarfish, NanoCount) | `-x map-ont -N 100` | The minimap2 names `lr:hq`, `lr:hq-fast`, `map-ont` and `lr` are accepted for `dna`, and `splice` for `cdna`. PacBio, short-read and assembly presets are not supported. @@ -21,6 +22,7 @@ build an index once: bonefish -d ref.raw --idx-raw ref.fa # dna; loads in under a second (8.5 GB for GRCh38) bonefish -x cdna -d cdna.raw --idx-raw ref.fa # spliced presets need their own index (k/w differ) bonefish -x rna -d rna.raw --idx-raw ref.fa +bonefish -x tx -d tx.raw --idx-raw transcripts.fa ``` Without `--idx-raw`, `-d` writes a smaller index that takes a few seconds to load. Indexes are diff --git a/docs/validation/2026-10-21-transcriptome/README.md b/docs/validation/2026-10-21-transcriptome/README.md new file mode 100644 index 0000000..01cc28b --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/README.md @@ -0,0 +1,90 @@ +# Transcriptome mapping for quantification (21 Oct 2026) + +Long-read quantifiers such as oarfish and NanoCount take reads aligned to a transcriptome, with +many secondary alignments per read. Their usual input is `minimap2 -ax map-ont -N 100`. This +report adds the `-x tx` preset for that workflow, and compares it with minimap2 and with +bonefish's `dna` preset. + +## Data and method + +- **Reads:** the first 200,000 HG002 direct-RNA (RNA004 sup) and cDNA (PCS114 hac) reads of the + [RNA report](../2026-10-21-rna-cdna/README.md). +- **Simulated reads:** Badread reads from 30,000 GENCODE transcripts, cDNA-like and + direct-RNA-like. Each read's source transcript and gene are known. +- **Reference:** GENCODE v49 transcripts, 533,740 sequences + (`gencode.v49.transcripts.fa.gz`, sha256 `08ee7ee8…`). +- **Commands:** + - minimap2 2.31: `-ax map-ont -N 100 --eqx`. + - bonefish: `-a --eqx`, with either `-x tx` or `-N 100` on the `dna` preset. + - Quantification: oarfish 0.10.3 with `--filter-group nanocount-filters`, compared with + `tx_quant_compare.py`. +- **Machine:** Core i5-12600K, 16 threads. + +**oarfish and bonefish BAMs:** oarfish only reads BAMs whose `@PG` header names an aligner it +has validated (minimap2, pbmm2, bramble or rammap). bonefish's BAMs were relabelled as rammap, +bonefish's upstream, for this test. oarfish otherwise only needs the standard `AS` tag. + +## The `tx` preset + +- **Seeding:** minimap2's `map-ont` seeding. k=15 and w=10, with no occurrence range and a 5 kb + query gap limit, suit short, noisy RNA reads. +- **Secondary alignments:** up to 100 per read (`-N 100`). +- **Everything else:** the `dna` preset's chaining and DP. There are no introns in transcriptome + mapping, so they apply unchanged. + +## Results + +| 200k reads | minimap2 `map-ont -N 100` | bonefish `dna -N 100` | **bonefish `-x tx`** | +|---|---:|---:|---:| +| cDNA: wall / CPU-s | 185 s / 2,285 | 146 s / 1,535 | **81 s / 1,149** | +| direct RNA: wall / CPU-s | 106 s / 1,026 | 102 s / 1,032 | **37 s / 485** | +| unmapped reads (cDNA / direct RNA) | 1,987 / 37,838 | 2,493 / 39,115 | **1,987 / 37,838** | +| oarfish genes vs minimap2: Pearson (cDNA / direct RNA) | | 0.99937 / 0.99968 | 0.99950 / 0.99997 | +| reads assigned to another gene (cDNA / direct RNA) | | 0.65% / 0.55% | 0.66% / 0.25% | +| reads assigned to another transcript (cDNA / direct RNA) | | 1.63% / 1.27% | 1.74% / 0.93% | +| simulated cDNA: primary on true transcript / gene | 93.73% / 99.53% | 93.89% / 99.48% | 93.88% / 99.53% | +| simulated direct RNA: mapped | 99.69% | 98.82% | 99.69% | +| simulated direct RNA: primary on true transcript / gene | 91.98% / 99.33% | 91.33% / 98.49% | 92.17% / 99.34% | +| simulated: true transcript among alignments (cDNA / direct RNA) | 99.89% / 99.68% | 99.85% / 98.80% | 99.89% / 99.68% | + +- **Sensitivity:** `-x tx` leaves exactly the same number of reads unmapped as minimap2. The `dna` + preset's sparser k=19 seeds missed some short reads. +- **Accuracy:** on simulated reads, `-x tx` puts at least as many primaries on the true + transcript as minimap2. The true transcript is among the alignments as often. +- **Quantification:** at gene level, quantification from `-x tx` agrees with minimap2's at + r = 0.9995 (cDNA) and 0.99997 (direct RNA). Assignment between isoforms, which is close to a + tie for many reads, differs for 0.9-1.7% of reads. +- **Speed:** `-x tx` is 2.3x (cDNA) and 2.8x (direct RNA) faster than minimap2 in wall time. + +## What made it fast + +The first `tx` build was no faster than minimap2 (`runs/tx1`: cDNA 184 s, direct RNA 102 s). +Profiling found three things: + +1. **A per-read scan of every reference sequence.** The code counted ALT contigs by looking at + all 533,740 transcript names twice per read. That was 32 of 35 post-chain thread-seconds on + 20k reads. On the 195-sequence human genome it cost nothing, so it had never shown up. The + count is now made once per index (`runs/tx3`: cDNA 85 s, direct RNA 39 s). Anything with many + reference sequences, such as metagenome assemblies, gains as well. +2. **Isolated anchors.** A read's k-mers hit hundreds of short transcripts, and most hits have no + other hit on the same transcript. They are now dropped before chaining, because they can only + form one-anchor chains, which are always discarded. Output is identical. Chaining is 8% + faster here, and DNA uses 1.2% less CPU (`runs/tx4`). +3. **Anchor copies.** Chain anchors are now copied only for regions that survive the secondary + filter. This made little difference on its own. + +For all three changes, output stayed identical: the DNA genome-wide sample has the same MD5 +(`9bc7ff3e…`), every golden-output test passes, and the transcriptome SAM is unchanged. + +## Files + +- `evidence/runs//`: GNU time, stderr, oarfish logs and metadata. + - `mm2`: minimap2. + - `dna_N100`: the `dna` preset with `-N 100`. + - `tx1`, `tx3`, `tx4`: the `tx` preset at each step above. +- `evidence/ledger.tsv`: one line per run. +- `evidence/scripts/`: `tx_bench.sh` (runs and scoring), `tx_quant_compare.py` (oarfish count + correlation) and `tx_sim_truth.py` (simulated-read truth). + +The simulated reads are those of the RNA report (`rna_sim_reads.sh`). `SHA256SUMS` covers +`evidence/`. Local paths are replaced by `${BENCH}`, `${DATA}`, `${REPO}` and `${HOME}`. diff --git a/docs/validation/2026-10-21-transcriptome/SHA256SUMS b/docs/validation/2026-10-21-transcriptome/SHA256SUMS new file mode 100644 index 0000000..ec4b308 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/SHA256SUMS @@ -0,0 +1,64 @@ +8025b129f3312ca17fe73a47a389f86649876e7f6fb1f42c2083200659764d51 evidence/ledger.tsv +8a94c61bec8aa2a21241e7a79ccdfa97d9e3850769a6e19ccc64d5b0c5afae71 evidence/runs/dna_N100/cdna.err +5384c3d8d1ec64e31df631b19d9792a0ae2a585e8ceeca07a548e2d380955ffc evidence/runs/dna_N100/cdna.oarfish.log +ae0c22b699d9b9b365e56343eae0764af4414ae3dee5260f7ba3afd7a40b28cd evidence/runs/dna_N100/cdna.oarfish.meta_info.json +aa686c327cfedea2aaea413c4a65acf04d9fb2078eaf607b3cb936d5f1f62d32 evidence/runs/dna_N100/cdna.time +f1cfed86480665e547d31ade9d1addfee042214245ab1fea370ddc5029b7b90e evidence/runs/dna_N100/drna.err +52f70a53a0de4aba88a9f649a1662d57c2763f7f925d80b493283acbffb066e8 evidence/runs/dna_N100/drna.oarfish.log +b304a3d889f7c44b94a1263e6ce0d53bee2cb867d58db265c0abecc1c213f882 evidence/runs/dna_N100/drna.oarfish.meta_info.json +7a16c5aaf7641b7099b6d3b6233975e12ca63c3c4e0b109102533c86e405aaba evidence/runs/dna_N100/drna.time +676f00c2a86110d49b69573ce54199ed0b3636c32dce34294bdb6344bb3a9ca9 evidence/runs/dna_N100/sim_cdna.err +6bcc8c7b54add0e7f260d17a0897e78180931a2677cf1a1930d877f3bd737401 evidence/runs/dna_N100/sim_cdna.time +6ad4f3cc8f53a2c4544e966162687ffde34ee697b98f70a12459dbd8ca639895 evidence/runs/dna_N100/sim_drna.err +f6f54b7110a7d4480cdaa3d368a4c24d7362b1c2eaf6cc3486dfc0e14a903a51 evidence/runs/dna_N100/sim_drna.time +8675dedfa639986dc63a71106095a1d24572504a8b7ca769f28bfdf3ccd98df4 evidence/runs/mm2/cdna.err +3e798946ee9e95db459bab6d17555d0e8a358870597c62a683357f06ab2fa27e evidence/runs/mm2/cdna.oarfish.log +c2b3f9683b91fd979bbd6ca433b32cbfe50977aa4d3a8c3d2ddb65e328794622 evidence/runs/mm2/cdna.oarfish.meta_info.json +01a9032811558cc0eb966bf812cafa2762b1b31964579e660f4152cd9777ff3a evidence/runs/mm2/cdna.time +db188083986318d5f4b11b3219bac3950f3c716b65792ce8caa9cff3d504e6ba evidence/runs/mm2/drna.err +7858d67f17738c43cf0eba5c58d2267247b848dcbc7aafd2b2224cff4ff68c06 evidence/runs/mm2/drna.oarfish.log +5bc3693c0290a5be9d64a3c65f0db3d227797efec42ad6d49c327a6d00402dc6 evidence/runs/mm2/drna.oarfish.meta_info.json +85795caf45eb55ab2f0fe50c51f40111a776c6726bd39d2fefed550e4864635f evidence/runs/mm2/drna.time +47062f5eeb8a9193e168c8f059776330566d7a78587541133c86a765f89beb1a evidence/runs/mm2/sim_cdna.err +846939527ce82e6ad1ba0a4828b12c67179ea19c93e10b792fe149959aa8bc22 evidence/runs/mm2/sim_cdna.time +d6ac3015dd0700bd6ff346e2b03d27437528859be43777fd0c2bb80b0b30981b evidence/runs/mm2/sim_drna.err +a21746be10b7ca2dec41bb0cf742856edad731c7aba4f039450e8055dc968947 evidence/runs/mm2/sim_drna.time +0c2a673b9249a77a4cd6206f49131cfdb4720dbbd050cbc10024e033dae919c0 evidence/runs/tx1/cdna.err +e4a40d6c6f2958cc6d615b10959adff5da5418f25419e9cde7638d26bfd594af evidence/runs/tx1/cdna.oarfish.log +0abc27adefbfe1988f65db2a601aecf467e93e0f137a61f46e03c9f8a106220c evidence/runs/tx1/cdna.oarfish.meta_info.json +57757b6946c5e7238648e6cd4c7372e94507ac849e3fdf0736fd0d86ccf8338b evidence/runs/tx1/cdna.time +016d1418e3ca4e5b1a4e24772034198ae9c2279aa135522919956b0583be00ab evidence/runs/tx1/drna.err +f42bffa9e81ee8673e6c00f3b3c46bc73c5cd85cd06242b97a8b2298d548a13c evidence/runs/tx1/drna.oarfish.log +5678688ca7d3e9fe6f0866ef6e3020d802f04f641d0f8e551f2652cb4ca08529 evidence/runs/tx1/drna.oarfish.meta_info.json +d545df618cc857b71c90df996908a0600489e4b2ae8ceb32987a85577709a826 evidence/runs/tx1/drna.time +1c01a286ea1b2ff8be5073795263cf3bf6d8d71cdc66a0e459288e4eb43821d8 evidence/runs/tx1/sim_cdna.err +80b6879f8343c872430b0182968b8fb719bb2e3235f5915ed52ca685e3068717 evidence/runs/tx1/sim_cdna.time +e3513a944b00257e10fc14b216b6bd4d239033018a0f570fec9e6abb0c15c2fa evidence/runs/tx1/sim_drna.err +38d7861805cd8427211aa9beb2f2b8d65f82df44643835e4bf0b5287ff2b6426 evidence/runs/tx1/sim_drna.time +e85197f9778db876cb7bf4a09e0144da7e0d7236a9f6b75b84098245d9c7f5dc evidence/runs/tx3/cdna.err +907cc2de423def8d4dba987d743554eecd6afa2e752fb38d3b4bdbf47c1d4961 evidence/runs/tx3/cdna.oarfish.log +e386672798db8c2eb8da0a21382105c354a52a07bf6d238c31982eb24153f52e evidence/runs/tx3/cdna.oarfish.meta_info.json +bb4eac6358534762ad0c4bc6a579a2c362f99ae430ba04175aa7a6137c6295af evidence/runs/tx3/cdna.time +f32a5330cc5daa7cfef5bfa46260f559dd43e8c387cac402c80e6b8fb2903225 evidence/runs/tx3/drna.err +099e092bd89e45135914cf94d2f2dcf4402d9d61249293976a44a8b40bd408ef evidence/runs/tx3/drna.oarfish.log +da99369453402c7ecaf23cf8151d7bbfbbb6e4bf483094f3a3549893264e8a27 evidence/runs/tx3/drna.oarfish.meta_info.json +7f28fd91c684f89bb2505db140be1107f439aa9f1984c7c22b592908906e35ff evidence/runs/tx3/drna.time +e411519b5c393bdacc7836f07ec08d3ff197f6803f01cc9ebaf1f774683bde68 evidence/runs/tx3/sim_cdna.err +455e0bfcb702bcfbcfd6ab33d609224a1a874ab1b9a1da2fc1d7324b6cabb9ca evidence/runs/tx3/sim_cdna.time +e117c27468b832ad60501d166d1e524899f5929b466a5c7bbade4acd16c87da7 evidence/runs/tx3/sim_drna.err +d3965e076553442935173a1c439c4e6f8a2c771adaef3c87a2b9550148f3527d evidence/runs/tx3/sim_drna.time +e1ebbd87d36b253f6d971ba227df5445d3f64a37aeda9bac1514344bef43bca2 evidence/runs/tx4/cdna.err +832bf3c3a53c950ce756bc9179eb94d80268348efe858b14b36f3470a986bb80 evidence/runs/tx4/cdna.oarfish.log +2034f3465326110e500bba277593d96f509ea5d876aada07df220214c0d5db48 evidence/runs/tx4/cdna.oarfish.meta_info.json +c4fe07dd2af4f6ad627de8f16b0790caf8646fddc9681c2dfea3b4bce6dc0092 evidence/runs/tx4/cdna.time +c9cee6c81876786f1989d410fe5af2d63d9035463a5b5ddbc3667c0e06084770 evidence/runs/tx4/drna.err +e407175f2c3ab8f01978b7fc68684bb0fc029c978ea735da9071abebd6dba513 evidence/runs/tx4/drna.oarfish.log +e384647a8d00e44c70d1435b7ebc290adb28f3cfe8b32253442c29aa889b8fbb evidence/runs/tx4/drna.oarfish.meta_info.json +68e7e11c61885ffb9fee25586c270191334b2819b4365c71af3b3c81591733e1 evidence/runs/tx4/drna.time +055ba99e622ccf003d22d45a768ed0f2faa32610d005339b6ac7f0a23e2083ab evidence/runs/tx4/sim_cdna.err +6fcac4c9e963b42056521450866a9793d4f1573c5b7a88bc6e2d8db63f743f75 evidence/runs/tx4/sim_cdna.time +46f044fbd53935795298e7403e71ce348c1289b05500998b4c3501ed12891c66 evidence/runs/tx4/sim_drna.err +27f7f92631e6130e985e562a9ef119bec22ba6458f78a0fb3bfd15125c7a7bf4 evidence/runs/tx4/sim_drna.time +ef45f763a905d70f277a281c987aa386f39d62cd121275390fcbda44ad0b339a evidence/scripts/tx_bench.sh +4feccc36aabe4eba92fd98829f9f3f85b48192f32d45bdb6992505fc19579bd1 evidence/scripts/tx_quant_compare.py +1814d4364452dbecdeb61898946e7449b162df210a7e1fd8935378a29e9cffcc evidence/scripts/tx_sim_truth.py diff --git a/docs/validation/2026-10-21-transcriptome/evidence/ledger.tsv b/docs/validation/2026-10-21-transcriptome/evidence/ledger.tsv new file mode 100755 index 0000000..d0ce0c7 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/ledger.tsv @@ -0,0 +1,5 @@ +mm2 cdna wall=184.67 cpu=2285.03 unmapped=1987 drna wall=105.69 cpu=1026.26 unmapped=37838 sim_cdna wall=47.19 cpu=441.83 mapped 99.89% primary on true transcript 93.73% on true gene 99.53% true transcript among alignments 99.89% sim_drna wall=38.16 cpu=396.32 mapped 99.69% primary on true transcript 91.98% on true gene 99.33% true transcript among alignments 99.68% +dna_N100 cdna wall=146.33 cpu=1535.12 unmapped=2493 transcripts: 21,271 with reads; reads 79,110 vs 78,331; Pearson 0.99893, log2 0.96622, Spearman 0.93590; reassigned 1,287 (1.63%) genes: 9,541 with reads; reads 79,110 vs 78,331; Pearson 0.99937, log2 0.99374, Spearman 0.99247; reassigned 515 (0.65%) drna wall=102.42 cpu=1032.07 unmapped=39115 transcripts: 27,353 with reads; reads 143,105 vs 141,931; Pearson 0.99900, log2 0.97835, Spearman 0.94488; reassigned 1,820 (1.27%) genes: 10,948 with reads; reads 143,105 vs 141,931; Pearson 0.99968, log2 0.99484, Spearman 0.99371; reassigned 792 (0.55%) sim_cdna wall=31.68 cpu=296.84 mapped 99.85% primary on true transcript 93.89% on true gene 99.48% true transcript among alignments 99.85% sim_drna wall=31.99 cpu=293.60 mapped 98.82% primary on true transcript 91.33% on true gene 98.49% true transcript among alignments 98.80% +tx1 cdna wall=183.68 cpu=2039.95 unmapped=1987 transcripts: 21,306 with reads; reads 79,110 vs 78,211; Pearson 0.99912, log2 0.96374, Spearman 0.92857; reassigned 1,380 (1.74%) genes: 9,532 with reads; reads 79,110 vs 78,211; Pearson 0.99950, log2 0.99472, Spearman 0.99366; reassigned 523 (0.66%) drna wall=101.63 cpu=1246.68 unmapped=37838 transcripts: 27,227 with reads; reads 143,105 vs 142,590; Pearson 0.99939, log2 0.98103, Spearman 0.95013; reassigned 1,334 (0.93%) genes: 10,925 with reads; reads 143,105 vs 142,590; Pearson 0.99997, log2 0.99755, Spearman 0.99693; reassigned 354 (0.25%) sim_cdna wall=45.58 cpu=418.40 mapped 99.89% primary on true transcript 93.88% on true gene 99.53% true transcript among alignments 99.89% sim_drna wall=37.63 cpu=363.29 mapped 99.69% primary on true transcript 92.17% on true gene 99.34% true transcript among alignments 99.68% +tx3 cdna wall=85.15 cpu=1193.59 unmapped=1987 transcripts: 21,306 with reads; reads 79,110 vs 78,211; Pearson 0.99912, log2 0.96374, Spearman 0.92856; reassigned 1,380 (1.74%) genes: 9,532 with reads; reads 79,110 vs 78,211; Pearson 0.99950, log2 0.99472, Spearman 0.99364; reassigned 523 (0.66%) drna wall=38.90 cpu=506.41 unmapped=37838 transcripts: 27,227 with reads; reads 143,105 vs 142,590; Pearson 0.99939, log2 0.98103, Spearman 0.95013; reassigned 1,334 (0.93%) genes: 10,925 with reads; reads 143,105 vs 142,590; Pearson 0.99997, log2 0.99755, Spearman 0.99693; reassigned 354 (0.25%) sim_cdna wall=20.41 cpu=227.59 mapped 99.89% primary on true transcript 93.88% on true gene 99.53% true transcript among alignments 99.89% sim_drna wall=18.53 cpu=183.47 mapped 99.69% primary on true transcript 92.17% on true gene 99.34% true transcript among alignments 99.68% +tx4 cdna wall=81.22 cpu=1148.77 unmapped=1987 transcripts: 21,306 with reads; reads 79,110 vs 78,211; Pearson 0.99912, log2 0.96374, Spearman 0.92856; reassigned 1,380 (1.74%) genes: 9,532 with reads; reads 79,110 vs 78,211; Pearson 0.99950, log2 0.99472, Spearman 0.99365; reassigned 523 (0.66%) drna wall=37.30 cpu=484.92 unmapped=37838 transcripts: 27,227 with reads; reads 143,105 vs 142,590; Pearson 0.99939, log2 0.98103, Spearman 0.95013; reassigned 1,334 (0.93%) genes: 10,925 with reads; reads 143,105 vs 142,590; Pearson 0.99997, log2 0.99755, Spearman 0.99693; reassigned 354 (0.25%) sim_cdna wall=19.80 cpu=219.74 mapped 99.89% primary on true transcript 93.88% on true gene 99.53% true transcript among alignments 99.89% sim_drna wall=18.09 cpu=175.25 mapped 99.69% primary on true transcript 92.17% on true gene 99.34% true transcript among alignments 99.68% diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/cdna.err b/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/cdna.err new file mode 100755 index 0000000..56bfa3b --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/cdna.err @@ -0,0 +1,3 @@ +[*] Loading index from ${DATA}/rna/tx/tx.bonefish.raw... +[*] Loaded part 1 in 0.058s — k: 19; w: 19; #seq: 533740; total len: 946309864 +[*] Mapped 200000 reads in 146.237s (1 index part) diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/cdna.oarfish.log b/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/cdna.oarfish.log new file mode 100755 index 0000000..adbe82b --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/cdna.oarfish.log @@ -0,0 +1,37 @@ +2026-10-21T22:19:01.762111Z  INFO oarfish: setting filters to nanocount defaults. +2026-10-21T22:19:01.966191Z  INFO oarfish::alignment_parser: read header from BAM file cdna.q.bam, contains 533,740 reference sequences. +2026-10-21T22:19:01.966394Z  INFO oarfish::alignment_parser: saw supported mapper rammap as a program in the header; proceeding. +2026-10-21T22:19:01.966413Z  INFO oarfish::util::digest_utils: calculating seqcol digest +2026-10-21T22:19:03.352014Z  INFO oarfish::util::digest_utils: done calculating seqcol digest +2026-10-21T22:19:03.399509Z  INFO oarfish: parsed reference information for 533,740 transcripts. +2026-10-21T22:19:05.936523Z  INFO oarfish::alignment_parser: the alignment file contained 2,493 unmapped read records. +2026-10-21T22:19:05.937428Z  INFO oarfish::bulk: Parsing of alignments from input took: 2s 537ms 727us 759ns +2026-10-21T22:19:05.937502Z  INFO oarfish::bulk: +discard_table: +╭─────────────────────────────────┬─────────╮ +│ reason │ count │ +├─────────────────────────────────┼─────────┤ +│ too far from 5' end │ 0 │ +│ too far from 3' end │ 461285 │ +│ score too low │ 455915 │ +│ aligned fraction too low │ 6057 │ +│ aligned length too short │ 270 │ +│ inconsistent orientation │ 1532440 │ +│ supplementary alignment │ 12166 │ +│ read had no mapping │ 0 │ +│ read had no valid alignment │ 113119 │ +│ reads with valid best alignment │ 78331 │ +╰─────────────────────────────────┴─────────╯ + +2026-10-21T22:19:05.937535Z  INFO oarfish::bulk: Total number of alignment records : 506,247 +2026-10-21T22:19:05.937556Z  INFO oarfish::bulk: number of aligned reads : 78,331 +2026-10-21T22:19:05.937575Z  INFO oarfish::bulk: number of unique alignments : 16,398 +2026-10-21T22:19:06.121495Z  INFO oarfish::em: iteration 100; rel diff 0.006687934315578949 +2026-10-21T22:19:06.309666Z  INFO oarfish::em: iteration 200; rel diff 0.0030421754005212145 +2026-10-21T22:19:06.500129Z  INFO oarfish::em: iteration 300; rel diff 0.0027460575988249723 +2026-10-21T22:19:06.683627Z  INFO oarfish::em: iteration 400; rel diff 0.0024077160823059766 +2026-10-21T22:19:06.865898Z  INFO oarfish::em: iteration 500; rel diff 0.0020442441605902054 +2026-10-21T22:19:07.052350Z  INFO oarfish::em: iteration 600; rel diff 0.0016790329612567613 +2026-10-21T22:19:07.234089Z  INFO oarfish::em: iteration 700; rel diff 0.0013360158155703073 +2026-10-21T22:19:07.411224Z  INFO oarfish::em: iteration 800; rel diff 0.0010336219741871734 +2026-10-21T22:19:07.778245Z  INFO oarfish: oarfish completed successfully. diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/cdna.oarfish.meta_info.json b/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/cdna.oarfish.meta_info.json new file mode 100755 index 0000000..cde72ca --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/cdna.oarfish.meta_info.json @@ -0,0 +1,61 @@ +{ + "prob_model": "no_coverage", + "alignment_source": "from_bam", + "alignment_time": { + "comment": "Time to parse (in alignment mode) or generate (in raw read mode) alignments, as well as apply filters, and compute conditional probabilities.", + "human_time": "2s 537ms 727us 759ns", + "seconds": 2.537727759 + }, + "bin_width": 100, + "filter_options": { + "five_prime_clip": 4294967295, + "three_prime_clip": 50, + "score_threshold": 0.949999988079071, + "min_aligned_fraction": 0.5, + "min_aligned_len": 50, + "which_strand": "Forward", + "model_coverage": false, + "logistic_growth_rate": 2.0, + "write_assignment_probs": false, + "write_assignment_probs_type": null, + "score_prob_denom": 5.0 + }, + "discard_table": { + "discard_5p": 0, + "discard_3p": 461285, + "discard_score": 455915, + "discard_aln_frac": 6057, + "discard_aln_len": 270, + "discard_ori": 1532440, + "discard_supp": 12166, + "no_mapping": 0, + "no_valid_aln": 113119, + "valid_best_aln": 78331 + }, + "alignments": "cdna.q.bam", + "output": "cdna.oarfish", + "verbose": false, + "single_cell": false, + "quiet": false, + "em_max_iter": 1000, + "em_convergence_thresh": 0.001, + "threads": 16, + "filter_group": "NanocountFilters", + "write_assignment_probs": null, + "short_quant": null, + "num_bootstraps": 0, + "digest": { + "bam_digest": { + "seqcol_digest": { + "level": 1, + "names": "JtpXbvlLRu6PIRtExJYCjpWt-HJCdlmY", + "lengths": "0af0dJPuoZAGSA1OxTC98WvOUDqyABTS", + "sorted_name_length_pairs": "obf7zAUChfwy1NNRVLcMGbXvxOeWlK7P" + }, + "sha256_digests": { + "sha256_names": "a36bb3e27e5844f319bb15d352c86e85df88a5e8f6cc82781434eac34ce725b5", + "sha256_seqs": null + } + } + } +} \ No newline at end of file diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/cdna.time b/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/cdna.time new file mode 100755 index 0000000..26296cb --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/cdna.time @@ -0,0 +1 @@ +146.33 1521.47 13.65 4282160 diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/drna.err b/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/drna.err new file mode 100755 index 0000000..14a47de --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/drna.err @@ -0,0 +1,3 @@ +[*] Loading index from ${DATA}/rna/tx/tx.bonefish.raw... +[*] Loaded part 1 in 0.057s — k: 19; w: 19; #seq: 533740; total len: 946309864 +[*] Mapped 200000 reads in 102.377s (1 index part) diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/drna.oarfish.log b/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/drna.oarfish.log new file mode 100755 index 0000000..c099e08 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/drna.oarfish.log @@ -0,0 +1,37 @@ +2026-10-21T22:20:58.905339Z  INFO oarfish: setting filters to nanocount defaults. +2026-10-21T22:20:59.455788Z  INFO oarfish::alignment_parser: read header from BAM file drna.q.bam, contains 533,740 reference sequences. +2026-10-21T22:20:59.456015Z  INFO oarfish::alignment_parser: saw supported mapper rammap as a program in the header; proceeding. +2026-10-21T22:20:59.456099Z  INFO oarfish::util::digest_utils: calculating seqcol digest +2026-10-21T22:21:01.037731Z  INFO oarfish::util::digest_utils: done calculating seqcol digest +2026-10-21T22:21:01.087342Z  INFO oarfish: parsed reference information for 533,740 transcripts. +2026-10-21T22:21:02.783703Z  INFO oarfish::alignment_parser: the alignment file contained 39,115 unmapped read records. +2026-10-21T22:21:02.784604Z  INFO oarfish::bulk: Parsing of alignments from input took: 1s 697ms 51us 619ns +2026-10-21T22:21:02.784683Z  INFO oarfish::bulk: +discard_table: +╭─────────────────────────────────┬────────╮ +│ reason │ count │ +├─────────────────────────────────┼────────┤ +│ too far from 5' end │ 0 │ +│ too far from 3' end │ 447021 │ +│ score too low │ 617833 │ +│ aligned fraction too low │ 3667 │ +│ aligned length too short │ 429 │ +│ inconsistent orientation │ 19775 │ +│ supplementary alignment │ 8963 │ +│ read had no mapping │ 0 │ +│ read had no valid alignment │ 15287 │ +│ reads with valid best alignment │ 141931 │ +╰─────────────────────────────────┴────────╯ + +2026-10-21T22:21:02.784719Z  INFO oarfish::bulk: Total number of alignment records : 902,842 +2026-10-21T22:21:02.784743Z  INFO oarfish::bulk: number of aligned reads : 141,931 +2026-10-21T22:21:02.784764Z  INFO oarfish::bulk: number of unique alignments : 33,138 +2026-10-21T22:21:03.051876Z  INFO oarfish::em: iteration 100; rel diff 0.010536816603753867 +2026-10-21T22:21:03.319086Z  INFO oarfish::em: iteration 200; rel diff 0.004419448920463556 +2026-10-21T22:21:03.603201Z  INFO oarfish::em: iteration 300; rel diff 0.0031571401822824063 +2026-10-21T22:21:03.896206Z  INFO oarfish::em: iteration 400; rel diff 0.0021467043918845007 +2026-10-21T22:21:04.185293Z  INFO oarfish::em: iteration 500; rel diff 0.0017786512610291755 +2026-10-21T22:21:04.478971Z  INFO oarfish::em: iteration 600; rel diff 0.0015069623348835158 +2026-10-21T22:21:04.758486Z  INFO oarfish::em: iteration 700; rel diff 0.0012990544237337647 +2026-10-21T22:21:05.030935Z  INFO oarfish::em: iteration 800; rel diff 0.0011007552935052263 +2026-10-21T22:21:05.526318Z  INFO oarfish: oarfish completed successfully. diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/drna.oarfish.meta_info.json b/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/drna.oarfish.meta_info.json new file mode 100755 index 0000000..7216635 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/drna.oarfish.meta_info.json @@ -0,0 +1,61 @@ +{ + "prob_model": "no_coverage", + "alignment_source": "from_bam", + "alignment_time": { + "comment": "Time to parse (in alignment mode) or generate (in raw read mode) alignments, as well as apply filters, and compute conditional probabilities.", + "human_time": "1s 697ms 51us 619ns", + "seconds": 1.697051619 + }, + "bin_width": 100, + "filter_options": { + "five_prime_clip": 4294967295, + "three_prime_clip": 50, + "score_threshold": 0.949999988079071, + "min_aligned_fraction": 0.5, + "min_aligned_len": 50, + "which_strand": "Forward", + "model_coverage": false, + "logistic_growth_rate": 2.0, + "write_assignment_probs": false, + "write_assignment_probs_type": null, + "score_prob_denom": 5.0 + }, + "discard_table": { + "discard_5p": 0, + "discard_3p": 447021, + "discard_score": 617833, + "discard_aln_frac": 3667, + "discard_aln_len": 429, + "discard_ori": 19775, + "discard_supp": 8963, + "no_mapping": 0, + "no_valid_aln": 15287, + "valid_best_aln": 141931 + }, + "alignments": "drna.q.bam", + "output": "drna.oarfish", + "verbose": false, + "single_cell": false, + "quiet": false, + "em_max_iter": 1000, + "em_convergence_thresh": 0.001, + "threads": 16, + "filter_group": "NanocountFilters", + "write_assignment_probs": null, + "short_quant": null, + "num_bootstraps": 0, + "digest": { + "bam_digest": { + "seqcol_digest": { + "level": 1, + "names": "JtpXbvlLRu6PIRtExJYCjpWt-HJCdlmY", + "lengths": "0af0dJPuoZAGSA1OxTC98WvOUDqyABTS", + "sorted_name_length_pairs": "obf7zAUChfwy1NNRVLcMGbXvxOeWlK7P" + }, + "sha256_digests": { + "sha256_names": "a36bb3e27e5844f319bb15d352c86e85df88a5e8f6cc82781434eac34ce725b5", + "sha256_seqs": null + } + } + } +} \ No newline at end of file diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/drna.time b/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/drna.time new file mode 100755 index 0000000..d428ad4 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/drna.time @@ -0,0 +1 @@ +102.42 1022.63 9.44 3816724 diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/sim_cdna.err b/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/sim_cdna.err new file mode 100755 index 0000000..f98c3eb --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/sim_cdna.err @@ -0,0 +1,3 @@ +[*] Loading index from ${DATA}/rna/tx/tx.bonefish.raw... +[*] Loaded part 1 in 0.072s — k: 19; w: 19; #seq: 533740; total len: 946309864 +[*] Mapped 44667 reads in 31.629s (1 index part) diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/sim_cdna.time b/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/sim_cdna.time new file mode 100755 index 0000000..0f132ad --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/sim_cdna.time @@ -0,0 +1 @@ +31.68 294.12 2.72 2966044 diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/sim_drna.err b/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/sim_drna.err new file mode 100755 index 0000000..6bb90bb --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/sim_drna.err @@ -0,0 +1,3 @@ +[*] Loading index from ${DATA}/rna/tx/tx.bonefish.raw... +[*] Loaded part 1 in 0.056s — k: 19; w: 19; #seq: 533740; total len: 946309864 +[*] Mapped 44787 reads in 31.935s (1 index part) diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/sim_drna.time b/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/sim_drna.time new file mode 100755 index 0000000..462835b --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/dna_N100/sim_drna.time @@ -0,0 +1 @@ +31.99 290.91 2.69 3017908 diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/cdna.err b/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/cdna.err new file mode 100755 index 0000000..b6fcd49 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/cdna.err @@ -0,0 +1,8 @@ +[M::main::1.479*0.93] loaded/built the index for 533740 target sequence(s) +[M::mm_mapopt_update::1.769*0.94] mid_occ = 321 +[M::mm_idx_stat] kmer size: 15; skip: 10; is_hpc: 0; #seq: 533740 +[M::mm_idx_stat::1.937*0.95] distinct minimizers: 24494645 (35.25% are singletons); average occurrences: 7.157; average spacing: 5.398; total length: 946309864 +[M::worker_pipeline::184.518*12.38] mapped 200000 sequences +[M::main] Version: 2.31-r1302 +[M::main] CMD: minimap2 -ax map-ont -N 100 --eqx -t 16 ${DATA}/rna/tx/tx.mm2.mmi ${DATA}/rna/cdna_hg002_200k.fq.gz +[M::main] Real time: 184.565 sec; CPU: 2284.936 sec; Peak RSS: 6.224 GB diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/cdna.oarfish.log b/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/cdna.oarfish.log new file mode 100755 index 0000000..c57d23d --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/cdna.oarfish.log @@ -0,0 +1,39 @@ +2026-10-21T22:12:45.930338Z  INFO oarfish: setting filters to nanocount defaults. +2026-10-21T22:12:47.255912Z  INFO oarfish::alignment_parser: read header from BAM file cdna.q.bam, contains 533,740 reference sequences. +2026-10-21T22:12:47.256039Z  INFO oarfish::alignment_parser: saw supported mapper minimap2 as a program in the header; proceeding. +2026-10-21T22:12:47.256069Z  INFO oarfish::util::digest_utils: calculating seqcol digest +2026-10-21T22:12:48.723814Z  INFO oarfish::util::digest_utils: done calculating seqcol digest +2026-10-21T22:12:48.778644Z  INFO oarfish: parsed reference information for 533,740 transcripts. +2026-10-21T22:12:51.457417Z  INFO oarfish::alignment_parser: the alignment file contained 1,987 unmapped read records. +2026-10-21T22:12:51.458428Z  INFO oarfish::bulk: Parsing of alignments from input took: 2s 679ms 677us 675ns +2026-10-21T22:12:51.458491Z  INFO oarfish::bulk: +discard_table: +╭─────────────────────────────────┬─────────╮ +│ reason │ count │ +├─────────────────────────────────┼─────────┤ +│ too far from 5' end │ 0 │ +│ too far from 3' end │ 462883 │ +│ score too low │ 490021 │ +│ aligned fraction too low │ 6006 │ +│ aligned length too short │ 512 │ +│ inconsistent orientation │ 1577719 │ +│ supplementary alignment │ 11758 │ +│ read had no mapping │ 0 │ +│ read had no valid alignment │ 112897 │ +│ reads with valid best alignment │ 79110 │ +╰─────────────────────────────────┴─────────╯ + +2026-10-21T22:12:51.458521Z  INFO oarfish::bulk: Total number of alignment records : 514,935 +2026-10-21T22:12:51.458538Z  INFO oarfish::bulk: number of aligned reads : 79,110 +2026-10-21T22:12:51.458554Z  INFO oarfish::bulk: number of unique alignments : 16,364 +2026-10-21T22:12:51.728608Z  INFO oarfish::em: iteration 100; rel diff 0.007166221619798012 +2026-10-21T22:12:52.000959Z  INFO oarfish::em: iteration 200; rel diff 0.0035408263878241334 +2026-10-21T22:12:52.293597Z  INFO oarfish::em: iteration 300; rel diff 0.0031280216458239142 +2026-10-21T22:12:52.556885Z  INFO oarfish::em: iteration 400; rel diff 0.0029273414618858477 +2026-10-21T22:12:52.808543Z  INFO oarfish::em: iteration 500; rel diff 0.00267875229554468 +2026-10-21T22:12:53.062457Z  INFO oarfish::em: iteration 600; rel diff 0.0023856855923007725 +2026-10-21T22:12:53.326400Z  INFO oarfish::em: iteration 700; rel diff 0.0020596900205139055 +2026-10-21T22:12:53.576518Z  INFO oarfish::em: iteration 800; rel diff 0.0017197054445243875 +2026-10-21T22:12:53.861055Z  INFO oarfish::em: iteration 900; rel diff 0.0013881675124502997 +2026-10-21T22:12:54.117372Z  INFO oarfish::em: iteration 1,000; rel diff 0.0010853914191092447 +2026-10-21T22:12:54.620657Z  INFO oarfish: oarfish completed successfully. diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/cdna.oarfish.meta_info.json b/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/cdna.oarfish.meta_info.json new file mode 100755 index 0000000..fbb4d79 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/cdna.oarfish.meta_info.json @@ -0,0 +1,61 @@ +{ + "prob_model": "no_coverage", + "alignment_source": "from_bam", + "alignment_time": { + "comment": "Time to parse (in alignment mode) or generate (in raw read mode) alignments, as well as apply filters, and compute conditional probabilities.", + "human_time": "2s 679ms 677us 675ns", + "seconds": 2.6796776749999998 + }, + "bin_width": 100, + "filter_options": { + "five_prime_clip": 4294967295, + "three_prime_clip": 50, + "score_threshold": 0.949999988079071, + "min_aligned_fraction": 0.5, + "min_aligned_len": 50, + "which_strand": "Forward", + "model_coverage": false, + "logistic_growth_rate": 2.0, + "write_assignment_probs": false, + "write_assignment_probs_type": null, + "score_prob_denom": 5.0 + }, + "discard_table": { + "discard_5p": 0, + "discard_3p": 462883, + "discard_score": 490021, + "discard_aln_frac": 6006, + "discard_aln_len": 512, + "discard_ori": 1577719, + "discard_supp": 11758, + "no_mapping": 0, + "no_valid_aln": 112897, + "valid_best_aln": 79110 + }, + "alignments": "cdna.q.bam", + "output": "cdna.oarfish", + "verbose": false, + "single_cell": false, + "quiet": false, + "em_max_iter": 1000, + "em_convergence_thresh": 0.001, + "threads": 16, + "filter_group": "NanocountFilters", + "write_assignment_probs": null, + "short_quant": null, + "num_bootstraps": 0, + "digest": { + "bam_digest": { + "seqcol_digest": { + "level": 1, + "names": "JtpXbvlLRu6PIRtExJYCjpWt-HJCdlmY", + "lengths": "0af0dJPuoZAGSA1OxTC98WvOUDqyABTS", + "sorted_name_length_pairs": "obf7zAUChfwy1NNRVLcMGbXvxOeWlK7P" + }, + "sha256_digests": { + "sha256_names": "a36bb3e27e5844f319bb15d352c86e85df88a5e8f6cc82781434eac34ce725b5", + "sha256_seqs": null + } + } + } +} \ No newline at end of file diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/cdna.time b/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/cdna.time new file mode 100755 index 0000000..162df5f --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/cdna.time @@ -0,0 +1 @@ +184.67 2274.58 10.45 6525996 diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/drna.err b/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/drna.err new file mode 100755 index 0000000..3c83301 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/drna.err @@ -0,0 +1,8 @@ +[M::main::1.568*0.90] loaded/built the index for 533740 target sequence(s) +[M::mm_mapopt_update::1.864*0.92] mid_occ = 321 +[M::mm_idx_stat] kmer size: 15; skip: 10; is_hpc: 0; #seq: 533740 +[M::mm_idx_stat::2.033*0.92] distinct minimizers: 24494645 (35.25% are singletons); average occurrences: 7.157; average spacing: 5.398; total length: 946309864 +[M::worker_pipeline::105.558*9.72] mapped 200000 sequences +[M::main] Version: 2.31-r1302 +[M::main] CMD: minimap2 -ax map-ont -N 100 --eqx -t 16 ${DATA}/rna/tx/tx.mm2.mmi ${DATA}/rna/drna_hg002_200k.fq.gz +[M::main] Real time: 105.607 sec; CPU: 1026.178 sec; Peak RSS: 4.956 GB diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/drna.oarfish.log b/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/drna.oarfish.log new file mode 100755 index 0000000..b6ab93c --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/drna.oarfish.log @@ -0,0 +1,37 @@ +2026-10-21T22:14:47.836713Z  INFO oarfish: setting filters to nanocount defaults. +2026-10-21T22:14:48.324842Z  INFO oarfish::alignment_parser: read header from BAM file drna.q.bam, contains 533,740 reference sequences. +2026-10-21T22:14:48.324939Z  INFO oarfish::alignment_parser: saw supported mapper minimap2 as a program in the header; proceeding. +2026-10-21T22:14:48.324964Z  INFO oarfish::util::digest_utils: calculating seqcol digest +2026-10-21T22:14:49.624816Z  INFO oarfish::util::digest_utils: done calculating seqcol digest +2026-10-21T22:14:49.679233Z  INFO oarfish: parsed reference information for 533,740 transcripts. +2026-10-21T22:14:51.559986Z  INFO oarfish::alignment_parser: the alignment file contained 37,838 unmapped read records. +2026-10-21T22:14:51.560861Z  INFO oarfish::bulk: Parsing of alignments from input took: 1s 881ms 500us 349ns +2026-10-21T22:14:51.560943Z  INFO oarfish::bulk: +discard_table: +╭─────────────────────────────────┬────────╮ +│ reason │ count │ +├─────────────────────────────────┼────────┤ +│ too far from 5' end │ 0 │ +│ too far from 3' end │ 448545 │ +│ score too low │ 685706 │ +│ aligned fraction too low │ 3709 │ +│ aligned length too short │ 1269 │ +│ inconsistent orientation │ 26528 │ +│ supplementary alignment │ 8810 │ +│ read had no mapping │ 0 │ +│ read had no valid alignment │ 15348 │ +│ reads with valid best alignment │ 143105 │ +╰─────────────────────────────────┴────────╯ + +2026-10-21T22:14:51.560976Z  INFO oarfish::bulk: Total number of alignment records : 915,447 +2026-10-21T22:14:51.561000Z  INFO oarfish::bulk: number of aligned reads : 143,105 +2026-10-21T22:14:51.561022Z  INFO oarfish::bulk: number of unique alignments : 32,841 +2026-10-21T22:14:51.927465Z  INFO oarfish::em: iteration 100; rel diff 0.01053681660375387 +2026-10-21T22:14:52.304491Z  INFO oarfish::em: iteration 200; rel diff 0.004420918597068916 +2026-10-21T22:14:52.694024Z  INFO oarfish::em: iteration 300; rel diff 0.0031571401822710642 +2026-10-21T22:14:53.099404Z  INFO oarfish::em: iteration 400; rel diff 0.0021467043918847344 +2026-10-21T22:14:53.497974Z  INFO oarfish::em: iteration 500; rel diff 0.001778651261029389 +2026-10-21T22:14:53.903052Z  INFO oarfish::em: iteration 600; rel diff 0.0015069623348832968 +2026-10-21T22:14:54.277297Z  INFO oarfish::em: iteration 700; rel diff 0.001299054423733763 +2026-10-21T22:14:54.650960Z  INFO oarfish::em: iteration 800; rel diff 0.001100755293505225 +2026-10-21T22:14:55.302345Z  INFO oarfish: oarfish completed successfully. diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/drna.oarfish.meta_info.json b/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/drna.oarfish.meta_info.json new file mode 100755 index 0000000..db96e1b --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/drna.oarfish.meta_info.json @@ -0,0 +1,61 @@ +{ + "prob_model": "no_coverage", + "alignment_source": "from_bam", + "alignment_time": { + "comment": "Time to parse (in alignment mode) or generate (in raw read mode) alignments, as well as apply filters, and compute conditional probabilities.", + "human_time": "1s 881ms 500us 349ns", + "seconds": 1.881500349 + }, + "bin_width": 100, + "filter_options": { + "five_prime_clip": 4294967295, + "three_prime_clip": 50, + "score_threshold": 0.949999988079071, + "min_aligned_fraction": 0.5, + "min_aligned_len": 50, + "which_strand": "Forward", + "model_coverage": false, + "logistic_growth_rate": 2.0, + "write_assignment_probs": false, + "write_assignment_probs_type": null, + "score_prob_denom": 5.0 + }, + "discard_table": { + "discard_5p": 0, + "discard_3p": 448545, + "discard_score": 685706, + "discard_aln_frac": 3709, + "discard_aln_len": 1269, + "discard_ori": 26528, + "discard_supp": 8810, + "no_mapping": 0, + "no_valid_aln": 15348, + "valid_best_aln": 143105 + }, + "alignments": "drna.q.bam", + "output": "drna.oarfish", + "verbose": false, + "single_cell": false, + "quiet": false, + "em_max_iter": 1000, + "em_convergence_thresh": 0.001, + "threads": 16, + "filter_group": "NanocountFilters", + "write_assignment_probs": null, + "short_quant": null, + "num_bootstraps": 0, + "digest": { + "bam_digest": { + "seqcol_digest": { + "level": 1, + "names": "JtpXbvlLRu6PIRtExJYCjpWt-HJCdlmY", + "lengths": "0af0dJPuoZAGSA1OxTC98WvOUDqyABTS", + "sorted_name_length_pairs": "obf7zAUChfwy1NNRVLcMGbXvxOeWlK7P" + }, + "sha256_digests": { + "sha256_names": "a36bb3e27e5844f319bb15d352c86e85df88a5e8f6cc82781434eac34ce725b5", + "sha256_seqs": null + } + } + } +} \ No newline at end of file diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/drna.time b/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/drna.time new file mode 100755 index 0000000..8a24812 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/drna.time @@ -0,0 +1 @@ +105.69 1019.72 6.54 5197236 diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/sim_cdna.err b/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/sim_cdna.err new file mode 100755 index 0000000..bed9aa5 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/sim_cdna.err @@ -0,0 +1,8 @@ +[M::main::2.493*0.75] loaded/built the index for 533740 target sequence(s) +[M::mm_mapopt_update::2.781*0.78] mid_occ = 321 +[M::mm_idx_stat] kmer size: 15; skip: 10; is_hpc: 0; #seq: 533740 +[M::mm_idx_stat::2.950*0.79] distinct minimizers: 24494645 (35.25% are singletons); average occurrences: 7.157; average spacing: 5.398; total length: 946309864 +[M::worker_pipeline::47.065*9.39] mapped 44667 sequences +[M::main] Version: 2.31-r1302 +[M::main] CMD: minimap2 -ax map-ont -N 100 --eqx -t 16 ${DATA}/rna/tx/tx.mm2.mmi ${DATA}/rna/sim/sim_cdna.fq.gz +[M::main] Real time: 47.091 sec; CPU: 441.736 sec; Peak RSS: 4.010 GB diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/sim_cdna.time b/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/sim_cdna.time new file mode 100755 index 0000000..b03c381 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/sim_cdna.time @@ -0,0 +1 @@ +47.19 437.68 4.15 4204928 diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/sim_drna.err b/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/sim_drna.err new file mode 100755 index 0000000..60b8c1e --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/sim_drna.err @@ -0,0 +1,8 @@ +[M::main::2.042*0.81] loaded/built the index for 533740 target sequence(s) +[M::mm_mapopt_update::2.395*0.84] mid_occ = 321 +[M::mm_idx_stat] kmer size: 15; skip: 10; is_hpc: 0; #seq: 533740 +[M::mm_idx_stat::2.584*0.85] distinct minimizers: 24494645 (35.25% are singletons); average occurrences: 7.157; average spacing: 5.398; total length: 946309864 +[M::worker_pipeline::38.039*10.42] mapped 44787 sequences +[M::main] Version: 2.31-r1302 +[M::main] CMD: minimap2 -ax map-ont -N 100 --eqx -t 16 ${DATA}/rna/tx/tx.mm2.mmi ${DATA}/rna/sim/sim_drna.fq.gz +[M::main] Real time: 38.073 sec; CPU: 396.242 sec; Peak RSS: 4.285 GB diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/sim_drna.time b/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/sim_drna.time new file mode 100755 index 0000000..4ec34bb --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/mm2/sim_drna.time @@ -0,0 +1 @@ +38.16 392.60 3.72 4492936 diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/cdna.err b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/cdna.err new file mode 100755 index 0000000..58780b0 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/cdna.err @@ -0,0 +1,3 @@ +[*] Loading index from ${DATA}/rna/tx/tx.tx.raw... +[*] Loaded part 1 in 0.082s — k: 15; w: 10; #seq: 533740; total len: 946309864 +[*] Mapped 200000 reads in 183.603s (1 index part) diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/cdna.oarfish.log b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/cdna.oarfish.log new file mode 100755 index 0000000..2b4fa35 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/cdna.oarfish.log @@ -0,0 +1,37 @@ +2026-10-21T22:25:30.849141Z  INFO oarfish: setting filters to nanocount defaults. +2026-10-21T22:25:31.487266Z  INFO oarfish::alignment_parser: read header from BAM file cdna.q.bam, contains 533,740 reference sequences. +2026-10-21T22:25:31.487375Z  INFO oarfish::alignment_parser: saw supported mapper rammap as a program in the header; proceeding. +2026-10-21T22:25:31.487397Z  INFO oarfish::util::digest_utils: calculating seqcol digest +2026-10-21T22:25:33.278578Z  INFO oarfish::util::digest_utils: done calculating seqcol digest +2026-10-21T22:25:33.331749Z  INFO oarfish: parsed reference information for 533,740 transcripts. +2026-10-21T22:25:36.080207Z  INFO oarfish::alignment_parser: the alignment file contained 1,987 unmapped read records. +2026-10-21T22:25:36.081394Z  INFO oarfish::bulk: Parsing of alignments from input took: 2s 749ms 534us 747ns +2026-10-21T22:25:36.081512Z  INFO oarfish::bulk: +discard_table: +╭─────────────────────────────────┬─────────╮ +│ reason │ count │ +├─────────────────────────────────┼─────────┤ +│ too far from 5' end │ 0 │ +│ too far from 3' end │ 484078 │ +│ score too low │ 470973 │ +│ aligned fraction too low │ 6842 │ +│ aligned length too short │ 597 │ +│ inconsistent orientation │ 1583343 │ +│ supplementary alignment │ 14706 │ +│ read had no mapping │ 0 │ +│ read had no valid alignment │ 112960 │ +│ reads with valid best alignment │ 78211 │ +╰─────────────────────────────────┴─────────╯ + +2026-10-21T22:25:36.081585Z  INFO oarfish::bulk: Total number of alignment records : 506,048 +2026-10-21T22:25:36.081603Z  INFO oarfish::bulk: number of aligned reads : 78,211 +2026-10-21T22:25:36.081615Z  INFO oarfish::bulk: number of unique alignments : 16,404 +2026-10-21T22:25:36.341782Z  INFO oarfish::em: iteration 100; rel diff 0.006687934315578947 +2026-10-21T22:25:36.593405Z  INFO oarfish::em: iteration 200; rel diff 0.0030421754005210814 +2026-10-21T22:25:36.852677Z  INFO oarfish::em: iteration 300; rel diff 0.002746057598825169 +2026-10-21T22:25:37.102096Z  INFO oarfish::em: iteration 400; rel diff 0.002407716082306436 +2026-10-21T22:25:37.361187Z  INFO oarfish::em: iteration 500; rel diff 0.0020442441605902067 +2026-10-21T22:25:37.597038Z  INFO oarfish::em: iteration 600; rel diff 0.001679032961256966 +2026-10-21T22:25:37.858003Z  INFO oarfish::em: iteration 700; rel diff 0.0013360158155701328 +2026-10-21T22:25:38.110912Z  INFO oarfish::em: iteration 800; rel diff 0.0010336219741874848 +2026-10-21T22:25:38.581135Z  INFO oarfish: oarfish completed successfully. diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/cdna.oarfish.meta_info.json b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/cdna.oarfish.meta_info.json new file mode 100755 index 0000000..94316b6 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/cdna.oarfish.meta_info.json @@ -0,0 +1,61 @@ +{ + "prob_model": "no_coverage", + "alignment_source": "from_bam", + "alignment_time": { + "comment": "Time to parse (in alignment mode) or generate (in raw read mode) alignments, as well as apply filters, and compute conditional probabilities.", + "human_time": "2s 749ms 534us 747ns", + "seconds": 2.7495347470000002 + }, + "bin_width": 100, + "filter_options": { + "five_prime_clip": 4294967295, + "three_prime_clip": 50, + "score_threshold": 0.949999988079071, + "min_aligned_fraction": 0.5, + "min_aligned_len": 50, + "which_strand": "Forward", + "model_coverage": false, + "logistic_growth_rate": 2.0, + "write_assignment_probs": false, + "write_assignment_probs_type": null, + "score_prob_denom": 5.0 + }, + "discard_table": { + "discard_5p": 0, + "discard_3p": 484078, + "discard_score": 470973, + "discard_aln_frac": 6842, + "discard_aln_len": 597, + "discard_ori": 1583343, + "discard_supp": 14706, + "no_mapping": 0, + "no_valid_aln": 112960, + "valid_best_aln": 78211 + }, + "alignments": "cdna.q.bam", + "output": "cdna.oarfish", + "verbose": false, + "single_cell": false, + "quiet": false, + "em_max_iter": 1000, + "em_convergence_thresh": 0.001, + "threads": 16, + "filter_group": "NanocountFilters", + "write_assignment_probs": null, + "short_quant": null, + "num_bootstraps": 0, + "digest": { + "bam_digest": { + "seqcol_digest": { + "level": 1, + "names": "JtpXbvlLRu6PIRtExJYCjpWt-HJCdlmY", + "lengths": "0af0dJPuoZAGSA1OxTC98WvOUDqyABTS", + "sorted_name_length_pairs": "obf7zAUChfwy1NNRVLcMGbXvxOeWlK7P" + }, + "sha256_digests": { + "sha256_names": "a36bb3e27e5844f319bb15d352c86e85df88a5e8f6cc82781434eac34ce725b5", + "sha256_seqs": null + } + } + } +} \ No newline at end of file diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/cdna.time b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/cdna.time new file mode 100755 index 0000000..d30eff8 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/cdna.time @@ -0,0 +1 @@ +183.68 2025.44 14.51 4993332 diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/drna.err b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/drna.err new file mode 100755 index 0000000..87f7ef3 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/drna.err @@ -0,0 +1,3 @@ +[*] Loading index from ${DATA}/rna/tx/tx.tx.raw... +[*] Loaded part 1 in 0.071s — k: 15; w: 10; #seq: 533740; total len: 946309864 +[*] Mapped 200000 reads in 101.582s (1 index part) diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/drna.oarfish.log b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/drna.oarfish.log new file mode 100755 index 0000000..aa97e69 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/drna.oarfish.log @@ -0,0 +1,37 @@ +2026-10-21T22:27:28.953675Z  INFO oarfish: setting filters to nanocount defaults. +2026-10-21T22:27:29.310599Z  INFO oarfish::alignment_parser: read header from BAM file drna.q.bam, contains 533,740 reference sequences. +2026-10-21T22:27:29.310694Z  INFO oarfish::alignment_parser: saw supported mapper rammap as a program in the header; proceeding. +2026-10-21T22:27:29.310714Z  INFO oarfish::util::digest_utils: calculating seqcol digest +2026-10-21T22:27:30.416792Z  INFO oarfish::util::digest_utils: done calculating seqcol digest +2026-10-21T22:27:30.463774Z  INFO oarfish: parsed reference information for 533,740 transcripts. +2026-10-21T22:27:32.205461Z  INFO oarfish::alignment_parser: the alignment file contained 37,838 unmapped read records. +2026-10-21T22:27:32.206340Z  INFO oarfish::bulk: Parsing of alignments from input took: 1s 742ms 477us 78ns +2026-10-21T22:27:32.206392Z  INFO oarfish::bulk: +discard_table: +╭─────────────────────────────────┬────────╮ +│ reason │ count │ +├─────────────────────────────────┼────────┤ +│ too far from 5' end │ 0 │ +│ too far from 3' end │ 472740 │ +│ score too low │ 654768 │ +│ aligned fraction too low │ 4076 │ +│ aligned length too short │ 1281 │ +│ inconsistent orientation │ 27194 │ +│ supplementary alignment │ 10887 │ +│ read had no mapping │ 0 │ +│ read had no valid alignment │ 15496 │ +│ reads with valid best alignment │ 142590 │ +╰─────────────────────────────────┴────────╯ + +2026-10-21T22:27:32.206423Z  INFO oarfish::bulk: Total number of alignment records : 908,590 +2026-10-21T22:27:32.206483Z  INFO oarfish::bulk: number of aligned reads : 142,590 +2026-10-21T22:27:32.206524Z  INFO oarfish::bulk: number of unique alignments : 33,000 +2026-10-21T22:27:32.556398Z  INFO oarfish::em: iteration 100; rel diff 0.010536816603753878 +2026-10-21T22:27:32.915011Z  INFO oarfish::em: iteration 200; rel diff 0.004420907414445859 +2026-10-21T22:27:33.290282Z  INFO oarfish::em: iteration 300; rel diff 0.0031571401822706093 +2026-10-21T22:27:33.709809Z  INFO oarfish::em: iteration 400; rel diff 0.0023764816476970715 +2026-10-21T22:27:34.119640Z  INFO oarfish::em: iteration 500; rel diff 0.0018667627046812466 +2026-10-21T22:27:34.508064Z  INFO oarfish::em: iteration 600; rel diff 0.0015069623348837274 +2026-10-21T22:27:34.894701Z  INFO oarfish::em: iteration 700; rel diff 0.0012990544237337618 +2026-10-21T22:27:35.218413Z  INFO oarfish::em: iteration 800; rel diff 0.0011007552935052244 +2026-10-21T22:27:36.037127Z  INFO oarfish: oarfish completed successfully. diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/drna.oarfish.meta_info.json b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/drna.oarfish.meta_info.json new file mode 100755 index 0000000..f1eb427 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/drna.oarfish.meta_info.json @@ -0,0 +1,61 @@ +{ + "prob_model": "no_coverage", + "alignment_source": "from_bam", + "alignment_time": { + "comment": "Time to parse (in alignment mode) or generate (in raw read mode) alignments, as well as apply filters, and compute conditional probabilities.", + "human_time": "1s 742ms 477us 78ns", + "seconds": 1.7424770779999998 + }, + "bin_width": 100, + "filter_options": { + "five_prime_clip": 4294967295, + "three_prime_clip": 50, + "score_threshold": 0.949999988079071, + "min_aligned_fraction": 0.5, + "min_aligned_len": 50, + "which_strand": "Forward", + "model_coverage": false, + "logistic_growth_rate": 2.0, + "write_assignment_probs": false, + "write_assignment_probs_type": null, + "score_prob_denom": 5.0 + }, + "discard_table": { + "discard_5p": 0, + "discard_3p": 472740, + "discard_score": 654768, + "discard_aln_frac": 4076, + "discard_aln_len": 1281, + "discard_ori": 27194, + "discard_supp": 10887, + "no_mapping": 0, + "no_valid_aln": 15496, + "valid_best_aln": 142590 + }, + "alignments": "drna.q.bam", + "output": "drna.oarfish", + "verbose": false, + "single_cell": false, + "quiet": false, + "em_max_iter": 1000, + "em_convergence_thresh": 0.001, + "threads": 16, + "filter_group": "NanocountFilters", + "write_assignment_probs": null, + "short_quant": null, + "num_bootstraps": 0, + "digest": { + "bam_digest": { + "seqcol_digest": { + "level": 1, + "names": "JtpXbvlLRu6PIRtExJYCjpWt-HJCdlmY", + "lengths": "0af0dJPuoZAGSA1OxTC98WvOUDqyABTS", + "sorted_name_length_pairs": "obf7zAUChfwy1NNRVLcMGbXvxOeWlK7P" + }, + "sha256_digests": { + "sha256_names": "a36bb3e27e5844f319bb15d352c86e85df88a5e8f6cc82781434eac34ce725b5", + "sha256_seqs": null + } + } + } +} \ No newline at end of file diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/drna.time b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/drna.time new file mode 100755 index 0000000..dbd421f --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/drna.time @@ -0,0 +1 @@ +101.63 1237.44 9.24 4765836 diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/sim_cdna.err b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/sim_cdna.err new file mode 100755 index 0000000..7c17635 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/sim_cdna.err @@ -0,0 +1,3 @@ +[*] Loading index from ${DATA}/rna/tx/tx.tx.raw... +[*] Loaded part 1 in 0.056s — k: 15; w: 10; #seq: 533740; total len: 946309864 +[*] Mapped 44667 reads in 45.543s (1 index part) diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/sim_cdna.time b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/sim_cdna.time new file mode 100755 index 0000000..d1e5ba1 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/sim_cdna.time @@ -0,0 +1 @@ +45.58 414.99 3.41 4006160 diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/sim_drna.err b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/sim_drna.err new file mode 100755 index 0000000..dd48a2d --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/sim_drna.err @@ -0,0 +1,3 @@ +[*] Loading index from ${DATA}/rna/tx/tx.tx.raw... +[*] Loaded part 1 in 0.073s — k: 15; w: 10; #seq: 533740; total len: 946309864 +[*] Mapped 44787 reads in 37.571s (1 index part) diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/sim_drna.time b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/sim_drna.time new file mode 100755 index 0000000..d554531 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx1/sim_drna.time @@ -0,0 +1 @@ +37.63 360.25 3.04 3998448 diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/cdna.err b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/cdna.err new file mode 100755 index 0000000..a8acaae --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/cdna.err @@ -0,0 +1,3 @@ +[*] Loading index from ${DATA}/rna/tx/tx.tx.raw... +[*] Loaded part 1 in 0.056s — k: 15; w: 10; #seq: 533740; total len: 946309864 +[*] Mapped 200000 reads in 85.094s (1 index part) diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/cdna.oarfish.log b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/cdna.oarfish.log new file mode 100755 index 0000000..0df72cb --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/cdna.oarfish.log @@ -0,0 +1,37 @@ +2026-10-21T23:13:49.362976Z  INFO oarfish: setting filters to nanocount defaults. +2026-10-21T23:13:49.971278Z  INFO oarfish::alignment_parser: read header from BAM file cdna.q.bam, contains 533,740 reference sequences. +2026-10-21T23:13:49.971511Z  INFO oarfish::alignment_parser: saw supported mapper rammap as a program in the header; proceeding. +2026-10-21T23:13:49.971540Z  INFO oarfish::util::digest_utils: calculating seqcol digest +2026-10-21T23:13:51.020221Z  INFO oarfish::util::digest_utils: done calculating seqcol digest +2026-10-21T23:13:51.070087Z  INFO oarfish: parsed reference information for 533,740 transcripts. +2026-10-21T23:13:53.654433Z  INFO oarfish::alignment_parser: the alignment file contained 1,987 unmapped read records. +2026-10-21T23:13:53.655444Z  INFO oarfish::bulk: Parsing of alignments from input took: 2s 585ms 262us 302ns +2026-10-21T23:13:53.655491Z  INFO oarfish::bulk: +discard_table: +╭─────────────────────────────────┬─────────╮ +│ reason │ count │ +├─────────────────────────────────┼─────────┤ +│ too far from 5' end │ 0 │ +│ too far from 3' end │ 484078 │ +│ score too low │ 470973 │ +│ aligned fraction too low │ 6842 │ +│ aligned length too short │ 597 │ +│ inconsistent orientation │ 1583343 │ +│ supplementary alignment │ 14706 │ +│ read had no mapping │ 0 │ +│ read had no valid alignment │ 112960 │ +│ reads with valid best alignment │ 78211 │ +╰─────────────────────────────────┴─────────╯ + +2026-10-21T23:13:53.655515Z  INFO oarfish::bulk: Total number of alignment records : 506,048 +2026-10-21T23:13:53.655527Z  INFO oarfish::bulk: number of aligned reads : 78,211 +2026-10-21T23:13:53.655539Z  INFO oarfish::bulk: number of unique alignments : 16,404 +2026-10-21T23:13:53.829419Z  INFO oarfish::em: iteration 100; rel diff 0.006687934315578945 +2026-10-21T23:13:54.002748Z  INFO oarfish::em: iteration 200; rel diff 0.0030421754005210836 +2026-10-21T23:13:54.183596Z  INFO oarfish::em: iteration 300; rel diff 0.0027460575988249736 +2026-10-21T23:13:54.358663Z  INFO oarfish::em: iteration 400; rel diff 0.00240771608230613 +2026-10-21T23:13:54.528410Z  INFO oarfish::em: iteration 500; rel diff 0.002044244160590207 +2026-10-21T23:13:54.697876Z  INFO oarfish::em: iteration 600; rel diff 0.0016790329612569657 +2026-10-21T23:13:54.869368Z  INFO oarfish::em: iteration 700; rel diff 0.0013360158155701321 +2026-10-21T23:13:55.040537Z  INFO oarfish::em: iteration 800; rel diff 0.0010336219741874848 +2026-10-21T23:13:55.412833Z  INFO oarfish: oarfish completed successfully. diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/cdna.oarfish.meta_info.json b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/cdna.oarfish.meta_info.json new file mode 100755 index 0000000..d1a3b6b --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/cdna.oarfish.meta_info.json @@ -0,0 +1,61 @@ +{ + "prob_model": "no_coverage", + "alignment_source": "from_bam", + "alignment_time": { + "comment": "Time to parse (in alignment mode) or generate (in raw read mode) alignments, as well as apply filters, and compute conditional probabilities.", + "human_time": "2s 585ms 262us 302ns", + "seconds": 2.585262302 + }, + "bin_width": 100, + "filter_options": { + "five_prime_clip": 4294967295, + "three_prime_clip": 50, + "score_threshold": 0.949999988079071, + "min_aligned_fraction": 0.5, + "min_aligned_len": 50, + "which_strand": "Forward", + "model_coverage": false, + "logistic_growth_rate": 2.0, + "write_assignment_probs": false, + "write_assignment_probs_type": null, + "score_prob_denom": 5.0 + }, + "discard_table": { + "discard_5p": 0, + "discard_3p": 484078, + "discard_score": 470973, + "discard_aln_frac": 6842, + "discard_aln_len": 597, + "discard_ori": 1583343, + "discard_supp": 14706, + "no_mapping": 0, + "no_valid_aln": 112960, + "valid_best_aln": 78211 + }, + "alignments": "cdna.q.bam", + "output": "cdna.oarfish", + "verbose": false, + "single_cell": false, + "quiet": false, + "em_max_iter": 1000, + "em_convergence_thresh": 0.001, + "threads": 16, + "filter_group": "NanocountFilters", + "write_assignment_probs": null, + "short_quant": null, + "num_bootstraps": 0, + "digest": { + "bam_digest": { + "seqcol_digest": { + "level": 1, + "names": "JtpXbvlLRu6PIRtExJYCjpWt-HJCdlmY", + "lengths": "0af0dJPuoZAGSA1OxTC98WvOUDqyABTS", + "sorted_name_length_pairs": "obf7zAUChfwy1NNRVLcMGbXvxOeWlK7P" + }, + "sha256_digests": { + "sha256_names": "a36bb3e27e5844f319bb15d352c86e85df88a5e8f6cc82781434eac34ce725b5", + "sha256_seqs": null + } + } + } +} \ No newline at end of file diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/cdna.time b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/cdna.time new file mode 100755 index 0000000..c18ecd6 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/cdna.time @@ -0,0 +1 @@ +85.15 1187.15 6.44 5136416 diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/drna.err b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/drna.err new file mode 100755 index 0000000..672c9f4 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/drna.err @@ -0,0 +1,3 @@ +[*] Loading index from ${DATA}/rna/tx/tx.tx.raw... +[*] Loaded part 1 in 0.053s — k: 15; w: 10; #seq: 533740; total len: 946309864 +[*] Mapped 200000 reads in 38.848s (1 index part) diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/drna.oarfish.log b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/drna.oarfish.log new file mode 100755 index 0000000..6dfbf7a --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/drna.oarfish.log @@ -0,0 +1,37 @@ +2026-10-21T23:14:42.280491Z  INFO oarfish: setting filters to nanocount defaults. +2026-10-21T23:14:42.711417Z  INFO oarfish::alignment_parser: read header from BAM file drna.q.bam, contains 533,740 reference sequences. +2026-10-21T23:14:42.711486Z  INFO oarfish::alignment_parser: saw supported mapper rammap as a program in the header; proceeding. +2026-10-21T23:14:42.711503Z  INFO oarfish::util::digest_utils: calculating seqcol digest +2026-10-21T23:14:43.843214Z  INFO oarfish::util::digest_utils: done calculating seqcol digest +2026-10-21T23:14:43.890772Z  INFO oarfish: parsed reference information for 533,740 transcripts. +2026-10-21T23:14:45.643414Z  INFO oarfish::alignment_parser: the alignment file contained 37,838 unmapped read records. +2026-10-21T23:14:45.644280Z  INFO oarfish::bulk: Parsing of alignments from input took: 1s 753ms 443us 140ns +2026-10-21T23:14:45.644351Z  INFO oarfish::bulk: +discard_table: +╭─────────────────────────────────┬────────╮ +│ reason │ count │ +├─────────────────────────────────┼────────┤ +│ too far from 5' end │ 0 │ +│ too far from 3' end │ 472740 │ +│ score too low │ 654768 │ +│ aligned fraction too low │ 4076 │ +│ aligned length too short │ 1281 │ +│ inconsistent orientation │ 27194 │ +│ supplementary alignment │ 10887 │ +│ read had no mapping │ 0 │ +│ read had no valid alignment │ 15496 │ +│ reads with valid best alignment │ 142590 │ +╰─────────────────────────────────┴────────╯ + +2026-10-21T23:14:45.644374Z  INFO oarfish::bulk: Total number of alignment records : 908,590 +2026-10-21T23:14:45.644386Z  INFO oarfish::bulk: number of aligned reads : 142,590 +2026-10-21T23:14:45.644398Z  INFO oarfish::bulk: number of unique alignments : 33,000 +2026-10-21T23:14:45.923126Z  INFO oarfish::em: iteration 100; rel diff 0.010536816603753874 +2026-10-21T23:14:46.182611Z  INFO oarfish::em: iteration 200; rel diff 0.004420907414445867 +2026-10-21T23:14:46.448334Z  INFO oarfish::em: iteration 300; rel diff 0.0031571401822707594 +2026-10-21T23:14:46.726713Z  INFO oarfish::em: iteration 400; rel diff 0.0023764816476970715 +2026-10-21T23:14:47.004955Z  INFO oarfish::em: iteration 500; rel diff 0.0018667627046812466 +2026-10-21T23:14:47.277195Z  INFO oarfish::em: iteration 600; rel diff 0.0015069623348837296 +2026-10-21T23:14:47.536203Z  INFO oarfish::em: iteration 700; rel diff 0.001299054423733764 +2026-10-21T23:14:47.791529Z  INFO oarfish::em: iteration 800; rel diff 0.0011007552935052257 +2026-10-21T23:14:48.271820Z  INFO oarfish: oarfish completed successfully. diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/drna.oarfish.meta_info.json b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/drna.oarfish.meta_info.json new file mode 100755 index 0000000..8ebc933 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/drna.oarfish.meta_info.json @@ -0,0 +1,61 @@ +{ + "prob_model": "no_coverage", + "alignment_source": "from_bam", + "alignment_time": { + "comment": "Time to parse (in alignment mode) or generate (in raw read mode) alignments, as well as apply filters, and compute conditional probabilities.", + "human_time": "1s 753ms 443us 140ns", + "seconds": 1.75344314 + }, + "bin_width": 100, + "filter_options": { + "five_prime_clip": 4294967295, + "three_prime_clip": 50, + "score_threshold": 0.949999988079071, + "min_aligned_fraction": 0.5, + "min_aligned_len": 50, + "which_strand": "Forward", + "model_coverage": false, + "logistic_growth_rate": 2.0, + "write_assignment_probs": false, + "write_assignment_probs_type": null, + "score_prob_denom": 5.0 + }, + "discard_table": { + "discard_5p": 0, + "discard_3p": 472740, + "discard_score": 654768, + "discard_aln_frac": 4076, + "discard_aln_len": 1281, + "discard_ori": 27194, + "discard_supp": 10887, + "no_mapping": 0, + "no_valid_aln": 15496, + "valid_best_aln": 142590 + }, + "alignments": "drna.q.bam", + "output": "drna.oarfish", + "verbose": false, + "single_cell": false, + "quiet": false, + "em_max_iter": 1000, + "em_convergence_thresh": 0.001, + "threads": 16, + "filter_group": "NanocountFilters", + "write_assignment_probs": null, + "short_quant": null, + "num_bootstraps": 0, + "digest": { + "bam_digest": { + "seqcol_digest": { + "level": 1, + "names": "JtpXbvlLRu6PIRtExJYCjpWt-HJCdlmY", + "lengths": "0af0dJPuoZAGSA1OxTC98WvOUDqyABTS", + "sorted_name_length_pairs": "obf7zAUChfwy1NNRVLcMGbXvxOeWlK7P" + }, + "sha256_digests": { + "sha256_names": "a36bb3e27e5844f319bb15d352c86e85df88a5e8f6cc82781434eac34ce725b5", + "sha256_seqs": null + } + } + } +} \ No newline at end of file diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/drna.time b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/drna.time new file mode 100755 index 0000000..ba98a20 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/drna.time @@ -0,0 +1 @@ +38.90 502.89 3.52 4842932 diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/sim_cdna.err b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/sim_cdna.err new file mode 100755 index 0000000..6d530ab --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/sim_cdna.err @@ -0,0 +1,3 @@ +[*] Loading index from ${DATA}/rna/tx/tx.tx.raw... +[*] Loaded part 1 in 0.058s — k: 15; w: 10; #seq: 533740; total len: 946309864 +[*] Mapped 44667 reads in 20.369s (1 index part) diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/sim_cdna.time b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/sim_cdna.time new file mode 100755 index 0000000..ec47479 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/sim_cdna.time @@ -0,0 +1 @@ +20.41 226.04 1.55 4006892 diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/sim_drna.err b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/sim_drna.err new file mode 100755 index 0000000..09481cc --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/sim_drna.err @@ -0,0 +1,3 @@ +[*] Loading index from ${DATA}/rna/tx/tx.tx.raw... +[*] Loaded part 1 in 0.056s — k: 15; w: 10; #seq: 533740; total len: 946309864 +[*] Mapped 44787 reads in 18.479s (1 index part) diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/sim_drna.time b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/sim_drna.time new file mode 100755 index 0000000..00ade84 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx3/sim_drna.time @@ -0,0 +1 @@ +18.53 181.98 1.49 3996852 diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/cdna.err b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/cdna.err new file mode 100755 index 0000000..ad2dcb6 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/cdna.err @@ -0,0 +1,3 @@ +[*] Loading index from ${DATA}/rna/tx/tx.tx.raw... +[*] Loaded part 1 in 0.054s — k: 15; w: 10; #seq: 533740; total len: 946309864 +[*] Mapped 200000 reads in 81.158s (1 index part) diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/cdna.oarfish.log b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/cdna.oarfish.log new file mode 100755 index 0000000..e9dbfa2 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/cdna.oarfish.log @@ -0,0 +1,37 @@ +2026-10-21T23:22:29.578604Z  INFO oarfish: setting filters to nanocount defaults. +2026-10-21T23:22:30.163537Z  INFO oarfish::alignment_parser: read header from BAM file cdna.q.bam, contains 533,740 reference sequences. +2026-10-21T23:22:30.163693Z  INFO oarfish::alignment_parser: saw supported mapper rammap as a program in the header; proceeding. +2026-10-21T23:22:30.163721Z  INFO oarfish::util::digest_utils: calculating seqcol digest +2026-10-21T23:22:31.127306Z  INFO oarfish::util::digest_utils: done calculating seqcol digest +2026-10-21T23:22:31.173077Z  INFO oarfish: parsed reference information for 533,740 transcripts. +2026-10-21T23:22:33.749556Z  INFO oarfish::alignment_parser: the alignment file contained 1,987 unmapped read records. +2026-10-21T23:22:33.750439Z  INFO oarfish::bulk: Parsing of alignments from input took: 2s 577ms 240us 24ns +2026-10-21T23:22:33.750504Z  INFO oarfish::bulk: +discard_table: +╭─────────────────────────────────┬─────────╮ +│ reason │ count │ +├─────────────────────────────────┼─────────┤ +│ too far from 5' end │ 0 │ +│ too far from 3' end │ 484078 │ +│ score too low │ 470973 │ +│ aligned fraction too low │ 6842 │ +│ aligned length too short │ 597 │ +│ inconsistent orientation │ 1583343 │ +│ supplementary alignment │ 14706 │ +│ read had no mapping │ 0 │ +│ read had no valid alignment │ 112960 │ +│ reads with valid best alignment │ 78211 │ +╰─────────────────────────────────┴─────────╯ + +2026-10-21T23:22:33.750528Z  INFO oarfish::bulk: Total number of alignment records : 506,048 +2026-10-21T23:22:33.750540Z  INFO oarfish::bulk: number of aligned reads : 78,211 +2026-10-21T23:22:33.750551Z  INFO oarfish::bulk: number of unique alignments : 16,404 +2026-10-21T23:22:33.978237Z  INFO oarfish::em: iteration 100; rel diff 0.006687934315579084 +2026-10-21T23:22:34.151374Z  INFO oarfish::em: iteration 200; rel diff 0.0030421754005210853 +2026-10-21T23:22:34.329616Z  INFO oarfish::em: iteration 300; rel diff 0.002746057598824974 +2026-10-21T23:22:34.499030Z  INFO oarfish::em: iteration 400; rel diff 0.00240771608230613 +2026-10-21T23:22:34.669623Z  INFO oarfish::em: iteration 500; rel diff 0.0020442441605902062 +2026-10-21T23:22:34.843810Z  INFO oarfish::em: iteration 600; rel diff 0.0016790329612569657 +2026-10-21T23:22:35.017640Z  INFO oarfish::em: iteration 700; rel diff 0.001336015815570133 +2026-10-21T23:22:35.191173Z  INFO oarfish::em: iteration 800; rel diff 0.001033621974187174 +2026-10-21T23:22:35.548327Z  INFO oarfish: oarfish completed successfully. diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/cdna.oarfish.meta_info.json b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/cdna.oarfish.meta_info.json new file mode 100755 index 0000000..4e886c7 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/cdna.oarfish.meta_info.json @@ -0,0 +1,61 @@ +{ + "prob_model": "no_coverage", + "alignment_source": "from_bam", + "alignment_time": { + "comment": "Time to parse (in alignment mode) or generate (in raw read mode) alignments, as well as apply filters, and compute conditional probabilities.", + "human_time": "2s 577ms 240us 24ns", + "seconds": 2.577240024 + }, + "bin_width": 100, + "filter_options": { + "five_prime_clip": 4294967295, + "three_prime_clip": 50, + "score_threshold": 0.949999988079071, + "min_aligned_fraction": 0.5, + "min_aligned_len": 50, + "which_strand": "Forward", + "model_coverage": false, + "logistic_growth_rate": 2.0, + "write_assignment_probs": false, + "write_assignment_probs_type": null, + "score_prob_denom": 5.0 + }, + "discard_table": { + "discard_5p": 0, + "discard_3p": 484078, + "discard_score": 470973, + "discard_aln_frac": 6842, + "discard_aln_len": 597, + "discard_ori": 1583343, + "discard_supp": 14706, + "no_mapping": 0, + "no_valid_aln": 112960, + "valid_best_aln": 78211 + }, + "alignments": "cdna.q.bam", + "output": "cdna.oarfish", + "verbose": false, + "single_cell": false, + "quiet": false, + "em_max_iter": 1000, + "em_convergence_thresh": 0.001, + "threads": 16, + "filter_group": "NanocountFilters", + "write_assignment_probs": null, + "short_quant": null, + "num_bootstraps": 0, + "digest": { + "bam_digest": { + "seqcol_digest": { + "level": 1, + "names": "JtpXbvlLRu6PIRtExJYCjpWt-HJCdlmY", + "lengths": "0af0dJPuoZAGSA1OxTC98WvOUDqyABTS", + "sorted_name_length_pairs": "obf7zAUChfwy1NNRVLcMGbXvxOeWlK7P" + }, + "sha256_digests": { + "sha256_names": "a36bb3e27e5844f319bb15d352c86e85df88a5e8f6cc82781434eac34ce725b5", + "sha256_seqs": null + } + } + } +} \ No newline at end of file diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/cdna.time b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/cdna.time new file mode 100755 index 0000000..d199741 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/cdna.time @@ -0,0 +1 @@ +81.22 1142.58 6.19 5041368 diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/drna.err b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/drna.err new file mode 100755 index 0000000..7c159e9 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/drna.err @@ -0,0 +1,3 @@ +[*] Loading index from ${DATA}/rna/tx/tx.tx.raw... +[*] Loaded part 1 in 0.058s — k: 15; w: 10; #seq: 533740; total len: 946309864 +[*] Mapped 200000 reads in 37.238s (1 index part) diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/drna.oarfish.log b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/drna.oarfish.log new file mode 100755 index 0000000..e5483de --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/drna.oarfish.log @@ -0,0 +1,37 @@ +2026-10-21T23:23:20.855165Z  INFO oarfish: setting filters to nanocount defaults. +2026-10-21T23:23:21.330634Z  INFO oarfish::alignment_parser: read header from BAM file drna.q.bam, contains 533,740 reference sequences. +2026-10-21T23:23:21.330782Z  INFO oarfish::alignment_parser: saw supported mapper rammap as a program in the header; proceeding. +2026-10-21T23:23:21.330799Z  INFO oarfish::util::digest_utils: calculating seqcol digest +2026-10-21T23:23:22.417163Z  INFO oarfish::util::digest_utils: done calculating seqcol digest +2026-10-21T23:23:22.465806Z  INFO oarfish: parsed reference information for 533,740 transcripts. +2026-10-21T23:23:24.210729Z  INFO oarfish::alignment_parser: the alignment file contained 37,838 unmapped read records. +2026-10-21T23:23:24.211650Z  INFO oarfish::bulk: Parsing of alignments from input took: 1s 745ms 639us 706ns +2026-10-21T23:23:24.211720Z  INFO oarfish::bulk: +discard_table: +╭─────────────────────────────────┬────────╮ +│ reason │ count │ +├─────────────────────────────────┼────────┤ +│ too far from 5' end │ 0 │ +│ too far from 3' end │ 472740 │ +│ score too low │ 654768 │ +│ aligned fraction too low │ 4076 │ +│ aligned length too short │ 1281 │ +│ inconsistent orientation │ 27194 │ +│ supplementary alignment │ 10887 │ +│ read had no mapping │ 0 │ +│ read had no valid alignment │ 15496 │ +│ reads with valid best alignment │ 142590 │ +╰─────────────────────────────────┴────────╯ + +2026-10-21T23:23:24.211750Z  INFO oarfish::bulk: Total number of alignment records : 908,590 +2026-10-21T23:23:24.211771Z  INFO oarfish::bulk: number of aligned reads : 142,590 +2026-10-21T23:23:24.211791Z  INFO oarfish::bulk: number of unique alignments : 33,000 +2026-10-21T23:23:24.462524Z  INFO oarfish::em: iteration 100; rel diff 0.010536816603753874 +2026-10-21T23:23:24.704591Z  INFO oarfish::em: iteration 200; rel diff 0.004420907414445402 +2026-10-21T23:23:24.964908Z  INFO oarfish::em: iteration 300; rel diff 0.0031571401822707555 +2026-10-21T23:23:25.235827Z  INFO oarfish::em: iteration 400; rel diff 0.0023764816476970715 +2026-10-21T23:23:25.500139Z  INFO oarfish::em: iteration 500; rel diff 0.0018667627046812466 +2026-10-21T23:23:25.756858Z  INFO oarfish::em: iteration 600; rel diff 0.0015069623348835139 +2026-10-21T23:23:26.014970Z  INFO oarfish::em: iteration 700; rel diff 0.0012990544237337638 +2026-10-21T23:23:26.260414Z  INFO oarfish::em: iteration 800; rel diff 0.0011007552935052253 +2026-10-21T23:23:26.738503Z  INFO oarfish: oarfish completed successfully. diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/drna.oarfish.meta_info.json b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/drna.oarfish.meta_info.json new file mode 100755 index 0000000..9caceba --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/drna.oarfish.meta_info.json @@ -0,0 +1,61 @@ +{ + "prob_model": "no_coverage", + "alignment_source": "from_bam", + "alignment_time": { + "comment": "Time to parse (in alignment mode) or generate (in raw read mode) alignments, as well as apply filters, and compute conditional probabilities.", + "human_time": "1s 745ms 639us 706ns", + "seconds": 1.745639706 + }, + "bin_width": 100, + "filter_options": { + "five_prime_clip": 4294967295, + "three_prime_clip": 50, + "score_threshold": 0.949999988079071, + "min_aligned_fraction": 0.5, + "min_aligned_len": 50, + "which_strand": "Forward", + "model_coverage": false, + "logistic_growth_rate": 2.0, + "write_assignment_probs": false, + "write_assignment_probs_type": null, + "score_prob_denom": 5.0 + }, + "discard_table": { + "discard_5p": 0, + "discard_3p": 472740, + "discard_score": 654768, + "discard_aln_frac": 4076, + "discard_aln_len": 1281, + "discard_ori": 27194, + "discard_supp": 10887, + "no_mapping": 0, + "no_valid_aln": 15496, + "valid_best_aln": 142590 + }, + "alignments": "drna.q.bam", + "output": "drna.oarfish", + "verbose": false, + "single_cell": false, + "quiet": false, + "em_max_iter": 1000, + "em_convergence_thresh": 0.001, + "threads": 16, + "filter_group": "NanocountFilters", + "write_assignment_probs": null, + "short_quant": null, + "num_bootstraps": 0, + "digest": { + "bam_digest": { + "seqcol_digest": { + "level": 1, + "names": "JtpXbvlLRu6PIRtExJYCjpWt-HJCdlmY", + "lengths": "0af0dJPuoZAGSA1OxTC98WvOUDqyABTS", + "sorted_name_length_pairs": "obf7zAUChfwy1NNRVLcMGbXvxOeWlK7P" + }, + "sha256_digests": { + "sha256_names": "a36bb3e27e5844f319bb15d352c86e85df88a5e8f6cc82781434eac34ce725b5", + "sha256_seqs": null + } + } + } +} \ No newline at end of file diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/drna.time b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/drna.time new file mode 100755 index 0000000..afd1d40 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/drna.time @@ -0,0 +1 @@ +37.30 481.26 3.66 4865160 diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/sim_cdna.err b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/sim_cdna.err new file mode 100755 index 0000000..1b5e67f --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/sim_cdna.err @@ -0,0 +1,3 @@ +[*] Loading index from ${DATA}/rna/tx/tx.tx.raw... +[*] Loaded part 1 in 0.054s — k: 15; w: 10; #seq: 533740; total len: 946309864 +[*] Mapped 44667 reads in 19.763s (1 index part) diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/sim_cdna.time b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/sim_cdna.time new file mode 100755 index 0000000..7551d96 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/sim_cdna.time @@ -0,0 +1 @@ +19.80 218.22 1.52 4003232 diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/sim_drna.err b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/sim_drna.err new file mode 100755 index 0000000..57d07d3 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/sim_drna.err @@ -0,0 +1,3 @@ +[*] Loading index from ${DATA}/rna/tx/tx.tx.raw... +[*] Loaded part 1 in 0.053s — k: 15; w: 10; #seq: 533740; total len: 946309864 +[*] Mapped 44787 reads in 18.054s (1 index part) diff --git a/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/sim_drna.time b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/sim_drna.time new file mode 100755 index 0000000..f5a2fda --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/runs/tx4/sim_drna.time @@ -0,0 +1 @@ +18.09 173.81 1.44 3999960 diff --git a/docs/validation/2026-10-21-transcriptome/evidence/scripts/tx_bench.sh b/docs/validation/2026-10-21-transcriptome/evidence/scripts/tx_bench.sh new file mode 100755 index 0000000..80f7782 --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/scripts/tx_bench.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Transcriptome-mapping benchmark: one build (BIN=minimap2 for the -x map-ont -N 100 --eqx baseline) +# on HG002 cDNA / direct RNA 200k (speed, oarfish counts vs minimap2's) and simulated reads (truth). +# Usage: tx_bench.sh LABEL BIN [extra args]; results in results/tx_quant/runs/LABEL, line in ledger.tsv. +set -uo pipefail +L=$1; BIN=$2; shift 2; EXTRA="$*" +R=${BENCH}; D=$R/data/rna; X=$D/tx; O=$R/results/tx_quant/runs/$L; B=$R/results/tx_quant/runs/mm2 +export PATH=$HOME/micromamba/envs/vareval/bin:$HOME/micromamba/envs/ontaln/bin:$HOME/micromamba/envs/quant/bin:$PATH +T=16; mkdir -p $O; cd $O +declare -A READS=([cdna]=$D/cdna_hg002_200k.fq.gz [drna]=$D/drna_hg002_200k.fq.gz [sim_cdna]=$D/sim/sim_cdna.fq.gz [sim_drna]=$D/sim/sim_drna.fq.gz) +if [[ $BIN == minimap2 ]]; then IDX=$X/tx.mm2.mmi; else IDX=${IDX:-$X/tx.tx.raw}; fi +cat $IDX > /dev/null +line="$L" +for s in cdna drna sim_cdna sim_drna; do + if [[ $BIN == minimap2 ]]; then + /usr/bin/time -f "%e %U %S %M" -o $s.time minimap2 -ax map-ont -N 100 --eqx -t $T $IDX ${READS[$s]} 2> $s.err | samtools view -b -o $s.bam - + else + /usr/bin/time -f "%e %U %S %M" -o $s.time $BIN -a --eqx -t $T $EXTRA $IDX ${READS[$s]} 2> $s.err | samtools view -b -o $s.bam - + fi + read -r w u sy m < $s.time + line+="\t$s wall=$w cpu=$(echo "$u + $sy" | bc)" + if [[ $s == sim_* ]]; then + line+=" $(python3 $R/bench/tx_sim_truth.py ${READS[$s]} $s.bam | cut -d: -f2- | sed 's/ */ /g')" + else + # oarfish accepts only BAMs whose @PG names a validated aligner; relabel bonefish as rammap (its upstream). + samtools view -H $s.bam | sed 's/^@PG\tID:bonefish\tPN:bonefish/@PG\tID:rammap\tPN:rammap/' > hdr.sam + samtools reheader hdr.sam $s.bam > $s.q.bam && rm hdr.sam + oarfish -j $T -a $s.q.bam -o $s.oarfish --filter-group nanocount-filters > $s.oarfish.log 2>&1; rm -f $s.q.bam + line+=" unmapped=$(samtools view -c -f 4 $s.bam)" + [[ -s $B/$s.oarfish.quant && $L != mm2 ]] && line+=" $(python3 $R/bench/tx_quant_compare.py $B/$s.oarfish.quant $s.oarfish.quant | tr '\n' ' ' | sed 's/ */ /g')" + fi +done +echo -e "$line" | tee -a $R/results/tx_quant/ledger.tsv diff --git a/docs/validation/2026-10-21-transcriptome/evidence/scripts/tx_quant_compare.py b/docs/validation/2026-10-21-transcriptome/evidence/scripts/tx_quant_compare.py new file mode 100644 index 0000000..85c1d9a --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/scripts/tx_quant_compare.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Compare oarfish quantifications from two aligners' transcriptome BAMs. + +Reads `.quant` (tname, len, num_reads) of each, and prints Pearson correlation of counts and +of log2(count + 1), Spearman correlation, and the share of reads assigned differently (half the sum +of absolute count differences / total), at transcript and gene level (gene = 2nd |-field of the +GENCODE transcript name). Only transcripts with at least one read in either run are compared. + +Usage: tx_quant_compare.py A.quant B.quant +""" +import math +import sys + + +def load(path): + d = {} + with open(path) as f: + header = f.readline().rstrip("\n").split("\t") + ci = header.index("num_reads") if "num_reads" in header else 2 + for line in f: + x = line.rstrip("\n").split("\t") + d[x[0]] = float(x[ci]) + return d + + +def pearson(x, y): + n = len(x) + mx, my = sum(x) / n, sum(y) / n + sxy = sum((a - mx) * (b - my) for a, b in zip(x, y)) + sx = math.sqrt(sum((a - mx) ** 2 for a in x)) + sy = math.sqrt(sum((b - my) ** 2 for b in y)) + return sxy / (sx * sy) if sx and sy else float("nan") + + +def ranks(v): + order = sorted(range(len(v)), key=lambda i: v[i]) + r = [0.0] * len(v) + i = 0 + while i < len(v): + j = i + while j + 1 < len(v) and v[order[j + 1]] == v[order[i]]: + j += 1 + for k in range(i, j + 1): + r[order[k]] = (i + j) / 2 + 1 + i = j + 1 + return r + + +def report(name, a, b): + keys = [k for k in set(a) | set(b) if a.get(k, 0) > 0 or b.get(k, 0) > 0] + x = [a.get(k, 0.0) for k in keys] + y = [b.get(k, 0.0) for k in keys] + tot = sum(x) + diff = sum(abs(p - q) for p, q in zip(x, y)) / 2 + lx = [math.log2(v + 1) for v in x] + ly = [math.log2(v + 1) for v in y] + print(f"{name}: {len(keys):,} with reads; reads {tot:,.0f} vs {sum(y):,.0f}; Pearson {pearson(x, y):.5f}, " + f"log2 {pearson(lx, ly):.5f}, Spearman {pearson(ranks(x), ranks(y)):.5f}; " + f"reassigned {diff:,.0f} ({100 * diff / max(tot, 1):.2f}%)") + + +def gene_level(d): + g = {} + for k, v in d.items(): + f = k.split("|") + gid = f[1] if len(f) > 1 else k + g[gid] = g.get(gid, 0.0) + v + return g + + +def main(): + a, b = load(sys.argv[1]), load(sys.argv[2]) + report("transcripts", a, b) + report("genes", gene_level(a), gene_level(b)) + + +if __name__ == "__main__": + main() diff --git a/docs/validation/2026-10-21-transcriptome/evidence/scripts/tx_sim_truth.py b/docs/validation/2026-10-21-transcriptome/evidence/scripts/tx_sim_truth.py new file mode 100644 index 0000000..46ed9aa --- /dev/null +++ b/docs/validation/2026-10-21-transcriptome/evidence/scripts/tx_sim_truth.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Score transcriptome alignments of Badread reads simulated from GENCODE transcripts. + +Read names carry the source transcript (`ENST...|type|GENE,+strand,a-b`); reference names are +GENCODE transcriptome headers (`ENST...|ENSG...|...|...|tx_name|GENE|len|type|`). Per BAM: reads +mapped, primary on the true transcript, primary on a transcript of the true gene, and the true +transcript among any of the read's alignments (what EM quantifiers use). + +Usage: tx_sim_truth.py READS.fq.gz A.bam [B.bam ...] +""" +import gzip +import subprocess +import sys + + +def main(): + truth = {} + with gzip.open(sys.argv[1], "rt") as fq: + for i, line in enumerate(fq): + if i % 4 == 0: + rid, _, desc = line[1:].rstrip("\n").partition(" ") + src = desc.split(",")[0].split("|") + truth[rid] = (src[0], src[2] if len(src) > 2 else "") + n = len(truth) + for bam in sys.argv[2:]: + prim = {} + anyhit = set() + p = subprocess.Popen(["samtools", "view", bam], stdout=subprocess.PIPE, text=True) + for line in p.stdout: + f = line.split("\t", 4) + flag = int(f[1]) + if flag & 4 or f[0] not in truth: + continue + ref = f[2].split("|") + tx, gene = ref[0], ref[5] if len(ref) > 5 else "" + if tx == truth[f[0]][0]: + anyhit.add(f[0]) + if not flag & 0x900: + prim[f[0]] = (tx, gene) + p.wait() + mapped = len(prim) + tx_ok = sum(prim[r][0] == truth[r][0] for r in prim) + gene_ok = sum(prim[r][1] == truth[r][1] for r in prim) + print(f"{bam}: mapped {100 * mapped / n:.2f}% primary on true transcript {100 * tx_ok / n:.2f}% " + f"on true gene {100 * gene_ok / n:.2f}% true transcript among alignments {100 * len(anyhit) / n:.2f}%") + + +if __name__ == "__main__": + main() diff --git a/docs/validation/README.md b/docs/validation/README.md index 20bcdfa..4356476 100644 --- a/docs/validation/README.md +++ b/docs/validation/README.md @@ -23,6 +23,7 @@ Each folder holds the README for one validation round, its evidence (logs, table | report | what | |---|---| | [2026-10-21-rna-cdna](2026-10-21-rna-cdna/README.md) | Direct RNA and cDNA (spliced presets) against minimap2, GENCODE and simulated reads | +| [2026-10-21-transcriptome](2026-10-21-transcriptome/README.md) | Transcriptome mapping for quantification (`-x tx`, oarfish) | | [2026-10-20-metagenomics-duplex-rdna](2026-10-20-metagenomics-duplex-rdna/README.md) | Metagenome (Zymo fecal), HG002 duplex reads, rDNA | ## Speed, platforms and output