diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index c360683..9c695f3 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -185,6 +185,8 @@ The project uses GitHub Actions with maturin for cross-platform wheel building: every x86 dispatch path runs regardless of runner hardware. `HEXHAMMING_EXPECT_X86_FEATURES` (checked by `tests/x86_cpu_features.rs`) fails the job if the emulated CPU does not report the expected features +- `pool-sanitizers`: the scan thread pool's tests (`par::`) under ThreadSanitizer and Miri + with `HEXHAMMING_NUM_THREADS=4` - `Benchmark PR` (`benchmark.yml`): benchmarks every PR against its base branch on x86 and comments the comparison - Builds wheels for Linux (manylinux), macOS, Windows diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 3f6acdf..698feeb 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -72,6 +72,33 @@ jobs: - name: Run Rust tests run: cargo test --no-default-features --release + pool-sanitizers: + name: Thread pool under ThreadSanitizer and Miri + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + # Enough workers to exercise the pool on any runner. + HEXHAMMING_NUM_THREADS: 4 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Install Rust nightly + uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1 + with: + toolchain: nightly + components: rust-src, miri + + - name: ThreadSanitizer + env: + RUSTFLAGS: -Zsanitizer=thread + run: cargo test -Zbuild-std --target x86_64-unknown-linux-gnu --no-default-features 'par::' + + - name: Miri + env: + # The pool's workers outlive the tests by design. + MIRIFLAGS: -Zmiri-disable-isolation -Zmiri-ignore-leaks -Zmiri-env-forward=HEXHAMMING_NUM_THREADS + run: cargo miri test --no-default-features 'par::' + x86-emulated: name: x86 tests under Intel SDE (${{ matrix.cpu }}) runs-on: ubuntu-latest @@ -136,10 +163,12 @@ jobs: - name: Python tests # Emulation makes wall-clock assertions meaningless, so skip the one - # timing test along with the benchmarks. + # timing test along with the benchmarks, and SDE cannot follow the + # execve of the test that runs Python in a subprocess. run: >- sde64 -${{ matrix.cpu }} -- python -m pytest -q -p no:cacheprovider test/ - -k "not bench and not test_check_hexstrings_within_dist_long_random_fast" + -k "not bench and not test_check_hexstrings_within_dist_long_random_fast + and not test_num_threads_environment_variable" build-and-test: name: Testing on ${{ matrix.os }} with Python ${{ matrix.python-version }} diff --git a/Cargo.lock b/Cargo.lock index d666b7a..d421612 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -202,7 +202,6 @@ dependencies = [ "criterion", "pyo3", "pyo3-build-config", - "rayon", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 4497c93..69fead5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,6 @@ python = ["pyo3", "pyo3-build-config"] [dependencies] pyo3 = { version = "0.29", features = ["extension-module"], optional = true } -rayon = "1.10" [build-dependencies] pyo3-build-config = { version = "0.29", optional = true } diff --git a/README.rst b/README.rst index 2785156..3c8ddc1 100644 --- a/README.rst +++ b/README.rst @@ -428,9 +428,8 @@ array all [100000×128, parallel] 99,266.0 ================================================ =========== On AArch64, LLVM's auto-vectorized native byte loop is faster than the -hand-written NEON byte kernel for these sizes, while hexadecimal strings still -use the packed NEON implementation. Large array workloads use four balanced -Rayon jobs to avoid oversubscribing the memory-bound scan. +hand-written NEON byte kernel for these sizes. Large array scans are split into +chunks that a small built-in thread pool claims dynamically. Python API (via PyO3) ~~~~~~~~~~~~~~~~~~~~~ @@ -505,10 +504,12 @@ dominates (roughly 30–40 ns on this machine). For large inputs (1024+ chars, 16384-element arrays), computation dominates and Python overhead is negligible. Byte operations release the GIL at 16 KiB, while immutable strings use a zero-copy detached path from 4 KiB. Array wrappers release the GIL -at 64 KiB; generic scans parallelize with Rayon at 5 MiB, while the optimized -16/32-byte NEON scanners use a measured 16 MiB crossover. The ``first`` variant -additionally short-circuits on the first hit, so a match near the start is much -faster than one near the end. +at 64 KiB. Array scans run on a built-in thread pool from 512 KiB for widths +with a block scanner (8/16/32/64 bytes) and from 64 KiB for other widths, the +measured crossovers on this machine. Set ``HEXHAMMING_NUM_THREADS`` to limit +the pool (``1`` disables it). The ``first`` and ``best`` variants scan a short +prefix serially and stop early, so a match near the start is much faster than +one near the end. Batch APIs vs. Python for-loops ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/src/api.rs b/src/api.rs index e7b8ff5..84ede46 100644 --- a/src/api.rs +++ b/src/api.rs @@ -11,13 +11,14 @@ use crate::par; use std::sync::atomic::{AtomicUsize, Ordering}; /// Scans of at least this many bytes are split across threads when each record -/// costs a kernel call (widths without a block scanner). Measured crossover -/// (Apple M4 Max, rayon): 1.6x at 1 MiB for 20-byte records. -pub(crate) const PAR_THRESHOLD_BYTES: usize = 1024 * 1024; +/// costs a kernel call (widths without a block scanner). Measured on an Apple +/// M4 Max with the scan pool: 20-byte records are 2.0-3.5x faster at 128 KiB +/// and ~3x at 256 KiB. +pub(crate) const PAR_THRESHOLD_BYTES: usize = 64 * 1024; /// Block scanners are cheaper per byte, so they need a larger scan before -/// waking the thread pool pays off: 0.6x at 2 MiB, 1.2-1.3x at 4 MiB and 1.8x -/// at 8 MiB for 16- and 64-byte records. -pub(crate) const FIXED_WIDTH_PAR_THRESHOLD_BYTES: usize = 4 * 1024 * 1024; +/// splitting pays off: 16- and 64-byte records are 0.8x at 256 KiB, 1.4-1.7x +/// at 512 KiB and 2.5x at 1 MiB. +pub(crate) const FIXED_WIDTH_PAR_THRESHOLD_BYTES: usize = 512 * 1024; /// `first` and `best` scan a serial prefix before going parallel, so early /// matches (and early exact matches for `best`) never pay for waking workers. /// It is 1/64 of the scan, clamped to 16-256 KiB, which keeps its cost on scans @@ -239,7 +240,7 @@ impl<'a> Scan<'a> { let prefix_bytes = (records.len() / SERIAL_PREFIX_SHARE) .clamp(SERIAL_PREFIX_MIN_BYTES, SERIAL_PREFIX_MAX_BYTES); let prefix = (prefix_bytes / width).next_multiple_of(16).min(count); - par::plan(count - prefix, width, 0).map(|plan| (prefix, plan)) + par::split(count - prefix, width, min_bytes).map(|plan| (prefix, plan)) } /// `first_parallel` with an explicit minimum scan size for splitting. @@ -690,7 +691,7 @@ mod tests { } // ----------------------------------------------------------------------- - // Wave 2b: rayon parallelization regression tests + // Parallel scan regression tests // ----------------------------------------------------------------------- /// Build a big array of `num_elements` chunks of size `elem_size`, all filled diff --git a/src/catalog.rs b/src/catalog.rs index 84620b6..d8bd6e0 100644 --- a/src/catalog.rs +++ b/src/catalog.rs @@ -33,17 +33,17 @@ const MAX_PROBE_RADIUS: usize = 4; // (Apple M4 Max, block scanners): a bucket probe, including verifying its // ~N/2^s records, costs 1.2-5.6 ns, rising with index size as probes miss // cache and TLB. A linear scan costs about 0.11 ns per record or 0.0133 ns -// per byte, whichever is larger; split across threads it runs about 1.2x -// faster at 4 MiB, 1.8x at 8 MiB and ~3x from 16 MiB. Slower linear scans -// elsewhere only make the planner more conservative. +// per byte, whichever is larger; split across threads it runs about 1.4x +// faster at 512 KiB, 2.5x at 1 MiB and 4-6x from 2 MiB (modelled as up to 4x). +// Slower linear scans elsewhere only make the planner more conservative. const QUERY_OVERHEAD_NS: f64 = 50.0; const PROBE_BASE_NS: f64 = 1.5; const PROBE_CACHE_MISS_NS: f64 = 4.0; const PROBE_MISS_SATURATION_BYTES: f64 = (128u64 << 20) as f64; const SCAN_NS_PER_RECORD: f64 = 0.11; const SCAN_NS_PER_BYTE: f64 = 0.0133; -const PARALLEL_SPEEDUP_BYTES: f64 = (5u64 << 20) as f64; -const MAX_PARALLEL_SPEEDUP: f64 = 3.0; +const PARALLEL_SPEEDUP_BYTES: f64 = (400u64 << 10) as f64; +const MAX_PARALLEL_SPEEDUP: f64 = 4.0; /// Estimated cost of scanning `n` records of `width` bytes linearly. fn linear_scan_ns(n: usize, width: usize) -> f64 { diff --git a/src/par.rs b/src/par.rs index 93d4cda..b77cbbe 100644 --- a/src/par.rs +++ b/src/par.rs @@ -3,24 +3,294 @@ //! A scan is split into chunks of whole records, and workers claim chunks //! dynamically, so faster cores take more of them. Callers only see //! [`map_chunks`] and [`find_first_chunk`], which return results in chunk -//! order; the executor behind them is an implementation detail. +//! order. +//! +//! The executor is a small persistent pool rather than a general work-stealing +//! scheduler. Idle workers spin for a moment before parking, so back-to-back +//! scans start in 0.5-3 us (more chunks bring in more workers) instead of the +//! 15-40 us it took to wake rayon's workers. A job only draws in as many +//! workers as it has chunks for, and each worker flags that it has joined on +//! its own cache line, so starting and finishing a job involves no contended +//! read-modify-writes beyond claiming chunks. A job lives on the caller's +//! stack and the caller works on it too, so it never waits for a parked +//! worker: it only waits for workers that have joined the job. The pool runs +//! one job at a time; concurrent or nested callers run their chunks inline. A +//! forked child gets a fresh pool. -use rayon::prelude::*; +use std::any::Any; +use std::mem::MaybeUninit; +use std::panic::{self, AssertUnwindSafe}; +use std::ptr; +use std::sync::atomic::{AtomicBool, AtomicPtr, AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Condvar, Mutex, OnceLock}; +use std::time::{Duration, Instant}; -/// Target bytes scanned per chunk: large enough that claiming a chunk is cheap -/// next to scanning it, small enough to balance load across cores. -const CHUNK_BYTES: usize = 256 * 1024; +/// A scan at the parallel threshold is split into this many chunks; larger +/// scans get proportionally more, up to `CHUNKS_PER_THREAD` per thread. +const CHUNKS_AT_THRESHOLD: usize = 8; +/// Smallest chunk worth claiming. +const MIN_CHUNK_BYTES: usize = 4 * 1024; /// Chunks per worker thread, so a slow core cannot hold up the whole scan. const CHUNKS_PER_THREAD: usize = 4; /// Record counts per chunk are rounded to this so block scanners (16 records /// per block) see no partial blocks except at the very end. const RECORD_ALIGN: usize = 16; +/// How long an idle worker spins before parking. Back-to-back scans (the +/// common case in loops) then start without a wake-up. +const SPIN: Duration = Duration::from_micros(50); +/// Threads that work on a job, including the caller. `HEXHAMMING_NUM_THREADS` +/// overrides the default (available parallelism); `RAYON_NUM_THREADS` is +/// honored too, since it configured the previous executor. #[inline] pub(crate) fn threads() -> usize { - rayon::current_num_threads() + static THREADS: OnceLock = OnceLock::new(); + *THREADS.get_or_init(configured_threads) +} + +fn configured_threads() -> usize { + ["HEXHAMMING_NUM_THREADS", "RAYON_NUM_THREADS"] + .iter() + .find_map(|var| std::env::var(var).ok()?.trim().parse::().ok()) + .filter(|&n| n > 0) + .unwrap_or_else(|| std::thread::available_parallelism().map_or(1, |n| n.get())) +} + +// --------------------------------------------------------------------------- +// Pool +// --------------------------------------------------------------------------- + +/// Keeps a hot atomic on its own cache line (128 bytes covers Apple and +/// x86 prefetch pairs). +#[repr(align(128))] +struct Padded(T); + +/// A job published to the workers. It lives on the caller's stack for the +/// whole of `Pool::run`. +struct Job { + chunks: usize, + next: Padded, + stop: AtomicBool, + task: *const (dyn Fn(usize) + Sync), + panic: Mutex>>, +} + +impl Job { + /// Claim and run chunks until none are left. After a panic the remaining + /// chunks are claimed without running, so the job still completes. + fn work(&self) { + loop { + let chunk = self.next.0.fetch_add(1, Ordering::Relaxed); + if chunk >= self.chunks { + return; + } + if !self.stop.load(Ordering::Relaxed) { + // SAFETY: `task` outlives the job (see `Pool::run`). + let task = unsafe { &*self.task }; + if let Err(payload) = panic::catch_unwind(AssertUnwindSafe(|| task(chunk))) { + self.stop.store(true, Ordering::Relaxed); + let mut slot = self.panic.lock().unwrap_or_else(|e| e.into_inner()); + slot.get_or_insert(payload); + } + } + } + } +} + +struct Pool { + /// Workers that started; they are numbered from 1. + workers: AtomicUsize, + pid: u32, + busy: AtomicBool, + generation: AtomicU64, + job: AtomicPtr, + /// Workers numbered above this sit the current job out, so a job with a + /// few chunks does not pull every worker's cache traffic into it. + helpers: AtomicUsize, + /// Per worker, by number (slot 0, the caller's, stays clear): set while + /// the worker may be reading `job`. Separate lines keep joining and + /// leaving a job free of contended read-modify-writes. + active: Box<[Padded]>, + parked: AtomicUsize, + wake: Mutex, + wake_cv: Condvar, +} + +impl Pool { + fn start(threads: usize) -> &'static Pool { + let pool: &'static Pool = Box::leak(Box::new(Pool { + workers: AtomicUsize::new(0), + pid: std::process::id(), + busy: AtomicBool::new(false), + generation: AtomicU64::new(0), + job: AtomicPtr::new(ptr::null_mut()), + helpers: AtomicUsize::new(0), + active: (0..threads) + .map(|_| Padded(AtomicBool::new(false))) + .collect(), + parked: AtomicUsize::new(0), + wake: Mutex::new(0), + wake_cv: Condvar::new(), + })); + for id in 1..threads { + let spawned = std::thread::Builder::new() + .name(format!("hexhamming-{id}")) + .spawn(move || pool.worker(id)); + if spawned.is_err() { + break; + } + pool.workers.fetch_add(1, Ordering::Relaxed); + } + pool + } + + fn worker(&'static self, id: usize) { + let active = &self.active[id].0; + let mut seen = self.generation.load(Ordering::Acquire); + loop { + seen = self.wait_for_job(seen); + if id > self.helpers.load(Ordering::Relaxed) { + continue; + } + active.store(true, Ordering::SeqCst); + let job = self.job.load(Ordering::SeqCst); + if !job.is_null() { + // SAFETY: the caller keeps the job alive until it has cleared + // `job` and seen every worker's `active` clear. + unsafe { (*job).work() }; + } + active.store(false, Ordering::Release); + } + } + + /// Spin, then park, until the generation moves past `seen`. + fn wait_for_job(&self, seen: u64) -> u64 { + let start = Instant::now(); + let mut spins = 0u32; + loop { + let generation = self.generation.load(Ordering::Acquire); + if generation != seen { + return generation; + } + spins = spins.wrapping_add(1); + if spins % 64 == 0 && start.elapsed() > SPIN { + let mut published = self.wake.lock().unwrap_or_else(|e| e.into_inner()); + self.parked.fetch_add(1, Ordering::SeqCst); + while *published == seen { + published = self + .wake_cv + .wait(published) + .unwrap_or_else(|e| e.into_inner()); + } + self.parked.fetch_sub(1, Ordering::SeqCst); + return *published; + } + std::hint::spin_loop(); + } + } + + /// Run `task(chunk)` for every chunk on the caller and the workers. Returns + /// `false` without running anything if the pool is busy. + fn run(&self, chunks: usize, task: &(dyn Fn(usize) + Sync)) -> bool { + if self + .busy + .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) + .is_err() + { + return false; + } + // SAFETY: only the lifetime is erased; `task` outlives this call and + // the job is unpublished (and unused) before it returns. + let task: *const (dyn Fn(usize) + Sync) = unsafe { std::mem::transmute(task) }; + let job = Job { + chunks, + next: Padded(AtomicUsize::new(0)), + stop: AtomicBool::new(false), + task, + panic: Mutex::new(None), + }; + let helpers = chunks + .saturating_sub(1) + .min(self.workers.load(Ordering::Relaxed)); + self.helpers.store(helpers, Ordering::Relaxed); + self.job + .store(&job as *const Job as *mut Job, Ordering::SeqCst); + let generation = self.generation.fetch_add(1, Ordering::Release) + 1; + { + // Workers register as parked under this lock, so checking here + // cannot miss one that is about to wait. + let mut published = self.wake.lock().unwrap_or_else(|e| e.into_inner()); + *published = generation; + if self.parked.load(Ordering::SeqCst) > 0 { + self.wake_cv.notify_all(); + } + } + + // Every chunk is claimed once this returns; each claimed by a worker + // is done once that worker's `active` is clear. A worker that has not + // set `active` by the time it is read here sees `job` cleared (both + // sides are SeqCst), so it never touches the job. + job.work(); + self.job.store(ptr::null_mut(), Ordering::SeqCst); + for slot in self.active.iter() { + let mut spins = 0u32; + while slot.0.load(Ordering::SeqCst) { + spins = spins.wrapping_add(1); + if spins % 1024 == 0 { + std::thread::yield_now(); + } else { + std::hint::spin_loop(); + } + } + } + self.busy.store(false, Ordering::Release); + + if let Some(payload) = job.panic.into_inner().unwrap_or_else(|e| e.into_inner()) { + panic::resume_unwind(payload); + } + true + } +} + +static POOL: Mutex> = Mutex::new(None); + +/// The pool for this process, started on first use, or `None` when only one +/// thread is configured. A forked child, whose copy has no worker threads, +/// starts its own. +fn pool() -> Option<&'static Pool> { + let mut slot = POOL.lock().unwrap_or_else(|e| e.into_inner()); + let pid = std::process::id(); + match *slot { + Some(pool) if pool.pid == pid => {} + _ => { + let threads = configured_threads(); + if threads < 2 { + return None; + } + *slot = Some(Pool::start(threads)); + } + } + slot.filter(|pool| pool.workers.load(Ordering::Relaxed) > 0) +} + +/// Run `task` for every chunk, in parallel when the pool is free. +fn for_each_chunk(chunks: usize, task: &(dyn Fn(usize) + Sync)) { + if chunks > 1 { + if let Some(pool) = pool() { + if pool.run(chunks, task) { + return; + } + } + } + for chunk in 0..chunks { + task(chunk); + } } +// --------------------------------------------------------------------------- +// Plans and chunked execution +// --------------------------------------------------------------------------- + /// How to split `records` records across chunks. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) struct Plan { @@ -40,12 +310,26 @@ impl Plan { /// Chunk plan for scanning `records` records of `width` bytes, or `None` when /// the scan (under `min_bytes`) is too small to be worth splitting. pub(crate) fn plan(records: usize, width: usize, min_bytes: usize) -> Option { + if records.saturating_mul(width) < min_bytes { + return None; + } + split(records, width, min_bytes) +} + +/// Chunk plan sized for a parallel threshold of `min_bytes` (a scan of that +/// size gets `CHUNKS_AT_THRESHOLD` chunks), without requiring this scan to +/// reach it. `None` if there is too little to split or only one thread. +pub(crate) fn split(records: usize, width: usize, min_bytes: usize) -> Option { let bytes = records.saturating_mul(width); + if records < 2 * RECORD_ALIGN { + return None; + } let threads = threads(); - if bytes < min_bytes || threads < 2 || records < 2 * RECORD_ALIGN { + if threads < 2 { return None; } - let chunks = (bytes / CHUNK_BYTES).clamp(2, threads * CHUNKS_PER_THREAD); + let chunk_bytes = (min_bytes / CHUNKS_AT_THRESHOLD).max(MIN_CHUNK_BYTES); + let chunks = (bytes / chunk_bytes).clamp(2, threads * CHUNKS_PER_THREAD); let records_per_chunk = records.div_ceil(chunks).next_multiple_of(RECORD_ALIGN); Some(Plan { records_per_chunk, @@ -56,11 +340,16 @@ pub(crate) fn plan(records: usize, width: usize, min_bytes: usize) -> Option Option { + let bytes = items.saturating_mul(bytes_per_item); + if items < 2 || bytes < min_bytes { + return None; + } let threads = threads(); - if items < 2 || threads < 2 || items.saturating_mul(bytes_per_item) < min_bytes { + if threads < 2 { return None; } - let chunks = items.min(threads * CHUNKS_PER_THREAD); + let chunk_bytes = (min_bytes / CHUNKS_AT_THRESHOLD).max(MIN_CHUNK_BYTES); + let chunks = (bytes / chunk_bytes).clamp(2, items.min(threads * CHUNKS_PER_THREAD)); let per_chunk = items.div_ceil(chunks); Some(Plan { records_per_chunk: per_chunk, @@ -68,17 +357,47 @@ pub(crate) fn plan_items(items: usize, bytes_per_item: usize, min_bytes: usize) }) } +/// Result slots written by exactly one chunk each. +struct Slots(Vec>); + +// SAFETY: every slot is written by exactly one chunk and read only after the +// job has completed. +unsafe impl Sync for Slots {} + +impl Slots { + fn new(len: usize) -> Self { + let mut slots = Vec::with_capacity(len); + slots.resize_with(len, MaybeUninit::uninit); + Self(slots) + } + + /// SAFETY: each index must be written exactly once, and not concurrently + /// with reads. + unsafe fn write(&self, index: usize, value: R) { + (self.0.as_ptr().add(index) as *mut MaybeUninit) + .as_mut() + .unwrap_unchecked() + .write(value); + } + + /// SAFETY: every slot must have been written. + unsafe fn into_vec(self) -> Vec { + self.0.into_iter().map(|slot| slot.assume_init()).collect() + } +} + /// `f(chunk)` for every chunk, in chunk order. pub(crate) fn map_chunks(chunks: usize, f: F) -> Vec where R: Send, F: Fn(usize) -> R + Sync, { - (0..chunks) - .into_par_iter() - .with_max_len(1) - .map(&f) - .collect() + let slots = Slots::new(chunks); + // SAFETY: chunk indices are claimed exactly once each. + for_each_chunk(chunks, &|chunk| unsafe { slots.write(chunk, f(chunk)) }); + // SAFETY: `for_each_chunk` returns only after every chunk ran (a panic + // unwinds past this point instead). + unsafe { slots.into_vec() } } /// The result of the lowest-numbered chunk for which `f` returns `Some`. @@ -88,10 +407,20 @@ where R: Send, F: Fn(usize) -> Option + Sync, { - (0..chunks) - .into_par_iter() - .with_max_len(1) - .find_map_first(&f) + let found = AtomicUsize::new(usize::MAX); + map_chunks(chunks, |chunk| { + if found.load(Ordering::Relaxed) < chunk { + return None; + } + let result = f(chunk); + if result.is_some() { + found.fetch_min(chunk, Ordering::Relaxed); + } + result + }) + .into_iter() + .flatten() + .next() } #[cfg(test)] @@ -121,6 +450,9 @@ mod tests { #[test] fn item_plans_cover_every_item_exactly_once() { + if threads() < 2 { + return; + } for items in [2usize, 3, 17, 64, 1000] { let plan = plan_items(items, 1 << 20, 0).unwrap(); let covered: usize = (0..plan.chunks) @@ -134,6 +466,74 @@ mod tests { assert_eq!(plan_items(1, 1 << 30, 0), None); } + #[test] + fn panics_propagate_and_the_pool_stays_usable() { + let result = std::panic::catch_unwind(|| { + map_chunks(64, |chunk| { + if chunk == 37 { + panic!("chunk 37"); + } + chunk + }) + }); + let payload = result.expect_err("panic must reach the caller"); + assert_eq!(payload.downcast_ref::<&str>(), Some(&"chunk 37")); + assert_eq!(map_chunks(64, |c| c + 1), (1..=64).collect::>()); + } + + #[test] + fn concurrent_and_nested_callers_get_correct_results() { + let (threads, rounds) = if cfg!(miri) { (3, 3) } else { (8, 50) }; + std::thread::scope(|scope| { + for t in 0..threads { + scope.spawn(move || { + for round in 0..rounds { + let chunks = 1 + (t * 7 + round) % 97; + let got = map_chunks(chunks, |c| { + // Nested jobs run inline instead of deadlocking. + let inner: usize = map_chunks(3, |i| i + c).into_iter().sum(); + (c, inner) + }); + let want: Vec<_> = (0..chunks).map(|c| (c, 3 * c + 3)).collect(); + assert_eq!(got, want); + } + }); + } + }); + } + + #[test] + fn back_to_back_jobs_complete() { + // Exercises the spin/park hand-off: some jobs start while workers + // spin, others after they have parked. + let rounds = if cfg!(miri) { 12 } else { 2000 }; + for round in 0..rounds { + let chunks = 2 + round % 61; + let sum: usize = map_chunks(chunks, |c| c).into_iter().sum(); + assert_eq!(sum, chunks * (chunks - 1) / 2); + if round % 500 == 499 || (cfg!(miri) && round % 4 == 3) { + std::thread::sleep(SPIN * 3); + } + } + } + + #[test] + fn every_chunk_runs_once_as_job_sizes_change() { + // Jobs of a few chunks leave most workers out; the next, larger job + // must draw them back in even if they saw a stale helper count. + let rounds = if cfg!(miri) { 16 } else { 3000 }; + for round in 0..rounds { + let chunks = [2, 64, 3, 17, 2, 2, 40, 5][round % 8]; + let runs: Vec = (0..chunks).map(|_| AtomicUsize::new(0)).collect(); + let out = map_chunks(chunks, |c| { + runs[c].fetch_add(1, Ordering::Relaxed); + c * 3 + }); + assert_eq!(out, (0..chunks).map(|c| c * 3).collect::>()); + assert!(runs.iter().all(|r| r.load(Ordering::Relaxed) == 1)); + } + } + #[test] fn chunk_helpers_keep_order() { assert_eq!( diff --git a/test/test_hexhamming.py b/test/test_hexhamming.py index 688ef5a..adae262 100644 --- a/test/test_hexhamming.py +++ b/test/test_hexhamming.py @@ -543,6 +543,67 @@ def test_fixed_width_array_apis_randomized_oracle(width): assert check_bytes_arrays_all_within_dist(array, needle, max_dist) == expected +def _planted_scan_catalog(width=16, count=200_000, at=123_456): + rng = random.Random(3) + records = bytearray(rng.randbytes(width * count)) + query = bytes(rng.randbytes(width)) + records[at * width : (at + 1) * width] = query + return bytes(records), query, at + + +@pytest.mark.skipif(not hasattr(__import__("os"), "fork"), reason="needs os.fork") +def test_parallel_scans_work_in_a_forked_child(): + """A forked child has no copy of the parent's scan workers; it must start + its own pool (or scan serially) instead of waiting on them.""" + import os + import signal + import warnings + + records, query, at = _planted_scan_catalog() + # 3.2 MB: large enough to split, and it starts the pool in the parent. + assert check_bytes_arrays_first_within_dist(records, query, 0) == at + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + pid = os.fork() + if pid == 0: + signal.alarm(30) + ok = ( + check_bytes_arrays_first_within_dist(records, query, 0) == at + and check_bytes_arrays_best_within_dist(records, query, 0) == (0, at) + and check_bytes_arrays_all_within_dist(records, query, 0) == [(0, at)] + ) + os._exit(0 if ok else 1) + _, status = os.waitpid(pid, 0) + assert os.WIFEXITED(status), status + assert os.WEXITSTATUS(status) == 0 + + +@pytest.mark.parametrize("threads", ("1", "3")) +def test_num_threads_environment_variable(threads): + """HEXHAMMING_NUM_THREADS sizes the scan pool; results never depend on it.""" + import os + import subprocess + import sys + + code = ( + "import random, hexhamming as h\n" + "rng = random.Random(3)\n" + "r = bytearray(rng.randbytes(16 * 200_000)); q = bytes(rng.randbytes(16))\n" + "r[123_456 * 16 : 123_457 * 16] = q\n" + "print(h.check_bytes_arrays_best_within_dist(bytes(r), q, 0))\n" + ) + result = subprocess.run( + [sys.executable, "-c", code], + env={**os.environ, "HEXHAMMING_NUM_THREADS": threads}, + capture_output=True, + text=True, + timeout=120, + check=False, + ) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "(0, 123456)" + + ############################ # Benchmarks #