From 445bbd69e1a43699c638c576a5aeb0814508c717 Mon Sep 17 00:00:00 2001 From: Defnull <879658+define-null@users.noreply.github.com> Date: Tue, 15 Sep 2026 10:47:25 +0200 Subject: [PATCH] fix(assignments): allow non-monotone top directories --- crates/assignments/src/builder.rs | 7 +- .../tests/test_non_monotone_tops.rs | 110 ++++++++++++++++++ .../tests/test_split_assignments.rs | 10 +- 3 files changed, 117 insertions(+), 10 deletions(-) create mode 100644 crates/assignments/tests/test_non_monotone_tops.rs diff --git a/crates/assignments/src/builder.rs b/crates/assignments/src/builder.rs index 52a9433..b7d6af6 100644 --- a/crates/assignments/src/builder.rs +++ b/crates/assignments/src/builder.rs @@ -831,8 +831,8 @@ impl Drop for WorkerDatasetBuilder<'_, Rng> { } } -/// A searched run column is only sound if it starts at chunk 0 and ascends; otherwise the reader's -/// "last run at or before this chunk" has nothing to land on. +/// Top runs must start at chunk 0 and strictly ascend by `first_chunk_index` so the reader can +/// find the last run at or before a chunk. Top directory values may decrease or recur. fn check_top_runs(runs: &[assignment_fb::TopRun]) -> anyhow::Result<()> { let Some(first) = runs.first() else { anyhow::bail!("a dataset with chunks must have at least one top run"); @@ -843,7 +843,6 @@ fn check_top_runs(runs: &[assignment_fb::TopRun]) -> anyhow::Result<()> { pair[0].first_chunk_index() < pair[1].first_chunk_index(), "top runs must strictly ascend by first_chunk_index" ); - anyhow::ensure!(pair[0].top() < pair[1].top(), "numeric tops must strictly ascend"); } Ok(()) } @@ -1110,7 +1109,7 @@ struct PortalDatasetColumns { block_deltas: Vec, hashes: Vec, /// Appended only when a chunk's top differs from the previous one's, which is what makes the - /// runs start at 0 and strictly ascend. + /// runs start at 0 and strictly ascend by `first_chunk_index`, regardless of top values. tops: Vec, /// Absolute epoch milliseconds, as given; 0 for a chunk that was staged without one, which is /// the only thing a column can say about a timestamp it never got. diff --git a/crates/assignments/tests/test_non_monotone_tops.rs b/crates/assignments/tests/test_non_monotone_tops.rs new file mode 100644 index 0000000..584bb0f --- /dev/null +++ b/crates/assignments/tests/test_non_monotone_tops.rs @@ -0,0 +1,110 @@ +//! Correction chunks can use top directories that decrease or recur in block order. + +mod common; + +#[cfg(all(feature = "builder", feature = "reader"))] +mod non_monotone_tops { + use rand::{rngs::StdRng, SeedableRng}; + use sqd_assignments::{ + PortalAssignment, PortalAssignmentBuilder, WorkerAssignment, WorkerAssignmentBuilder, + WorkerStatus, + }; + + use super::common; + + /// (id, first block, last block, version). + type Chunk = (&'static str, u64, u64, u32); + + /// The new partition plus the head: contiguous blocks, tops 0, 16, 0. + const APPLIED: [Chunk; 3] = [ + ("0000000000/0000000000-0000000015-rrrr1", 0, 15, 1), + ("0000000016/0000000016-0000000020-rrrr2", 16, 20, 1), + ("0000000000/0000000021-0000000030-ccccc", 21, 30, 0), + ]; + + /// Old and new partitions interleaved by first block, as a pending batch is published to + /// workers: tops 0, 0, 0, 16, 0. Overlaps are legal there (continuity is off). + const PENDING: [Chunk; 5] = [ + ("0000000000/0000000000-0000000010-aaaaa", 0, 10, 0), + ("0000000000/0000000000-0000000015-rrrr1", 0, 15, 1), + ("0000000000/0000000011-0000000020-bbbbb", 11, 20, 0), + ("0000000016/0000000016-0000000020-rrrr2", 16, 20, 1), + ("0000000000/0000000021-0000000030-ccccc", 21, 30, 0), + ]; + + /// With continuity off the builder appends a non-contiguous chunk and only reports it. + fn appended(result: anyhow::Result<()>) -> anyhow::Result<()> { + match result { + Err(e) if e.to_string().starts_with("Chunks in the dataset must be contiguous") => { + Ok(()) + } + other => other, + } + } + + fn worker_bytes(chunks: &[Chunk]) -> anyhow::Result> { + let mut builder = + WorkerAssignmentBuilder::new_with_rng("test-secret", StdRng::seed_from_u64(0)) + .check_continuity(false); + builder.register_write_schema(1, &["blocks"])?; + let mut dataset = builder.new_dataset("s3://ds", "https://ds.example"); + dataset.register_generation(1, "https://ds.example/bf")?; + for (id, first, last, version) in chunks { + appended( + dataset + .new_chunk() + .id(id) + .block_range(*first..=*last) + .size(1) + .write_schema_id(1) + .version(*version) + .worker_indexes(&[0]) + .finish(), + )?; + } + dataset.finish()?; + builder.add_worker(common::get_test_keypair().public().to_peer_id(), WorkerStatus::Ok); + Ok(builder.finish()) + } + + fn portal_bytes(chunks: &[Chunk]) -> anyhow::Result> { + let mut builder = PortalAssignmentBuilder::new(); + let mut dataset = builder.new_dataset("s3://ds", 1); + for (id, first, last, version) in chunks { + dataset + .new_chunk() + .id(id) + .block_range(*first..=*last) + .version(*version) + .worker_indexes(&[0]) + .finish()?; + } + dataset.finish(None)?; + builder.add_worker(common::get_test_keypair().public().to_peer_id(), WorkerStatus::Ok); + Ok(builder.finish()) + } + + fn ids(chunks: &[Chunk]) -> Vec { + chunks.iter().map(|(id, ..)| (*id).to_owned()).collect() + } + + #[test] + fn worker_assignment_accepts_tops_that_do_not_ascend() { + for (label, chunks) in [("applied", &APPLIED[..]), ("pending", &PENDING[..])] { + let bytes = worker_bytes(chunks).unwrap_or_else(|e| panic!("{label}: {e:#}")); + let assignment = WorkerAssignment::from_owned(bytes).unwrap(); + let dataset = assignment.get_dataset("s3://ds").unwrap(); + let read: Vec = dataset.chunks().map(|c| c.id().unwrap()).collect(); + assert_eq!(read, ids(chunks), "{label}: every chunk resolves its own top"); + } + } + + #[test] + fn portal_assignment_accepts_tops_that_do_not_ascend() { + let bytes = portal_bytes(&APPLIED).unwrap_or_else(|e| panic!("applied: {e:#}")); + let assignment = PortalAssignment::from_owned(bytes).unwrap(); + let dataset = assignment.get_dataset("s3://ds").unwrap(); + let read: Vec = dataset.chunks().map(|c| c.id().unwrap()).collect(); + assert_eq!(read, ids(&APPLIED), "every chunk resolves its own top"); + } +} diff --git a/crates/assignments/tests/test_split_assignments.rs b/crates/assignments/tests/test_split_assignments.rs index 053682c..bf87653 100644 --- a/crates/assignments/tests/test_split_assignments.rs +++ b/crates/assignments/tests/test_split_assignments.rs @@ -664,7 +664,7 @@ fn test_portal_tops_are_runs_and_hashes_keep_their_length() { #[cfg(feature = "builder")] #[test] -fn test_worker_rejects_descending_numeric_tops() { +fn test_worker_accepts_descending_numeric_tops() { let mut builder = test_builder(); builder.register_write_schema(7, &["blocks"]).unwrap(); let mut dataset = test_dataset(&mut builder); @@ -682,13 +682,12 @@ fn test_worker_rejects_descending_numeric_tops() { .unwrap(); } - let error = dataset.finish().unwrap_err(); - assert_eq!(error.to_string(), "numeric tops must strictly ascend"); + dataset.finish().unwrap(); } #[cfg(feature = "builder")] #[test] -fn test_portal_rejects_descending_numeric_tops() { +fn test_portal_accepts_descending_numeric_tops() { use sqd_assignments::PortalAssignmentBuilder; let mut builder = PortalAssignmentBuilder::new(); @@ -700,8 +699,7 @@ fn test_portal_rejects_descending_numeric_tops() { dataset.new_chunk().id(id).block_range(range).finish().unwrap(); } - let error = dataset.finish(None).unwrap_err(); - assert_eq!(error.to_string(), "numeric tops must strictly ascend"); + dataset.finish(None).unwrap(); } /// The routing column is flattened and staged through a buffer the builder reuses, so a chunk