Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 3 additions & 2 deletions crates/assignments/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
[package]
name = "sqd-assignments"
license = "AGPL-3.0-or-later"
version = "0.4.0"
version = "0.5.0"
edition = "2021"

[features]
reader = ["serde_json", "ouroboros"]
builder = ["base64", "curve25519-dalek", "hmac", "serde_json", "url"]
builder = ["base64", "curve25519-dalek", "hmac", "serde_json", "sqd-data-chunk", "url"]

[dependencies]
anyhow = { version = "1.0" }
Expand All @@ -20,6 +20,7 @@ ouroboros = { version = "0.18.5", optional = true }
serde = { version = "1", features = ["derive"] }
serde_json = { version = "1.0", optional = true }
sha2 = { version = "0.10.8" }
sqd-data-chunk = { path = "../data-chunk", optional = true }
thiserror = "2.0"
tracing = "0.1.41"
url = { version = "2.5.0", optional = true }
Expand Down
12 changes: 6 additions & 6 deletions crates/assignments/benches/assignment.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ use criterion::{black_box, criterion_group, criterion_main, BatchSize, Criterion
use libp2p_identity::{Keypair, PeerId};
use rand::{rngs::StdRng, Rng, SeedableRng};
use sqd_assignments::{
Assignment, PortalAssignment, PortalAssignmentBuilder, WorkerAssignment,
Assignment, DataChunk, PortalAssignment, PortalAssignmentBuilder, WorkerAssignment,
WorkerAssignmentBuilder, WorkerStatus,
};

Expand Down Expand Up @@ -40,9 +40,9 @@ fn dataset_id(index: usize) -> String {
format!("s3://chain-{index:04}-mainnet")
}

fn chunk_id(chunk: usize, first: u64, last: u64) -> String {
fn chunk_id(chunk: usize, first: u64, last: u64) -> DataChunk {
let top = (chunk / CHUNKS_PER_TOP) as u64 * CHUNKS_PER_TOP as u64 * BLOCKS_PER_CHUNK;
format!("{top:010}/{first:010}-{last:010}-{:08x}", first ^ 0x274f02d8)
DataChunk::new(top, first, last, &format!("{:08x}", first ^ 0x274f02d8)).expect("valid id")
}

fn timestamp(dataset: usize, chunk: usize) -> u64 {
Expand Down Expand Up @@ -99,7 +99,7 @@ fn build() -> Fixture {

let mut staged = legacy
.new_chunk()
.id(&cid)
.id(cid)
.dataset_id(&id)
.dataset_base_url(&base_url)
.block_range(first..=last)
Expand All @@ -115,7 +115,7 @@ fn build() -> Fixture {

worker_dataset
.new_chunk()
.id(&cid)
.id(cid)
.block_range(first..=last)
.size(1_000_000)
.write_schema_id(dataset as u32 + 1)
Expand All @@ -125,7 +125,7 @@ fn build() -> Fixture {

portal_dataset
.new_chunk()
.id(&cid)
.id(cid)
.block_range(first..=last)
.last_block_timestamp(ts)
.worker_indexes(&indexes)
Expand Down
31 changes: 15 additions & 16 deletions crates/assignments/examples/convert_assignment.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ use std::{

use anyhow::Context as _;
use sqd_assignments::{
Assignment, PortalAssignment, PortalAssignmentBuilder, WorkerAssignment,
Assignment, DataChunk, PortalAssignment, PortalAssignmentBuilder, WorkerAssignment,
WorkerAssignmentBuilder,
};

Expand Down Expand Up @@ -187,10 +187,11 @@ fn build_worker(legacy: &Assignment) -> anyhow::Result<Vec<u8>> {
let mut staging = builder.new_dataset(dataset.id(), chunks.get(0).dataset_base_url());
for chunk in chunks.iter() {
let tables = tables_of(&chunk);
let id = id_of(&chunk)?;
let mut staged = staging
.new_chunk()
.id(chunk.id())
.block_range(chunk.first_block()..=last_block_of(&chunk)?)
.id(id)
.block_range(id.first_block()..=id.last_block())
.size(chunk.size())
.write_schema_id(write_schema_id)
.worker_indexes(&chunk.worker_indexes().iter().collect::<Vec<_>>());
Expand Down Expand Up @@ -227,10 +228,11 @@ fn build_portal(legacy: &Assignment) -> anyhow::Result<Vec<u8>> {
let (mut dataset_zeros, mut dataset_descents) = (0usize, 0usize);
let mut staging = builder.new_dataset(dataset.id(), schema_id(index));
for chunk in dataset.chunks().iter() {
let id = id_of(&chunk)?;
let mut staged = staging
.new_chunk()
.id(chunk.id())
.block_range(chunk.first_block()..=last_block_of(&chunk)?)
.id(id)
.block_range(id.first_block()..=id.last_block())
.worker_indexes(&chunk.worker_indexes().iter().collect::<Vec<_>>());
if let Some(timestamp) = chunk.last_block_timestamp() {
if timestamp == 0 {
Expand Down Expand Up @@ -300,7 +302,7 @@ fn verify(

for (i, source_chunk) in chunks.iter().enumerate() {
let chunk_id = source_chunk.id();
let last_block = last_block_of(&source_chunk)?;
let last_block = id_of(&source_chunk)?.last_block();
anyhow::ensure!(
source_chunk.dataset_id() == id,
"{chunk_id}: chunk names dataset '{}' but is filed under '{id}'",
Expand Down Expand Up @@ -389,20 +391,17 @@ fn verify(
}

/// Legacy chunks carry no end block; the id does, and it agrees with `first_block` throughout.
fn last_block_of(chunk: &sqd_assignments::fb::Chunk<'_>) -> anyhow::Result<u64> {
let id = chunk.id();
let (_, rest) = id.split_once('/').with_context(|| format!("chunk id '{id}' has no top"))?;
let mut parts = rest.splitn(3, '-');
let (Some(first), Some(last), Some(_)) = (parts.next(), parts.next(), parts.next()) else {
anyhow::bail!("chunk id '{id}' is not <top>/<first>-<last>-<hash>");
};
let first: u64 = first.parse().with_context(|| format!("chunk id '{id}'"))?;
fn id_of(chunk: &sqd_assignments::fb::Chunk<'_>) -> anyhow::Result<DataChunk> {
let id: DataChunk = chunk
.id()
.parse()
.map_err(|()| anyhow::anyhow!("chunk id '{}' does not parse", chunk.id()))?;
anyhow::ensure!(
first == chunk.first_block(),
id.first_block() == chunk.first_block(),
"chunk id '{id}' disagrees with first_block {}",
chunk.first_block()
);
last.parse().with_context(|| format!("chunk id '{id}'"))
Ok(id)
}

/// Ordinals are 1-based, leaving 0 free to mean "unset".
Expand Down
105 changes: 33 additions & 72 deletions crates/assignments/src/builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ use crypto_box::{
};
use flatbuffers::{self as fb, WIPOffset};
use libp2p_identity::PeerId;
use sqd_data_chunk::DataChunk;

use crate::{
common,
Expand Down Expand Up @@ -286,8 +287,8 @@ impl<'b, Rng: CryptoRngCore> ChunkBuilder<'b, Rng> {
}

#[must_use = "a chunk is staged by `finish`; a builder that is dropped stages nothing"]
pub fn id(mut self, id: &str) -> Self {
self.id = Some(self.p.builder.create_string(id));
pub fn id(mut self, id: DataChunk) -> Self {
self.id = Some(self.p.builder.create_string(&id.to_string()));
self
}

Expand Down Expand Up @@ -851,9 +852,7 @@ pub struct WorkerAssignmentChunkBuilder<'b, Rng: CryptoRngCore> {
p: &'b mut WorkerAssignmentBuilder<Rng>,

block_range: Option<RangeInclusive<u64>>,
/// Parsed by [`Self::id`], since nothing downstream wants the string back. The error is held
/// rather than returned so it still surfaces at `finish`.
parsed_id: Option<anyhow::Result<ParsedChunkId>>,
id: Option<DataChunk>,
size: Option<u32>,
version: u32,
write_schema_id: Option<u32>,
Expand All @@ -869,21 +868,19 @@ impl<'b, Rng: CryptoRngCore> WorkerAssignmentChunkBuilder<'b, Rng> {
Self {
p: parent,
block_range: None,
parsed_id: None,
id: None,
size: None,
version: 0,
write_schema_id: None,
tables_present: None,
}
}

/// e.g. `"0221000000/0221000000-0221000649-9QgFD"`. Split into the `tops`, `first_blocks`,
/// `block_deltas` and `hashes` columns and rebuilt on read, so it must parse — here rather
/// than at `finish`, which keeps the string out of the staged chunk. A malformed id is
/// reported by [`Self::finish`] all the same.
/// Split into the `tops`, `first_blocks`, `block_deltas` and `hashes` columns and rebuilt on
/// read.
#[must_use = "a chunk is staged by `finish`; a builder that is dropped stages nothing"]
pub fn id(mut self, id: &str) -> Self {
self.parsed_id = Some(parse_chunk_id(id));
pub fn id(mut self, id: DataChunk) -> Self {
self.id = Some(id);
self
}

Expand Down Expand Up @@ -975,19 +972,19 @@ impl<'b, Rng: CryptoRngCore> WorkerAssignmentChunkBuilder<'b, Rng> {
///
/// # Errors
///
/// If the id is unset or malformed, or disagrees with `block_range`; if `write_schema_id` is
/// If the id is unset, or disagrees with `block_range`; if `write_schema_id` is
/// unset or names an unregistered schema; if a table isn't in that schema's roster; if a
/// non-zero `version` has no generation registered for this dataset; if the chunk names a
/// different base url than the dataset's other chunks; or if it breaks block continuity (see
/// [`WorkerAssignmentBuilder::check_continuity`]).
pub fn finish(self) -> anyhow::Result<()> {
let block_range = self.block_range.expect("Block range must be set");
let parsed = self.parsed_id.context("Chunk id must be set")??;
let id = self.id.context("Chunk id must be set")?;
anyhow::ensure!(
parsed.first_block == *block_range.start() && parsed.last_block == *block_range.end(),
id.first_block() == *block_range.start() && id.last_block() == *block_range.end(),
"chunk id names blocks {}-{}, but the chunk covers {}-{}",
parsed.first_block,
parsed.last_block,
id.first_block(),
id.last_block(),
block_range.start(),
block_range.end()
);
Expand All @@ -1014,9 +1011,9 @@ impl<'b, Rng: CryptoRngCore> WorkerAssignmentChunkBuilder<'b, Rng> {
}

self.p.push_chunk(PushChunk {
top: parsed.top,
top: id.top(),
block_range,
hash: parsed.hash,
hash: chunk_hash(&id),
size: self.size.expect("Size must be set"),
write_schema_id,
version: self.version,
Expand Down Expand Up @@ -1365,52 +1362,18 @@ impl Drop for PortalDatasetBuilder<'_> {
}
}

/// The pieces a chunk id is made of; the reader reassembles them the same way.
struct ParsedChunkId {
top: u64,
first_block: u64,
last_block: u64,
hash: assignment_fb::ChunkHash,
}

/// Splits `"0221000000/0221000000-0221000649-9QgFD"` into the parts the columns hold.
fn parse_chunk_id(id: &str) -> anyhow::Result<ParsedChunkId> {
let (top, rest) = id
.split_once('/')
.with_context(|| format!("chunk id '{id}' has no top directory"))?;
let mut parts = rest.splitn(3, '-');
let (Some(first_block), Some(last_block), Some(hash)) =
(parts.next(), parts.next(), parts.next())
else {
anyhow::bail!("chunk id '{id}' is not <top>/<first_block>-<last_block>-<hash>");
};
// Portal chunk ids require a 5-to-8 character hash.
anyhow::ensure!(
(5..=8).contains(&hash.len())
&& hash.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_'),
"chunk id '{id}' has a hash that is not 5 to 8 word characters"
);
fn chunk_hash(id: &DataChunk) -> assignment_fb::ChunkHash {
let hash = id.last_hash().as_bytes();
let mut bytes = [0u8; 8];
bytes[..hash.len()].copy_from_slice(hash.as_bytes());
Ok(ParsedChunkId {
top: top.parse().with_context(|| format!("chunk id '{id}' has a non-numeric top"))?,
first_block: first_block
.parse()
.with_context(|| format!("chunk id '{id}' has a non-numeric first block"))?,
last_block: last_block
.parse()
.with_context(|| format!("chunk id '{id}' has a non-numeric last block"))?,
hash: assignment_fb::ChunkHash::new(&bytes),
})
bytes[..hash.len()].copy_from_slice(hash);
assignment_fb::ChunkHash::new(&bytes)
}

pub struct PortalAssignmentChunkBuilder<'b> {
p: &'b mut PortalAssignmentBuilder,

block_range: Option<RangeInclusive<u64>>,
/// Parsed by [`Self::id`], since nothing downstream wants the string back. The error is held
/// rather than returned so it still surfaces at `finish`.
parsed_id: Option<anyhow::Result<ParsedChunkId>>,
id: Option<DataChunk>,
version: u32,
last_block_timestamp: u64,
}
Expand All @@ -1421,19 +1384,17 @@ impl<'b> PortalAssignmentChunkBuilder<'b> {
Self {
p: parent,
block_range: None,
parsed_id: None,
id: None,
version: 0,
last_block_timestamp: 0,
}
}

/// The chunk id, e.g. `"0221000000/0221000000-0221000649-9QgFD"`. Not stored as such: it is
/// split into the `tops`, `first_blocks`, `block_deltas` and `hashes` columns and reassembled
/// on read, so it must parse. Parsing happens here rather than at `finish`, which keeps the
/// string out of the staged chunk; a malformed id is still reported by [`Self::finish`].
/// Split into the `tops`, `first_blocks`, `block_deltas` and `hashes` columns and rebuilt on
/// read.
#[must_use = "a chunk is staged by `finish`; a builder that is dropped stages nothing"]
pub fn id(mut self, id: &str) -> Self {
self.parsed_id = Some(parse_chunk_id(id));
pub fn id(mut self, id: DataChunk) -> Self {
self.id = Some(id);
self
}

Expand Down Expand Up @@ -1473,24 +1434,24 @@ impl<'b> PortalAssignmentChunkBuilder<'b> {
///
/// # Errors
///
/// If the id is unset or malformed, if it disagrees with `block_range` — the two encode the
/// If the id is unset, if it disagrees with `block_range` — the two encode the
/// same block numbers, and the id is rebuilt from the range on read — or if the chunk breaks
/// block continuity (see [`PortalAssignmentBuilder::check_continuity`]).
pub fn finish(self) -> anyhow::Result<()> {
let block_range = self.block_range.expect("Block range must be set");
let parsed = self.parsed_id.context("Chunk id must be set")??;
let id = self.id.context("Chunk id must be set")?;
anyhow::ensure!(
parsed.first_block == *block_range.start() && parsed.last_block == *block_range.end(),
id.first_block() == *block_range.start() && id.last_block() == *block_range.end(),
"chunk id names blocks {}-{}, but the chunk covers {}-{}",
parsed.first_block,
parsed.last_block,
id.first_block(),
id.last_block(),
block_range.start(),
block_range.end()
);
self.p.push_chunk(
parsed.top,
id.top(),
block_range,
parsed.hash,
chunk_hash(&id),
self.version,
self.last_block_timestamp,
)
Expand Down
2 changes: 2 additions & 0 deletions crates/assignments/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,8 @@ pub use builder::{
PortalAssignmentBuilder, PortalAssignmentChunkBuilder, PortalDatasetBuilder,
WorkerAssignmentBuilder, WorkerAssignmentChunkBuilder, WorkerDatasetBuilder,
};
#[cfg(feature = "builder")]
pub use sqd_data_chunk::DataChunk;

#[cfg(feature = "reader")]
pub use reader::{
Expand Down
Loading
Loading