From 65ba1374af2526f8fe9d950d1a946056fc7bdb9d Mon Sep 17 00:00:00 2001 From: t Date: Thu, 9 Jul 2026 21:43:38 -0500 Subject: [PATCH 001/124] feat(node,git): cap concurrent served git ops with a 503 load-shed (#62) PR3 of the #62 served-git hardening stack (timeout #165 and teardown wiring #150 are merged). A bounded semaphore caps how many upload-pack / receive-pack / info-refs operations run at once; past the cap a request is shed with a clean 503 + Retry-After before spawning another git subprocess, instead of exhausting the PID/thread table. A permit is acquired at the top of each of the three handlers and held for the whole op, releasing on return. The cap is a portable backstop: the compose pids_limit is absent on Fly, whose 500-connection cap is a different axis. Size --max-concurrent-git-ops (GITLAWB_MAX_CONCURRENT_GIT_OPS, default 128) below the process budget. Range 1..=1_048_576 so 0 (shed everything) and an oversized value that would panic tokio's Semaphore at boot are clean CLI errors. Known gap, tracked separately: info/refs and the withheld-blob (upload_pack_excluding) path are not duration-bounded and do not reap their git child on client disconnect, so a hung git on those two paths holds its slot until it exits and live git can briefly exceed the cap. The main pack path (run_git_service) tears its group down on drop. Tests: Overloaded maps to 503 + Retry-After; the config knob defaults and rejects out-of-range; git_permit sheds at capacity and releases; and each of the three endpoints sheds with 503 when the semaphore is exhausted (load-bearing: drop the permit line and the endpoint test goes red). --- crates/gitlawb-node/src/api/repos.rs | 37 ++++++++ crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/config.rs | 61 ++++++++++++ crates/gitlawb-node/src/error.rs | 31 +++++- crates/gitlawb-node/src/main.rs | 1 + crates/gitlawb-node/src/state.rs | 4 + crates/gitlawb-node/src/test_support.rs | 119 ++++++++++++++++++++++++ 7 files changed, 253 insertions(+), 1 deletion(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index b9cbc352..4bc1f737 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -507,6 +507,9 @@ pub async fn git_info_refs( headers: axum::http::HeaderMap, auth: Option>, ) -> Result { + // Shed with a 503 before spawning git when the concurrency cap is saturated; + // held for the whole op (incl. the smart_http call), released on return. + let _permit = git_permit(&state.git_semaphore)?; let name = smart_http_repo_name(&repo)?; tracing::info!(owner = %owner, repo = %name, "info/refs request"); let record = state @@ -589,6 +592,22 @@ pub async fn git_info_refs( }) } +/// Acquire a permit from the served-git concurrency semaphore, or shed the +/// request with a 503 + Retry-After when every slot is in use. Bind the returned +/// permit to a named local so it is held for the whole git op (it releases on +/// drop); a bare `_` would release it immediately. +fn git_permit( + sem: &std::sync::Arc, +) -> Result { + sem.clone().try_acquire_owned().map_err(|_| { + // Surface the shed so operators can see the cap engaging, mirroring the + // receive-pack rate-limit warn above. A silent 503 makes a saturated or + // misconfigured cap look like a client problem instead of a capacity one. + tracing::warn!("served-git concurrency cap reached; shedding request with 503"); + AppError::Overloaded("git service at capacity, retry shortly".into()) + }) +} + /// Map an error from a `smart_http` git service call to the right `AppError`: /// [`smart_http::GitServiceTimeout`] to 504, a malformed client request to 400, /// anything else to a 500 git error. Pure (no logging) so it is unit-testable; @@ -616,6 +635,9 @@ pub async fn git_upload_pack( auth: Option>, body: Bytes, ) -> Result { + // Shed with a 503 before spawning git when the concurrency cap is saturated; + // held for the whole op (incl. the smart_http call), released on return. + let _permit = git_permit(&state.git_semaphore)?; let name = smart_http_repo_name(&repo)?; let record = state .db @@ -853,6 +875,10 @@ pub async fn git_receive_pack( Extension(auth): Extension, body: Bytes, ) -> Result { + // Shed with a 503 before spawning git when the concurrency cap is saturated. + // Acquired at the very top so it wraps the write-guard below; held for the + // whole op (incl. the smart_http call), released on return. + let _permit = git_permit(&state.git_semaphore)?; let name = smart_http_repo_name(&repo)?; tracing::info!(owner = %owner, repo = %name, "receive-pack request"); let record = state @@ -1862,6 +1888,17 @@ mod tests { assert!(matches!(git_service_app_error(&other), AppError::Git(_))); } + #[test] + fn git_permit_sheds_at_capacity_and_releases() { + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + let p1 = git_permit(&sem).expect("first acquire succeeds"); + // At capacity the next request is shed with Overloaded (-> 503), not queued. + assert!(matches!(git_permit(&sem), Err(AppError::Overloaded(_)))); + // Releasing the permit frees the slot for the next request. + drop(p1); + assert!(git_permit(&sem).is_ok()); + } + fn repo_owned_by(owner_did: &str) -> crate::db::RepoRecord { let now = chrono::Utc::now(); crate::db::RepoRecord { diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 720fb3ae..b541b57b 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -520,6 +520,7 @@ mod tests { sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), shutdown_tx: tokio::sync::watch::channel(false).0, + git_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), } } diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index fc2247d9..688e99f4 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -234,6 +234,40 @@ pub struct Config { value_parser = clap::value_parser!(u64).range(1..) )] pub db_retry_max_secs: u64, + + /// Maximum number of served git operations (upload-pack / receive-pack / + /// info-refs) allowed to run concurrently. Beyond this the node sheds the + /// request with a clean 503 + Retry-After instead of spawning another git + /// subprocess and risking PID/thread exhaustion. Portable backstop: the + /// compose `pids_limit` is not present on Fly, whose connection-concurrency + /// cap is a different axis (500 connections each fan out to git + + /// pack-objects + threads). Size below the process budget with headroom. + /// + /// A permit is held for the whole op. upload-pack/receive-pack are + /// duration-bounded by `git_service_timeout_secs`, but the `info/refs` + /// advertisement and the withheld-blob (`upload_pack_excluding`) path are + /// not, so a hung git on those two paths holds its slot until it exits, so a + /// stuck advertisement permanently costs one unit of capacity. Bounding + /// those paths' duration is tracked as separate follow-up. + /// + /// The same two paths have the mirror gap on cancellation: the permit frees + /// when the handler future drops (client disconnect), but neither reaps its + /// git child — `info/refs` runs a bare `Command`, and `upload_pack_excluding` + /// a blocking `spawn_blocking` a dropped future cannot cancel — so the child + /// runs to completion after its slot is freed and live git can briefly exceed + /// the cap. The main pack path (`run_git_service`) tears its process group + /// down on drop, so this is confined to the same two follow-up paths. + /// + /// Default: 128. Must be between 1 and 1_048_576; the ceiling keeps the value + /// well under tokio's `Semaphore` permit limit so an oversized value is a + /// clean CLI error rather than a boot-time panic. + #[arg( + long, + env = "GITLAWB_MAX_CONCURRENT_GIT_OPS", + default_value_t = 128, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub max_concurrent_git_ops: usize, } impl Config { @@ -272,4 +306,31 @@ mod tests { Config::try_parse_from(["gitlawb-node", "--git-service-timeout-secs", "0"]).is_err() ); } + + #[test] + fn max_concurrent_git_ops_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).max_concurrent_git_ops, + 128 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-git-ops", "8"]) + .max_concurrent_git_ops, + 8 + ); + // 0 permits would shed every served-git request with a 503; clap must reject it. + assert!(Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-ops", "0"]).is_err()); + // Above the ceiling would panic tokio's Semaphore::new at boot (permits > + // usize::MAX >> 3); clap must reject it as a clean CLI error instead. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-ops", "1048577"]) + .is_err() + ); + // The ceiling itself is accepted. + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-git-ops", "1048576"]) + .max_concurrent_git_ops, + 1_048_576 + ); + } } diff --git a/crates/gitlawb-node/src/error.rs b/crates/gitlawb-node/src/error.rs index bdd5aec9..a07235fa 100644 --- a/crates/gitlawb-node/src/error.rs +++ b/crates/gitlawb-node/src/error.rs @@ -47,6 +47,9 @@ pub enum AppError { #[error("git service timed out: {0}")] Timeout(String), + #[error("server overloaded: {0}")] + Overloaded(String), + #[error("database error: {0}")] Db(#[from] sqlx::Error), @@ -147,6 +150,12 @@ impl IntoResponse for AppError { DB_UNAVAILABLE_CODE, DB_UNAVAILABLE_MESSAGE.into(), ), + // 503 with a Retry-After (attached after this match — the shared tail + // can't carry per-variant headers). This is the single place Overloaded + // becomes a response, so it can never ship a 503 without the retry hint. + AppError::Overloaded(msg) => { + (StatusCode::SERVICE_UNAVAILABLE, "overloaded", msg.clone()) + } AppError::Db(e) => (StatusCode::INTERNAL_SERVER_ERROR, "db_error", e.to_string()), AppError::Internal(e) => ( StatusCode::INTERNAL_SERVER_ERROR, @@ -160,7 +169,17 @@ impl IntoResponse for AppError { "message": message, })); - (status, body).into_response() + let mut resp = (status, body).into_response(); + // Overloaded advertises when to retry. It rides the shared tail above for + // its body/status, so the header is attached here rather than in a bespoke + // early return — keeping the variant handled in exactly one place. + if matches!(self, AppError::Overloaded(_)) { + resp.headers_mut().insert( + axum::http::header::RETRY_AFTER, + axum::http::HeaderValue::from_static("1"), + ); + } + resp } } @@ -182,4 +201,14 @@ mod tests { StatusCode::INTERNAL_SERVER_ERROR ); } + + #[test] + fn overloaded_maps_to_503_with_retry_after() { + let resp = AppError::Overloaded("x".into()).into_response(); + assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + resp.headers().get("retry-after").unwrap().to_str().unwrap(), + "1" + ); + } } diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index aa0483db..ee36c783 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -378,6 +378,7 @@ async fn main() -> Result<()> { sync_trigger_rate_limiter, peer_write_rate_limiter, shutdown_tx: shutdown_tx.clone(), + git_semaphore: Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_git_ops)), }; // Periodic peer-count poll for the metrics gauge. If p2p is disabled diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 6a84b3c2..e6e69ea6 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -88,6 +88,10 @@ pub struct AppState { /// * the libp2p swarm task /// * the gossip, sync, operator heartbeat, and rate-limit cleanup loops pub shutdown_tx: tokio::sync::watch::Sender, + /// Bounds concurrent served git operations. A handler acquires a permit + /// before spawning git and holds it for the op; when none are free the + /// request is shed with a 503 rather than exhausting the PID/thread table. + pub git_semaphore: Arc, } impl AppState { diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 4fe52f64..cc322a67 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -82,6 +82,8 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), shutdown_tx: tokio::sync::watch::channel(false).0, + // Generous — no test drives the handler-level shed (git_permit is unit-tested). + git_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), } } @@ -187,6 +189,123 @@ mod tests { ); } + /// PR3 (#62): the served-git concurrency cap sheds at the HTTP layer. One test + /// per git handler drives the `let _permit = git_permit(...)` wiring end to end + /// (this one plus the git_upload_pack / git_receive_pack siblings below); the + /// git_permit unit test covers the helper in isolation. DB-free: an exhausted + /// semaphore sheds before any DB/disk access, so a lazy state works. Remove the + /// permit line from git_info_refs and this goes red (the request falls through + /// to the DB and returns something other than 503). + #[tokio::test] + async fn git_info_refs_sheds_with_503_when_semaphore_exhausted() { + let mut state = test_state_lazy(); + state.git_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + + let router = Router::new() + .route( + "/{owner}/{repo}/info/refs", + axum::routing::get(crate::api::repos::git_info_refs), + ) + .with_state(state); + let resp = router + .oneshot(anon_get( + "/alice/repo.git/info/refs?service=git-upload-pack", + )) + .await + .unwrap(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted git semaphore must shed info/refs with 503 before touching the DB" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + + /// PR3 (#62) sibling of the info/refs shed test: git-upload-pack also acquires a + /// permit at the top, so an exhausted semaphore must shed it with a 503 before + /// any DB/disk work. Anonymous-reachable, so no auth injection is needed. Remove + /// the permit line from git_upload_pack and this goes red. + #[tokio::test] + async fn git_upload_pack_sheds_with_503_when_semaphore_exhausted() { + let mut state = test_state_lazy(); + state.git_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + + let router = Router::new() + .route( + "/{owner}/{repo}/git-upload-pack", + axum::routing::post(crate::api::repos::git_upload_pack), + ) + .with_state(state); + let req = Request::builder() + .method(Method::POST) + .uri("/alice/repo.git/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + let resp = router.oneshot(req).await.unwrap(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted git semaphore must shed git-upload-pack with 503 before touching the DB" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + + /// PR3 (#62) sibling for the push path: git-receive-pack requires an + /// AuthenticatedDid extension (production: require_signature injects it), so the + /// request carries one via signed_request_as — without it the Extension + /// extractor 500s before the handler body reaches git_permit. The permit is the + /// first statement, so an exhausted semaphore still sheds 503 before any DB + /// work. Remove the permit line from git_receive_pack and this goes red. + #[tokio::test] + async fn git_receive_pack_sheds_with_503_when_semaphore_exhausted() { + let mut state = test_state_lazy(); + state.git_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + + let router = Router::new() + .route( + "/{owner}/{repo}/git-receive-pack", + axum::routing::post(crate::api::repos::git_receive_pack), + ) + .with_state(state); + let owner = "did:key:zRECVSHEDOWNERAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let resp = router + .oneshot(signed_request_as( + owner, + Method::POST, + "/alice/repo.git/git-receive-pack", + Body::from(&b"0000"[..]), + )) + .await + .unwrap(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted git semaphore must shed git-receive-pack with 503 before touching the DB" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + /// N7: merge_pr is owner-only. A non-owner is rejected by require_repo_owner /// before any git work (so no on-disk repo is needed for the rejection). #[sqlx::test] From db4de844add8e1eeb61f2e5b5118751a677ca6f3 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 10 Jul 2026 12:18:07 -0500 Subject: [PATCH 002/124] feat(node,config): add write-pool and per-caller read-cap knobs (#62) Add max_concurrent_git_pushes (default 32) and max_concurrent_reads_per_caller (default 16), both clap range(1..=1_048_576) so an oversized value is a clean CLI error, not a Semaphore::new boot panic. The per-caller knob documents that per-source-IP keying is only as granular as GITLAWB_TRUSTED_PROXY. Wiring lands in the following commits; these are the config surface for the #174 concurrency-fairness fix. Resolves jatmn P1a/P1b groundwork on #174. --- .env.example | 15 +++++ crates/gitlawb-node/src/config.rs | 96 +++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+) diff --git a/.env.example b/.env.example index bbd9a342..d7c6a30b 100644 --- a/.env.example +++ b/.env.example @@ -113,6 +113,21 @@ GITLAWB_MAX_PACK_BYTES=2147483648 # advertisement or the withheld-blob path, which remain unbounded. Default 600. GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600 +# Max concurrent git-receive-pack (push) operations, in a pool separate from the +# read pool (GITLAWB_MAX_CONCURRENT_GIT_OPS) so anonymous reads cannot shed an +# authenticated push at admission. Over-cap sheds a 503 + Retry-After. Default 32. +GITLAWB_MAX_CONCURRENT_GIT_PUSHES=32 + +# Max concurrent read ops (upload-pack + info/refs advertisements) a single caller +# may hold, so one caller cannot monopolize the read pool. Keyed per-DID when +# signed, else per-source-IP. The per-IP key is only as granular as +# GITLAWB_TRUSTED_PROXY below: left unset, a node behind an edge/NAT keys all +# anonymous callers on the edge IP and this collapses to one global anonymous cap. +# Set GITLAWB_TRUSTED_PROXY for per-client keying; a high-fanout caller (CI behind +# one NAT) should authenticate for a per-DID budget or the operator raises this. +# Default 16. +GITLAWB_MAX_CONCURRENT_READS_PER_CALLER=16 + # ── Push rate limiting (git-receive-pack flood brake) ───────────────────── # Max receive-pack requests (info/refs advertisement + push POST) per client # IP per hour. 0 disables. Default 600. diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 688e99f4..ff2c2556 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -268,6 +268,42 @@ pub struct Config { value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) )] pub max_concurrent_git_ops: usize, + + /// Maximum number of concurrent `git-receive-pack` (push) operations. Pushes + /// draw from this dedicated pool, separate from `max_concurrent_git_ops` + /// (reads), so a flood of anonymous reads cannot shed an authenticated push at + /// admission (#174). Beyond this a push sheds a clean 503 + Retry-After. + /// + /// Default: 32. Must be between 1 and 1_048_576 (the ceiling keeps the value + /// under tokio's `Semaphore` permit limit so an oversized value is a clean CLI + /// error rather than a boot-time panic). + #[arg( + long, + env = "GITLAWB_MAX_CONCURRENT_GIT_PUSHES", + default_value_t = 32, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub max_concurrent_git_pushes: usize, + + /// Maximum concurrent read operations (`upload-pack` and the `info/refs` + /// advertisements) a single caller may hold at once, so one caller cannot + /// monopolize the `max_concurrent_git_ops` read pool (#174). Callers are keyed + /// per-DID when signed, else per-source-IP. IMPORTANT: the per-source-IP key is + /// only as granular as `GITLAWB_TRUSTED_PROXY`. Left unset (the default), a node + /// behind an edge/NAT keys all anonymous callers on the edge IP, so this cap + /// collapses to a single global anonymous cap rather than per-client. Set + /// `GITLAWB_TRUSTED_PROXY` to key on the real client; a known high-fanout caller + /// (a CI fleet behind one NAT) should authenticate for a per-DID budget or the + /// operator raises this. Over-cap for a caller sheds a clean 503 + Retry-After. + /// + /// Default: 16. Must be between 1 and 1_048_576. + #[arg( + long, + env = "GITLAWB_MAX_CONCURRENT_READS_PER_CALLER", + default_value_t = 16, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub max_concurrent_reads_per_caller: usize, } impl Config { @@ -333,4 +369,64 @@ mod tests { 1_048_576 ); } + + #[test] + fn max_concurrent_git_pushes_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).max_concurrent_git_pushes, + 32 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "8"]) + .max_concurrent_git_pushes, + 8 + ); + // 0 permits would shed every push with a 503; clap must reject it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "0"]).is_err() + ); + // Above the ceiling would panic tokio's Semaphore::new at boot; clap rejects it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "1048577"]) + .is_err() + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "1048576"]) + .max_concurrent_git_pushes, + 1_048_576 + ); + } + + #[test] + fn max_concurrent_reads_per_caller_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).max_concurrent_reads_per_caller, + 16 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-reads-per-caller", "4"]) + .max_concurrent_reads_per_caller, + 4 + ); + // 0 would shed every read from a keyed caller; clap must reject it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-reads-per-caller", "0"]) + .is_err() + ); + assert!(Config::try_parse_from([ + "gitlawb-node", + "--max-concurrent-reads-per-caller", + "1048577" + ]) + .is_err()); + assert_eq!( + Config::parse_from([ + "gitlawb-node", + "--max-concurrent-reads-per-caller", + "1048576" + ]) + .max_concurrent_reads_per_caller, + 1_048_576 + ); + } } From 2434dde42f1ac776da08df8186b7f4a3b54073ae Mon Sep 17 00:00:00 2001 From: t Date: Fri, 10 Jul 2026 12:29:30 -0500 Subject: [PATCH 003/124] fix(node,git): isolate authed pushes in a dedicated write pool (#62) git-receive-pack now draws from a separate git_write_semaphore (max_concurrent_git_pushes) instead of the shared pool, so a flood of anonymous reads can no longer shed an authenticated push at admission (jatmn P1a). The shared field is renamed git_read_semaphore and continues to gate upload-pack and both info/refs advertisements. The write permit stays above acquire_write so it precedes the Tigris fresh-acquire (INV-10). Handler-layer tests: write-pool shed (503), and a cross-boundary proof that an exhausted read pool does NOT shed a push; both mutation-checked (routing receive-pack back to the read pool flips each RED). 497 tests pass. Part of #174. --- crates/gitlawb-node/src/api/repos.rs | 12 ++++--- crates/gitlawb-node/src/auth/mod.rs | 3 +- crates/gitlawb-node/src/main.rs | 5 ++- crates/gitlawb-node/src/state.rs | 14 +++++--- crates/gitlawb-node/src/test_support.rs | 47 ++++++++++++++++++++++--- 5 files changed, 65 insertions(+), 16 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 4bc1f737..dd030a36 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -509,7 +509,7 @@ pub async fn git_info_refs( ) -> Result { // Shed with a 503 before spawning git when the concurrency cap is saturated; // held for the whole op (incl. the smart_http call), released on return. - let _permit = git_permit(&state.git_semaphore)?; + let _permit = git_permit(&state.git_read_semaphore)?; let name = smart_http_repo_name(&repo)?; tracing::info!(owner = %owner, repo = %name, "info/refs request"); let record = state @@ -637,7 +637,7 @@ pub async fn git_upload_pack( ) -> Result { // Shed with a 503 before spawning git when the concurrency cap is saturated; // held for the whole op (incl. the smart_http call), released on return. - let _permit = git_permit(&state.git_semaphore)?; + let _permit = git_permit(&state.git_read_semaphore)?; let name = smart_http_repo_name(&repo)?; let record = state .db @@ -876,9 +876,11 @@ pub async fn git_receive_pack( body: Bytes, ) -> Result { // Shed with a 503 before spawning git when the concurrency cap is saturated. - // Acquired at the very top so it wraps the write-guard below; held for the - // whole op (incl. the smart_http call), released on return. - let _permit = git_permit(&state.git_semaphore)?; + // Pushes draw from the dedicated WRITE pool, separate from reads, so a flood of + // anonymous reads cannot shed an authenticated push (#174). Acquired at the very + // top so it wraps the write-guard below (and precedes the Tigris acquire_write, + // bounding concurrent fresh acquires — INV-10); held for the whole op. + let _permit = git_permit(&state.git_write_semaphore)?; let name = smart_http_repo_name(&repo)?; tracing::info!(owner = %owner, repo = %name, "receive-pack request"); let record = state diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index b541b57b..1d169e1c 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -520,7 +520,8 @@ mod tests { sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), shutdown_tx: tokio::sync::watch::channel(false).0, - git_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), } } diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index ee36c783..b8f687b1 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -378,7 +378,10 @@ async fn main() -> Result<()> { sync_trigger_rate_limiter, peer_write_rate_limiter, shutdown_tx: shutdown_tx.clone(), - git_semaphore: Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_git_ops)), + git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_git_ops)), + git_write_semaphore: Arc::new(tokio::sync::Semaphore::new( + config.max_concurrent_git_pushes, + )), }; // Periodic peer-count poll for the metrics gauge. If p2p is disabled diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index e6e69ea6..b43f4de2 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -88,10 +88,16 @@ pub struct AppState { /// * the libp2p swarm task /// * the gossip, sync, operator heartbeat, and rate-limit cleanup loops pub shutdown_tx: tokio::sync::watch::Sender, - /// Bounds concurrent served git operations. A handler acquires a permit - /// before spawning git and holds it for the op; when none are free the - /// request is shed with a 503 rather than exhausting the PID/thread table. - pub git_semaphore: Arc, + /// Bounds concurrent served git READ operations (upload-pack + both info/refs + /// advertisements). A read handler acquires a permit before spawning git and + /// holds it for the op; when none are free the request is shed with a 503. + /// Writes draw from `git_write_semaphore` so a read flood cannot shed an + /// authenticated push at admission (#174). + pub git_read_semaphore: Arc, + /// Bounds concurrent `git-receive-pack` (push) operations, a pool separate + /// from `git_read_semaphore` so anonymous reads can never shed an authenticated + /// push (#174). Sized by `max_concurrent_git_pushes`. + pub git_write_semaphore: Arc, } impl AppState { diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index cc322a67..fd68c8e6 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -83,7 +83,8 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), shutdown_tx: tokio::sync::watch::channel(false).0, // Generous — no test drives the handler-level shed (git_permit is unit-tested). - git_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), } } @@ -199,7 +200,7 @@ mod tests { #[tokio::test] async fn git_info_refs_sheds_with_503_when_semaphore_exhausted() { let mut state = test_state_lazy(); - state.git_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + state.git_read_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); let router = Router::new() .route( @@ -235,7 +236,7 @@ mod tests { #[tokio::test] async fn git_upload_pack_sheds_with_503_when_semaphore_exhausted() { let mut state = test_state_lazy(); - state.git_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + state.git_read_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); let router = Router::new() .route( @@ -273,7 +274,7 @@ mod tests { #[tokio::test] async fn git_receive_pack_sheds_with_503_when_semaphore_exhausted() { let mut state = test_state_lazy(); - state.git_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + state.git_write_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); let router = Router::new() .route( @@ -295,7 +296,7 @@ mod tests { assert_eq!( resp.status(), StatusCode::SERVICE_UNAVAILABLE, - "an exhausted git semaphore must shed git-receive-pack with 503 before touching the DB" + "an exhausted write pool must shed git-receive-pack with 503 before touching the DB" ); assert_eq!( resp.headers() @@ -306,6 +307,42 @@ mod tests { ); } + /// #174 (SC1, load-bearing): a saturated READ pool must NOT shed an + /// authenticated push — the write pool is a separate budget. Read pool at zero, + /// write pool with capacity: the push proceeds PAST admission (it then errors on + /// the placeholder DB, but crucially it is not a 503). Route git-receive-pack + /// back to the read pool and this goes red — that is the isolation proof. + #[tokio::test] + async fn git_receive_pack_not_shed_by_exhausted_read_pool() { + let mut state = test_state_lazy(); + // Read pool exhausted as if a flood of anonymous clones held every slot. + state.git_read_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + // Write pool keeps its default capacity from test_state_lazy. + + let router = Router::new() + .route( + "/{owner}/{repo}/git-receive-pack", + axum::routing::post(crate::api::repos::git_receive_pack), + ) + .with_state(state); + let owner = "did:key:zRECVCROSSBOUNDARYAAAAAAAAAAAAAAAAAAAAA"; + let resp = router + .oneshot(signed_request_as( + owner, + Method::POST, + "/alice/repo.git/git-receive-pack", + Body::from(&b"0000"[..]), + )) + .await + .unwrap(); + + assert_ne!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted READ pool must not shed a push — the write pool is a separate budget (#174)" + ); + } + /// N7: merge_pr is owner-only. A non-owner is rejected by require_repo_owner /// before any git work (so no on-disk repo is needed for the rejection). #[sqlx::test] From fc4c206c3a1ccf38a433b2a4d8255b83910f4357 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 10 Jul 2026 12:50:46 -0500 Subject: [PATCH 004/124] fix(node,git): per-caller concurrency sub-cap on the read pool (#62) Adds PerCallerConcurrency, a bounded-keyed in-flight limiter (distinct from the request-rate RateLimiter) so no single caller monopolizes the served-git read pool. Each caller (per-DID when signed via optional_signature, else per-source-IP via client_key) may hold at most max_concurrent_reads_per_caller concurrent reads; over that it sheds 503. The key map is self-bounding (a key is dropped when its in-flight count hits zero) with a reject-before-insert max_keys backstop so a key farm can't grow it (INV-15). Applied in git_upload_pack and both info/refs advertisements, acquired after the visibility gate so a denied request never consumes a slot (KTD7). Primitive unit-tested (cap + self-bounding + reject-before-insert) and mutation-checked. Handler-layer SC2: same-caller sheds while a different caller passes, proven on BOTH git_info_refs and git_upload_pack with independent mutation probes; plus a None-key bypass test. Per-source-IP keying is trust-config dependent, documented on the config knob. 502 tests pass. Part of #174. --- crates/gitlawb-node/src/api/repos.rs | 210 ++++++++++++++++++++++++ crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/main.rs | 3 + crates/gitlawb-node/src/rate_limit.rs | 142 ++++++++++++++++ crates/gitlawb-node/src/state.rs | 5 + crates/gitlawb-node/src/test_support.rs | 1 + 6 files changed, 362 insertions(+) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index dd030a36..e51b62e6 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -566,6 +566,28 @@ pub async fn git_info_refs( } } + // Per-caller read sub-cap (#174): acquired AFTER the visibility + push-rate + // gates (KTD7) so a denied or rate-limited request never consumes a caller's + // scarce read slot. Applies to BOTH advertisements. Held for the whole op. + let caller_key = read_caller_key( + auth.as_ref().map(|e| e.0 .0.as_str()), + &headers, + peer, + state.push_limiter_trust, + ); + let _caller_permit = match &caller_key { + Some(k) => match state.git_read_per_caller.try_acquire(k) { + Some(p) => Some(p), + None => { + tracing::warn!(repo = %name, caller = %k, "per-caller read cap reached; shedding info/refs with 503"); + return Err(AppError::Overloaded( + "git read service at capacity for this caller, retry shortly".into(), + )); + } + }, + None => None, + }; + // For receive-pack (push), download the latest from Tigris so the client // sees the same refs that acquire_write() will operate on. let disk_path = if service == "git-receive-pack" { @@ -608,6 +630,23 @@ fn git_permit( }) } +/// Resolve the per-caller key for the read sub-cap (#174): the authenticated DID +/// when the caller signed (via `optional_signature`), else the trusted client IP +/// (`client_key`). `None` when neither is available — such a request is bounded by +/// the global read pool only, never a 500. The per-source-IP key is only as +/// granular as `trust`; see the `max_concurrent_reads_per_caller` config doc. +fn read_caller_key( + caller_did: Option<&str>, + headers: &axum::http::HeaderMap, + peer: Option, + trust: crate::rate_limit::TrustedProxy, +) -> Option { + match caller_did { + Some(did) => Some(did.to_string()), + None => crate::rate_limit::client_key(headers, peer, trust), + } +} + /// Map an error from a `smart_http` git service call to the right `AppError`: /// [`smart_http::GitServiceTimeout`] to 504, a malformed client request to 400, /// anything else to a 500 git error. Pure (no logging) so it is unit-testable; @@ -633,6 +672,8 @@ pub async fn git_upload_pack( State(state): State, Path((owner, repo)): Path<(String, String)>, auth: Option>, + crate::rate_limit::PeerAddr(peer): crate::rate_limit::PeerAddr, + headers: axum::http::HeaderMap, body: Bytes, ) -> Result { // Shed with a 503 before spawning git when the concurrency cap is saturated; @@ -658,6 +699,23 @@ pub async fn git_upload_pack( return Err(AppError::RepoNotFound(format!("{owner}/{name}"))); } + // Per-caller read sub-cap (#174): after the visibility gate (KTD7) so a + // visibility-denied caller never consumes a scarce read slot. Keyed per-DID + // when signed, else per-source-IP; no resolvable key -> global read pool only. + let caller_key = read_caller_key(caller, &headers, peer, state.push_limiter_trust); + let _caller_permit = match &caller_key { + Some(k) => match state.git_read_per_caller.try_acquire(k) { + Some(p) => Some(p), + None => { + tracing::warn!(repo = %name, caller = %k, "per-caller read cap reached; shedding upload-pack with 503"); + return Err(AppError::Overloaded( + "git read service at capacity for this caller, retry shortly".into(), + )); + } + }, + None => None, + }; + let disk_path = state .repo_store .acquire(&record.owner_did, &record.name) @@ -2618,6 +2676,158 @@ mod tests { ); } + /// #174 SC2 (info_refs probe): the per-caller read sub-cap sheds a caller that + /// is already at its concurrency budget on the upload-pack advertisement, while + /// a DIFFERENT caller still enters. Remove the sub-cap from `git_info_refs` and + /// the same-caller assertion goes green-not-503 — this is the info_refs half of + /// the two-handler mutation probe. + #[sqlx::test] + async fn info_refs_per_caller_cap_sheds_one_caller_not_others(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6pcadv", "pc", "/tmp/pc-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.31:5000".parse().unwrap(); + // Fill this caller's single read slot (a clone shares the Arc-backed map). + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first slot for this caller"); + + // Same caller (IP) at its cap -> shed 503 before the git/Tigris work. + let router = crate::server::build_router(state.clone()); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6pcadv/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + assert_eq!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a caller already at its per-caller read cap must shed the advertisement with 503" + ); + + // A DIFFERENT caller (IP) has its own budget -> not shed by the per-caller cap. + let other: SocketAddr = "203.0.113.32:5000".parse().unwrap(); + let router2 = crate::server::build_router(state.clone()); + let mut req2 = Request::builder() + .method(Method::GET) + .uri("/z6pcadv/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req2.extensions_mut().insert(ConnectInfo(other)); + assert_ne!( + router2.oneshot(req2).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a different caller must not be shed by another caller's saturated budget" + ); + } + + /// #174 SC2 (upload_pack probe): the same per-caller shed on the POST + /// upload-pack path. Remove the sub-cap from `git_upload_pack` and this goes + /// green-not-503 — the upload_pack half of the two-handler mutation probe. + #[sqlx::test] + async fn upload_pack_per_caller_cap_sheds_one_caller_not_others(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6pcupl", "pc", "/tmp/pc-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.41:5000".parse().unwrap(); + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first slot for this caller"); + + let router = crate::server::build_router(state.clone()); + let mut req = Request::builder() + .method(Method::POST) + .uri("/z6pcupl/pc/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + assert_eq!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a caller already at its per-caller read cap must shed upload-pack with 503" + ); + + let other: SocketAddr = "203.0.113.42:5000".parse().unwrap(); + let router2 = crate::server::build_router(state.clone()); + let mut req2 = Request::builder() + .method(Method::POST) + .uri("/z6pcupl/pc/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + req2.extensions_mut().insert(ConnectInfo(other)); + assert_ne!( + router2.oneshot(req2).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a different caller must not be shed by another caller's saturated budget" + ); + } + + /// #174 SC2 (None-key): a request with no resolvable caller key (no ConnectInfo, + /// no trusted header) must NOT be shed by the per-caller cap even when another + /// caller's budget is full — it is bounded by the global read pool only. A None + /// key never keys into the map, so it never 503s from the per-caller sub-cap. + #[sqlx::test] + async fn info_refs_none_key_bypasses_per_caller_cap(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::http::{Method, Request, StatusCode}; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6pcnone", "pc", "/tmp/pc-nonexistent", None, false) + .await + .unwrap(); + // Saturate an unrelated caller's budget; the None-key request must be + // unaffected because it never keys into the per-caller map. + let _slot = state + .git_read_per_caller + .try_acquire("203.0.113.99") + .expect("hold an unrelated caller's slot"); + + // No ConnectInfo inserted -> PeerAddr is None -> no per-caller key. + let router = crate::server::build_router(state.clone()); + let req = Request::builder() + .method(Method::GET) + .uri("/z6pcnone/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + assert_ne!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a request with no resolvable caller key must not be shed by the per-caller cap" + ); + } + /// Repo creation must be throttled by the per-IP creation limiter BEFORE /// signature verification — otherwise a DID farm (one throwaway did:key per /// repo, each carrying a valid but machine-solved iCaptcha proof) walks past diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 1d169e1c..253b3210 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -522,6 +522,7 @@ mod tests { shutdown_tx: tokio::sync::watch::channel(false).0, git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), } } diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index b8f687b1..c617b434 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -382,6 +382,9 @@ async fn main() -> Result<()> { git_write_semaphore: Arc::new(tokio::sync::Semaphore::new( config.max_concurrent_git_pushes, )), + git_read_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( + config.max_concurrent_reads_per_caller, + ), }; // Periodic peer-count poll for the metrics gauge. If p2p is disabled diff --git a/crates/gitlawb-node/src/rate_limit.rs b/crates/gitlawb-node/src/rate_limit.rs index d40e2691..d505f1bb 100644 --- a/crates/gitlawb-node/src/rate_limit.rs +++ b/crates/gitlawb-node/src/rate_limit.rs @@ -132,6 +132,91 @@ impl RateLimiter { } } +/// A bounded per-caller CONCURRENCY limiter — distinct from [`RateLimiter`], which +/// caps request RATE. Each caller key may hold at most `per_caller` in-flight +/// permits at once; beyond that [`try_acquire`](Self::try_acquire) returns `None` +/// and the caller sheds. Used to stop one caller (a single anonymous source-IP or +/// DID) monopolizing the served-git read pool (#174). +/// +/// The key map is self-bounding: a key is removed the instant its in-flight count +/// reaches zero, so it never holds more keys than there are concurrently-active +/// callers (itself bounded by the read semaphore). A `max_keys` reject-before-insert +/// backstop guarantees a key farm can never grow the map even if that invariant +/// weakened — a NEW key at the cap is rejected WITHOUT allocating an entry (INV-15). +/// +/// Uses a `std::sync::Mutex` (not the file's `tokio::sync::Mutex`) because the +/// permit's `Drop` must release synchronously; the critical section holds no await. +#[derive(Clone)] +pub struct PerCallerConcurrency { + state: Arc>>, + per_caller: usize, + max_keys: usize, +} + +/// RAII permit from [`PerCallerConcurrency::try_acquire`]. On drop it decrements +/// the caller's in-flight count and removes the key when it reaches zero. +pub struct PerCallerPermit { + state: Arc>>, + key: String, +} + +impl PerCallerConcurrency { + pub fn new(per_caller: usize, max_keys: usize) -> Self { + Self { + state: Arc::new(std::sync::Mutex::new(HashMap::new())), + per_caller: per_caller.max(1), + max_keys: max_keys.max(1), + } + } + + /// Convenience constructor with the default key bound. + pub fn with_default_max_keys(per_caller: usize) -> Self { + Self::new(per_caller, DEFAULT_MAX_KEYS) + } + + /// `Some(permit)` when the caller is under its cap and the map has room; + /// `None` (shed) otherwise. Reject-before-insert: a new key at `max_keys` is + /// rejected without allocating. + pub fn try_acquire(&self, key: &str) -> Option { + let mut map = self.state.lock().expect("per-caller mutex poisoned"); + match map.get_mut(key) { + Some(count) => { + if *count >= self.per_caller { + return None; + } + *count += 1; + } + None => { + if map.len() >= self.max_keys { + return None; + } + map.insert(key.to_string(), 1); + } + } + Some(PerCallerPermit { + state: self.state.clone(), + key: key.to_string(), + }) + } + + #[cfg(test)] + pub fn tracked_keys(&self) -> usize { + self.state.lock().unwrap().len() + } +} + +impl Drop for PerCallerPermit { + fn drop(&mut self) { + let mut map = self.state.lock().expect("per-caller mutex poisoned"); + if let Some(count) = map.get_mut(&self.key) { + *count -= 1; + if *count == 0 { + map.remove(&self.key); + } + } + } +} + pub async fn rate_limit_by_did(request: Request, next: Next) -> Response { let limiter = request.extensions().get::().cloned(); @@ -288,6 +373,63 @@ pub async fn rate_limit_by_ip(request: Request, next: Next) -> Response { mod tests { use super::*; + #[test] + fn per_caller_concurrency_caps_one_caller_and_frees_on_drop() { + let lim = PerCallerConcurrency::new(2, 100); + let p1 = lim.try_acquire("did:key:zA").expect("first under cap"); + let p2 = lim.try_acquire("did:key:zA").expect("second under cap"); + assert!( + lim.try_acquire("did:key:zA").is_none(), + "a third in-flight op for the same caller sheds (over the per-caller cap)" + ); + // A DIFFERENT caller is unaffected — the cap is per-caller, not global. + let _other = lim + .try_acquire("did:key:zB") + .expect("a different caller has its own budget"); + drop(p1); + assert!( + lim.try_acquire("did:key:zA").is_some(), + "freeing one in-flight slot lets the same caller back in" + ); + drop(p2); + } + + #[test] + fn per_caller_concurrency_map_is_self_bounding_and_reject_before_insert() { + // Self-bounding: acquire+drop many distinct keys — the map never grows + // because a key is removed the instant its in-flight count hits zero. + let lim = PerCallerConcurrency::new(4, 3); + for i in 0..50 { + let _p = lim.try_acquire(&format!("k{i}")); + } + assert_eq!( + lim.tracked_keys(), + 0, + "keys with zero in-flight ops are removed, so an acquire+drop flood leaves the map empty" + ); + // Reject-before-insert: HOLD max_keys distinct keys, then a new key sheds + // WITHOUT growing the map past the cap (INV-15 — a rejected request never + // allocates an entry). + let held: Vec<_> = (0..3) + .map(|i| lim.try_acquire(&format!("h{i}")).unwrap()) + .collect(); + assert_eq!( + lim.tracked_keys(), + 3, + "three distinct callers held concurrently" + ); + assert!( + lim.try_acquire("h3").is_none(), + "a new key at max_keys is rejected" + ); + assert_eq!( + lim.tracked_keys(), + 3, + "the rejected new key did not allocate an entry (reject-before-insert)" + ); + drop(held); + } + #[tokio::test] async fn allows_within_limit() { let limiter = RateLimiter::new(3, Duration::from_secs(60)); diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index b43f4de2..01007ac4 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -98,6 +98,11 @@ pub struct AppState { /// from `git_read_semaphore` so anonymous reads can never shed an authenticated /// push (#174). Sized by `max_concurrent_git_pushes`. pub git_write_semaphore: Arc, + /// Per-caller concurrency sub-cap on the read pool: each caller (per-DID when + /// signed, else per-source-IP) may hold at most `max_concurrent_reads_per_caller` + /// in-flight read ops, so one caller cannot monopolize `git_read_semaphore` + /// (#174). Applied by `git_upload_pack` and both `info/refs` advertisements. + pub git_read_per_caller: crate::rate_limit::PerCallerConcurrency, } impl AppState { diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index fd68c8e6..7745d233 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -85,6 +85,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { // Generous — no test drives the handler-level shed (git_permit is unit-tested). git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), } } From 2c264e08e391f5675cbfd3edeff0238434e9bb15 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 10 Jul 2026 13:09:34 -0500 Subject: [PATCH 005/124] fix(node,git): bound and reap the info/refs advertisement (#62) info_refs ran a bare Command::output() with no timeout and no process-group teardown, so a hung git pinned its concurrency slot indefinitely and a client disconnect orphaned the child (jatmn P1b). Extract the timeout + process_group(0) + KillGroupOnDrop core from run_git_service into a shared drive_git_child, and route info_refs through it with an injectable git_bin. A hung advertisement now aborts with GitServiceTimeout (mapped to 504); disconnect reaps the group. run_git_service's teardown tests all pass through the shared core (proving the group teardown info_refs inherits), the real-git filter tests cover the advertisement happy path, and a new watchdog-bounded test proves a hung advertisement times out. 503 tests pass. Part of #174. --- crates/gitlawb-node/src/api/repos.rs | 15 +++- crates/gitlawb-node/src/git/smart_http.rs | 87 +++++++++++++++++++---- 2 files changed, 85 insertions(+), 17 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index e51b62e6..c42a6989 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -606,11 +606,20 @@ pub async fn git_info_refs( AppError::Git(e.to_string()) })?; - smart_http::info_refs(&disk_path, &service) + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + smart_http::info_refs("git", &service, &disk_path, git_timeout) .await .map_err(|e| { - tracing::error!(repo = %name, service = %service, err = %e, "info_refs git failed"); - AppError::Git(e.to_string()) + let app = git_service_app_error(&e); + match &app { + AppError::Timeout(_) => { + tracing::warn!(repo = %name, service = %service, "info/refs advertisement timed out") + } + _ => { + tracing::error!(repo = %name, service = %service, err = %e, "info_refs git failed") + } + } + app }) } diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index eeb35b99..b4e05a01 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -14,21 +14,29 @@ use tokio::process::Command; /// or `?service=git-receive-pack` /// /// This is the ref advertisement — the first step of a clone or push. -pub async fn info_refs(repo_path: &Path, service: &str) -> Result { +/// +/// `git_bin` is injectable purely so the process-group teardown can be driven by a +/// fake `git` in tests (production passes `"git"`). `timeout` bounds the whole child +/// interaction: previously the advertisement ran a bare `Command::output()` with no +/// deadline and no teardown, so a hung git pinned its concurrency slot indefinitely +/// and a client disconnect orphaned the child (#174). It now shares +/// [`drive_git_child`]'s timeout + `process_group(0)` + [`KillGroupOnDrop`] teardown. +pub async fn info_refs( + git_bin: &str, + service: &str, + repo_path: &Path, + timeout: Duration, +) -> Result { validate_service(service)?; - let output = Command::new("git") + let mut command = Command::new(git_bin); + command .arg(service_to_command(service)) .arg("--stateless-rpc") .arg("--advertise-refs") - .arg(repo_path) - .output() - .await?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - bail!("git {service} --advertise-refs failed: {stderr}"); - } + .arg(repo_path); + // No request body — advertise-refs does not read stdin. + let stdout = drive_git_child(command, Bytes::new(), timeout, "advertise-refs").await?; let content_type = format!("application/x-{service}-advertisement"); @@ -38,7 +46,7 @@ pub async fn info_refs(repo_path: &Path, service: &str) -> Result { let mut body = Vec::new(); body.extend_from_slice(&pkt_service); body.extend_from_slice(flush); - body.extend_from_slice(&output.stdout); + body.extend_from_slice(&stdout); Ok(Response::builder() .status(StatusCode::OK) @@ -207,7 +215,24 @@ async fn run_git_service( command .arg(service_to_command(service)) .arg("--stateless-rpc") - .arg(repo_path) + .arg(repo_path); + drive_git_child(command, input, timeout, service).await +} + +/// Drive a spawned git child under `timeout` with process-group teardown, returning +/// its stdout. Shared core for [`run_git_service`] and [`info_refs`]: the caller +/// passes a `Command` with its args set; this adds piped stdio and `process_group(0)`. +/// On the deadline the whole group is torn down and reaped before returning +/// [`GitServiceTimeout`]; on a dropped future (client disconnect) the +/// [`KillGroupOnDrop`] guard fires. `input` is written to the child's stdin (empty +/// for the advertise-refs path, which has no request body); `what` labels errors. +async fn drive_git_child( + mut command: Command, + input: Bytes, + timeout: Duration, + what: &str, +) -> Result> { + command .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -294,7 +319,7 @@ async fn run_git_service( // the real cause. if !status.success() { let stderr = String::from_utf8_lossy(&err); - bail!("{service} failed: {stderr}"); + bail!("{what} failed: {stderr}"); } write_result.context("failed to write to git stdin")?; @@ -650,7 +675,9 @@ mod tests { axum::extract::Query(q): axum::extract::Query>, ) -> Response { let service = q.get("service").cloned().unwrap_or_default(); - info_refs(&st.repo, &service).await.unwrap() + info_refs("git", &service, &st.repo, Duration::from_secs(30)) + .await + .unwrap() } async fn pack_handler( @@ -1225,6 +1252,38 @@ mod tests { ); } + // #174 (SC3): the info/refs advertisement is now duration-bounded — previously + // it ran a bare `Command::output()` with no deadline, so a hung git pinned its + // concurrency slot forever. A hung fake git must abort with GitServiceTimeout + // (which the handler maps to 504). The outer watchdog turns a missing timeout + // into a loud failure instead of hanging the suite. info_refs shares + // drive_git_child, so its disconnect/group-teardown is the same code proven by + // run_git_service_tears_down_group_when_future_dropped above. + #[cfg(unix)] + #[tokio::test] + async fn info_refs_times_out_a_hung_advertisement() { + let tmp = tempfile::TempDir::new().unwrap(); + // A fake git that hangs forever instead of advertising refs. + let git_bin = write_fake_git(tmp.path(), "#!/bin/sh\nsleep 300\n"); + + let result = tokio::time::timeout( + Duration::from_secs(10), + info_refs( + git_bin.to_str().unwrap(), + "git-upload-pack", + tmp.path(), + Duration::from_millis(200), + ), + ) + .await + .expect("info_refs must return within the watchdog — its own timeout must fire"); + let err = result.expect_err("a hung advertisement must return an error, not hang"); + assert!( + err.downcast_ref::().is_some(), + "a hung advertisement must abort with GitServiceTimeout (-> 504), got: {err}" + ); + } + // A request that runs to completion must DISARM the guard after reaping, so // no stray group SIGTERM fires. The fake exits non-zero (surfacing as Err) // but leaves a grandchild alive; the grandchild must survive. Goes RED if the From 57c6e01c26ac1e3032cb06b57ebc640e18582a85 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 10 Jul 2026 13:16:41 -0500 Subject: [PATCH 006/124] fix(node,git): bound and reap the withheld-blob pack build (#62) The filtered-pack path ran the whole rev-list + pack-objects build inside a spawn_blocking, so an outer tokio timeout could not cancel the blocking thread and a client disconnect orphaned the git child while the permit freed (jatmn P1b, the second gap path). Split it: rev-list enumeration stays blocking off the runtime (rev_list_keep), but the streaming pack-objects stage now runs under the shared drive_git_child on the async side, so it is duration-bounded (GitServiceTimeout -> 504) and its process group is reaped on disconnect. build_filtered_pack becomes async and takes a git_bin seam + timeout; upload_pack_excluding threads the git_service_timeout through. A watchdog-bounded test proves a hung pack-objects times out (rev-list fast, pack-objects hangs). The refactor's happy path is covered by the existing filtered-pack correctness and real-git partial-clone/fetch tests, all still green; disconnect/group-teardown is the shared drive_git_child code proven by the run_git_service tests. 504 tests pass. Part of #174. --- crates/gitlawb-node/src/api/repos.rs | 2 +- crates/gitlawb-node/src/git/smart_http.rs | 145 ++++++++++++++-------- 2 files changed, 95 insertions(+), 52 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index c42a6989..dba8ea71 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -765,7 +765,7 @@ pub async fn git_upload_pack( smart_http::upload_pack(&disk_path, body, git_timeout).await } else { tracing::info!(repo = %name, caller = ?caller, withheld = withheld.len(), "serving filtered pack"); - smart_http::upload_pack_excluding(&disk_path, body, &withheld).await + smart_http::upload_pack_excluding(&disk_path, body, &withheld, git_timeout).await } } .map_err(|e| { diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index b4e05a01..5586a2c7 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -349,12 +349,17 @@ fn pkt_line(data: &str) -> Vec { format!("{len:04x}{data}").into_bytes() } -/// Build a packfile containing every object reachable from all refs EXCEPT the -/// given blob OIDs. Commits and trees are always included, so SHAs stay intact; -/// only the named blobs are dropped. -pub fn build_filtered_pack(repo_path: &Path, withheld: &HashSet) -> Result> { - // All reachable objects as "oid [path]" lines. - let rev = std::process::Command::new("git") +/// Run `rev-list --objects --all` and return the reachable object ids minus the +/// withheld blobs. Blocking (shells out to git); the caller runs it off the async +/// runtime. Kept separate from the pack-objects stage so that stage can live on the +/// async side under [`drive_git_child`] (`git_bin` is injectable for the same +/// fake-git testing reason as `run_git_service`). +fn rev_list_keep( + git_bin: &str, + repo_path: &Path, + withheld: &HashSet, +) -> Result> { + let rev = std::process::Command::new(git_bin) .args(["rev-list", "--objects", "--all"]) .current_dir(repo_path) .output()?; @@ -372,39 +377,39 @@ pub fn build_filtered_pack(repo_path: &Path, withheld: &HashSet) -> Resu } keep.push(oid.to_string()); } - let mut child = std::process::Command::new("git") - .args(["pack-objects", "--stdout"]) - .current_dir(repo_path) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; - // Feed the object ids on stdin, but always reap the child afterward even if - // the write fails or stdin is missing, so an error can't drop the Child - // unwaited and leak a zombie (#53). - let write_result: std::io::Result<()> = { - use std::io::Write as _; - match child.stdin.take() { - Some(mut stdin) => { - let mut data = keep.join("\n").into_bytes(); - data.push(b'\n'); - stdin.write_all(&data) - } - None => Err(std::io::Error::new( - std::io::ErrorKind::BrokenPipe, - "git pack-objects stdin unavailable", - )), - } + Ok(keep) +} + +/// Build a packfile containing every object reachable from all refs EXCEPT the +/// given blob OIDs. Commits and trees are always included, so SHAs stay intact; +/// only the named blobs are dropped. +/// +/// The rev-list enumeration runs blocking off the async runtime; the streaming +/// `pack-objects` stage runs under [`drive_git_child`], so a hung/slow pack build is +/// duration-bounded and its process group is reaped on client disconnect. An outer +/// `tokio::time::timeout` around a `spawn_blocking` cannot cancel the blocking +/// thread, so the streaming stage MUST live on the async side (#174, KTD5). +pub async fn build_filtered_pack( + git_bin: &str, + repo_path: &Path, + withheld: &HashSet, + timeout: Duration, +) -> Result> { + let keep = { + let git_bin = git_bin.to_string(); + let repo_path = repo_path.to_path_buf(); + let withheld = withheld.clone(); + tokio::task::spawn_blocking(move || rev_list_keep(&git_bin, &repo_path, &withheld)) + .await + .context("rev-list task panicked")?? }; - let out = child.wait_with_output()?; - write_result.context("failed to write object ids to git pack-objects stdin")?; - if !out.status.success() { - bail!( - "git pack-objects failed: {}", - String::from_utf8_lossy(&out.stderr) - ); - } - Ok(out.stdout) + let mut data = keep.join("\n").into_bytes(); + data.push(b'\n'); + let mut command = Command::new(git_bin); + command + .args(["pack-objects", "--stdout"]) + .current_dir(repo_path); + drive_git_child(command, Bytes::from(data), timeout, "pack-objects").await } /// Serve a clone/fetch with the withheld blobs removed from the response pack. @@ -434,17 +439,13 @@ pub async fn upload_pack_excluding( repo_path: &Path, request_body: Bytes, withheld: &HashSet, + timeout: Duration, ) -> Result { - // build_filtered_pack shells out to git (rev-list, pack-objects) with - // blocking std::process I/O; run it off the async worker so a large repo's - // pack build does not stall the tokio runtime. - let pack = { - let repo_path = repo_path.to_path_buf(); - let withheld = withheld.clone(); - tokio::task::spawn_blocking(move || build_filtered_pack(&repo_path, &withheld)) - .await - .context("filtered-pack build task panicked")?? - }; + // The rev-list enumeration runs blocking off the runtime; the streaming + // pack-objects stage is duration-bounded and its process group is reaped on + // disconnect via drive_git_child (#174), so a hung build no longer pins its + // concurrency slot and a client disconnect no longer orphans the git child. + let pack = build_filtered_pack("git", repo_path, withheld, timeout).await?; // The client lists its capabilities on the first `want` line. Honor // side-band-64k when offered (every modern smart-HTTP client offers it); @@ -563,7 +564,9 @@ mod tests { let mut withheld = std::collections::HashSet::new(); withheld.insert(secret.clone()); - let pack = build_filtered_pack(&bare, &withheld).unwrap(); + let pack = build_filtered_pack("git", &bare, &withheld, Duration::from_secs(30)) + .await + .unwrap(); let ids = pack_object_ids(&pack); assert!(ids.contains(&public), "public blob must be in the pack"); assert!( @@ -625,7 +628,9 @@ mod tests { b"0098want 0000000000000000000000000000000000000000 \ side-band-64k ofs-delta agent=git/2\n00000009done\n", ); - let resp = upload_pack_excluding(&bare, req, &withheld).await.unwrap(); + let resp = upload_pack_excluding(&bare, req, &withheld, Duration::from_secs(30)) + .await + .unwrap(); let body = to_bytes(resp.into_body(), usize::MAX).await.unwrap(); let ids = pack_object_ids(&extract_pack(&body)); assert!( @@ -684,7 +689,7 @@ mod tests { axum::extract::State(st): axum::extract::State>, body: Bytes, ) -> Response { - upload_pack_excluding(&st.repo, body, &st.withheld) + upload_pack_excluding(&st.repo, body, &st.withheld, Duration::from_secs(30)) .await .unwrap() } @@ -1284,6 +1289,44 @@ mod tests { ); } + // #174 (SC3, KTD5): the filtered-pack build's streaming pack-objects stage is + // duration-bounded on the ASYNC side. The old build ran the whole thing in a + // spawn_blocking, so an outer tokio timeout could not cancel the blocking thread + // and a disconnect orphaned the git child. A fake git that returns objects fast + // on rev-list but hangs on pack-objects must now abort with GitServiceTimeout; + // the watchdog turns a missing bound into a loud failure. The stage runs under + // drive_git_child, so its disconnect/group-teardown is the same code proven by + // run_git_service_tears_down_group_when_future_dropped. + #[cfg(unix)] + #[tokio::test] + async fn build_filtered_pack_times_out_a_hung_pack_objects() { + let tmp = tempfile::TempDir::new().unwrap(); + // rev-list returns one oid fast; pack-objects hangs forever. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) echo deadbeefdeadbeefdeadbeefdeadbeefdeadbeef ;;\n pack-objects) sleep 300 ;;\n *) exit 1 ;;\nesac\n"; + let git_bin = write_fake_git(tmp.path(), body); + let withheld = HashSet::new(); + + let result = tokio::time::timeout( + Duration::from_secs(10), + build_filtered_pack( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_millis(200), + ), + ) + .await + .expect( + "build_filtered_pack must return within the watchdog — the pack-objects \ + stage must be timeout-bounded, not an uncancellable spawn_blocking", + ); + let err = result.expect_err("a hung pack-objects must return an error, not hang"); + assert!( + err.downcast_ref::().is_some(), + "a hung pack-objects must abort with GitServiceTimeout, got: {err}" + ); + } + // A request that runs to completion must DISARM the guard after reaping, so // no stray group SIGTERM fires. The fake exits non-zero (surfacing as Err) // but leaves a grandchild alive; the grandchild must survive. Goes RED if the From 6336657efcdaf3babc8234224e78e5867d2ebf25 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 10 Jul 2026 13:20:18 -0500 Subject: [PATCH 007/124] docs(node,config): reconcile the concurrency-cap comments with the fix (#62) The max_concurrent_git_ops and git_service_timeout_secs doc-comments (and .env.example) described the info/refs and withheld-blob paths as unbounded follow-up gaps. Both are now closed (#174): the comments reflect the read/write pool split, the per-caller sub-cap, and that every capped path is duration-bounded with process-group teardown. Verified the pattern-doc pre-ship checklist: every git_permit / write-permit / per-caller site holds only a timeout+teardown git path, and all three size knobs are range(1..=1_048_576). Closes the #174 work. No behavior change. --- .env.example | 4 ++-- crates/gitlawb-node/src/config.rs | 32 +++++++++++++++---------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.env.example b/.env.example index d7c6a30b..69865920 100644 --- a/.env.example +++ b/.env.example @@ -109,8 +109,8 @@ GITLAWB_MAX_PACK_BYTES=2147483648 # Max seconds a served git upload-pack / receive-pack (clone / push) may run # before it is aborted with a 504. Bounds a hung git that would otherwise pin a -# worker and, on push, the repo write lock. Does NOT cover the info/refs -# advertisement or the withheld-blob path, which remain unbounded. Default 600. +# worker and, on push, the repo write lock. Also bounds the info/refs +# advertisement and the withheld-blob pack build (#174). Default 600. GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600 # Max concurrent git-receive-pack (push) operations, in a pool separate from the diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index ff2c2556..93c49449 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -172,9 +172,9 @@ pub struct Config { /// its process group torn down, in seconds. Bounds a git that neither /// finishes nor disconnects. Must be positive; set it very large to /// effectively disable the bound. Default: 600s (10 min), generous for large - /// clones. Does not cover the ref advertisement (`info/refs`) or the - /// withheld-blob fetch path (`upload_pack_excluding`, a blocking - /// `spawn_blocking` a tokio timeout cannot cancel); both remain unbounded. + /// clones. Also bounds the ref advertisement (`info/refs`) and the withheld-blob + /// pack build (`upload_pack_excluding`'s pack-objects stage), which now share the + /// same timeout + process-group teardown (#174). #[arg( long, env = "GITLAWB_GIT_SERVICE_TIMEOUT_SECS", @@ -243,20 +243,20 @@ pub struct Config { /// cap is a different axis (500 connections each fan out to git + /// pack-objects + threads). Size below the process budget with headroom. /// - /// A permit is held for the whole op. upload-pack/receive-pack are - /// duration-bounded by `git_service_timeout_secs`, but the `info/refs` - /// advertisement and the withheld-blob (`upload_pack_excluding`) path are - /// not, so a hung git on those two paths holds its slot until it exits, so a - /// stuck advertisement permanently costs one unit of capacity. Bounding - /// those paths' duration is tracked as separate follow-up. + /// This is the READ pool (`git_read_semaphore`): upload-pack and both info/refs + /// advertisements. Authenticated pushes draw from a separate write pool + /// (`max_concurrent_git_pushes`) and each caller is additionally bounded by + /// `max_concurrent_reads_per_caller`, so an anonymous flood can neither shed a + /// push nor monopolize reads (#174). /// - /// The same two paths have the mirror gap on cancellation: the permit frees - /// when the handler future drops (client disconnect), but neither reaps its - /// git child — `info/refs` runs a bare `Command`, and `upload_pack_excluding` - /// a blocking `spawn_blocking` a dropped future cannot cancel — so the child - /// runs to completion after its slot is freed and live git can briefly exceed - /// the cap. The main pack path (`run_git_service`) tears its process group - /// down on drop, so this is confined to the same two follow-up paths. + /// A permit is held for the whole op, and every capped path is now + /// duration-bounded and reaps its process group on disconnect: upload-pack, + /// receive-pack, and both info/refs advertisements run under + /// `git_service_timeout_secs` with `process_group(0)` teardown, and the + /// withheld-blob (`upload_pack_excluding`) pack-objects stage runs on the async + /// side under the same teardown. So a hung git can no longer pin a slot and a + /// client disconnect no longer orphans its git child (#174 closed the two + /// duration/cancellation gaps this comment previously tracked as follow-up). /// /// Default: 128. Must be between 1 and 1_048_576; the ceiling keeps the value /// well under tokio's `Semaphore` permit limit so an oversized value is a From 5069cd13ca18eeaf7f6be59ace288fa30ca35472 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 10 Jul 2026 13:45:38 -0500 Subject: [PATCH 008/124] fix(node,concurrency): resolve #174 code-review findings Review of the served-git concurrency cap found no P0/P1; these are the verified P2 follow-ups. config: the max_concurrent_git_ops doc overclaimed that "every capped path is duration-bounded." The rev-list object enumeration in the withheld-blob path still runs in an uncancellable spawn_blocking, so a stuck rev-list can hold its slot until git exits. Scope the guarantee to the streaming stages and name the residual. Also tighten the fairness claim: the receive-pack advertisement shares the read pool (a shed advertisement is a cheap retryable GET); only the push POST is on the isolated write pool. api/repos: add info_refs_per_caller_cap_keys_on_did_not_ip, the missing handler proof that a signed caller is keyed by its DID, not its source IP. Filling the DID slot sheds a request from a free IP; collapsing read_caller_key to its IP arm turns the assertion green-not-503 (mutation-verified RED). api/repos: extract acquire_read_caller_permit so both read handlers share one shed path instead of a duplicated match block. rate_limit: recover from a poisoned PerCallerConcurrency mutex instead of panicking. The critical section is pure counter arithmetic and cannot poison the lock, but a panic there would brick the limiter for every caller. 505 tests pass; clippy -D warnings and fmt clean. Part of #174. --- crates/gitlawb-node/src/api/repos.rs | 127 +++++++++++++++++++++----- crates/gitlawb-node/src/config.rs | 20 ++-- crates/gitlawb-node/src/rate_limit.rs | 9 +- 3 files changed, 122 insertions(+), 34 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index dba8ea71..f739c04b 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -575,18 +575,12 @@ pub async fn git_info_refs( peer, state.push_limiter_trust, ); - let _caller_permit = match &caller_key { - Some(k) => match state.git_read_per_caller.try_acquire(k) { - Some(p) => Some(p), - None => { - tracing::warn!(repo = %name, caller = %k, "per-caller read cap reached; shedding info/refs with 503"); - return Err(AppError::Overloaded( - "git read service at capacity for this caller, retry shortly".into(), - )); - } - }, - None => None, - }; + let _caller_permit = acquire_read_caller_permit( + &state.git_read_per_caller, + caller_key.as_deref(), + name, + "info/refs", + )?; // For receive-pack (push), download the latest from Tigris so the client // sees the same refs that acquire_write() will operate on. @@ -656,6 +650,30 @@ fn read_caller_key( } } +/// Acquire the per-caller read sub-cap permit (#174), or shed with a 503. `key` is +/// `None` when no caller key resolves — that request is bounded by the global read +/// pool only and is never shed here (returns `Ok(None)`). `handler` labels the shed +/// log line. Shared by both read handlers so the two acquire sites cannot drift. +fn acquire_read_caller_permit( + limiter: &crate::rate_limit::PerCallerConcurrency, + key: Option<&str>, + repo: &str, + handler: &str, +) -> Result> { + match key { + Some(k) => match limiter.try_acquire(k) { + Some(p) => Ok(Some(p)), + None => { + tracing::warn!(repo = %repo, caller = %k, handler, "per-caller read cap reached; shedding with 503"); + Err(AppError::Overloaded( + "git read service at capacity for this caller, retry shortly".into(), + )) + } + }, + None => Ok(None), + } +} + /// Map an error from a `smart_http` git service call to the right `AppError`: /// [`smart_http::GitServiceTimeout`] to 504, a malformed client request to 400, /// anything else to a 500 git error. Pure (no logging) so it is unit-testable; @@ -712,18 +730,12 @@ pub async fn git_upload_pack( // visibility-denied caller never consumes a scarce read slot. Keyed per-DID // when signed, else per-source-IP; no resolvable key -> global read pool only. let caller_key = read_caller_key(caller, &headers, peer, state.push_limiter_trust); - let _caller_permit = match &caller_key { - Some(k) => match state.git_read_per_caller.try_acquire(k) { - Some(p) => Some(p), - None => { - tracing::warn!(repo = %name, caller = %k, "per-caller read cap reached; shedding upload-pack with 503"); - return Err(AppError::Overloaded( - "git read service at capacity for this caller, retry shortly".into(), - )); - } - }, - None => None, - }; + let _caller_permit = acquire_read_caller_permit( + &state.git_read_per_caller, + caller_key.as_deref(), + name, + "upload-pack", + )?; let disk_path = state .repo_store @@ -2798,6 +2810,73 @@ mod tests { ); } + /// #174 SC2 (per-DID key): a SIGNED caller is keyed by its DID, not its source + /// IP. Fill the DID's single read slot, then drive a request carrying that + /// `AuthenticatedDid` from an IP whose own slot is free — it must shed 503, + /// proving the key is the DID. A second request from the SAME IP but a DIFFERENT + /// DID is not shed. Collapse `read_caller_key` to its IP arm and the first + /// assertion goes green-not-503 (the IP slot is free) — this is the per-DID half + /// of the keying proof that the per-IP probes above do not cover. + #[sqlx::test] + async fn info_refs_per_caller_cap_keys_on_did_not_ip(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6pcdid", "pc", "/tmp/pc-nonexistent", None, false) + .await + .unwrap(); + + let did_a = "did:key:z6MkPerCallerKeyingProofDidAAAAAAAAAAAAAAAA"; + let did_b = "did:key:z6MkPerCallerKeyingProofDidBBBBBBBBBBBBBBBB"; + let peer: SocketAddr = "203.0.113.51:5000".parse().unwrap(); + + // Fill DID_A's single read slot; the IP's own slot is left untouched. + let _slot = state + .git_read_per_caller + .try_acquire(did_a) + .expect("first slot for this DID"); + + // Signed as DID_A, from `peer` (whose IP slot is free): keyed by DID -> shed. + let router = crate::server::build_router(state.clone()); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6pcdid/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req.extensions_mut() + .insert(crate::auth::AuthenticatedDid(did_a.to_string())); + assert_eq!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a signed caller at its per-DID read cap must shed with 503 (keyed by DID, not the free IP slot)" + ); + + // Same IP, DIFFERENT DID -> its own budget -> not shed by DID_A's saturation. + let router2 = crate::server::build_router(state.clone()); + let mut req2 = Request::builder() + .method(Method::GET) + .uri("/z6pcdid/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req2.extensions_mut().insert(ConnectInfo(peer)); + req2.extensions_mut() + .insert(crate::auth::AuthenticatedDid(did_b.to_string())); + assert_ne!( + router2.oneshot(req2).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a different DID from the same IP must not be shed by another DID's saturated budget" + ); + } + /// #174 SC2 (None-key): a request with no resolvable caller key (no ConnectInfo, /// no trusted header) must NOT be shed by the per-caller cap even when another /// caller's budget is full — it is bounded by the global read pool only. A None diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 93c49449..90a061e0 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -244,19 +244,23 @@ pub struct Config { /// pack-objects + threads). Size below the process budget with headroom. /// /// This is the READ pool (`git_read_semaphore`): upload-pack and both info/refs - /// advertisements. Authenticated pushes draw from a separate write pool - /// (`max_concurrent_git_pushes`) and each caller is additionally bounded by - /// `max_concurrent_reads_per_caller`, so an anonymous flood can neither shed a - /// push nor monopolize reads (#174). + /// advertisements. The authenticated push POST draws from a separate write pool + /// (`max_concurrent_git_pushes`) that anonymous reads can never reach, and each + /// caller is additionally bounded by `max_concurrent_reads_per_caller`, so an + /// anonymous flood cannot shed the actual push nor monopolize reads (#174). (The + /// receive-pack advertisement itself shares the read pool; a shed advertisement + /// is a cheap retryable GET, and the write POST it precedes always has capacity.) /// - /// A permit is held for the whole op, and every capped path is now + /// A permit is held for the whole op. Every git subprocess that STREAMS is /// duration-bounded and reaps its process group on disconnect: upload-pack, /// receive-pack, and both info/refs advertisements run under /// `git_service_timeout_secs` with `process_group(0)` teardown, and the /// withheld-blob (`upload_pack_excluding`) pack-objects stage runs on the async - /// side under the same teardown. So a hung git can no longer pin a slot and a - /// client disconnect no longer orphans its git child (#174 closed the two - /// duration/cancellation gaps this comment previously tracked as follow-up). + /// side under the same teardown (#174 closed the two duration/cancellation gaps + /// this comment previously tracked). The one remaining blocking stage is the + /// `rev-list` object enumeration in the withheld-blob path — a bounded walk that + /// terminates, run off the async runtime; it is not process-group-reaped on + /// disconnect, so a stuck rev-list can still hold its slot until git exits. /// /// Default: 128. Must be between 1 and 1_048_576; the ceiling keeps the value /// well under tokio's `Semaphore` permit limit so an oversized value is a diff --git a/crates/gitlawb-node/src/rate_limit.rs b/crates/gitlawb-node/src/rate_limit.rs index d505f1bb..22281691 100644 --- a/crates/gitlawb-node/src/rate_limit.rs +++ b/crates/gitlawb-node/src/rate_limit.rs @@ -178,7 +178,12 @@ impl PerCallerConcurrency { /// `None` (shed) otherwise. Reject-before-insert: a new key at `max_keys` is /// rejected without allocating. pub fn try_acquire(&self, key: &str) -> Option { - let mut map = self.state.lock().expect("per-caller mutex poisoned"); + // Recover from a poisoned lock rather than panicking: the critical section + // is pure counter arithmetic and cannot itself panic, so a poisoned mutex + // would only ever come from an unrelated abort, and a slightly-off count + // self-heals as permits drop. A panic here would instead brick the limiter + // for every caller (each subsequent lock re-panics). + let mut map = self.state.lock().unwrap_or_else(|e| e.into_inner()); match map.get_mut(key) { Some(count) => { if *count >= self.per_caller { @@ -207,7 +212,7 @@ impl PerCallerConcurrency { impl Drop for PerCallerPermit { fn drop(&mut self) { - let mut map = self.state.lock().expect("per-caller mutex poisoned"); + let mut map = self.state.lock().unwrap_or_else(|e| e.into_inner()); if let Some(count) = map.get_mut(&self.key) { *count -= 1; if *count == 0 { From 3b1fa54b73c2af9280b89042946cb4c7b26892e4 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 10 Jul 2026 16:05:29 -0500 Subject: [PATCH 009/124] docs(config): add GITLAWB_MAX_CONCURRENT_GIT_OPS to .env.example (#174) The read-pool knob was referenced by the push and per-caller entries' comments but had no example line of its own, so operators couldn't discover it from the template. Add it with the config default (128). --- .env.example | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.env.example b/.env.example index 69865920..cbf3db3a 100644 --- a/.env.example +++ b/.env.example @@ -113,6 +113,13 @@ GITLAWB_MAX_PACK_BYTES=2147483648 # advertisement and the withheld-blob pack build (#174). Default 600. GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600 +# Max concurrent git read ops (upload-pack + info/refs advertisements) served at +# once, a global pool separate from the push pool below. Over-cap sheds a clean +# 503 + Retry-After. Anonymous reads draw from here, so pair it with +# GITLAWB_MAX_CONCURRENT_READS_PER_CALLER (below) so one caller cannot monopolize +# the pool. Default 128. +GITLAWB_MAX_CONCURRENT_GIT_OPS=128 + # Max concurrent git-receive-pack (push) operations, in a pool separate from the # read pool (GITLAWB_MAX_CONCURRENT_GIT_OPS) so anonymous reads cannot shed an # authenticated push at admission. Over-cap sheds a 503 + Retry-After. Default 32. From 362ee3462e84015913c3f57012e29a972e6c1d8d Mon Sep 17 00:00:00 2001 From: t Date: Sat, 11 Jul 2026 13:02:24 -0500 Subject: [PATCH 010/124] fix(node,git): bound the rev-list stage under drive_git_child so a hung enumeration can't pin a git permit (#174) --- crates/gitlawb-node/src/git/smart_http.rs | 108 +++++++++++++++------- 1 file changed, 77 insertions(+), 31 deletions(-) diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index 5586a2c7..206e7b27 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -6,7 +6,7 @@ use bytes::Bytes; use std::collections::HashSet; use std::path::Path; use std::process::Stdio; -use std::time::Duration; +use std::time::{Duration, Instant}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::process::Command; @@ -349,28 +349,27 @@ fn pkt_line(data: &str) -> Vec { format!("{len:04x}{data}").into_bytes() } -/// Run `rev-list --objects --all` and return the reachable object ids minus the -/// withheld blobs. Blocking (shells out to git); the caller runs it off the async -/// runtime. Kept separate from the pack-objects stage so that stage can live on the -/// async side under [`drive_git_child`] (`git_bin` is injectable for the same -/// fake-git testing reason as `run_git_service`). -fn rev_list_keep( +/// Run `rev-list --objects --all` under `timeout` and return the reachable object +/// ids minus the withheld blobs. Runs under [`drive_git_child`] (async, with the +/// `tokio::time::timeout` + `process_group(0)` + [`KillGroupOnDrop`] teardown), so a +/// hung/slow enumeration is duration-bounded and reaped on client disconnect just +/// like the pack-objects stage. A bare blocking `Command::output()` inside a +/// `spawn_blocking` is uncancellable, so a hung rev-list would pin the endpoint's +/// concurrency permit for the whole hang (#174). `git_bin` is injectable for the +/// same fake-git testing reason as `run_git_service`. +async fn rev_list_keep( git_bin: &str, repo_path: &Path, withheld: &HashSet, + timeout: Duration, ) -> Result> { - let rev = std::process::Command::new(git_bin) + let mut command = Command::new(git_bin); + command .args(["rev-list", "--objects", "--all"]) - .current_dir(repo_path) - .output()?; - if !rev.status.success() { - bail!( - "git rev-list failed: {}", - String::from_utf8_lossy(&rev.stderr) - ); - } + .current_dir(repo_path); + let stdout = drive_git_child(command, Bytes::new(), timeout, "rev-list").await?; let mut keep = Vec::new(); - for line in String::from_utf8_lossy(&rev.stdout).lines() { + for line in String::from_utf8_lossy(&stdout).lines() { let oid = line.split_whitespace().next().unwrap_or(""); if oid.is_empty() || withheld.contains(oid) { continue; @@ -384,32 +383,41 @@ fn rev_list_keep( /// given blob OIDs. Commits and trees are always included, so SHAs stay intact; /// only the named blobs are dropped. /// -/// The rev-list enumeration runs blocking off the async runtime; the streaming -/// `pack-objects` stage runs under [`drive_git_child`], so a hung/slow pack build is -/// duration-bounded and its process group is reaped on client disconnect. An outer -/// `tokio::time::timeout` around a `spawn_blocking` cannot cancel the blocking -/// thread, so the streaming stage MUST live on the async side (#174, KTD5). +/// Both git stages — the `rev-list` enumeration and the streaming `pack-objects` +/// build — run under [`drive_git_child`] sharing one deadline, so a hung/slow git at +/// either stage is duration-bounded and its process group is reaped on client +/// disconnect. An outer `tokio::time::timeout` around a `spawn_blocking` cannot +/// cancel the blocking thread, so neither stage may live off the async side +/// (#174, KTD5). pub async fn build_filtered_pack( git_bin: &str, repo_path: &Path, withheld: &HashSet, timeout: Duration, ) -> Result> { - let keep = { - let git_bin = git_bin.to_string(); - let repo_path = repo_path.to_path_buf(); - let withheld = withheld.clone(); - tokio::task::spawn_blocking(move || rev_list_keep(&git_bin, &repo_path, &withheld)) - .await - .context("rev-list task panicked")?? - }; + // One deadline spans both git stages so a slow rev-list eats into the pack + // budget rather than granting each stage a fresh `timeout` (2x the permit hold). + let deadline = Instant::now() + timeout; + let keep = rev_list_keep( + git_bin, + repo_path, + withheld, + deadline.saturating_duration_since(Instant::now()), + ) + .await?; let mut data = keep.join("\n").into_bytes(); data.push(b'\n'); let mut command = Command::new(git_bin); command .args(["pack-objects", "--stdout"]) .current_dir(repo_path); - drive_git_child(command, Bytes::from(data), timeout, "pack-objects").await + drive_git_child( + command, + Bytes::from(data), + deadline.saturating_duration_since(Instant::now()), + "pack-objects", + ) + .await } /// Serve a clone/fetch with the withheld blobs removed from the response pack. @@ -1327,6 +1335,44 @@ mod tests { ); } + // #174 (SC3, KTD5): the filtered-pack build's rev-list ENUMERATION stage is now + // duration-bounded on the ASYNC side too. It previously ran inside a + // spawn_blocking via a bare `Command::output()`, which an outer tokio timeout + // cannot cancel, so a hung/slow rev-list pinned the endpoint's concurrency permit + // for the whole hang. A fake git that hangs on rev-list must now abort with + // GitServiceTimeout; the watchdog turns a missing bound into a loud failure. + // rev-list runs under drive_git_child, so its disconnect/group-teardown is the + // same code proven by run_git_service_tears_down_group_when_future_dropped. + #[cfg(unix)] + #[tokio::test] + async fn build_filtered_pack_times_out_a_hung_rev_list() { + let tmp = tempfile::TempDir::new().unwrap(); + // rev-list hangs forever; pack-objects would return fast if it were reached. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) sleep 300 ;;\n pack-objects) printf '' ;;\n *) exit 1 ;;\nesac\n"; + let git_bin = write_fake_git(tmp.path(), body); + let withheld = HashSet::new(); + + let result = tokio::time::timeout( + Duration::from_secs(10), + build_filtered_pack( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_millis(200), + ), + ) + .await + .expect( + "build_filtered_pack must return within the watchdog — the rev-list \ + stage must be timeout-bounded, not an uncancellable spawn_blocking", + ); + let err = result.expect_err("a hung rev-list must return an error, not hang"); + assert!( + err.downcast_ref::().is_some(), + "a hung rev-list must abort with GitServiceTimeout, got: {err}" + ); + } + // A request that runs to completion must DISARM the guard after reaping, so // no stray group SIGTERM fires. The fake exits non-zero (surfacing as Err) // but leaves a grandchild alive; the grandchild must survive. Goes RED if the From 4983b854d1a25c43354d08f420490555e4bce728 Mon Sep 17 00:00:00 2001 From: t Date: Sat, 11 Jul 2026 23:44:13 -0500 Subject: [PATCH 011/124] fix(node): key the per-caller read cap on source IP, not the signed DID (#174) read_caller_key returned the authenticated DID when a caller signed, dropping the source-IP key. Public read routes accept any valid did:key via optional_signature with no admission step, so one host could mint N disposable DIDs and hold max_concurrent_reads_per_caller slots under each, multiplying its budget N-fold and filling the global read pool, while the same host unsigned was capped on its IP. Key the read sub-cap on the resolved source IP for every caller, signed or not, mirroring the push path's IpRateLimiter which already throttles on source IP for this exact DID-farm reason. Drops the now-unused caller_did parameter at both call sites (git_info_refs, git_upload_pack). Inverts info_refs_per_caller_cap_keys_on_did_not_ip into info_refs_per_caller_cap_keys_on_ip_not_did: fill one source IP's slot, then two requests signed under different DIDs from that same IP both shed 503 (farm defeated), while a signed request from a different IP keeps its own budget. RED on the DID-keyed tree, GREEN after. --- crates/gitlawb-node/src/api/repos.rs | 91 ++++++++++++++++------------ 1 file changed, 53 insertions(+), 38 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index f739c04b..e95910ef 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -569,12 +569,7 @@ pub async fn git_info_refs( // Per-caller read sub-cap (#174): acquired AFTER the visibility + push-rate // gates (KTD7) so a denied or rate-limited request never consumes a caller's // scarce read slot. Applies to BOTH advertisements. Held for the whole op. - let caller_key = read_caller_key( - auth.as_ref().map(|e| e.0 .0.as_str()), - &headers, - peer, - state.push_limiter_trust, - ); + let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); let _caller_permit = acquire_read_caller_permit( &state.git_read_per_caller, caller_key.as_deref(), @@ -633,21 +628,21 @@ fn git_permit( }) } -/// Resolve the per-caller key for the read sub-cap (#174): the authenticated DID -/// when the caller signed (via `optional_signature`), else the trusted client IP -/// (`client_key`). `None` when neither is available — such a request is bounded by -/// the global read pool only, never a 500. The per-source-IP key is only as -/// granular as `trust`; see the `max_concurrent_reads_per_caller` config doc. +/// Resolve the per-caller key for the read sub-cap (#174): always the resolved +/// source IP (`client_key`), never the signed DID. Public read routes accept any +/// valid `did:key` via `optional_signature` with no admission step, so keying on +/// the DID would let one host mint disposable DIDs to multiply its per-source +/// budget; the push path already throttles on the resolved source IP for exactly +/// this DID-farm reason (`rate_limit.rs`, `IpRateLimiter`). `None` when no key +/// resolves (no trusted header and no peer): such a request is bounded by the +/// global read pool only, never a 500. The per-source-IP key is only as granular +/// as `trust`; see the `max_concurrent_reads_per_caller` config doc. fn read_caller_key( - caller_did: Option<&str>, headers: &axum::http::HeaderMap, peer: Option, trust: crate::rate_limit::TrustedProxy, ) -> Option { - match caller_did { - Some(did) => Some(did.to_string()), - None => crate::rate_limit::client_key(headers, peer, trust), - } + crate::rate_limit::client_key(headers, peer, trust) } /// Acquire the per-caller read sub-cap permit (#174), or shed with a 503. `key` is @@ -727,9 +722,10 @@ pub async fn git_upload_pack( } // Per-caller read sub-cap (#174): after the visibility gate (KTD7) so a - // visibility-denied caller never consumes a scarce read slot. Keyed per-DID - // when signed, else per-source-IP; no resolvable key -> global read pool only. - let caller_key = read_caller_key(caller, &headers, peer, state.push_limiter_trust); + // visibility-denied caller never consumes a scarce read slot. Keyed on the + // resolved source IP (never the signed DID, #174 U1); no resolvable key -> + // global read pool only. + let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); let _caller_permit = acquire_read_caller_permit( &state.git_read_per_caller, caller_key.as_deref(), @@ -2810,15 +2806,16 @@ mod tests { ); } - /// #174 SC2 (per-DID key): a SIGNED caller is keyed by its DID, not its source - /// IP. Fill the DID's single read slot, then drive a request carrying that - /// `AuthenticatedDid` from an IP whose own slot is free — it must shed 503, - /// proving the key is the DID. A second request from the SAME IP but a DIFFERENT - /// DID is not shed. Collapse `read_caller_key` to its IP arm and the first - /// assertion goes green-not-503 (the IP slot is free) — this is the per-DID half - /// of the keying proof that the per-IP probes above do not cover. + /// #174 SC2 (per-source key, U1): the per-caller read sub-cap keys on the + /// resolved source IP, NOT the signed DID, so a disposable-DID farm cannot + /// multiply its budget. Fill the source IP's single read slot, then drive two + /// requests signed under DIFFERENT DIDs from that SAME IP: both must shed 503 + /// (keyed by the saturated IP, not their own free DID slots). A signed request + /// from a DIFFERENT source IP keeps its own budget. Revert `read_caller_key` to + /// prefer the DID and the same-IP assertions go green-not-503 (each fresh DID + /// gets a free slot) -- the farm-defeat mutation probe. #[sqlx::test] - async fn info_refs_per_caller_cap_keys_on_did_not_ip(pool: sqlx::PgPool) { + async fn info_refs_per_caller_cap_keys_on_ip_not_did(pool: sqlx::PgPool) { use axum::body::Body; use axum::extract::ConnectInfo; use axum::http::{Method, Request, StatusCode}; @@ -2830,7 +2827,7 @@ mod tests { state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; state .db - .upsert_mirror_repo("z6pcdid", "pc", "/tmp/pc-nonexistent", None, false) + .upsert_mirror_repo("z6pcip", "pc", "/tmp/pc-nonexistent", None, false) .await .unwrap(); @@ -2838,17 +2835,17 @@ mod tests { let did_b = "did:key:z6MkPerCallerKeyingProofDidBBBBBBBBBBBBBBBB"; let peer: SocketAddr = "203.0.113.51:5000".parse().unwrap(); - // Fill DID_A's single read slot; the IP's own slot is left untouched. + // Fill the SOURCE IP's single read slot; both DIDs' own slots stay free. let _slot = state .git_read_per_caller - .try_acquire(did_a) - .expect("first slot for this DID"); + .try_acquire(&peer.ip().to_string()) + .expect("first slot for this source IP"); - // Signed as DID_A, from `peer` (whose IP slot is free): keyed by DID -> shed. + // Signed as DID_A from `peer`: keyed by the saturated source IP -> shed 503. let router = crate::server::build_router(state.clone()); let mut req = Request::builder() .method(Method::GET) - .uri("/z6pcdid/pc/info/refs?service=git-upload-pack") + .uri("/z6pcip/pc/info/refs?service=git-upload-pack") .body(Body::empty()) .unwrap(); req.extensions_mut().insert(ConnectInfo(peer)); @@ -2857,23 +2854,41 @@ mod tests { assert_eq!( router.oneshot(req).await.unwrap().status(), StatusCode::SERVICE_UNAVAILABLE, - "a signed caller at its per-DID read cap must shed with 503 (keyed by DID, not the free IP slot)" + "a signed caller must be keyed by its source IP, not its DID: the saturated IP must shed it 503" ); - // Same IP, DIFFERENT DID -> its own budget -> not shed by DID_A's saturation. + // Same IP, a DIFFERENT DID: still keyed by the same saturated IP -> also shed. + // The farm defeat: minting a fresh DID buys no fresh per-source budget. let router2 = crate::server::build_router(state.clone()); let mut req2 = Request::builder() .method(Method::GET) - .uri("/z6pcdid/pc/info/refs?service=git-upload-pack") + .uri("/z6pcip/pc/info/refs?service=git-upload-pack") .body(Body::empty()) .unwrap(); req2.extensions_mut().insert(ConnectInfo(peer)); req2.extensions_mut() .insert(crate::auth::AuthenticatedDid(did_b.to_string())); - assert_ne!( + assert_eq!( router2.oneshot(req2).await.unwrap().status(), StatusCode::SERVICE_UNAVAILABLE, - "a different DID from the same IP must not be shed by another DID's saturated budget" + "a second DID from the same source IP must also shed 503: a DID farm cannot multiply the per-source budget" + ); + + // A signed caller from a DIFFERENT source IP keeps its own budget -> not shed. + let other: SocketAddr = "203.0.113.52:5000".parse().unwrap(); + let router3 = crate::server::build_router(state.clone()); + let mut req3 = Request::builder() + .method(Method::GET) + .uri("/z6pcip/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req3.extensions_mut().insert(ConnectInfo(other)); + req3.extensions_mut() + .insert(crate::auth::AuthenticatedDid(did_a.to_string())); + assert_ne!( + router3.oneshot(req3).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a signed caller from a different source IP must keep its own per-source budget" ); } From e444bc0ae5f3813c93c7e57b147c6ff974f5f497 Mon Sep 17 00:00:00 2001 From: t Date: Sat, 11 Jul 2026 23:54:21 -0500 Subject: [PATCH 012/124] fix(node): draw the receive-pack advertisement from the write pool (#174) git_info_refs acquired git_read_semaphore for BOTH services, so the push handshake (GET /info/refs?service=git-receive-pack) competed in the global read pool. An anonymous clone flood could exhaust that pool and shed a legitimate push with 503 during its required advertisement phase, before it ever reached git_write_semaphore on the POST. The write pool exists precisely so anonymous reads cannot shed an authenticated push, but only the POST drew from it. Select the pool by service: the receive-pack advertisement (phase one of a push) now draws from git_write_semaphore, like the git-receive-pack POST, so a saturated read pool cannot starve it. The per-IP push_rate_limiter that already brakes the advertisement stays as the anti-flood control, and the advertisement stays reader-visible with no new auth requirement. Because the receive-pack branch is now a write-path op, it no longer consumes a read per-caller slot. Handler-layer proofs: with the read pool at zero the receive-pack advertisement survives while the upload-pack advertisement sheds; with the write pool at zero the receive-pack advertisement sheds while upload-pack is unaffected; and a receive-pack advertisement from an IP whose read per-caller budget is full still gets through (mutation-checked, RED when the skip is neutralized). --- crates/gitlawb-node/src/api/repos.rs | 164 ++++++++++++++++++++++++--- 1 file changed, 148 insertions(+), 16 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index e95910ef..3be94bfc 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -507,10 +507,21 @@ pub async fn git_info_refs( headers: axum::http::HeaderMap, auth: Option>, ) -> Result { - // Shed with a 503 before spawning git when the concurrency cap is saturated; - // held for the whole op (incl. the smart_http call), released on return. - let _permit = git_permit(&state.git_read_semaphore)?; let name = smart_http_repo_name(&repo)?; + let service = query + .service + .ok_or_else(|| AppError::BadRequest("missing ?service= parameter".into()))?; + // Shed with a 503 before spawning git when the concurrency cap is saturated; + // held for the whole op (incl. the smart_http call), released on return. The + // receive-pack advertisement is phase one of a push, so it draws from the WRITE + // pool (like the git-receive-pack POST), not the global read pool an anonymous + // clone flood can exhaust and thereby starve the push handshake (#174 U2). The + // upload-pack advertisement stays on the read pool with its per-caller sub-cap. + let _permit = if service == "git-receive-pack" { + git_permit(&state.git_write_semaphore)? + } else { + git_permit(&state.git_read_semaphore)? + }; tracing::info!(owner = %owner, repo = %name, "info/refs request"); let record = state .db @@ -524,9 +535,6 @@ pub async fn git_info_refs( return Err(AppError::RepoNotFound(format!("{owner}/{name}"))); } - let service = query - .service - .ok_or_else(|| AppError::BadRequest("missing ?service= parameter".into()))?; tracing::debug!(service = %service, repo = %name, "info/refs service"); // Enforce read visibility on the ref advertisement, for BOTH services. The @@ -566,16 +574,23 @@ pub async fn git_info_refs( } } - // Per-caller read sub-cap (#174): acquired AFTER the visibility + push-rate - // gates (KTD7) so a denied or rate-limited request never consumes a caller's - // scarce read slot. Applies to BOTH advertisements. Held for the whole op. - let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); - let _caller_permit = acquire_read_caller_permit( - &state.git_read_per_caller, - caller_key.as_deref(), - name, - "info/refs", - )?; + // Per-caller read sub-cap (#174): upload-pack advertisement only. The + // receive-pack advertisement is a write-path op (it drew from git_write_semaphore + // above and is braked per-IP by push_rate_limiter), so it must NOT consume a + // read caller's scarce slot (#174 U2). Acquired AFTER the visibility + push-rate + // gates (KTD7) so a denied or rate-limited request never consumes a slot. Held + // for the whole op. + let _caller_permit = if service == "git-receive-pack" { + None + } else { + let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); + acquire_read_caller_permit( + &state.git_read_per_caller, + caller_key.as_deref(), + name, + "info/refs", + )? + }; // For receive-pack (push), download the latest from Tigris so the client // sees the same refs that acquire_write() will operate on. @@ -2693,6 +2708,123 @@ mod tests { ); } + /// #174 U2: the receive-pack advertisement (`GET info/refs?service=git-receive-pack`) + /// is phase one of a push, so it draws from the WRITE pool, not the global read + /// pool an anonymous clone flood can exhaust. Proven at the handler by saturating + /// each pool to zero and checking who shares it (INV-10). Revert the branch to the + /// read pool and the read-saturated receive-pack assertion goes 503 (the exact + /// push-handshake starvation jatmn flagged). + #[sqlx::test] + async fn receive_pack_advertisement_draws_from_write_pool(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + // Build a fresh state with the two pools sized independently, then drive one + // info/refs advertisement for `service` and return its handler status. + async fn advert_status( + pool: &sqlx::PgPool, + read_permits: usize, + write_permits: usize, + service: &str, + ) -> StatusCode { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.git_read_semaphore = Arc::new(Semaphore::new(read_permits)); + state.git_write_semaphore = Arc::new(Semaphore::new(write_permits)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6wpadv", "wp", "/tmp/wp-nonexistent", None, false) + .await + .unwrap(); + let peer: SocketAddr = "203.0.113.61:6000".parse().unwrap(); + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/z6wpadv/wp/info/refs?service={service}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + router.oneshot(req).await.unwrap().status() + } + + // Read pool saturated, write pool free: the push handshake SURVIVES. + assert_ne!( + advert_status(&pool, 0, 8, "git-receive-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "receive-pack advertisement must draw from the write pool: a saturated read pool must not shed it" + ); + // Write pool saturated, read pool free: the receive-pack advertisement SHEDS, + // proving it now consumes the write pool (not the read pool). + assert_eq!( + advert_status(&pool, 8, 0, "git-receive-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "receive-pack advertisement draws from the write pool, so a saturated write pool sheds it 503" + ); + // Read pool saturated: the upload-pack advertisement still SHEDS (unchanged). + assert_eq!( + advert_status(&pool, 0, 8, "git-upload-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "upload-pack advertisement stays on the read pool: a saturated read pool sheds it 503" + ); + // Write pool saturated, read pool free: the upload-pack advertisement is + // UNAFFECTED, proving reads never touch the write pool in either direction. + assert_ne!( + advert_status(&pool, 8, 0, "git-upload-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "upload-pack advertisement never touches the write pool" + ); + } + + /// #174 U2: the receive-pack advertisement is a write-path op, so it must not be + /// shed by the READ per-caller sub-cap even when the caller's source IP has + /// exhausted its read budget (e.g. concurrent clones from the same host). Fill + /// the IP's read per-caller slot, then the receive-pack advertisement from that + /// same IP must still get through. Restore the unconditional read-cap acquire on + /// the receive-pack branch and this goes 503. + #[sqlx::test] + async fn receive_pack_advertisement_ignores_read_per_caller_cap(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6wpc", "wp", "/tmp/wp-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.71:6000".parse().unwrap(); + // Exhaust the source IP's single READ per-caller slot, as concurrent clones + // from the same host would. + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("fill the IP's read per-caller slot"); + + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6wpc/wp/info/refs?service=git-receive-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + assert_ne!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "receive-pack advertisement must not be shed by the read per-caller cap: it is a write-path op" + ); + } + /// #174 SC2 (info_refs probe): the per-caller read sub-cap sheds a caller that /// is already at its concurrency budget on the upload-pack advertisement, while /// a DIFFERENT caller still enters. Remove the sub-cap from `git_info_refs` and From 71389e621a233a30c97f959b9d5ccc5c9f1bd57b Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 00:14:39 -0500 Subject: [PATCH 013/124] fix(node): bound the withheld-blob walk at the shared blob_paths seam (#174) The withheld-blob classification walk (blob_paths) fanned out blocking git children with no deadline and no process-group teardown: git for-each-ref, git cat-file, git rev-list, a git ls-tree per commit, and an uncounted git rev-parse (via store::head_commit). A hung or pathologically slow child pinned the caller's served-git permit for the whole hang, and on client disconnect the spawn_blocking task and its git children ran on, orphaned. blob_paths is the shared core of five callers: the upload-pack serve path (holds a read permit) AND, inside git_receive_pack, the post-push replication and encrypt-then-pin walks (hold the write permit U2 reserves for pushes). So the same unbounded walk could pin either pool, and leaving the write-side twin unbounded would have made U2's reservation a claim that does not match behavior. Bound every git child at the blob_paths spawn seam on the blocking side: each child runs in its own process group with a watchdog thread that SIGTERMs (then SIGKILLs) the group on one shared deadline spanning the whole walk, and retains admission until the group is reaped. This is the blocking-side counterpart of smart_http::drive_git_child (spawn_blocking cannot be cancelled by an async timeout). blob_paths stays sync, so all five callers keep their signatures and the 32 classification tests are unchanged; because every caller funnels through blob_paths, one seam bounds both the serve and replication paths. The previously unbounded store::head_commit child becomes a bounded git rev-parse inside the walk. A walk that hits its deadline carries GitServiceTimeout, which the serve handler now maps to 504 rather than a generic 500. Proof: a fake git that hangs on rev-list makes blob_paths return GitServiceTimeout within the watchdog budget (not block on the child) and the recorded process-group leader is reaped, not orphaned; neutralizing the watchdog kill makes it hang past the budget (RED). The 32 real-git classification tests stay green through the refactor, including detached-HEAD, non-standard-ref, and deleted-in-history cases. --- crates/gitlawb-node/src/api/repos.rs | 4 +- .../gitlawb-node/src/git/visibility_pack.rs | 333 +++++++++++++----- 2 files changed, 242 insertions(+), 95 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 3be94bfc..7a8aa5c2 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -781,7 +781,9 @@ pub async fn git_upload_pack( }) .await .map_err(|e| AppError::Git(e.to_string()))? - .map_err(|e| AppError::Git(e.to_string()))? + // A walk that hit its deadline carries GitServiceTimeout; map it to 504 + // like the smart_http paths, not a generic 500 (#174 U3). + .map_err(|e| git_service_app_error(&e))? }; if withheld.is_empty() { diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index cb70e39c..7b9bda69 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -4,11 +4,122 @@ //! content is held back. use crate::db::VisibilityRule; -use crate::git::store; use crate::visibility::{visibility_check, Decision}; use anyhow::{Context, Result}; use std::collections::{BTreeSet, HashMap, HashSet}; use std::path::Path; +use std::process::Stdio; +use std::sync::mpsc; +use std::time::{Duration, Instant}; + +/// Fixed budget bounding the whole withheld-blob classification walk (#174 U3). +/// The walk is fast for a real repo; this bound exists to reap a hung or +/// pathologically slow git child so it cannot pin a served-git permit (the read +/// permit on the upload-pack serve path, the write permit on the receive-pack +/// post-push replication path) past the deadline. Every caller funnels through +/// `blob_paths`, so bounding here bounds both paths at one seam. +const WALK_TIMEOUT: Duration = Duration::from_secs(600); + +/// Run one git child under a shared `deadline` with process-group teardown, +/// BLOCKING, and return its stdout. The child runs in its own process group; a +/// watchdog thread SIGTERMs (lets git clean up its `*.lock` files), then SIGKILLs, +/// the whole group if the deadline passes before the child is reaped, so a hung or +/// slow git can pin neither a served-git permit nor a blocking thread past the +/// deadline (jatmn's "retain admission until they are reaped"). This is the +/// blocking-side counterpart of `smart_http::drive_git_child`, needed because the +/// walk's callers run it inside `spawn_blocking`, which an async timeout cannot +/// cancel. Returns [`crate::git::smart_http::GitServiceTimeout`] on the deadline so +/// the serve handler maps it to 504. `git_bin` is injectable so a fake `git` can +/// drive the teardown in tests without mutating the process-global PATH; +/// `stdin_bytes` feeds children that read stdin (empty for the arg-only children). +#[cfg(unix)] +fn run_bounded_git( + git_bin: &str, + args: &[&str], + repo_path: &Path, + stdin_bytes: &[u8], + deadline: Instant, +) -> Result> { + use std::io::{Read, Write}; + use std::os::unix::process::CommandExt; + use std::sync::mpsc::RecvTimeoutError; + + let label = args.first().copied().unwrap_or("git"); + let mut child = std::process::Command::new(git_bin) + .args(args) + .current_dir(repo_path) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .process_group(0) + .spawn() + .with_context(|| format!("failed to spawn git {label}"))?; + // With process_group(0) the child leads its own group, so pgid == its pid. + let pgid = child.id() as i32; + + // Watchdog: on the deadline, SIGTERM the whole group, poll until it exits, + // escalate to SIGKILL after a grace, and report whether it killed. Signalled to + // stand down when the child is reaped in time. Kept off the main thread because + // the main thread's stdout drain is exactly what blocks until a hung child is + // torn down. + let (done_tx, done_rx) = mpsc::channel::<()>(); + let watchdog = std::thread::spawn(move || -> bool { + let wait = deadline.saturating_duration_since(Instant::now()); + match done_rx.recv_timeout(wait) { + Ok(()) | Err(RecvTimeoutError::Disconnected) => false, + Err(RecvTimeoutError::Timeout) => { + // SAFETY: kill(2) takes only integers and borrows no Rust memory; + // ESRCH on an already-gone group is ignored. + unsafe { libc::kill(-pgid, libc::SIGTERM) }; + for step in 0..200u32 { + if unsafe { libc::kill(-pgid, 0) } != 0 { + return true; // ESRCH: every group member has exited + } + if step == 100 { + unsafe { libc::kill(-pgid, libc::SIGKILL) }; + } + std::thread::sleep(Duration::from_millis(10)); + } + true + } + } + }); + + // Feed stdin on a writer thread and drain stderr on a reader thread so the main + // thread can drain stdout concurrently; writing all of stdin (or draining one + // pipe) before the others can deadlock once a pipe buffer fills. + let mut stdin = child.stdin.take(); + let input = stdin_bytes.to_vec(); + let writer = std::thread::spawn(move || { + if let Some(mut s) = stdin.take() { + let _ = s.write_all(&input); + } + }); + let mut stderr = child.stderr.take().context("git stderr was not piped")?; + let err_reader = std::thread::spawn(move || { + let mut err = Vec::new(); + let _ = stderr.read_to_end(&mut err); + err + }); + let mut stdout = child.stdout.take().context("git stdout was not piped")?; + let mut out = Vec::new(); + let read_result = stdout.read_to_end(&mut out); + let status = child.wait().context("git wait failed")?; + let err = err_reader.join().unwrap_or_default(); + let _ = writer.join(); + + // Reaped: stand the watchdog down and learn whether it killed on the deadline. + let _ = done_tx.send(()); + let killed = watchdog.join().unwrap_or(false); + if killed { + return Err(crate::git::smart_http::GitServiceTimeout.into()); + } + read_result.context("failed to read git stdout")?; + if !status.success() { + anyhow::bail!("git {label} failed: {}", String::from_utf8_lossy(&err)); + } + Ok(out) +} /// Fail closed unless every ref ultimately resolves to a commit (a ref pointing /// directly at a blob or tree, or an annotated tag — even a nested one — of such @@ -21,19 +132,15 @@ use std::path::Path; /// Full peeling is why this is not `for-each-ref %(*objecttype)`, which /// dereferences only one tag level and so misclassifies a tag-of-a-tag-of-a- /// commit as a non-commit. -fn assert_all_refs_are_commits(repo_path: &Path) -> Result<()> { - let refs = std::process::Command::new("git") - .args(["for-each-ref", "--format=%(refname)"]) - .current_dir(repo_path) - .output() - .context("git for-each-ref failed")?; - if !refs.status.success() { - anyhow::bail!( - "git for-each-ref failed: {}", - String::from_utf8_lossy(&refs.stderr) - ); - } - let refs_stdout = String::from_utf8_lossy(&refs.stdout); +fn assert_all_refs_are_commits(repo_path: &Path, git_bin: &str, deadline: Instant) -> Result<()> { + let refs_out = run_bounded_git( + git_bin, + &["for-each-ref", "--format=%(refname)"], + repo_path, + b"", + deadline, + )?; + let refs_stdout = String::from_utf8_lossy(&refs_out); let refnames: Vec<&str> = refs_stdout .lines() .map(str::trim) @@ -43,59 +150,25 @@ fn assert_all_refs_are_commits(repo_path: &Path) -> Result<()> { return Ok(()); } - // Peel every ref in one `git cat-file --batch-check` pass: one - // `^{}` query per line, one output line per input line, in order. - // The stdin write runs on a separate thread so this thread can drain stdout - // concurrently. cat-file echoes the full query on a ` missing` line, - // so output scales with refname length (not a fixed size per ref); writing - // all of stdin before reading any stdout would deadlock both pipes once the - // child's stdout buffer fills. Dropping `stdin` at the end of the closure - // sends EOF. + // Peel every ref in one `git cat-file --batch-check` pass: one `^{}` + // query per line, one output line per input line, in order. cat-file echoes the + // full query on a ` missing` line, so output scales with refname length; + // run_bounded_git drains stdout concurrently with the stdin write, so the pipe + // cannot deadlock, and the whole peel is bounded by the shared walk deadline. let queries = refnames .iter() .map(|r| format!("{r}^{{}}")) .collect::>() .join("\n"); - use std::io::Write; - let mut child = std::process::Command::new("git") - .args(["cat-file", "--batch-check=%(objecttype)"]) - .current_dir(repo_path) - .stdin(std::process::Stdio::piped()) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) - .spawn() - .context("failed to spawn git cat-file")?; - // Feed stdin on a writer thread so this thread can drain stdout via - // wait_with_output concurrently; a None handle (the pipe vanished) becomes a - // broken-pipe write error. wait_with_output reaps the child unconditionally - // before any error is surfaced, so no path drops it unwaited (#53), and the - // writer is joined only after the drain so the join cannot deadlock. - let writer = child - .stdin - .take() - .map(|mut stdin| std::thread::spawn(move || stdin.write_all(queries.as_bytes()))); - let peel_result = child.wait_with_output(); - let write_result = match writer { - Some(handle) => handle - .join() - .map_err(|_| anyhow::anyhow!("git cat-file stdin writer thread panicked"))?, - None => Err(std::io::Error::new( - std::io::ErrorKind::BrokenPipe, - "git cat-file stdin unavailable", - )), - }; - // Surface a write error only if the process didn't already fail with a - // clearer status. - let peel = peel_result.context("git cat-file failed")?; - if !peel.status.success() { - anyhow::bail!( - "git cat-file --batch-check failed: {}", - String::from_utf8_lossy(&peel.stderr) - ); - } - write_result.context("failed to write to git cat-file stdin")?; - - let peel_stdout = String::from_utf8_lossy(&peel.stdout); + let peel_out = run_bounded_git( + git_bin, + &["cat-file", "--batch-check=%(objecttype)"], + repo_path, + queries.as_bytes(), + deadline, + )?; + + let peel_stdout = String::from_utf8_lossy(&peel_out); let types: Vec<&str> = peel_stdout.lines().map(str::trim).collect(); // A short read means at least one ref went unclassified — fail closed. if types.len() != refnames.len() { @@ -147,47 +220,46 @@ fn assert_all_refs_are_commits(repo_path: &Path) -> Result<()> { /// Fails closed: if commit enumeration or any tree walk fails, returns an error so /// the caller aborts the serve/pin rather than producing a partial (under-withheld) /// set. -fn blob_paths(repo_path: &Path) -> Result> { - assert_all_refs_are_commits(repo_path)?; +fn blob_paths(repo_path: &Path, git_bin: &str, timeout: Duration) -> Result> { + // One deadline spans the whole walk (the ref check, the HEAD probe, rev-list, + // and every per-commit ls-tree), so a slow or hung walk is bounded as a unit + // rather than granting each git child a fresh timeout. + let deadline = Instant::now() + timeout; + assert_all_refs_are_commits(repo_path, git_bin, deadline)?; // Enumerate every reachable commit, not just ref tips. `--all` walks all refs; // append HEAD so a detached HEAD (reachable by rev-list/upload-pack but in no // ref) is still classified. When HEAD does not resolve (unborn branch on an - // empty repo) `--all` alone yields nothing, which is correct — no objects exist. - let head = store::head_commit(repo_path).context("resolve HEAD failed")?; + // empty repo) `--all` alone yields nothing, which is correct: no objects exist. + // The HEAD probe is a bounded `git rev-parse --verify HEAD` (a clean exit means + // HEAD resolves), replacing the previously unbounded `store::head_commit` child. + let head_resolves = run_bounded_git( + git_bin, + &["rev-parse", "--verify", "HEAD"], + repo_path, + b"", + deadline, + ) + .is_ok(); let mut rev_args = vec!["rev-list", "--all"]; - if head.is_some() { + if head_resolves { rev_args.push("HEAD"); } - let commits = std::process::Command::new("git") - .args(&rev_args) - .current_dir(repo_path) - .output() - .context("git rev-list --all failed")?; - if !commits.status.success() { - anyhow::bail!( - "git rev-list --all failed: {}", - String::from_utf8_lossy(&commits.stderr) - ); - } - let commits_stdout = String::from_utf8_lossy(&commits.stdout); + let commits_out = run_bounded_git(git_bin, &rev_args, repo_path, b"", deadline)?; + let commits_stdout = String::from_utf8_lossy(&commits_out); let mut out: HashSet<(String, String)> = HashSet::new(); for commit in commits_stdout.lines() { let commit = commit.trim(); if commit.is_empty() { continue; } - let listing = std::process::Command::new("git") - .args(["ls-tree", "-rz", commit]) - .current_dir(repo_path) - .output() - .context("git ls-tree -rz failed")?; - if !listing.status.success() { - anyhow::bail!( - "git ls-tree -rz {commit} failed: {}", - String::from_utf8_lossy(&listing.stderr) - ); - } + let listing_out = run_bounded_git( + git_bin, + &["ls-tree", "-rz", commit], + repo_path, + b"", + deadline, + )?; // `-z` NUL-delimits records and emits paths raw; plain `git ls-tree -r` // C-quotes any path with non-ASCII or special bytes (e.g. café.txt becomes // "secret/caf\303\251.txt"), and that quoted literal would not match a @@ -198,7 +270,7 @@ fn blob_paths(repo_path: &Path) -> Result> { // path (e.g. a non-UTF-8 directory name) with U+FFFD, and the mangled string // would no longer match its deny rule — the same under-withholding class, one // layer down. Fail closed instead so the caller aborts rather than leaks. - let Ok(listing_stdout) = std::str::from_utf8(&listing.stdout) else { + let Ok(listing_stdout) = std::str::from_utf8(&listing_out) else { anyhow::bail!( "git ls-tree -rz {commit} returned a non-UTF-8 path; \ refusing to produce a partial (under-withheld) set" @@ -236,7 +308,7 @@ pub fn withheld_blob_oids( owner_did: &str, caller: Option<&str>, ) -> Result> { - let pairs = blob_paths(repo_path)?; + let pairs = blob_paths(repo_path, "git", WALK_TIMEOUT)?; Ok(withheld_from_pairs( &pairs, rules, is_public, owner_did, caller, )) @@ -332,7 +404,7 @@ pub fn allowed_blob_set_for_caller( owner_did: &str, caller: Option<&str>, ) -> Result> { - let pairs = blob_paths(repo_path)?; + let pairs = blob_paths(repo_path, "git", WALK_TIMEOUT)?; let mut allowed = HashSet::new(); for (oid, path) in &pairs { if visibility_check(rules, is_public, owner_did, caller, path) == Decision::Allow { @@ -372,7 +444,7 @@ pub fn withheld_blob_recipients( owner_did: &str, ) -> Result>> { // One history walk feeds both the withheld set and the recipient mapping. - let pairs = blob_paths(repo_path)?; + let pairs = blob_paths(repo_path, "git", WALK_TIMEOUT)?; let withheld = withheld_from_pairs(&pairs, rules, is_public, owner_did, None); if withheld.is_empty() { return Ok(HashMap::new()); @@ -402,6 +474,79 @@ pub fn withheld_blob_recipients( #[cfg(test)] mod tests { use super::*; + + /// Write an executable fake `git` shell script into `dir` and return its path, + /// so a test can drive the walk's process-group teardown without a real git and + /// without mutating the process-global PATH (the crate's only injection seam). + #[cfg(unix)] + fn write_fake_git(dir: &Path, body: &str) -> String { + use std::os::unix::fs::PermissionsExt; + let p = dir.join("fakegit"); + std::fs::write(&p, body).unwrap(); + let mut perm = std::fs::metadata(&p).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&p, perm).unwrap(); + p.to_str().unwrap().to_string() + } + + /// #174 U3: the withheld-blob walk is bounded at the shared `blob_paths` seam, so + /// a hung git child cannot pin the caller's permit past the deadline. A fake git + /// that hangs on `rev-list` must make `blob_paths` return `GitServiceTimeout` + /// within the watchdog budget (not block for the child's lifetime), and the + /// child's process group must be reaped (its recorded leader PID gone). Every + /// caller (upload-pack serve, receive-pack replication) funnels through + /// `blob_paths`, so this seam-level proof covers both permit pools. Neutralize + /// the watchdog SIGTERM and this hangs past the recv budget (RED). + #[cfg(unix)] + #[test] + fn blob_paths_times_out_and_reaps_a_hung_walk() { + use std::time::Duration; + let tmp = TempDir::new().unwrap(); + // Fast on every stage except rev-list, which records its own (group-leader) + // PID and then hangs. `sleep 30` bounds the worst case if the watchdog is + // ever broken, so a regression cannot wedge the suite for 300s. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) echo $$ > revlist.pid ; sleep 30 ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + + // Run the walk on a thread with a short budget; the recv_timeout succeeding + // is itself proof the walk did not block on the hung child. + let (tx, rx) = mpsc::channel(); + let path = tmp.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(blob_paths(&path, &git_bin, Duration::from_millis(200))); + }); + let result = rx.recv_timeout(Duration::from_secs(10)).expect( + "blob_paths must return within the watchdog budget, not hang on a stuck git child", + ); + let err = result.expect_err("a hung rev-list must abort the walk with an error"); + assert!( + err.downcast_ref::() + .is_some(), + "a hung walk must abort with GitServiceTimeout (mapped to 504), got: {err}" + ); + + // The recorded process-group leader must be gone: the watchdog reaps the + // whole group before blob_paths returns, so no orphaned git lingers. + let pid: i32 = std::fs::read_to_string(tmp.path().join("revlist.pid")) + .expect("the fake git must have recorded its rev-list PID") + .trim() + .parse() + .expect("recorded PID must parse"); + let mut gone = false; + for _ in 0..200 { + // SAFETY: kill(2) with signal 0 only probes existence; ESRCH (-1) means + // the process is gone. Borrows no Rust memory. + if unsafe { libc::kill(pid, 0) } != 0 { + gone = true; + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + assert!( + gone, + "the hung git child (pid {pid}) must be reaped, not orphaned, after the walk aborts" + ); + } use crate::db::VisibilityMode; use chrono::Utc; use std::process::Command; From ab0ea240fcca3087bb7e3c08302885aede32627a Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 00:20:07 -0500 Subject: [PATCH 014/124] docs(node): correct the git-service-timeout coverage note (#174) GITLAWB_GIT_SERVICE_TIMEOUT_SECS bounds the info/refs advertisement too: smart_http::info_refs drives it through drive_git_child under this timeout, with a passing test proving the 504. The old note claimed it does not. It also claimed the withheld-blob path is unbounded; after the blob_paths seam bound (this PR) the walk is bounded and reaped, by a fixed internal deadline rather than this env var, so the line now states that precisely instead of overclaiming this setting covers it. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 57ce2885..d02c4a0b 100644 --- a/README.md +++ b/README.md @@ -343,7 +343,7 @@ Important node settings: | `GITLAWB_REQUIRE_SIGNED_PEER_WRITES` | Require signed peer announce/sync writes. | | `GITLAWB_AUTO_SYNC` | Enable automatic sync from known peers. | | `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. | -| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack/receive-pack may run before it is aborted (504). Default 600. Does not bound `info/refs` or the withheld-blob path. | +| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. The withheld-blob classification walk is also bounded and reaped, by a fixed internal deadline rather than this setting. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | | `GITLAWB_IRYS_URL` | Optional Irys/Arweave permanent anchoring. | From 87a7e40513b018b335d8f3a2e46084d323484798 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 00:36:42 -0500 Subject: [PATCH 015/124] fix(review): close the withheld-walk watchdog reused-pgid race (#174) Code review found run_bounded_git's watchdog could return a spurious 504 and signal a recycled process group. The watchdog runs off a wall clock on its own thread; done_tx.send() only fires after child.wait() reaps the leader, so a walk that finished within microseconds of the deadline took the watchdog's Timeout branch, discarded a fully-captured successful result, and returned GitServiceTimeout (a 504 for a walk that actually completed). Worse, the Timeout branch SIGTERMed -pgid unconditionally after the leader was reaped, so a recycled pgid could be signalled, the exact hazard smart_http guards via disarm-after-wait. Set a reaped AtomicBool the instant the main thread reaps the child; the watchdog checks it before every kill and stands down if the leader is already reaped. Gate the timeout verdict on !status.success(), so a child that exited on its own is never reported as a timeout even if the watchdog fired late. Add the survived-SIGKILL warn smart_http's reap already emits, for operator visibility on a wedged (D-state) git. The hung-walk test stays green (a killed child exits by signal, not success, so it still surfaces GitServiceTimeout and reaps the group) and the 32 real-git classification tests stay green (a fast walk is never spuriously killed). --- .../gitlawb-node/src/git/visibility_pack.rs | 66 +++++++++++++------ 1 file changed, 45 insertions(+), 21 deletions(-) diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index 7b9bda69..0d6661e4 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -63,27 +63,46 @@ fn run_bounded_git( // the main thread's stdout drain is exactly what blocks until a hung child is // torn down. let (done_tx, done_rx) = mpsc::channel::<()>(); - let watchdog = std::thread::spawn(move || -> bool { - let wait = deadline.saturating_duration_since(Instant::now()); - match done_rx.recv_timeout(wait) { - Ok(()) | Err(RecvTimeoutError::Disconnected) => false, - Err(RecvTimeoutError::Timeout) => { - // SAFETY: kill(2) takes only integers and borrows no Rust memory; - // ESRCH on an already-gone group is ignored. - unsafe { libc::kill(-pgid, libc::SIGTERM) }; - for step in 0..200u32 { - if unsafe { libc::kill(-pgid, 0) } != 0 { - return true; // ESRCH: every group member has exited + // Set true the instant the main thread reaps the child, so the watchdog never + // SIGTERMs a pgid whose leader is already reaped and whose pid the kernel may + // recycle (the reused-pgid hazard smart_http guards via disarm-after-wait). + let reaped = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let watchdog = { + let reaped = reaped.clone(); + std::thread::spawn(move || -> bool { + use std::sync::atomic::Ordering; + let wait = deadline.saturating_duration_since(Instant::now()); + match done_rx.recv_timeout(wait) { + Ok(()) | Err(RecvTimeoutError::Disconnected) => false, + Err(RecvTimeoutError::Timeout) => { + // The child was reaped right as the deadline fired: do not signal + // its (possibly-recycled) pgid. + if reaped.load(Ordering::SeqCst) { + return false; } - if step == 100 { - unsafe { libc::kill(-pgid, libc::SIGKILL) }; + // SAFETY: kill(2) takes only integers and borrows no Rust memory; + // ESRCH on an already-gone group is ignored. + unsafe { libc::kill(-pgid, libc::SIGTERM) }; + for step in 0..200u32 { + if reaped.load(Ordering::SeqCst) || unsafe { libc::kill(-pgid, 0) } != 0 { + return true; // reaped, or ESRCH: every group member has exited + } + if step == 100 { + unsafe { libc::kill(-pgid, libc::SIGKILL) }; + } + std::thread::sleep(Duration::from_millis(10)); } - std::thread::sleep(Duration::from_millis(10)); + // Survived SIGKILL past the cap: a wedged (D-state) git, the same + // condition smart_http's reap logs. Warn so an operator sees it. + tracing::warn!( + pgid, + "withheld-walk git survived SIGKILL past the watchdog cap (uninterruptible I/O?)" + ); + true } - true } - } - }); + }) + }; // Feed stdin on a writer thread and drain stderr on a reader thread so the main // thread can drain stdout concurrently; writing all of stdin (or draining one @@ -105,16 +124,21 @@ fn run_bounded_git( let mut out = Vec::new(); let read_result = stdout.read_to_end(&mut out); let status = child.wait().context("git wait failed")?; + // Reaped: bar the watchdog from signalling the now-reaped (possibly recycled) + // pgid before it can fire, then stand it down. + reaped.store(true, std::sync::atomic::Ordering::SeqCst); let err = err_reader.join().unwrap_or_default(); let _ = writer.join(); - - // Reaped: stand the watchdog down and learn whether it killed on the deadline. let _ = done_tx.send(()); let killed = watchdog.join().unwrap_or(false); - if killed { + read_result.context("failed to read git stdout")?; + // The watchdog runs off a wall clock that can race a child finishing right at the + // deadline. A child that exited on its own (success) is not a timeout even if the + // watchdog fired late; only a child that did not exit successfully is a genuine + // timeout, which keeps a walk completing at its budget from a spurious 504. + if killed && !status.success() { return Err(crate::git::smart_http::GitServiceTimeout.into()); } - read_result.context("failed to read git stdout")?; if !status.success() { anyhow::bail!("git {label} failed: {}", String::from_utf8_lossy(&err)); } From e750c3042dff5dc0aa8b1959f5bb72ca7be68137 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 00:56:24 -0500 Subject: [PATCH 016/124] fix(review): cap the receive-pack advertisement per source so anon cannot starve the write pool (#174) U2 moved the receive-pack info/refs advertisement onto git_write_semaphore to keep an anonymous read flood from starving the push handshake. But the advertisement is anon-reachable on public repos and holds its write permit across the slow acquire_fresh Tigris download, and the only per-source brake on it was the push RATE limiter, not a concurrency cap. So a multi-source flood of receive-pack advertisements could hold the write pool's slots across those downloads and shed authenticated pushes (both the advertisement and the owner-gated git-receive-pack POST draw from the same pool). U2 thus introduced the first anonymous consumer of the write pool the state doc promised anon could never reach; the plan's residual note (no worse than the POST) was wrong, because the POST is owner-gated and the advertisement is not. Add git_push_advert_per_caller, a per-source concurrency sub-cap on the receive-pack advertisement keyed on the resolved source IP (the same PerCallerConcurrency mechanism U1 uses for reads), sized to an eighth of the write pool so a single source holds at most that share and saturating the pool takes many distinct source IPs, each also braked by the per-IP push rate limiter. The upload-pack advertisement keeps its read-pool per-caller cap; the owner-gated POST is unchanged. Correct the state doc for git_write_semaphore accordingly. Handler-layer proof: a source at its receive-pack advertisement cap sheds 503 (RED before the acquisition, 500-not-503), while a different source and the upload-pack advertisement are unaffected. Full suite 510 green. --- crates/gitlawb-node/src/api/repos.rs | 101 +++++++++++++++++++++--- crates/gitlawb-node/src/auth/mod.rs | 2 + crates/gitlawb-node/src/main.rs | 7 ++ crates/gitlawb-node/src/state.rs | 22 ++++-- crates/gitlawb-node/src/test_support.rs | 3 + 5 files changed, 121 insertions(+), 14 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 7a8aa5c2..796558cf 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -574,16 +574,24 @@ pub async fn git_info_refs( } } - // Per-caller read sub-cap (#174): upload-pack advertisement only. The - // receive-pack advertisement is a write-path op (it drew from git_write_semaphore - // above and is braked per-IP by push_rate_limiter), so it must NOT consume a - // read caller's scarce slot (#174 U2). Acquired AFTER the visibility + push-rate - // gates (KTD7) so a denied or rate-limited request never consumes a slot. Held - // for the whole op. + // Per-source concurrency sub-cap (#174), keyed on the resolved source IP and + // acquired AFTER the visibility + push-rate gates (KTD7) so a denied or + // rate-limited request never consumes a slot; held for the whole op. The + // upload-pack advertisement is bounded on the read pool (git_read_per_caller). + // The receive-pack advertisement draws from the write pool, so it is bounded per + // source (git_push_advert_per_caller) instead: without this, an anonymous + // multi-source flood of push-handshake advertisements could hold the write pool's + // slots across acquire_fresh and shed authenticated pushes, since the per-IP push + // rate limiter caps rate, not concurrency (#174 review fix). + let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); let _caller_permit = if service == "git-receive-pack" { - None + acquire_read_caller_permit( + &state.git_push_advert_per_caller, + caller_key.as_deref(), + name, + "receive-pack advert", + )? } else { - let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); acquire_read_caller_permit( &state.git_read_per_caller, caller_key.as_deref(), @@ -676,7 +684,7 @@ fn acquire_read_caller_permit( None => { tracing::warn!(repo = %repo, caller = %k, handler, "per-caller read cap reached; shedding with 503"); Err(AppError::Overloaded( - "git read service at capacity for this caller, retry shortly".into(), + "git service at capacity for this caller, retry shortly".into(), )) } }, @@ -2827,6 +2835,81 @@ mod tests { ); } + /// #174 (review fix): the anon-reachable receive-pack advertisement draws from + /// the write pool, so it is bounded per source by `git_push_advert_per_caller` to + /// stop one source from monopolizing the write pool and shedding authenticated + /// pushes. Fill one source IP's advert slot; its next receive-pack advertisement + /// sheds 503, while a different source and the upload-pack advertisement are + /// unaffected. Remove the advert-cap acquisition and the same-source assertion + /// goes green-not-503. + #[sqlx::test] + async fn receive_pack_advertisement_capped_per_source(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_push_advert_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6advcap", "ac", "/tmp/ac-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.81:6000".parse().unwrap(); + // Fill this source IP's single receive-pack-advertisement slot. + let _slot = state + .git_push_advert_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first advert slot for this source IP"); + + // Same source: the receive-pack advertisement sheds 503 (advert cap full). + let router = crate::server::build_router(state.clone()); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6advcap/ac/info/refs?service=git-receive-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + assert_eq!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its receive-pack advertisement cap must shed 503, so it cannot monopolize the write pool" + ); + + // A DIFFERENT source keeps its own advert budget -> not shed. + let other: SocketAddr = "203.0.113.82:6000".parse().unwrap(); + let router2 = crate::server::build_router(state.clone()); + let mut req2 = Request::builder() + .method(Method::GET) + .uri("/z6advcap/ac/info/refs?service=git-receive-pack") + .body(Body::empty()) + .unwrap(); + req2.extensions_mut().insert(ConnectInfo(other)); + assert_ne!( + router2.oneshot(req2).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a different source must keep its own receive-pack advertisement budget" + ); + + // The upload-pack advertisement is NOT bounded by the receive-pack advert cap. + let router3 = crate::server::build_router(state); + let mut req3 = Request::builder() + .method(Method::GET) + .uri("/z6advcap/ac/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req3.extensions_mut().insert(ConnectInfo(peer)); + assert_ne!( + router3.oneshot(req3).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "the upload-pack advertisement must not be shed by the receive-pack advert cap" + ); + } + /// #174 SC2 (info_refs probe): the per-caller read sub-cap sheds a caller that /// is already at its concurrency budget on the upload-pack advertisement, while /// a DIFFERENT caller still enters. Remove the sub-cap from `git_info_refs` and diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 253b3210..b9e28e57 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -523,6 +523,8 @@ mod tests { git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), + git_push_advert_per_caller: + crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), } } diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index c617b434..0f33fde4 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -385,6 +385,13 @@ async fn main() -> Result<()> { git_read_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( config.max_concurrent_reads_per_caller, ), + // Per-source cap on the receive-pack advertisement, sized to an eighth of the + // write pool (min 1): a single source can hold at most this many write-pool + // slots via the anon advertisement, so saturating the pool takes ~8 distinct + // source IPs, each also rate-limited (#174). + git_push_advert_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( + (config.max_concurrent_git_pushes / 8).max(1), + ), }; // Periodic peer-count poll for the metrics gauge. If p2p is disabled diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 01007ac4..f55abeb7 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -95,14 +95,26 @@ pub struct AppState { /// authenticated push at admission (#174). pub git_read_semaphore: Arc, /// Bounds concurrent `git-receive-pack` (push) operations, a pool separate - /// from `git_read_semaphore` so anonymous reads can never shed an authenticated - /// push (#174). Sized by `max_concurrent_git_pushes`. + /// from `git_read_semaphore` so an anonymous READ flood can never shed an + /// authenticated push (#174). Sized by `max_concurrent_git_pushes`. Drawn from + /// by the `git-receive-pack` POST (owner-gated) and the receive-pack `info/refs` + /// advertisement (anon-reachable); the advertisement is additionally bounded per + /// source by `git_push_advert_per_caller` so no single source can monopolize this + /// pool and shed pushes. pub git_write_semaphore: Arc, - /// Per-caller concurrency sub-cap on the read pool: each caller (per-DID when - /// signed, else per-source-IP) may hold at most `max_concurrent_reads_per_caller` + /// Per-caller concurrency sub-cap on the read pool: each caller (keyed on the + /// resolved source IP, #174 U1) may hold at most `max_concurrent_reads_per_caller` /// in-flight read ops, so one caller cannot monopolize `git_read_semaphore` - /// (#174). Applied by `git_upload_pack` and both `info/refs` advertisements. + /// (#174). Applied by `git_upload_pack` and the upload-pack `info/refs` + /// advertisement. pub git_read_per_caller: crate::rate_limit::PerCallerConcurrency, + /// Per-source concurrency sub-cap on the anon-reachable receive-pack `info/refs` + /// advertisement: each source IP may hold at most a small share of the write + /// pool, so a multi-source flood of push-handshake advertisements cannot + /// saturate `git_write_semaphore` and shed authenticated pushes (#174). Sized as + /// a fraction of `max_concurrent_git_pushes`, so filling the write pool takes many + /// distinct source IPs (each also braked by the per-IP push rate limiter). + pub git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency, } impl AppState { diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 7745d233..03d3d5c4 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -86,6 +86,9 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), + git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( + 8, + ), } } From c35452ed41091ba38f1b294a96f44d1d7b827f74 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 02:12:08 -0500 Subject: [PATCH 017/124] test(node): vet the withheld-walk bound end to end; use the configured timeout (#174) Close the reasoned-not-run gaps from the code review by making the walk's git binary and timeout injectable, then driving the missing branches with a real handler and a fake git instead of reasoning about them. - Add state.git_bin and *_bounded variants of the walk entry points taking (git_bin, timeout); the served handlers (upload-pack serve, receive-pack replication and full-scan and encrypt-pin, and the ipfs gate) now pass the operator-configured GITLAWB_GIT_SERVICE_TIMEOUT_SECS, so the whole walk is bounded by the same budget as the other served-git ops rather than a fixed constant. The git_bin-less wrappers stay for the real-git classification tests. Newly vetted by execution (not reasoning): - receive-pack replication path is bounded: replication_withheld_set with an injected hung git returns within the budget and fails closed, so it cannot pin the write permit git_receive_pack holds across it. - a hung withheld-blob walk on the upload-pack POST returns 504 (real handler, real repo on disk, injected hung git), proving the GitServiceTimeout -> git_service_app_error wiring end to end. - the watchdog status-gate: a child that exits successfully is not reported as a timeout even when the watchdog fired (mutation-checked: drop the guard -> RED). - SIGKILL escalation: a SIGTERM-ignoring child is still reaped via SIGKILL and the group is gone; a truly uninterruptible D-state child (unreapable by any signal) is the documented residual, matching the async teardown. - the advertisement per-source cap sizing never derives 0. Full gitlawb-node suite 515 green. --- crates/gitlawb-node/src/api/ipfs.rs | 12 +- crates/gitlawb-node/src/api/repos.rs | 208 +++++++++++++++++- crates/gitlawb-node/src/auth/mod.rs | 1 + .../gitlawb-node/src/git/visibility_pack.rs | 173 ++++++++++++++- crates/gitlawb-node/src/main.rs | 1 + crates/gitlawb-node/src/state.rs | 5 + crates/gitlawb-node/src/test_support.rs | 1 + 7 files changed, 383 insertions(+), 18 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index f3de7570..e921c39f 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -27,7 +27,7 @@ use std::str::FromStr; use crate::auth::AuthenticatedDid; use crate::error::{AppError, Result}; use crate::git::store; -use crate::git::visibility_pack::{allowed_blob_set_for_caller, has_path_scoped_rule}; +use crate::git::visibility_pack::{allowed_blob_set_for_caller_bounded, has_path_scoped_rule}; use crate::state::AppState; use crate::visibility::{visibility_check, Decision}; @@ -142,10 +142,16 @@ pub async fn get_by_cid( let is_public = repo.is_public; let owner = repo.owner_did.clone(); let caller_for_walk = caller_owned.clone(); - // Full-history walk shells out to git — keep it off the async runtime. + let git_bin = state.git_bin.clone(); + let walk_timeout = + std::time::Duration::from_secs(state.config.git_service_timeout_secs); + // Full-history walk shells out to git — keep it off the async runtime, + // bounded and reaped like the served-git ops (#174). let walk = tokio::task::spawn_blocking(move || { - allowed_blob_set_for_caller( + allowed_blob_set_for_caller_bounded( &rp, + &git_bin, + walk_timeout, &r, is_public, &owner, diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 796558cf..1532be58 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -44,6 +44,8 @@ async fn replication_withheld_set( owner_did: &str, is_public: bool, disk_path: std::path::PathBuf, + git_bin: String, + timeout: std::time::Duration, ) -> (bool, Option>) { let announce = match &rules { Some(rules) => crate::visibility::listable_at_root(rules, is_public, owner_did, None), @@ -65,8 +67,8 @@ async fn replication_withheld_set( Some(rules) => { let owner_did = owner_did.to_string(); tokio::task::spawn_blocking(move || { - crate::git::visibility_pack::withheld_blob_oids( - &disk_path, &rules, is_public, &owner_did, None, + crate::git::visibility_pack::withheld_blob_oids_bounded( + &disk_path, &git_bin, timeout, &rules, is_public, &owner_did, None, ) }) .await @@ -106,10 +108,12 @@ async fn fail_closed_full_scan_objects( is_public: bool, owner_did: String, candidates: Vec, + git_bin: String, + timeout: std::time::Duration, ) -> Vec { tokio::task::spawn_blocking(move || -> anyhow::Result> { - let allowed = crate::git::visibility_pack::replicable_blob_set( - &disk_path, &rules, is_public, &owner_did, + let allowed = crate::git::visibility_pack::replicable_blob_set_bounded( + &disk_path, &git_bin, timeout, &rules, is_public, &owner_did, )?; let all_blobs = crate::git::push_delta::all_blob_oids(&disk_path)?; Ok(crate::git::visibility_pack::replicable_objects_fail_closed( @@ -778,9 +782,12 @@ pub async fn git_upload_pack( let owner_did = record.owner_did.clone(); let caller_owned = caller.map(str::to_string); let is_public = record.is_public; + let git_bin = state.git_bin.clone(); tokio::task::spawn_blocking(move || { - visibility_pack::withheld_blob_oids( + visibility_pack::withheld_blob_oids_bounded( &path, + &git_bin, + git_timeout, &rules, is_public, &owner_did, @@ -1185,6 +1192,8 @@ pub async fn git_receive_pack( &record.owner_did, record.is_public, disk_path.clone(), + state.git_bin.clone(), + std::time::Duration::from_secs(state.config.git_service_timeout_secs), ) .await; @@ -1219,6 +1228,8 @@ pub async fn git_receive_pack( record.is_public, record.owner_did.clone(), pin_set.candidates, + state.git_bin.clone(), + std::time::Duration::from_secs(state.config.git_service_timeout_secs), ) .await } else { @@ -1244,6 +1255,8 @@ pub async fn git_receive_pack( let node_did_str = state.node_did.to_string(); let node_seed = state.node_keypair.to_seed(); let repo_name = record.name.clone(); + let enc_git_bin = state.git_bin.clone(); + let enc_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); tokio::spawn(async move { let pinned = crate::ipfs_pin::pin_new_objects( &ipfs_api, @@ -1268,9 +1281,15 @@ pub async fn git_receive_pack( { let p = repo_path_clone.clone(); let owner = owner_did.clone(); + let git_bin = enc_git_bin.clone(); let recip = tokio::task::spawn_blocking(move || { - crate::git::visibility_pack::withheld_blob_recipients( - &p, &rules, is_public, &owner, + crate::git::visibility_pack::withheld_blob_recipients_bounded( + &p, + &git_bin, + enc_timeout, + &rules, + is_public, + &owner, ) }) .await; @@ -2031,16 +2050,39 @@ mod tests { let dummy = std::path::PathBuf::from("/nonexistent"); // Private: no rules at all. - let (announce, _) = replication_withheld_set(None, OWNER_DID, false, dummy.clone()).await; + let (announce, _) = replication_withheld_set( + None, + OWNER_DID, + false, + dummy.clone(), + "git".into(), + std::time::Duration::from_secs(600), + ) + .await; assert!(!announce, "private repo (no rules) must not announce"); // Private: empty rule set, is_public=false → still not listable at root. - let (announce, _) = - replication_withheld_set(Some(vec![]), OWNER_DID, false, dummy.clone()).await; + let (announce, _) = replication_withheld_set( + Some(vec![]), + OWNER_DID, + false, + dummy.clone(), + "git".into(), + std::time::Duration::from_secs(600), + ) + .await; assert!(!announce, "private repo (empty rules) must not announce"); // Public: empty rule set, is_public=true → listable at root, announces. - let (announce, _) = replication_withheld_set(Some(vec![]), OWNER_DID, true, dummy).await; + let (announce, _) = replication_withheld_set( + Some(vec![]), + OWNER_DID, + true, + dummy, + "git".into(), + std::time::Duration::from_secs(600), + ) + .await; assert!(announce, "public repo must announce"); } @@ -2132,6 +2174,150 @@ mod tests { } } + #[cfg(unix)] + fn write_fake_git(dir: &std::path::Path, body: &str) -> String { + use std::os::unix::fs::PermissionsExt; + let p = dir.join("fakegit"); + std::fs::write(&p, body).unwrap(); + let mut perm = std::fs::metadata(&p).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&p, perm).unwrap(); + p.to_str().unwrap().to_string() + } + + /// #174 (write-pool twin, vetted by execution not reasoning): the receive-pack + /// post-push replication walk is bounded. Drive `replication_withheld_set` with an + /// injected fake git that hangs on `rev-list` and a short budget: it must RETURN + /// within the budget (so `git_receive_pack` releases the write permit it holds + /// across this await, rather than pinning it for the hang) AND fail closed + /// (announce suppressed) because the walk could not be vetted. Proves this path + /// funnels through the bounded `blob_paths`, on the write-permit-holding side. + #[cfg(unix)] + #[tokio::test] + async fn replication_walk_is_bounded_and_fails_closed_on_a_hung_git() { + use std::time::Duration; + let tmp = tempfile::TempDir::new().unwrap(); + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) sleep 30 ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + // Public root (announceable) + a path-scoped rule, so the walk actually runs + // rather than taking the has_path_scoped_rule short-circuit. + let rules = Some(vec![vis_rule("/secret/**", &[])]); + + let result = tokio::time::timeout( + Duration::from_secs(10), + replication_withheld_set( + rules, + OWNER_DID, + true, + tmp.path().to_path_buf(), + git_bin, + Duration::from_millis(200), + ), + ) + .await + .expect( + "replication_withheld_set must return within the budget; a hung walk must \ + not pin the write permit git_receive_pack holds across it", + ); + assert_eq!( + result, + (false, None), + "a walk that could not be vetted must suppress the announce (fail closed)" + ); + } + + /// #174 (serve-path 504, vetted by execution): a hung withheld-blob walk on the + /// upload-pack POST maps to 504, not a generic 500. Real repo dir on disk (so + /// acquire's fast path returns it) + a path-scoped rule (so the walk runs) + + /// an injected fake git that hangs on rev-list. The handler must return 504, + /// proving git_upload_pack routes the walk's GitServiceTimeout through + /// git_service_app_error end to end. + #[cfg(unix)] + #[sqlx::test] + async fn upload_pack_hung_withheld_walk_returns_504(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) sleep 30 ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n"; + let fake = write_fake_git(tmp.path(), body); + + let mut state = crate::test_support::test_state(pool).await; + state.git_bin = fake; + let mut cfg = (*state.config).clone(); + cfg.git_service_timeout_secs = 1; + state.config = std::sync::Arc::new(cfg); + state + .db + .upsert_mirror_repo("z6srv504", "sv", "/tmp/z6srv504-sv", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6srv504", "sv").await.unwrap().unwrap(); + // Path-scoped rule so has_path_scoped_rule() is true and the walk runs; the + // public root still lets an anonymous caller past the "/" gate. + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &[], + OWNER_DID, + ) + .await + .unwrap(); + // acquire()'s fast path returns the local path when it exists on disk. + let disk = std::path::Path::new("/tmp/z6srv504/sv.git"); + std::fs::create_dir_all(disk).unwrap(); + + let peer: SocketAddr = "203.0.113.91:7000".parse().unwrap(); + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::POST) + .uri("/z6srv504/sv/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let status = router.oneshot(req).await.unwrap().status(); + let _ = std::fs::remove_dir_all("/tmp/z6srv504"); + assert_eq!( + status, + StatusCode::GATEWAY_TIMEOUT, + "a hung withheld-blob walk must surface as 504, not a generic 500" + ); + } + + /// #174 (F2 sizing edge, vetted by execution): the receive-pack advertisement + /// per-source cap is derived in main.rs as `(max_concurrent_git_pushes / 8).max(1)`, + /// so it is never 0 even at the minimum write-pool size (1). A 0 cap would make + /// PerCallerConcurrency shed EVERY receive-pack advertisement and break all pushes. + #[test] + fn advert_per_caller_cap_sizing_is_never_zero() { + let cap = |pushes: usize| (pushes / 8).max(1); + for pushes in [1usize, 4, 8, 32, 256] { + assert!( + cap(pushes) >= 1, + "advert cap must be >= 1 for pushes={pushes}" + ); + } + assert_eq!(cap(1), 1, "minimum write pool must derive cap 1, not 0"); + assert_eq!( + cap(32), + 4, + "default write pool 32 derives cap 4 (~8 source IPs to fill)" + ); + // A cap of 1 admits one and sheds the second from the same source. + let lim = crate::rate_limit::PerCallerConcurrency::new(cap(1), 100); + let _held = lim.try_acquire("src").expect("first advert admitted"); + assert!( + lim.try_acquire("src").is_none(), + "second advert from the same source is shed" + ); + } + #[test] fn fork_owner_full_did_with_path_rule_allowed() { // Owner reads everything (implicit reader), so nothing is withheld. diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index b9e28e57..5befa7f3 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -525,6 +525,7 @@ mod tests { git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), + git_bin: "git".to_string(), } } diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index 0d6661e4..09a80aba 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -17,7 +17,10 @@ use std::time::{Duration, Instant}; /// pathologically slow git child so it cannot pin a served-git permit (the read /// permit on the upload-pack serve path, the write permit on the receive-pack /// post-push replication path) past the deadline. Every caller funnels through -/// `blob_paths`, so bounding here bounds both paths at one seam. +/// `blob_paths`, so bounding here bounds both paths at one seam. Production callers +/// pass the operator-configured `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` instead; this +/// fixed budget only backs the `git_bin`-less test wrappers. +#[cfg(test)] const WALK_TIMEOUT: Duration = Duration::from_secs(600); /// Run one git child under a shared `deadline` with process-group teardown, @@ -122,6 +125,12 @@ fn run_bounded_git( }); let mut stdout = child.stdout.take().context("git stdout was not piped")?; let mut out = Vec::new(); + // Blocking drain, unblocked by the child closing stdout on exit. The watchdog's + // SIGTERM/SIGKILL is what makes a hung child exit; a git wedged in uninterruptible + // (D-state) I/O survives even SIGKILL, so this drain and the wait below can block + // until the kernel returns, pinning the walk thread and its permit. That residual + // is unreachable in userspace (no signal reaps a D-state process) and matches the + // async `reap_group_on_timeout`, which likewise only warns and gives up there. let read_result = stdout.read_to_end(&mut out); let status = child.wait().context("git wait failed")?; // Reaped: bar the watchdog from signalling the now-reaped (possibly recycled) @@ -325,6 +334,7 @@ fn blob_paths(repo_path: &Path, git_bin: &str, timeout: Duration) -> Result, ) -> Result> { - let pairs = blob_paths(repo_path, "git", WALK_TIMEOUT)?; + withheld_blob_oids_bounded( + repo_path, + "git", + WALK_TIMEOUT, + rules, + is_public, + owner_did, + caller, + ) +} + +/// [`withheld_blob_oids`] with an injectable `git_bin` and walk `timeout`. Served +/// handlers call this with the operator-configured git binary and +/// `GITLAWB_GIT_SERVICE_TIMEOUT_SECS`, so the whole walk is bounded by the same +/// budget as the other served-git ops and a fake `git` can drive its teardown in +/// tests. The `git_bin`-less wrapper above keeps the fixed [`WALK_TIMEOUT`] for the +/// classification tests that run against real git. +pub fn withheld_blob_oids_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> Result> { + let pairs = blob_paths(repo_path, git_bin, timeout)?; Ok(withheld_from_pairs( &pairs, rules, is_public, owner_did, caller, )) @@ -400,6 +436,7 @@ pub fn replicable_objects(all: Vec, withheld: &HashSet) -> Vec Result> { + allowed_blob_set_for_caller_bounded( + repo_path, git_bin, timeout, rules, is_public, owner_did, None, + ) +} + /// Reachable blob OIDs that visibility ALLOWS `caller` at some path. The /// caller-aware generalization of `replicable_blob_set` (which is the anonymous /// `caller = None` case). Used by `GET /ipfs/{cid}` to gate fail-closed against @@ -421,6 +473,7 @@ pub fn replicable_blob_set( /// elsewhere (its content is readable to this caller elsewhere). Trees and /// commits are NOT included here; the caller decides per object type whether /// the allow-set applies (it does not for trees/commits — KTD3). +#[cfg(test)] pub fn allowed_blob_set_for_caller( repo_path: &Path, rules: &[VisibilityRule], @@ -428,7 +481,29 @@ pub fn allowed_blob_set_for_caller( owner_did: &str, caller: Option<&str>, ) -> Result> { - let pairs = blob_paths(repo_path, "git", WALK_TIMEOUT)?; + allowed_blob_set_for_caller_bounded( + repo_path, + "git", + WALK_TIMEOUT, + rules, + is_public, + owner_did, + caller, + ) +} + +/// [`allowed_blob_set_for_caller`] with an injectable `git_bin` and walk `timeout`, +/// for the `GET /ipfs/{cid}` gate. +pub fn allowed_blob_set_for_caller_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> Result> { + let pairs = blob_paths(repo_path, git_bin, timeout)?; let mut allowed = HashSet::new(); for (oid, path) in &pairs { if visibility_check(rules, is_public, owner_did, caller, path) == Decision::Allow { @@ -461,14 +536,28 @@ pub fn replicable_objects_fail_closed( /// owner plus any reader DID that `visibility_check` Allows at some path the /// blob appears at. Least-privilege: a reader of one private subtree is not a /// recipient of a blob that only lives in another. +#[cfg(test)] pub fn withheld_blob_recipients( repo_path: &Path, rules: &[VisibilityRule], is_public: bool, owner_did: &str, +) -> Result>> { + withheld_blob_recipients_bounded(repo_path, "git", WALK_TIMEOUT, rules, is_public, owner_did) +} + +/// [`withheld_blob_recipients`] with an injectable `git_bin` and walk `timeout`, for +/// the receive-pack encrypt-then-pin path. +pub fn withheld_blob_recipients_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, ) -> Result>> { // One history walk feeds both the withheld set and the recipient mapping. - let pairs = blob_paths(repo_path, "git", WALK_TIMEOUT)?; + let pairs = blob_paths(repo_path, git_bin, timeout)?; let withheld = withheld_from_pairs(&pairs, rules, is_public, owner_did, None); if withheld.is_empty() { return Ok(HashMap::new()); @@ -571,6 +660,82 @@ mod tests { "the hung git child (pid {pid}) must be reaped, not orphaned, after the walk aborts" ); } + + /// #174 (F1 status-gate, vetted by execution): a child that exits SUCCESSFULLY is + /// never reported as a timeout even when the watchdog fires, so a walk finishing + /// right at its deadline is not a spurious 504. The fake only exits when signalled + /// and exits 0 on SIGTERM, so with a deadline already elapsed the watchdog always + /// reaches its kill path (killed == true) yet the child's status is success. + /// Drop the `!status.success()` guard and this returns GitServiceTimeout (RED). + #[cfg(unix)] + #[test] + fn run_bounded_git_success_at_the_deadline_is_not_a_timeout() { + use std::time::{Duration, Instant}; + let tmp = TempDir::new().unwrap(); + let body = "#!/bin/sh\ntrap 'exit 0' TERM\nsleep 30 &\nwait\n"; + let git_bin = write_fake_git(tmp.path(), body); + let out = run_bounded_git( + &git_bin, + &["rev-list"], + tmp.path(), + b"", + Instant::now() + Duration::from_millis(100), + ); + assert!( + out.is_ok(), + "a child that exited successfully must not be reported as a timeout even if the watchdog fired: {out:?}" + ); + } + + /// #174 (F3, vetted by execution): a child that IGNORES SIGTERM is still reaped + /// via the watchdog's SIGKILL escalation, so it cannot pin the walk thread or its + /// permit. The fake traps SIGTERM and keeps sleeping; run_bounded_git must still + /// return (via SIGKILL at the grace step) with a timeout error and the group must + /// be gone. (A truly uninterruptible D-state child, which no signal can reap, is + /// the documented residual this teardown, like the async twin, cannot cover.) + #[cfg(unix)] + #[test] + fn run_bounded_git_reaps_a_sigterm_ignoring_child_via_sigkill() { + use std::time::{Duration, Instant}; + let tmp = TempDir::new().unwrap(); + let body = "#!/bin/sh\ntrap '' TERM\necho $$ > pid\nwhile true; do sleep 1; done\n"; + let git_bin = write_fake_git(tmp.path(), body); + let (tx, rx) = std::sync::mpsc::channel(); + let path = tmp.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(run_bounded_git( + &git_bin, + &["rev-list"], + &path, + b"", + Instant::now() + Duration::from_millis(100), + )); + }); + let out = rx + .recv_timeout(Duration::from_secs(10)) + .expect("run_bounded_git must return via SIGKILL even for a SIGTERM-ignoring child"); + assert!( + out.is_err(), + "a SIGTERM-ignoring child killed by SIGKILL is a timeout, not a success: {out:?}" + ); + let pid: i32 = std::fs::read_to_string(tmp.path().join("pid")) + .unwrap() + .trim() + .parse() + .unwrap(); + let mut gone = false; + for _ in 0..300 { + if unsafe { libc::kill(pid, 0) } != 0 { + gone = true; + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + assert!( + gone, + "the SIGTERM-ignoring child (pid {pid}) must be reaped via SIGKILL, not left running" + ); + } use crate::db::VisibilityMode; use chrono::Utc; use std::process::Command; diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 0f33fde4..9b646c2b 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -392,6 +392,7 @@ async fn main() -> Result<()> { git_push_advert_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( (config.max_concurrent_git_pushes / 8).max(1), ), + git_bin: "git".to_string(), }; // Periodic peer-count poll for the metrics gauge. If p2p is disabled diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index f55abeb7..94952120 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -115,6 +115,11 @@ pub struct AppState { /// a fraction of `max_concurrent_git_pushes`, so filling the write pool takes many /// distinct source IPs (each also braked by the per-IP push rate limiter). pub git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency, + /// The `git` executable the served-git withheld-blob walk spawns. Production is + /// `"git"` (resolved via PATH); injectable so a fake `git` can drive the walk's + /// process-group teardown in handler tests without mutating the process-global + /// PATH (#174). + pub git_bin: String, } impl AppState { diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 03d3d5c4..a43a79a0 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -89,6 +89,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( 8, ), + git_bin: "git".to_string(), } } From 454e59ba533535168a6eb763447275ef107564ad Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 02:12:37 -0500 Subject: [PATCH 018/124] docs(node): the withheld-walk is bounded by the service timeout, not a fixed const (#174) Follow-up to threading the configured timeout into the walk: the walk now honors GITLAWB_GIT_SERVICE_TIMEOUT_SECS on both the serve and replication paths, so the README no longer says a fixed internal deadline. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d02c4a0b..0f760449 100644 --- a/README.md +++ b/README.md @@ -343,7 +343,7 @@ Important node settings: | `GITLAWB_REQUIRE_SIGNED_PEER_WRITES` | Require signed peer announce/sync writes. | | `GITLAWB_AUTO_SYNC` | Enable automatic sync from known peers. | | `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. | -| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. The withheld-blob classification walk is also bounded and reaped, by a fixed internal deadline rather than this setting. | +| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths), which is reaped at the deadline. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | | `GITLAWB_IRYS_URL` | Optional Irys/Arweave permanent anchoring. | From 79b9ee5c89070f2c9b8d534d28195803110385a1 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 14:32:24 -0500 Subject: [PATCH 019/124] fix(node): shed the served-git pool before the DB and cap per-source before it (#174) CodeRabbit review: the held git_permit was acquired at the top of git_info_refs and git_upload_pack, before the per-source cap, so one source could occupy the global pool during the DB/visibility window before its sub-cap rejected the excess. Move the held git_permit to after acquire_read_caller_permit (still before acquire_fresh/git, so INV-10's bound on the fresh Tigris acquire holds), and add a cheap pre-DB early shed (available_permits() == 0 -> 503, holds no permit) so the #62 shed-before-DB property is preserved without holding a permit across the DB work. The receive-pack POST is unchanged (owner-only; its top-of-handler git_permit is deliberate for acquire_write bounding). Tests: the two #62 shed-before-DB tests now exercise the explicit early check (load-bearing: disable it and they fall through to the DB); three new *_per_source_cap_sheds_with_global_capacity tests prove the per-source sub-cap sheds a capped source even with global capacity free (load-bearing on the caller cap). Full workspace green. --- crates/gitlawb-node/src/api/repos.rs | 253 ++++++++++++++++++++++-- crates/gitlawb-node/src/test_support.rs | 62 ++++-- 2 files changed, 290 insertions(+), 25 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 1532be58..85eaa227 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -515,17 +515,27 @@ pub async fn git_info_refs( let service = query .service .ok_or_else(|| AppError::BadRequest("missing ?service= parameter".into()))?; - // Shed with a 503 before spawning git when the concurrency cap is saturated; - // held for the whole op (incl. the smart_http call), released on return. The - // receive-pack advertisement is phase one of a push, so it draws from the WRITE - // pool (like the git-receive-pack POST), not the global read pool an anonymous - // clone flood can exhaust and thereby starve the push handshake (#174 U2). The - // upload-pack advertisement stays on the read pool with its per-caller sub-cap. - let _permit = if service == "git-receive-pack" { - git_permit(&state.git_write_semaphore)? - } else { - git_permit(&state.git_read_semaphore)? - }; + // #62 cheap pre-DB load shed: if the pool this service draws from is already + // saturated, shed with 503 before any DB/disk work. Best-effort (holds no + // permit); the authoritative hold is `git_permit` below, after the per-source + // cap. Restores the shed-before-DB property the reordered held acquire alone + // would drop, while the reorder still prevents one source from occupying global + // slots during the DB/visibility window. + { + let pool = if service == "git-receive-pack" { + &state.git_write_semaphore + } else { + &state.git_read_semaphore + }; + if pool.available_permits() == 0 { + tracing::warn!( + "served-git concurrency cap reached; shedding request with 503 (pre-DB)" + ); + return Err(AppError::Overloaded( + "git service at capacity, retry shortly".into(), + )); + } + } tracing::info!(owner = %owner, repo = %name, "info/refs request"); let record = state .db @@ -604,6 +614,22 @@ pub async fn git_info_refs( )? }; + // Shed with a 503 before spawning git when the concurrency cap is saturated; + // held for the whole op (incl. the smart_http call), released on return. Taken + // AFTER the per-source cap above so one source cannot occupy global slots it + // would be sub-cap-denied for during the DB/visibility window and starve other + // sources; still before acquire_fresh/git so it bounds the fresh Tigris acquire + // and git exec (INV-10). The receive-pack advertisement is phase one of a push, + // so it draws from the WRITE pool (like the git-receive-pack POST), not the + // global read pool an anonymous clone flood can exhaust and thereby starve the + // push handshake (#174 U2). The upload-pack advertisement stays on the read + // pool with its per-caller sub-cap. + let _permit = if service == "git-receive-pack" { + git_permit(&state.git_write_semaphore)? + } else { + git_permit(&state.git_read_semaphore)? + }; + // For receive-pack (push), download the latest from Tigris so the client // sees the same refs that acquire_write() will operate on. let disk_path = if service == "git-receive-pack" { @@ -725,9 +751,15 @@ pub async fn git_upload_pack( headers: axum::http::HeaderMap, body: Bytes, ) -> Result { - // Shed with a 503 before spawning git when the concurrency cap is saturated; - // held for the whole op (incl. the smart_http call), released on return. - let _permit = git_permit(&state.git_read_semaphore)?; + // #62 cheap pre-DB load shed (see git_info_refs): shed before DB when the read + // pool is saturated; the authoritative hold is `git_permit` below, after the + // per-source cap. + if state.git_read_semaphore.available_permits() == 0 { + tracing::warn!("served-git concurrency cap reached; shedding request with 503 (pre-DB)"); + return Err(AppError::Overloaded( + "git service at capacity, retry shortly".into(), + )); + } let name = smart_http_repo_name(&repo)?; let record = state .db @@ -760,6 +792,14 @@ pub async fn git_upload_pack( "upload-pack", )?; + // Shed with a 503 before spawning git when the concurrency cap is saturated; + // held for the whole op (incl. the smart_http call), released on return. Taken + // AFTER the per-source cap above so one source cannot occupy global slots it + // would be sub-cap-denied for during the DB/visibility window and starve other + // sources; still before acquire/git so it bounds the Tigris acquire and git + // exec (INV-10). + let _permit = git_permit(&state.git_read_semaphore)?; + let disk_path = state .repo_store .acquire(&record.owner_did, &record.name) @@ -3209,6 +3249,191 @@ mod tests { ); } + /// #174 (review fix): the per-source caller cap is an independent brake that + /// sheds a capped source even when the global pool has free capacity — the + /// sub-cap is not a mere pre-filter for pool exhaustion. Proven by leaving the + /// global read pool with capacity (so the pre-DB early shed passes) AND + /// pre-holding the source's upload-pack read sub-cap: the request reaches the + /// caller cap and sheds there, so its 503 body reads "for this caller". Remove + /// the `acquire_read_caller_permit` call and the capped source falls through to + /// the git op instead of shedding with "for this caller" — this is the + /// caller-cap acquire probe for the info/refs upload-pack branch. + #[sqlx::test] + async fn info_refs_upload_pack_per_source_cap_sheds_with_global_capacity(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + // Global read pool has free capacity (early shed passes); source pre-held at + // its per-caller cap so it sheds on the caller cap, not the global pool. + state.git_read_semaphore = Arc::new(Semaphore::new(4)); + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6ordir", "oi", "/tmp/oi-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.91:5000".parse().unwrap(); + // Pin this source at its single upload-pack read slot. + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first read slot for this source IP"); + + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6ordir/oi/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its read sub-cap must shed 503 even with global pool capacity" + ); + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .expect("body bytes"); + let body = String::from_utf8_lossy(&bytes); + assert!( + body.contains("for this caller"), + "the per-source cap is an independent brake: with global capacity free, the capped source must still shed with the caller-cap body, got {body}" + ); + } + + /// #174 (review fix): same independent-brake guarantee for the receive-pack + /// advertisement branch of info/refs — its per-source cap + /// (`git_push_advert_per_caller`) sheds a capped source even when the global + /// write pool has capacity. Leave the global write pool with capacity (so the + /// pre-DB early shed passes) and pre-hold the source's advert slot: the request + /// reaches the caller cap, so the 503 body reads "for this caller". Remove the + /// caller-cap acquire and the capped source falls through instead of shedding + /// with "for this caller". The push rate limiter is left permissive so the + /// request reaches the caller cap. + #[sqlx::test] + async fn info_refs_receive_pack_per_source_cap_sheds_with_global_capacity(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + // Global write pool has free capacity (early shed passes); source pre-held at + // its advert sub-cap so it sheds on the caller cap, not the global pool. + state.git_write_semaphore = Arc::new(Semaphore::new(4)); + state.git_push_advert_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + // Permissive push rate limiter so the advertisement passes the rate gate and + // reaches the per-source concurrency cap. + state.push_rate_limiter = crate::rate_limit::RateLimiter::new(100, Duration::from_secs(60)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6ordrp", "or", "/tmp/or-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.92:5000".parse().unwrap(); + // Pin this source at its single receive-pack advertisement slot. + let _slot = state + .git_push_advert_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first advert slot for this source IP"); + + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6ordrp/or/info/refs?service=git-receive-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its advert sub-cap must shed 503 even with global write pool capacity" + ); + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .expect("body bytes"); + let body = String::from_utf8_lossy(&bytes); + assert!( + body.contains("for this caller"), + "the per-source advert cap is an independent brake: with global write capacity free, the capped source must still shed with the caller-cap body, got {body}" + ); + } + + /// #174 (review fix): same independent-brake guarantee for the POST upload-pack + /// handler — its per-source read cap sheds a capped source even when the global + /// read pool has capacity. Leave the global read pool with capacity (so the + /// pre-DB early shed passes) and pre-hold the source's read slot: the request + /// reaches the caller cap, so the 503 body reads "for this caller". Remove the + /// caller-cap acquire and the capped source falls through instead of shedding + /// with "for this caller". + #[sqlx::test] + async fn upload_pack_per_source_cap_sheds_with_global_capacity(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + // Global read pool has free capacity (early shed passes); source pre-held at + // its per-caller cap so it sheds on the caller cap, not the global pool. + state.git_read_semaphore = Arc::new(Semaphore::new(4)); + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6ordup", "ou", "/tmp/ou-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.93:5000".parse().unwrap(); + // Pin this source at its single read slot. + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first read slot for this source IP"); + + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::POST) + .uri("/z6ordup/ou/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its read sub-cap must shed 503 even with global pool capacity" + ); + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .expect("body bytes"); + let body = String::from_utf8_lossy(&bytes); + assert!( + body.contains("for this caller"), + "the per-source cap is an independent brake: with global capacity free, the capped source must still shed with the caller-cap body, got {body}" + ); + } + /// #174 SC2 (per-source key, U1): the per-caller read sub-cap keys on the /// resolved source IP, NOT the signed DID, so a disposable-DID farm cannot /// multiply its budget. Fill the source IP's single read slot, then drive two diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index a43a79a0..765545f1 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -195,13 +195,14 @@ mod tests { ); } - /// PR3 (#62): the served-git concurrency cap sheds at the HTTP layer. One test - /// per git handler drives the `let _permit = git_permit(...)` wiring end to end - /// (this one plus the git_upload_pack / git_receive_pack siblings below); the - /// git_permit unit test covers the helper in isolation. DB-free: an exhausted - /// semaphore sheds before any DB/disk access, so a lazy state works. Remove the - /// permit line from git_info_refs and this goes red (the request falls through - /// to the DB and returns something other than 503). + /// PR3 (#62): the served-git concurrency cap sheds at the HTTP layer before the + /// DB. The held `git_permit` acquire now sits after the per-source cap, so the + /// shed-before-DB property is carried by an explicit `available_permits() == 0` + /// early check at the top of the handler (the held permit remains the + /// authoritative bound further down). DB-free: an exhausted semaphore sheds + /// before any DB/disk access, so a lazy state works. Remove the early-shed block + /// from git_info_refs and this goes red (the request falls through to the DB and + /// returns something other than 503). #[tokio::test] async fn git_info_refs_sheds_with_503_when_semaphore_exhausted() { let mut state = test_state_lazy(); @@ -234,10 +235,11 @@ mod tests { ); } - /// PR3 (#62) sibling of the info/refs shed test: git-upload-pack also acquires a - /// permit at the top, so an exhausted semaphore must shed it with a 503 before - /// any DB/disk work. Anonymous-reachable, so no auth injection is needed. Remove - /// the permit line from git_upload_pack and this goes red. + /// PR3 (#62) sibling of the info/refs shed test: git-upload-pack carries the same + /// explicit `available_permits() == 0` early-shed check at the top, so an + /// exhausted semaphore must shed it with a 503 before any DB/disk work. + /// Anonymous-reachable, so no auth injection is needed. Remove the early-shed + /// block from git_upload_pack and this goes red. #[tokio::test] async fn git_upload_pack_sheds_with_503_when_semaphore_exhausted() { let mut state = test_state_lazy(); @@ -270,6 +272,44 @@ mod tests { ); } + /// PR3 (#62) receive-pack sibling of the info/refs shed test: the early shed + /// selects the WRITE pool for a git-receive-pack advertisement (phase one of a + /// push, #174 U2), so an exhausted write pool sheds the advert with 503 before + /// any DB/disk work — even while the read pool is free (only the write pool is + /// zeroed here). Flip the pool selection to the read pool, or remove the + /// early-shed block, and this goes red. + #[tokio::test] + async fn git_info_refs_receive_pack_sheds_with_503_when_write_pool_exhausted() { + let mut state = test_state_lazy(); + state.git_write_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + + let router = Router::new() + .route( + "/{owner}/{repo}/info/refs", + axum::routing::get(crate::api::repos::git_info_refs), + ) + .with_state(state); + let resp = router + .oneshot(anon_get( + "/alice/repo.git/info/refs?service=git-receive-pack", + )) + .await + .unwrap(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted WRITE pool must shed the receive-pack advertisement with 503 before touching the DB" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + /// PR3 (#62) sibling for the push path: git-receive-pack requires an /// AuthenticatedDid extension (production: require_signature injects it), so the /// request carries one via signed_request_as — without it the Extension From f19f2813a2b73c326001e5724e0ae175d0b9d72c Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 09:44:53 -0500 Subject: [PATCH 020/124] fix(node): give the receive-pack advert its own pool so it can't shed authed pushes (#174) jatmn P1: the anon-reachable GET info/refs?service=git-receive-pack drew from the same git_write_semaphore as the authenticated git-receive-pack POST. The per-source advert cap reserves no global slots, so ~8 sources (each within the per-IP push rate limit) could hold all 32 write permits across acquire_fresh + the advertisement, and the next authenticated push then 503s at admission. That is an anonymous caller shedding an authenticated push across the auth boundary (INV-10). Add a dedicated git_push_advert_semaphore, disjoint from the write pool. Both the pre-DB early-shed peek and the held acquire for the receive-pack advertisement now draw from it; the write pool is left exclusively for the POST. An advert flood can at worst exhaust the advert pool (each source still capped by git_push_advert_per_caller and the per-IP push rate limiter); the authenticated push pool is untouched. Test receive_pack_advertisement_draws_from_dedicated_advert_pool proves at the handler layer that the advert sheds when its own pool is saturated AND survives when the write pool is saturated (the reservation). Both flip under revert, verified independently for the pre-DB peek and the held acquire. --- crates/gitlawb-node/src/api/repos.rs | 74 ++++++++++++++++--------- crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/main.rs | 7 +++ crates/gitlawb-node/src/state.rs | 16 ++++-- crates/gitlawb-node/src/test_support.rs | 17 +++--- 5 files changed, 76 insertions(+), 39 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 85eaa227..719246e0 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -522,8 +522,11 @@ pub async fn git_info_refs( // would drop, while the reorder still prevents one source from occupying global // slots during the DB/visibility window. { + // The receive-pack advertisement peeks its DEDICATED advert pool, not the + // write pool the authenticated POST uses (#174) — matching the held acquire + // below, so the pre-DB shed and the authoritative hold agree on the pool. let pool = if service == "git-receive-pack" { - &state.git_write_semaphore + &state.git_push_advert_semaphore } else { &state.git_read_semaphore }; @@ -620,12 +623,14 @@ pub async fn git_info_refs( // would be sub-cap-denied for during the DB/visibility window and starve other // sources; still before acquire_fresh/git so it bounds the fresh Tigris acquire // and git exec (INV-10). The receive-pack advertisement is phase one of a push, - // so it draws from the WRITE pool (like the git-receive-pack POST), not the - // global read pool an anonymous clone flood can exhaust and thereby starve the - // push handshake (#174 U2). The upload-pack advertisement stays on the read - // pool with its per-caller sub-cap. + // but it is ANON-reachable, so it draws from the dedicated advert pool + // (`git_push_advert_semaphore`), NOT the write pool the authenticated POST uses: + // an advert flood can at worst exhaust the advert pool, never a permit a push + // POST needs at admission (#174 U2). A clone flood on the read pool likewise + // can't touch either. The upload-pack advertisement stays on the read pool with + // its per-caller sub-cap. let _permit = if service == "git-receive-pack" { - git_permit(&state.git_write_semaphore)? + git_permit(&state.git_push_advert_semaphore)? } else { git_permit(&state.git_read_semaphore)? }; @@ -2944,14 +2949,19 @@ mod tests { ); } - /// #174 U2: the receive-pack advertisement (`GET info/refs?service=git-receive-pack`) - /// is phase one of a push, so it draws from the WRITE pool, not the global read - /// pool an anonymous clone flood can exhaust. Proven at the handler by saturating - /// each pool to zero and checking who shares it (INV-10). Revert the branch to the - /// read pool and the read-saturated receive-pack assertion goes 503 (the exact - /// push-handshake starvation jatmn flagged). + /// #174 (jatmn P1): the anon-reachable receive-pack advertisement + /// (`GET info/refs?service=git-receive-pack`) draws from a DEDICATED advert pool + /// (`git_push_advert_semaphore`), NOT the write pool the authenticated POST uses. + /// Proven at the handler by saturating each pool to zero and checking who shares + /// it (INV-10, across the auth boundary). The load-bearing pair: + /// * advert pool at 0 -> the advert SHEDS 503 (it is bound to that pool); + /// * write pool at 0 -> the advert SURVIVES (it can NOT consume a permit the + /// authenticated POST needs — the reservation jatmn asked for). + /// Revert the branch to `git_write_semaphore` and BOTH flip: the advert-pool-0 + /// case stops shedding and the write-pool-0 case starts shedding (the exact + /// anon-sheds-authed-push starvation). #[sqlx::test] - async fn receive_pack_advertisement_draws_from_write_pool(pool: sqlx::PgPool) { + async fn receive_pack_advertisement_draws_from_dedicated_advert_pool(pool: sqlx::PgPool) { use axum::body::Body; use axum::extract::ConnectInfo; use axum::http::{Method, Request, StatusCode}; @@ -2960,17 +2970,19 @@ mod tests { use tokio::sync::Semaphore; use tower::ServiceExt; - // Build a fresh state with the two pools sized independently, then drive one + // Build a fresh state with the three pools sized independently, then drive one // info/refs advertisement for `service` and return its handler status. async fn advert_status( pool: &sqlx::PgPool, read_permits: usize, write_permits: usize, + advert_permits: usize, service: &str, ) -> StatusCode { let mut state = crate::test_support::test_state(pool.clone()).await; state.git_read_semaphore = Arc::new(Semaphore::new(read_permits)); state.git_write_semaphore = Arc::new(Semaphore::new(write_permits)); + state.git_push_advert_semaphore = Arc::new(Semaphore::new(advert_permits)); state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; state .db @@ -2988,31 +3000,39 @@ mod tests { router.oneshot(req).await.unwrap().status() } - // Read pool saturated, write pool free: the push handshake SURVIVES. + // Advert pool saturated (read + write free): the receive-pack advert SHEDS, + // proving it is bound to the dedicated advert pool. + assert_eq!( + advert_status(&pool, 8, 8, 0, "git-receive-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "receive-pack advertisement draws from the dedicated advert pool: a saturated advert pool sheds it 503" + ); + // WRITE pool saturated (advert + read free): the advert SURVIVES. This is the + // reservation — an advert flood can never occupy a permit the authenticated + // push POST relies on at admission. assert_ne!( - advert_status(&pool, 0, 8, "git-receive-pack").await, + advert_status(&pool, 8, 0, 8, "git-receive-pack").await, StatusCode::SERVICE_UNAVAILABLE, - "receive-pack advertisement must draw from the write pool: a saturated read pool must not shed it" + "receive-pack advertisement must NOT draw from the write pool: a saturated write pool must not shed it" ); - // Write pool saturated, read pool free: the receive-pack advertisement SHEDS, - // proving it now consumes the write pool (not the read pool). - assert_eq!( - advert_status(&pool, 8, 0, "git-receive-pack").await, + // Read pool saturated (advert + write free): the advert SURVIVES (never on the read pool). + assert_ne!( + advert_status(&pool, 0, 8, 8, "git-receive-pack").await, StatusCode::SERVICE_UNAVAILABLE, - "receive-pack advertisement draws from the write pool, so a saturated write pool sheds it 503" + "receive-pack advertisement must not draw from the read pool" ); // Read pool saturated: the upload-pack advertisement still SHEDS (unchanged). assert_eq!( - advert_status(&pool, 0, 8, "git-upload-pack").await, + advert_status(&pool, 0, 8, 8, "git-upload-pack").await, StatusCode::SERVICE_UNAVAILABLE, "upload-pack advertisement stays on the read pool: a saturated read pool sheds it 503" ); - // Write pool saturated, read pool free: the upload-pack advertisement is - // UNAFFECTED, proving reads never touch the write pool in either direction. + // Write + advert pools saturated, read free: the upload-pack advertisement is + // UNAFFECTED, proving reads never touch either write-side pool. assert_ne!( - advert_status(&pool, 8, 0, "git-upload-pack").await, + advert_status(&pool, 8, 0, 0, "git-upload-pack").await, StatusCode::SERVICE_UNAVAILABLE, - "upload-pack advertisement never touches the write pool" + "upload-pack advertisement never touches the write or advert pool" ); } diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 5befa7f3..5d827794 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -522,6 +522,7 @@ mod tests { shutdown_tx: tokio::sync::watch::channel(false).0, git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 9b646c2b..18019395 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -382,6 +382,13 @@ async fn main() -> Result<()> { git_write_semaphore: Arc::new(tokio::sync::Semaphore::new( config.max_concurrent_git_pushes, )), + // Anon receive-pack advertisements get their OWN pool, same size as the + // write pool but disjoint, so filling it (which takes many source IPs, each + // capped by git_push_advert_per_caller) never occupies a permit the + // authenticated POST needs (#174). + git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new( + config.max_concurrent_git_pushes, + )), git_read_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( config.max_concurrent_reads_per_caller, ), diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 94952120..eb07a3e4 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -97,11 +97,19 @@ pub struct AppState { /// Bounds concurrent `git-receive-pack` (push) operations, a pool separate /// from `git_read_semaphore` so an anonymous READ flood can never shed an /// authenticated push (#174). Sized by `max_concurrent_git_pushes`. Drawn from - /// by the `git-receive-pack` POST (owner-gated) and the receive-pack `info/refs` - /// advertisement (anon-reachable); the advertisement is additionally bounded per - /// source by `git_push_advert_per_caller` so no single source can monopolize this - /// pool and shed pushes. + /// by the `git-receive-pack` POST (owner-gated) ONLY. The anon-reachable + /// receive-pack `info/refs` advertisement draws from the SEPARATE + /// `git_push_advert_semaphore` below, never this pool, so a multi-source flood + /// of push-handshake advertisements can never occupy a permit an authenticated + /// POST needs at admission (#174). pub git_write_semaphore: Arc, + /// Bounds concurrent anon-reachable `git-receive-pack` `info/refs` + /// advertisements — a pool SEPARATE from `git_write_semaphore` so adverts (which + /// hold a permit across `acquire_fresh` + `info/refs`) can never consume a slot + /// the authenticated POST relies on. A per-source flood can at worst exhaust this + /// advert pool (each source also capped by `git_push_advert_per_caller` and the + /// per-IP push rate limiter), and the reserved POST pool is untouched (#174). + pub git_push_advert_semaphore: Arc, /// Per-caller concurrency sub-cap on the read pool: each caller (keyed on the /// resolved source IP, #174 U1) may hold at most `max_concurrent_reads_per_caller` /// in-flight read ops, so one caller cannot monopolize `git_read_semaphore` diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 765545f1..0805fec1 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -85,6 +85,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { // Generous — no test drives the handler-level shed (git_permit is unit-tested). git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( 8, @@ -273,15 +274,15 @@ mod tests { } /// PR3 (#62) receive-pack sibling of the info/refs shed test: the early shed - /// selects the WRITE pool for a git-receive-pack advertisement (phase one of a - /// push, #174 U2), so an exhausted write pool sheds the advert with 503 before - /// any DB/disk work — even while the read pool is free (only the write pool is - /// zeroed here). Flip the pool selection to the read pool, or remove the - /// early-shed block, and this goes red. + /// selects the dedicated ADVERT pool for a git-receive-pack advertisement (#174), + /// so an exhausted advert pool sheds the advert with 503 before any DB/disk work + /// — while the write pool (reserved for authenticated POSTs) is left free here. + /// Flip the pool selection back to the write pool, or remove the early-shed + /// block, and this goes red. #[tokio::test] - async fn git_info_refs_receive_pack_sheds_with_503_when_write_pool_exhausted() { + async fn git_info_refs_receive_pack_sheds_with_503_when_advert_pool_exhausted() { let mut state = test_state_lazy(); - state.git_write_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + state.git_push_advert_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); let router = Router::new() .route( @@ -299,7 +300,7 @@ mod tests { assert_eq!( resp.status(), StatusCode::SERVICE_UNAVAILABLE, - "an exhausted WRITE pool must shed the receive-pack advertisement with 503 before touching the DB" + "an exhausted ADVERT pool must shed the receive-pack advertisement with 503 before touching the DB" ); assert_eq!( resp.headers() From 473924dcfec11c22b8ea5da5f125478bbcd5f845 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 09:49:33 -0500 Subject: [PATCH 021/124] fix(node): compile the bounded visibility git runner on non-Unix targets (#174) jatmn P1: run_bounded_git was gated #[cfg(unix)], but its callers (assert_all_refs_are_commits, blob_paths) and the module are compiled on all targets, so a non-Unix build (the continue-on-error Windows release target) could not resolve the function. Add a #[cfg(not(unix))] twin with the same signature and result semantics. It bounds a single child (no process-group teardown, which is Unix-only): threads feed stdin and drain stderr, the main thread drains stdout, and a watchdog thread kills the child at the deadline. The child is shared with the watchdog behind a mutex the main thread does not hold while draining, so the watchdog can always acquire it to kill, and killing closes stdout to unblock the drain. Best-effort (reaps only the direct child), which is why the hardened group-aware path stays Unix-only. Verified the twin type-checks by compiling its (portable-std) body on the native target; the full non-Unix link is CI's Windows job. --- .../gitlawb-node/src/git/visibility_pack.rs | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index 09a80aba..0eaeed2d 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -154,6 +154,98 @@ fn run_bounded_git( Ok(out) } +/// Non-Unix fallback for [`run_bounded_git`]. Windows and other non-Unix targets +/// have no process-group teardown (`process_group(0)` / `kill(-pgid)` are Unix-only), +/// so this bounds a single child on its own: threads feed stdin and drain stderr +/// while the main thread drains stdout, and a watchdog thread kills the child at the +/// deadline (which closes stdout and unblocks the drain). The child is shared with +/// the watchdog behind a mutex that the main thread does NOT hold while draining, so +/// the watchdog can always acquire it to kill. Best-effort — it reaps only the direct +/// child, not a descendant group — which is why the hardened, group-aware path above +/// is gated to Unix, the only target the served node actually runs on (the Windows +/// release binary is best-effort / `continue-on-error` in CI). Kept in lockstep with +/// the Unix version's signature and result semantics so every caller compiles on all +/// targets (#174). +#[cfg(not(unix))] +fn run_bounded_git( + git_bin: &str, + args: &[&str], + repo_path: &Path, + stdin_bytes: &[u8], + deadline: Instant, +) -> Result> { + use std::io::{Read, Write}; + use std::sync::mpsc::RecvTimeoutError; + + let label = args.first().copied().unwrap_or("git"); + let mut child = std::process::Command::new(git_bin) + .args(args) + .current_dir(repo_path) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .with_context(|| format!("failed to spawn git {label}"))?; + + let mut stdin = child.stdin.take(); + let input = stdin_bytes.to_vec(); + let writer = std::thread::spawn(move || { + if let Some(mut s) = stdin.take() { + let _ = s.write_all(&input); + } + }); + let mut stderr = child.stderr.take().context("git stderr was not piped")?; + let err_reader = std::thread::spawn(move || { + let mut err = Vec::new(); + let _ = stderr.read_to_end(&mut err); + err + }); + let mut stdout = child.stdout.take().context("git stdout was not piped")?; + + // Share the child with the watchdog. The main thread drains stdout WITHOUT + // holding this lock, so the watchdog can always acquire it to kill on timeout; + // killing closes stdout and unblocks the drain below. + let child = std::sync::Arc::new(std::sync::Mutex::new(child)); + let (done_tx, done_rx) = mpsc::channel::<()>(); + let watchdog = { + let child = child.clone(); + std::thread::spawn(move || -> bool { + let wait = deadline.saturating_duration_since(Instant::now()); + match done_rx.recv_timeout(wait) { + Ok(()) | Err(RecvTimeoutError::Disconnected) => false, + Err(RecvTimeoutError::Timeout) => { + if let Ok(mut c) = child.lock() { + let _ = c.kill(); + } + true + } + } + }) + }; + + let mut out = Vec::new(); + let read_result = stdout.read_to_end(&mut out); + // The drain has returned (child exited or was killed), so taking the lock here + // cannot deadlock against the watchdog. + let status = child + .lock() + .expect("git child mutex poisoned") + .wait() + .context("git wait failed")?; + let _ = done_tx.send(()); + let killed = watchdog.join().unwrap_or(false); + let err = err_reader.join().unwrap_or_default(); + let _ = writer.join(); + read_result.context("failed to read git stdout")?; + if killed && !status.success() { + return Err(crate::git::smart_http::GitServiceTimeout.into()); + } + if !status.success() { + anyhow::bail!("git {label} failed: {}", String::from_utf8_lossy(&err)); + } + Ok(out) +} + /// Fail closed unless every ref ultimately resolves to a commit (a ref pointing /// directly at a blob or tree, or an annotated tag — even a nested one — of such /// an object is refused). `git rev-list --all` silently *skips* such refs, but From 7a6bd359eb915285ba4e459ae3339f0ed8716891 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 10:11:41 -0500 Subject: [PATCH 022/124] fix(node): SIGKILL a SIGTERM-ignoring member of the visibility-walk process group (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit jatmn/CodeRabbit P1: the walk watchdog stood down as soon as the group LEADER was reaped. A leader that exits cleanly on SIGTERM while a background member ignores SIGTERM and closes its inherited pipes let the main drain unblock, the leader be reaped, and the watchdog skip its SIGKILL escalation — so the handler returned while the descendant kept running. jatmn's suggested "continue until ESRCH" could not work: the main thread reaping the leader is what frees the pgid, so continuing to kill(-pgid) after that races a recycled group. Restructure the coordination instead: the main thread now defers reaping the leader until the watchdog thread returns. The watchdog always escalates through an unconditional group SIGKILL (no leader-reap short-circuit), and because the leader is still unreaped while every kill(-pgid) fires, the pgid cannot have been recycled. The grace before SIGKILL is a fixed budget (only paid on the exceptional timeout path). Regression run_bounded_git_sigkills_a_sigterm_ignoring_descendant_after_leader_exits: a leader that exits 0 on SIGTERM spawns an sh -c descendant (its own $$, not a ( ) subshell's parent pid) that ignores SIGTERM and closes its pipes; the descendant must be dead when the runner returns. RED on the old teardown (survives), GREEN now. The existing leader-ignores-SIGTERM and hung-walk teardown tests still pass. --- .../gitlawb-node/src/git/visibility_pack.rs | 162 +++++++++++++----- 1 file changed, 118 insertions(+), 44 deletions(-) diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index 0eaeed2d..ec55423e 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -23,6 +23,12 @@ use std::time::{Duration, Instant}; #[cfg(test)] const WALK_TIMEOUT: Duration = Duration::from_secs(600); +/// How long the process-group watchdog waits after SIGTERM before escalating to +/// SIGKILL, giving a well-behaved git child time to clean up its `*.lock` files. Only +/// paid on a timeout (already the exceptional path). +#[cfg(unix)] +const WATCHDOG_TERM_GRACE: Duration = Duration::from_secs(1); + /// Run one git child under a shared `deadline` with process-group teardown, /// BLOCKING, and return its stdout. The child runs in its own process group; a /// watchdog thread SIGTERMs (lets git clean up its `*.lock` files), then SIGKILLs, @@ -60,52 +66,46 @@ fn run_bounded_git( // With process_group(0) the child leads its own group, so pgid == its pid. let pgid = child.id() as i32; - // Watchdog: on the deadline, SIGTERM the whole group, poll until it exits, - // escalate to SIGKILL after a grace, and report whether it killed. Signalled to - // stand down when the child is reaped in time. Kept off the main thread because - // the main thread's stdout drain is exactly what blocks until a hung child is - // torn down. + // Watchdog: on the deadline, tear the WHOLE process group down — SIGTERM, a grace + // for a well-behaved child to clean up its `*.lock` files, then an UNCONDITIONAL + // SIGKILL of the group. It never stands down on leader-reap alone: a group member + // that ignores SIGTERM while the leader exits cleanly would otherwise escape the + // SIGKILL and keep running past the deadline (finding 3, #174). The main thread + // defers reaping the leader until this thread returns (see below), so the leader's + // pid is still unreaped while every `kill(-pgid)` fires and the pgid cannot have + // been recycled — which is why this no longer needs the old `reaped` short-circuit. + // Kept off the main thread because the main thread's stdout drain is exactly what + // blocks until a hung child is torn down. let (done_tx, done_rx) = mpsc::channel::<()>(); - // Set true the instant the main thread reaps the child, so the watchdog never - // SIGTERMs a pgid whose leader is already reaped and whose pid the kernel may - // recycle (the reused-pgid hazard smart_http guards via disarm-after-wait). - let reaped = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); - let watchdog = { - let reaped = reaped.clone(); - std::thread::spawn(move || -> bool { - use std::sync::atomic::Ordering; - let wait = deadline.saturating_duration_since(Instant::now()); - match done_rx.recv_timeout(wait) { - Ok(()) | Err(RecvTimeoutError::Disconnected) => false, - Err(RecvTimeoutError::Timeout) => { - // The child was reaped right as the deadline fired: do not signal - // its (possibly-recycled) pgid. - if reaped.load(Ordering::SeqCst) { - return false; - } - // SAFETY: kill(2) takes only integers and borrows no Rust memory; - // ESRCH on an already-gone group is ignored. - unsafe { libc::kill(-pgid, libc::SIGTERM) }; - for step in 0..200u32 { - if reaped.load(Ordering::SeqCst) || unsafe { libc::kill(-pgid, 0) } != 0 { - return true; // reaped, or ESRCH: every group member has exited - } - if step == 100 { - unsafe { libc::kill(-pgid, libc::SIGKILL) }; - } - std::thread::sleep(Duration::from_millis(10)); - } - // Survived SIGKILL past the cap: a wedged (D-state) git, the same - // condition smart_http's reap logs. Warn so an operator sees it. + let watchdog = std::thread::spawn(move || -> bool { + let wait = deadline.saturating_duration_since(Instant::now()); + match done_rx.recv_timeout(wait) { + Ok(()) | Err(RecvTimeoutError::Disconnected) => false, + Err(RecvTimeoutError::Timeout) => { + // SAFETY: kill(2) takes only integers and borrows no Rust memory; + // ESRCH on an already-gone group is ignored. + unsafe { libc::kill(-pgid, libc::SIGTERM) }; + // Fixed grace: because the main thread defers the leader's reap, a + // fully-exited group still shows a zombie leader here, so polling for + // ESRCH cannot detect early completion — just wait the grace, then + // SIGKILL. On a group of only zombies the SIGKILL is a harmless no-op; + // on a SIGTERM-ignoring member it is what actually kills it. + std::thread::sleep(WATCHDOG_TERM_GRACE); + unsafe { libc::kill(-pgid, libc::SIGKILL) }; + // Brief settle so the SIGKILL is delivered before the main thread + // reaps the leader and frees the pgid. A wedged (D-state) member + // survives even SIGKILL — the documented residual, as in smart_http. + std::thread::sleep(Duration::from_millis(20)); + if unsafe { libc::kill(-pgid, 0) } == 0 { tracing::warn!( pgid, "withheld-walk git survived SIGKILL past the watchdog cap (uninterruptible I/O?)" ); - true } + true } - }) - }; + } + }); // Feed stdin on a writer thread and drain stderr on a reader thread so the main // thread can drain stdout concurrently; writing all of stdin (or draining one @@ -132,14 +132,18 @@ fn run_bounded_git( // is unreachable in userspace (no signal reaps a D-state process) and matches the // async `reap_group_on_timeout`, which likewise only warns and gives up there. let read_result = stdout.read_to_end(&mut out); + // The drain has returned: either the child exited on its own, or the watchdog's + // teardown closed its pipes. Tell the watchdog the drain is done, then WAIT for it + // to return BEFORE reaping the leader. On the timeout path the watchdog runs the + // full SIGTERM -> grace -> SIGKILL teardown; joining it before `child.wait()` keeps + // the leader's pid unreaped while every `kill(-pgid)` fires (so the pgid can't be + // recycled), and guarantees a SIGTERM-ignoring group member has already been + // SIGKILLed rather than left running past the deadline (finding 3, #174). + let _ = done_tx.send(()); + let killed = watchdog.join().unwrap_or(false); let status = child.wait().context("git wait failed")?; - // Reaped: bar the watchdog from signalling the now-reaped (possibly recycled) - // pgid before it can fire, then stand it down. - reaped.store(true, std::sync::atomic::Ordering::SeqCst); let err = err_reader.join().unwrap_or_default(); let _ = writer.join(); - let _ = done_tx.send(()); - let killed = watchdog.join().unwrap_or(false); read_result.context("failed to read git stdout")?; // The watchdog runs off a wall clock that can race a child finishing right at the // deadline. A child that exited on its own (success) is not a timeout even if the @@ -828,6 +832,76 @@ mod tests { "the SIGTERM-ignoring child (pid {pid}) must be reaped via SIGKILL, not left running" ); } + + /// #174 finding 3 (jatmn/CodeRabbit): a group MEMBER that ignores SIGTERM must + /// still be SIGKILLed even when the group LEADER exits cleanly on SIGTERM. The + /// leader traps SIGTERM to exit 0, but first spawns a descendant (`sh -c`, so its + /// `$$` is its OWN pid — a `( )` subshell's `$$` is the parent's) that ignores + /// SIGTERM and closes its inherited stdout/stderr. When the watchdog SIGTERMs the + /// group, the leader exits, its stdout closes, the main drain unblocks, and the + /// leader is reaped — the exact window a `reaped`-gated watchdog stands down in, + /// before escalating to SIGKILL. The descendant must be dead when run_bounded_git + /// returns; a teardown that stands down on leader-reap leaves it running (RED). + #[cfg(unix)] + #[test] + fn run_bounded_git_sigkills_a_sigterm_ignoring_descendant_after_leader_exits() { + use std::time::{Duration, Instant}; + let tmp = TempDir::new().unwrap(); + // Both loops are bounded (~30s) so a broken teardown cannot leak a permanent + // orphan or wedge the suite; the assertion fires well before then. + let body = "#!/bin/sh\n\ +case \"$1\" in\n\ + rev-list)\n\ + sh -c 'trap \"\" TERM; echo $$ > desc.pid; exec 1>&- 2>&-; i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done' &\n\ + trap 'exit 0' TERM\n\ + i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;;\n\ + *) : ;;\n\ +esac\n"; + let git_bin = write_fake_git(tmp.path(), body); + let (tx, rx) = std::sync::mpsc::channel(); + let path = tmp.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(run_bounded_git( + &git_bin, + &["rev-list"], + &path, + b"", + Instant::now() + Duration::from_millis(100), + )); + }); + let _ = rx + .recv_timeout(Duration::from_secs(10)) + .expect("run_bounded_git must return within the watchdog budget"); + + // Wait for the descendant to record its OWN pid, then assert it is gone. + let desc_pid_path = tmp.path().join("desc.pid"); + let mut desc: Option = None; + for _ in 0..200 { + if let Some(p) = std::fs::read_to_string(&desc_pid_path) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + desc = Some(p); + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + let desc = desc.expect("the fake leader must have spawned and recorded a descendant"); + let mut gone = false; + for _ in 0..300 { + if unsafe { libc::kill(desc, 0) } != 0 { + gone = true; + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + // Kill it regardless so a RED run leaks no orphan. + unsafe { libc::kill(desc, libc::SIGKILL) }; + assert!( + gone, + "a SIGTERM-ignoring descendant (pid {desc}) must be SIGKILLed even after the leader exits cleanly, not orphaned" + ); + } use crate::db::VisibilityMode; use chrono::Utc; use std::process::Command; From 9412aa9c5bdc56d5c27cbf5be3dfebea995e7039 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 10:34:44 -0500 Subject: [PATCH 023/124] fix(node): bound and reap push-candidate git children under the write permit (#174) Route push_delta's candidate-discovery git children (cat-file / rev-list) through the bounded run_bounded_git (deadline + process-group SIGKILL teardown) instead of bare Command::output(). A hung scan no longer pins the git-receive-pack write permit past the deadline or a client disconnect. Threads git_bin + a shared per-scan deadline (git_service_timeout_secs) through resolve_candidates_for_push and all_blob_oids. --- crates/gitlawb-node/src/api/repos.rs | 4 +- crates/gitlawb-node/src/git/push_delta.rs | 278 ++++++++++++------ .../gitlawb-node/src/git/visibility_pack.rs | 11 +- 3 files changed, 193 insertions(+), 100 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 719246e0..ea6f126e 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -115,7 +115,7 @@ async fn fail_closed_full_scan_objects( let allowed = crate::git::visibility_pack::replicable_blob_set_bounded( &disk_path, &git_bin, timeout, &rules, is_public, &owner_did, )?; - let all_blobs = crate::git::push_delta::all_blob_oids(&disk_path)?; + let all_blobs = crate::git::push_delta::all_blob_oids(&disk_path, &git_bin, std::time::Instant::now() + timeout)?; Ok(crate::git::visibility_pack::replicable_objects_fail_closed( candidates, &allowed, &all_blobs, )) @@ -1264,6 +1264,8 @@ pub async fn git_receive_pack( disk_path.clone(), new_tips, old_tips, + state.git_bin.clone(), + std::time::Duration::from_secs(state.config.git_service_timeout_secs), ) .await; if pin_set.full_scan { diff --git a/crates/gitlawb-node/src/git/push_delta.rs b/crates/gitlawb-node/src/git/push_delta.rs index 7ab00816..51b6f836 100644 --- a/crates/gitlawb-node/src/git/push_delta.rs +++ b/crates/gitlawb-node/src/git/push_delta.rs @@ -29,9 +29,9 @@ use std::collections::HashSet; use std::path::{Path, PathBuf}; -use std::process::Command; +use std::time::{Duration, Instant}; -use anyhow::{bail, Context, Result}; +use anyhow::Result; /// Env var that forces the push path to always full-scan, bypassing the delta /// optimization (KTD7 kill-switch). Reuses the already-tested fallback branch @@ -64,11 +64,21 @@ pub enum PinCandidates { /// Fail-closed: any condition where the introduced set cannot be safely /// determined returns [`PinCandidates::FullScanRequired`] rather than a partial /// set, so the caller full-scans instead of silently under-pinning. -pub fn resolve_push_delta(repo_path: &Path, new_tips: &[&str], old_tips: &[&str]) -> PinCandidates { - // KTD7 kill-switch: force the (already-tested) full-scan fallback. The env - // read is split out from the pure logic so the resolver stays unit-testable - // without touching process-global state. - resolve_push_delta_inner(repo_path, new_tips, old_tips, force_full_scan()) +pub fn resolve_push_delta( + repo_path: &Path, + new_tips: &[&str], + old_tips: &[&str], + git_bin: &str, + deadline: Instant, +) -> PinCandidates { + resolve_push_delta_inner( + repo_path, + new_tips, + old_tips, + force_full_scan(), + git_bin, + deadline, + ) } /// Whether the force-full-scan kill-switch env var is set. @@ -83,6 +93,8 @@ fn resolve_push_delta_inner( new_tips: &[&str], old_tips: &[&str], force_full_scan: bool, + git_bin: &str, + deadline: Instant, ) -> PinCandidates { if force_full_scan { tracing::debug!("{FORCE_FULL_SCAN_ENV} set — forcing full scan"); @@ -102,7 +114,7 @@ fn resolve_push_delta_inner( // commit). Bare `cat-file -t` returns `tag` for an annotated tag, and // `for-each-ref %(*objecttype)` peels only one level — neither is correct. for tip in new_tips { - match peeled_object_type(repo_path, tip) { + match peeled_object_type(repo_path, tip, git_bin, deadline) { Some(t) if t == "commit" => {} other => { tracing::debug!( @@ -115,7 +127,7 @@ fn resolve_push_delta_inner( } } - match rev_list_delta(repo_path, new_tips, old_tips) { + match rev_list_delta(repo_path, new_tips, old_tips, git_bin, deadline) { Ok(oids) => PinCandidates::Delta(oids), Err(e) => { tracing::debug!(err = %e, "push-delta rev-list failed — forcing full scan"); @@ -126,41 +138,43 @@ fn resolve_push_delta_inner( /// Return the fully-peeled object type of `sha` (e.g. `commit`, `tree`, /// `blob`), or `None` if the object is missing/unpeelable or git errored. -fn peeled_object_type(repo_path: &Path, sha: &str) -> Option { - let output = Command::new("git") - .args(["cat-file", "-t", &format!("{sha}^{{}}")]) - .current_dir(repo_path) - .output() - .ok()?; - if !output.status.success() { - return None; - } - Some(String::from_utf8_lossy(&output.stdout).trim().to_string()) +fn peeled_object_type( + repo_path: &Path, + sha: &str, + git_bin: &str, + deadline: Instant, +) -> Option { + let peel = format!("{sha}^{{}}"); + let out = crate::git::visibility_pack::run_bounded_git( + git_bin, + &["cat-file", "-t", &peel], + repo_path, + b"", + deadline, + ) + .ok()?; + Some(String::from_utf8_lossy(&out).trim().to_string()) } /// Run `git rev-list --objects --no-object-names --not ` and return /// the bare OID set. Decides on `status.success()` *before* parsing stdout, so /// a walk that prints a valid prefix then errors mid-walk is discarded. -fn rev_list_delta(repo_path: &Path, new_tips: &[&str], old_tips: &[&str]) -> Result> { +fn rev_list_delta( + repo_path: &Path, + new_tips: &[&str], + old_tips: &[&str], + git_bin: &str, + deadline: Instant, +) -> Result> { let mut args: Vec<&str> = vec!["rev-list", "--objects", "--no-object-names"]; args.extend_from_slice(new_tips); if !old_tips.is_empty() { args.push("--not"); args.extend_from_slice(old_tips); } - - let output = Command::new("git") - .args(&args) - .current_dir(repo_path) - .output() - .context("failed to run git rev-list for push delta")?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - bail!("git rev-list failed: {stderr}"); - } - - let stdout = String::from_utf8_lossy(&output.stdout); + let out = + crate::git::visibility_pack::run_bounded_git(git_bin, &args, repo_path, b"", deadline)?; + let stdout = String::from_utf8_lossy(&out); Ok(stdout .lines() .map(|l| l.trim().to_string()) @@ -175,23 +189,19 @@ fn rev_list_delta(repo_path: &Path, new_tips: &[&str], old_tips: &[&str]) -> Res /// reconciliation sweep relies on. It returns *all* objects (including /// unreachable/dangling ones), which is what the sweep needs to catch /// stragglers — do not swap it for a reachability walk. -pub fn list_all_objects(repo_path: &Path) -> Result> { - let output = Command::new("git") - .args([ +pub fn list_all_objects(repo_path: &Path, git_bin: &str, deadline: Instant) -> Result> { + let out = crate::git::visibility_pack::run_bounded_git( + git_bin, + &[ "cat-file", "--batch-all-objects", "--batch-check=%(objectname)", - ]) - .current_dir(repo_path) - .output() - .context("failed to run git cat-file")?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - bail!("git cat-file failed: {stderr}"); - } - - let stdout = String::from_utf8_lossy(&output.stdout); + ], + repo_path, + b"", + deadline, + )?; + let stdout = String::from_utf8_lossy(&out); Ok(stdout .lines() .map(|l| l.trim().to_string()) @@ -203,23 +213,23 @@ pub fn list_all_objects(repo_path: &Path) -> Result> { /// `--batch-check='%(objectname) %(objecttype)'`. The pin path's fail-closed /// filter needs to tell blobs (content, withholdable) from commits/trees /// (structural, never withheld) without typing the candidate list itself. -pub fn list_all_objects_with_type(repo_path: &Path) -> Result> { - let output = Command::new("git") - .args([ +pub fn list_all_objects_with_type( + repo_path: &Path, + git_bin: &str, + deadline: Instant, +) -> Result> { + let out = crate::git::visibility_pack::run_bounded_git( + git_bin, + &[ "cat-file", "--batch-all-objects", "--batch-check=%(objectname) %(objecttype)", - ]) - .current_dir(repo_path) - .output() - .context("failed to run git cat-file")?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - bail!("git cat-file failed: {stderr}"); - } - - let stdout = String::from_utf8_lossy(&output.stdout); + ], + repo_path, + b"", + deadline, + )?; + let stdout = String::from_utf8_lossy(&out); Ok(stdout .lines() .filter_map(|l| { @@ -235,8 +245,12 @@ pub fn list_all_objects_with_type(repo_path: &Path) -> Result Result> { - Ok(list_all_objects_with_type(repo_path)? +pub fn all_blob_oids( + repo_path: &Path, + git_bin: &str, + deadline: Instant, +) -> Result> { + Ok(list_all_objects_with_type(repo_path, git_bin, deadline)? .into_iter() .filter(|(_, ty)| ty == "blob") .map(|(oid, _)| oid) @@ -272,18 +286,22 @@ pub async fn resolve_candidates_for_push( repo_path: PathBuf, new_tips: Vec, old_tips: Vec, + git_bin: String, + timeout: Duration, ) -> PinCandidateSet { tokio::task::spawn_blocking(move || { + // ONE shared deadline for the whole scan, per jatmn ("the same deadline"). + let deadline = Instant::now() + timeout; let new_refs: Vec<&str> = new_tips.iter().map(String::as_str).collect(); let old_refs: Vec<&str> = old_tips.iter().map(String::as_str).collect(); - match resolve_push_delta(&repo_path, &new_refs, &old_refs) { + match resolve_push_delta(&repo_path, &new_refs, &old_refs, &git_bin, deadline) { PinCandidates::Delta(objs) => { tracing::info!(delta = objs.len(), repo = %repo_path.display(), "pin candidate set from push delta"); PinCandidateSet { candidates: objs, full_scan: false } } PinCandidates::FullScanRequired => { tracing::warn!(repo = %repo_path.display(), "pin delta unavailable (non-commit tip, git error, or force-full-scan) — full-scan fallback"); - match list_all_objects(&repo_path) { + match list_all_objects(&repo_path, &git_bin, deadline) { Ok(objs) => PinCandidateSet { candidates: objs, full_scan: true }, Err(e) => { tracing::warn!(repo = %repo_path.display(), err = %e, "full-scan fallback failed; pinning nothing this push (reconciliation sweep backstops)"); @@ -304,8 +322,13 @@ pub async fn resolve_candidates_for_push( mod tests { use super::*; use std::collections::HashSet; + use std::process::Command; use tempfile::TempDir; + fn td() -> std::time::Instant { + std::time::Instant::now() + std::time::Duration::from_secs(600) + } + /// Minimal git helper for building test repos. struct Repo { _td: TempDir, @@ -363,9 +386,10 @@ mod tests { let repo = Repo::new(); let c1 = repo.commit_file("a.txt", "one\n"); let c2 = repo.commit_file("b.txt", "two\n"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&c2], &[&c1])) - .into_iter() - .collect(); + let got: HashSet = + delta(resolve_push_delta(&repo.path, &[&c2], &[&c1], "git", td())) + .into_iter() + .collect(); // The new blob b.txt and commit c2 are in the delta; the old blob a.txt // and commit c1 are not. let new_blob = repo.rev("HEAD:b.txt"); @@ -383,7 +407,7 @@ mod tests { // genuinely new objects, never fewer. let repo = Repo::new(); let c1 = repo.commit_file("a.txt", "one\n"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&c1], &[])) + let got: HashSet = delta(resolve_push_delta(&repo.path, &[&c1], &[], "git", td())) .into_iter() .collect(); assert!(got.contains(&c1)); @@ -399,9 +423,15 @@ mod tests { // Rewrite history: reset to base, commit a different file. repo.git(&["reset", "-q", "--hard", &base]); let new_tip = repo.commit_file("c.txt", "three\n"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&new_tip], &[&old_tip])) - .into_iter() - .collect(); + let got: HashSet = delta(resolve_push_delta( + &repo.path, + &[&new_tip], + &[&old_tip], + "git", + td(), + )) + .into_iter() + .collect(); assert!(got.contains(&new_tip), "new tip in delta"); assert!(got.contains(&repo.rev(&format!("{new_tip}:c.txt")))); // No error; force-push computes new-minus-old cleanly. @@ -413,7 +443,7 @@ mod tests { repo.commit_file("a.txt", "one\n"); // All updates were deletions => new_tips empty after the caller strips zeros. assert_eq!( - resolve_push_delta(&repo.path, &[], &[ZERO]), + resolve_push_delta(&repo.path, &[], &[ZERO], "git", td()), PinCandidates::Delta(Vec::new()) ); } @@ -424,7 +454,7 @@ mod tests { repo.commit_file("a.txt", "one\n"); let blob = repo.rev("HEAD:a.txt"); assert_eq!( - resolve_push_delta(&repo.path, &[&blob], &[]), + resolve_push_delta(&repo.path, &[&blob], &[], "git", td()), PinCandidates::FullScanRequired, "a blob tip must force full scan (rev-list would exit 0 and walk it)" ); @@ -436,7 +466,7 @@ mod tests { repo.commit_file("a.txt", "one\n"); let tree = repo.rev("HEAD^{tree}"); assert_eq!( - resolve_push_delta(&repo.path, &[&tree], &[]), + resolve_push_delta(&repo.path, &[&tree], &[], "git", td()), PinCandidates::FullScanRequired ); } @@ -449,7 +479,7 @@ mod tests { repo.git(&["tag", "-a", "treetag", "-m", "x", &tree]); let tag = repo.rev("treetag"); assert_eq!( - resolve_push_delta(&repo.path, &[&tag], &[]), + resolve_push_delta(&repo.path, &[&tag], &[], "git", td()), PinCandidates::FullScanRequired, "annotated tag peeling to a tree must force full scan" ); @@ -465,7 +495,7 @@ mod tests { repo.git(&["tag", "-a", "t2", "-m", "x", &t1]); let t2 = repo.rev("t2"); assert_eq!( - resolve_push_delta(&repo.path, &[&t2], &[]), + resolve_push_delta(&repo.path, &[&t2], &[], "git", td()), PinCandidates::FullScanRequired ); } @@ -480,7 +510,7 @@ mod tests { let c1 = repo.commit_file("a.txt", "one\n"); repo.git(&["tag", "-a", "rel", "-m", "release", &c1]); let tag = repo.rev("rel"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&tag], &[])) + let got: HashSet = delta(resolve_push_delta(&repo.path, &[&tag], &[], "git", td())) .into_iter() .collect(); assert!( @@ -501,7 +531,7 @@ mod tests { let t1 = repo.rev("t1"); repo.git(&["tag", "-a", "t2", "-m", "x", &t1]); let t2 = repo.rev("t2"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&t2], &[])) + let got: HashSet = delta(resolve_push_delta(&repo.path, &[&t2], &[], "git", td())) .into_iter() .collect(); assert!( @@ -516,9 +546,14 @@ mod tests { let repo = Repo::new(); let c1 = repo.commit_file("a.txt", "one\n"); let c2 = repo.commit_file("b.txt", "two\n"); - let set = - resolve_candidates_for_push(repo.path.clone(), vec![c2.clone()], vec![c1.clone()]) - .await; + let set = resolve_candidates_for_push( + repo.path.clone(), + vec![c2.clone()], + vec![c1.clone()], + "git".to_string(), + std::time::Duration::from_secs(600), + ) + .await; assert!(!set.full_scan, "happy-path delta is not a full scan"); let got: HashSet = set.candidates.into_iter().collect(); let new_blob = repo.rev("HEAD:b.txt"); @@ -536,8 +571,18 @@ mod tests { let repo = Repo::new(); repo.commit_file("a.txt", "one\n"); let blob = repo.rev("HEAD:a.txt"); - let all: HashSet = list_all_objects(&repo.path).unwrap().into_iter().collect(); - let set = resolve_candidates_for_push(repo.path.clone(), vec![blob], vec![]).await; + let all: HashSet = list_all_objects(&repo.path, "git", td()) + .unwrap() + .into_iter() + .collect(); + let set = resolve_candidates_for_push( + repo.path.clone(), + vec![blob], + vec![], + "git".to_string(), + std::time::Duration::from_secs(600), + ) + .await; assert!(set.full_scan, "non-commit tip is signalled as a full scan"); let got: HashSet = set.candidates.into_iter().collect(); assert_eq!(got, all, "non-commit tip falls back to full repo scan"); @@ -549,7 +594,7 @@ mod tests { repo.commit_file("a.txt", "one\n"); let bogus = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"; assert_eq!( - resolve_push_delta(&repo.path, &[bogus], &[]), + resolve_push_delta(&repo.path, &[bogus], &[], "git", td()), PinCandidates::FullScanRequired, "a missing/corrupt tip OID must force full scan" ); @@ -564,7 +609,7 @@ mod tests { let c1 = repo.commit_file("a.txt", "one\n"); let c2 = repo.commit_file("b.txt", "two\n"); let old_tree = repo.rev(&format!("{c1}^{{tree}}")); - let result = resolve_push_delta(&repo.path, &[&c2], &[&old_tree]); + let result = resolve_push_delta(&repo.path, &[&c2], &[&old_tree], "git", td()); match result { PinCandidates::FullScanRequired => {} // safe: caller full-scans PinCandidates::Delta(objs) => { @@ -583,10 +628,15 @@ mod tests { // branch2 from base advances independently repo.git(&["checkout", "-q", "-b", "branch2", &base]); let b2 = repo.commit_file("c.txt", "three\n"); - let got: HashSet = - delta(resolve_push_delta(&repo.path, &[&b1, &b2], &[&base, &base])) - .into_iter() - .collect(); + let got: HashSet = delta(resolve_push_delta( + &repo.path, + &[&b1, &b2], + &[&base, &base], + "git", + td(), + )) + .into_iter() + .collect(); assert!(got.contains(&b1), "branch1 new commit"); assert!(got.contains(&b2), "branch2 new commit"); } @@ -595,7 +645,7 @@ mod tests { fn empty_repo_no_tips_yields_empty_delta() { let repo = Repo::new(); assert_eq!( - resolve_push_delta(&repo.path, &[], &[]), + resolve_push_delta(&repo.path, &[], &[], "git", td()), PinCandidates::Delta(Vec::new()) ); } @@ -609,12 +659,12 @@ mod tests { let repo = Repo::new(); let c1 = repo.commit_file("a.txt", "one\n"); assert_eq!( - resolve_push_delta_inner(&repo.path, &[&c1], &[], true), + resolve_push_delta_inner(&repo.path, &[&c1], &[], true, "git", td()), PinCandidates::FullScanRequired ); // And with the flag off, the same push yields a Delta. assert!(matches!( - resolve_push_delta_inner(&repo.path, &[&c1], &[], false), + resolve_push_delta_inner(&repo.path, &[&c1], &[], false, "git", td()), PinCandidates::Delta(_) )); } @@ -624,7 +674,7 @@ mod tests { let repo = Repo::new(); repo.commit_file("a.txt", "one\n"); repo.commit_file("b.txt", "two\n"); - let all = list_all_objects(&repo.path).unwrap(); + let all = list_all_objects(&repo.path, "git", td()).unwrap(); // 2 commits + 2 trees + 2 blobs = 6 objects. assert_eq!(all.len(), 6, "got: {all:?}"); } @@ -642,7 +692,7 @@ mod tests { std::fs::write(repo.path.join("orphan.bin"), b"dangling\n").unwrap(); let dangling = repo.git(&["hash-object", "-w", "orphan.bin"]); - let blobs = all_blob_oids(&repo.path).unwrap(); + let blobs = all_blob_oids(&repo.path, "git", td()).unwrap(); assert!(blobs.contains(&reachable_blob), "reachable blob present"); assert!( blobs.contains(&dangling), @@ -652,7 +702,7 @@ mod tests { assert!(!blobs.contains(&tree), "tree is not a blob"); // The typed lister tags each object; spot-check the dangling blob's type. - let typed = list_all_objects_with_type(&repo.path).unwrap(); + let typed = list_all_objects_with_type(&repo.path, "git", td()).unwrap(); assert!( typed .iter() @@ -660,4 +710,40 @@ mod tests { "dangling object is typed as a blob" ); } + + #[cfg(unix)] + #[test] + fn list_all_objects_times_out_on_a_hung_git_instead_of_blocking() { + use std::os::unix::fs::PermissionsExt; + use std::time::{Duration, Instant}; + let dir = tempfile::TempDir::new().unwrap(); + // Fake git: hang on cat-file (bounded to 30s so a broken test can't wedge). + let fake = dir.path().join("fakegit"); + std::fs::write( + &fake, + "#!/bin/sh\ncase \"$1\" in\n cat-file) i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;;\n *) : ;;\nesac\n", + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + let git_bin = fake.to_str().unwrap().to_string(); + + let (tx, rx) = std::sync::mpsc::channel(); + let path = dir.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(list_all_objects( + &path, + &git_bin, + Instant::now() + Duration::from_millis(150), + )); + }); + let res = rx.recv_timeout(Duration::from_secs(10)).expect( + "list_all_objects must return within the watchdog budget, not block on a hung git", + ); + assert!( + res.is_err(), + "a hung git must make list_all_objects error out, not hang" + ); + } } diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index ec55423e..9f40cd4b 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -42,7 +42,7 @@ const WATCHDOG_TERM_GRACE: Duration = Duration::from_secs(1); /// drive the teardown in tests without mutating the process-global PATH; /// `stdin_bytes` feeds children that read stdin (empty for the arg-only children). #[cfg(unix)] -fn run_bounded_git( +pub(crate) fn run_bounded_git( git_bin: &str, args: &[&str], repo_path: &Path, @@ -171,7 +171,7 @@ fn run_bounded_git( /// the Unix version's signature and result semantics so every caller compiles on all /// targets (#174). #[cfg(not(unix))] -fn run_bounded_git( +pub(crate) fn run_bounded_git( git_bin: &str, args: &[&str], repo_path: &Path, @@ -1235,7 +1235,12 @@ esac\n"; String::from_utf8_lossy(&out.stdout).trim().to_string() }; - let all_blobs = crate::git::push_delta::all_blob_oids(&work).unwrap(); + let all_blobs = crate::git::push_delta::all_blob_oids( + &work, + "git", + std::time::Instant::now() + std::time::Duration::from_secs(600), + ) + .unwrap(); assert!( all_blobs.contains(&dangling_oid), "precondition: the dangling blob is in the all-objects universe" From 4410b834df04163c6121629e3e3b05b227501e27 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 10:46:24 -0500 Subject: [PATCH 024/124] docs(node): correct concurrency + timeout docs to match the final git routing (#174) jatmn P2 (INV-24): the config help, .env.example, and README described routing the handler no longer does. - The read pool serves upload-pack and the UPLOAD-PACK info/refs advertisement only; the anon receive-pack info/refs advertisement draws from its own dedicated pool (sized like the write pool but disjoint), not the read pool and not the write pool. - The write pool is the authenticated push POST only. - The per-caller read cap is keyed on the resolved source IP, never the DID: a signature does not move a caller off it (the docs claimed a per-DID budget, which would tell a NATed authenticated client it escapes the shared cap when it does not). - git_service_timeout_secs now also bounds the push-side pin-candidate discovery (rev-list / cat-file), reaped via process-group teardown, alongside the info/refs advertisements and the withheld-blob walk. --- .env.example | 40 ++++++++++++--------- README.md | 2 +- crates/gitlawb-node/src/config.rs | 60 +++++++++++++++++-------------- 3 files changed, 57 insertions(+), 45 deletions(-) diff --git a/.env.example b/.env.example index cbf3db3a..0ef193f8 100644 --- a/.env.example +++ b/.env.example @@ -109,29 +109,35 @@ GITLAWB_MAX_PACK_BYTES=2147483648 # Max seconds a served git upload-pack / receive-pack (clone / push) may run # before it is aborted with a 504. Bounds a hung git that would otherwise pin a -# worker and, on push, the repo write lock. Also bounds the info/refs -# advertisement and the withheld-blob pack build (#174). Default 600. +# worker and, on push, the repo write lock. Also bounds both info/refs +# advertisements, the withheld-blob pack build, and the push-side candidate +# discovery (rev-list / cat-file), all reaped via process-group teardown (#174). +# Default 600. GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600 -# Max concurrent git read ops (upload-pack + info/refs advertisements) served at -# once, a global pool separate from the push pool below. Over-cap sheds a clean -# 503 + Retry-After. Anonymous reads draw from here, so pair it with -# GITLAWB_MAX_CONCURRENT_READS_PER_CALLER (below) so one caller cannot monopolize -# the pool. Default 128. +# Max concurrent git READ ops (upload-pack + the upload-pack info/refs +# advertisement) served at once, a global pool separate from the push pool below. +# The anon receive-pack info/refs advertisement has its OWN pool (see below), not +# this one. Over-cap sheds a clean 503 + Retry-After. Anonymous reads draw from +# here, so pair it with GITLAWB_MAX_CONCURRENT_READS_PER_CALLER (below) so one +# caller cannot monopolize the pool. Default 128. GITLAWB_MAX_CONCURRENT_GIT_OPS=128 -# Max concurrent git-receive-pack (push) operations, in a pool separate from the -# read pool (GITLAWB_MAX_CONCURRENT_GIT_OPS) so anonymous reads cannot shed an -# authenticated push at admission. Over-cap sheds a 503 + Retry-After. Default 32. +# Max concurrent git-receive-pack (push) POST operations, in a pool separate from +# the read pool (GITLAWB_MAX_CONCURRENT_GIT_OPS) so anonymous reads cannot shed an +# authenticated push at admission. The anon receive-pack info/refs advertisement +# runs in a SEPARATE pool of the same size (disjoint from this one), so an +# advertisement flood cannot shed a push either. Over-cap sheds a 503 + +# Retry-After. Default 32. GITLAWB_MAX_CONCURRENT_GIT_PUSHES=32 -# Max concurrent read ops (upload-pack + info/refs advertisements) a single caller -# may hold, so one caller cannot monopolize the read pool. Keyed per-DID when -# signed, else per-source-IP. The per-IP key is only as granular as -# GITLAWB_TRUSTED_PROXY below: left unset, a node behind an edge/NAT keys all -# anonymous callers on the edge IP and this collapses to one global anonymous cap. -# Set GITLAWB_TRUSTED_PROXY for per-client keying; a high-fanout caller (CI behind -# one NAT) should authenticate for a per-DID budget or the operator raises this. +# Max concurrent read ops (upload-pack + the upload-pack info/refs advertisement) +# a single caller may hold, so one caller cannot monopolize the read pool. Keyed +# on the resolved SOURCE IP, never the DID: a signature does not move a caller off +# this cap. The source-IP key is only as granular as GITLAWB_TRUSTED_PROXY below: +# left unset, a node behind an edge/NAT keys all callers on the edge IP and this +# collapses to one global cap. Set GITLAWB_TRUSTED_PROXY for per-client keying; a +# high-fanout caller (CI behind one NAT) then needs the operator to raise this. # Default 16. GITLAWB_MAX_CONCURRENT_READS_PER_CALLER=16 diff --git a/README.md b/README.md index 0f760449..542a988e 100644 --- a/README.md +++ b/README.md @@ -343,7 +343,7 @@ Important node settings: | `GITLAWB_REQUIRE_SIGNED_PEER_WRITES` | Require signed peer announce/sync writes. | | `GITLAWB_AUTO_SYNC` | Enable automatic sync from known peers. | | `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. | -| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths), which is reaped at the deadline. | +| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | | `GITLAWB_IRYS_URL` | Optional Irys/Arweave permanent anchoring. | diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 90a061e0..d0a4c319 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -243,24 +243,25 @@ pub struct Config { /// cap is a different axis (500 connections each fan out to git + /// pack-objects + threads). Size below the process budget with headroom. /// - /// This is the READ pool (`git_read_semaphore`): upload-pack and both info/refs - /// advertisements. The authenticated push POST draws from a separate write pool - /// (`max_concurrent_git_pushes`) that anonymous reads can never reach, and each - /// caller is additionally bounded by `max_concurrent_reads_per_caller`, so an - /// anonymous flood cannot shed the actual push nor monopolize reads (#174). (The - /// receive-pack advertisement itself shares the read pool; a shed advertisement - /// is a cheap retryable GET, and the write POST it precedes always has capacity.) + /// This is the READ pool (`git_read_semaphore`): upload-pack and the UPLOAD-PACK + /// `info/refs` advertisement only. The authenticated push POST draws from a + /// separate write pool (`max_concurrent_git_pushes`) that anonymous reads can + /// never reach, and each read caller is additionally bounded by + /// `max_concurrent_reads_per_caller`, so an anonymous flood cannot shed the actual + /// push nor monopolize reads (#174). The anon-reachable RECEIVE-PACK `info/refs` + /// advertisement draws from its OWN dedicated pool (sized like the write pool but + /// disjoint), so an advertisement flood can never occupy a permit the + /// authenticated push POST needs at admission (#174). /// /// A permit is held for the whole op. Every git subprocess that STREAMS is /// duration-bounded and reaps its process group on disconnect: upload-pack, /// receive-pack, and both info/refs advertisements run under /// `git_service_timeout_secs` with `process_group(0)` teardown, and the - /// withheld-blob (`upload_pack_excluding`) pack-objects stage runs on the async - /// side under the same teardown (#174 closed the two duration/cancellation gaps - /// this comment previously tracked). The one remaining blocking stage is the - /// `rev-list` object enumeration in the withheld-blob path — a bounded walk that - /// terminates, run off the async runtime; it is not process-group-reaped on - /// disconnect, so a stuck rev-list can still hold its slot until git exits. + /// withheld-blob (`upload_pack_excluding`) pack-objects stage plus the push-side + /// candidate-discovery children (`rev-list` / `cat-file`) now run under the same + /// bounded runner with process-group teardown, so a stuck git child no longer + /// holds its slot indefinitely (#174 closed the duration/cancellation gaps this + /// comment previously tracked). /// /// Default: 128. Must be between 1 and 1_048_576; the ceiling keeps the value /// well under tokio's `Semaphore` permit limit so an oversized value is a @@ -273,10 +274,14 @@ pub struct Config { )] pub max_concurrent_git_ops: usize, - /// Maximum number of concurrent `git-receive-pack` (push) operations. Pushes - /// draw from this dedicated pool, separate from `max_concurrent_git_ops` - /// (reads), so a flood of anonymous reads cannot shed an authenticated push at - /// admission (#174). Beyond this a push sheds a clean 503 + Retry-After. + /// Maximum number of concurrent `git-receive-pack` (push) operations. The + /// authenticated push POST draws from this dedicated pool, separate from + /// `max_concurrent_git_ops` (reads), so a flood of anonymous reads cannot shed an + /// authenticated push at admission (#174). The anon-reachable receive-pack + /// `info/refs` advertisement runs in a SEPARATE pool of the same size (derived + /// from this knob), disjoint from this one, so an advertisement flood cannot + /// occupy a POST's slot either (#174). Beyond this a push sheds a clean 503 + + /// Retry-After. /// /// Default: 32. Must be between 1 and 1_048_576 (the ceiling keeps the value /// under tokio's `Semaphore` permit limit so an oversized value is a clean CLI @@ -289,16 +294,17 @@ pub struct Config { )] pub max_concurrent_git_pushes: usize, - /// Maximum concurrent read operations (`upload-pack` and the `info/refs` - /// advertisements) a single caller may hold at once, so one caller cannot - /// monopolize the `max_concurrent_git_ops` read pool (#174). Callers are keyed - /// per-DID when signed, else per-source-IP. IMPORTANT: the per-source-IP key is - /// only as granular as `GITLAWB_TRUSTED_PROXY`. Left unset (the default), a node - /// behind an edge/NAT keys all anonymous callers on the edge IP, so this cap - /// collapses to a single global anonymous cap rather than per-client. Set - /// `GITLAWB_TRUSTED_PROXY` to key on the real client; a known high-fanout caller - /// (a CI fleet behind one NAT) should authenticate for a per-DID budget or the - /// operator raises this. Over-cap for a caller sheds a clean 503 + Retry-After. + /// Maximum concurrent read operations (`upload-pack` and the upload-pack + /// `info/refs` advertisement) a single caller may hold at once, so one caller + /// cannot monopolize the `max_concurrent_git_ops` read pool (#174). Callers are + /// keyed on the RESOLVED SOURCE IP, never the DID — a signature does not move a + /// caller off this cap, so an authenticated client cannot mint DIDs to escape it. + /// IMPORTANT: the source-IP key is only as granular as `GITLAWB_TRUSTED_PROXY`. + /// Left unset (the default), a node behind an edge/NAT keys all callers on the + /// edge IP, so this cap collapses to a single global cap rather than per-client. + /// Set `GITLAWB_TRUSTED_PROXY` to key on the real client; a high-fanout caller (a + /// CI fleet behind one NAT) then needs the operator to raise this. Over-cap for a + /// caller sheds a clean 503 + Retry-After. /// /// Default: 16. Must be between 1 and 1_048_576. #[arg( From cf97405bc188e67beaff37c6a708694dda04fe94 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 10:55:57 -0500 Subject: [PATCH 025/124] test(node): execution-cover the delta-path push-scan timeout wiring (#174) The finding-2 wiring test covered only list_all_objects (the full-scan fallback). Add delta_path_exec_fns_time_out_on_a_hung_git covering the common push path: peeled_object_type (cat-file), rev_list_delta (rev-list), and list_all_objects_with_type (cat-file, which all_blob_oids delegates to). Each must return within the watchdog budget on a hung git; reverting any to a bare Command::output() blocks past the recv budget (RED-verified). --- crates/gitlawb-node/src/git/push_delta.rs | 56 +++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/crates/gitlawb-node/src/git/push_delta.rs b/crates/gitlawb-node/src/git/push_delta.rs index 51b6f836..962c9c65 100644 --- a/crates/gitlawb-node/src/git/push_delta.rs +++ b/crates/gitlawb-node/src/git/push_delta.rs @@ -746,4 +746,60 @@ mod tests { "a hung git must make list_all_objects error out, not hang" ); } + + /// #174 finding 2: the DELTA-path children — `peeled_object_type` (cat-file), + /// `rev_list_delta` (rev-list) — and `list_all_objects_with_type` (cat-file) are + /// the remaining candidate-discovery execs (the common push path). Each must + /// return within the watchdog budget on a hung git rather than block and pin the + /// write permit. Revert any one to a bare `Command::output()` and its arm blocks + /// past the recv budget (RED). + #[cfg(unix)] + #[test] + fn delta_path_exec_fns_time_out_on_a_hung_git() { + use std::os::unix::fs::PermissionsExt; + use std::time::{Duration, Instant}; + let dir = tempfile::TempDir::new().unwrap(); + // Fake git hangs on BOTH cat-file and rev-list (bounded 30s so a broken + // test can't leak a permanent orphan or wedge the suite). + let fake = dir.path().join("fakegit"); + std::fs::write( + &fake, + "#!/bin/sh\ncase \"$1\" in\n cat-file|rev-list) i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;;\n *) : ;;\nesac\n", + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + let git_bin = fake.to_str().unwrap().to_string(); + let path = dir.path().to_path_buf(); + + // Run `f` on a thread and require it to RETURN (not block) within 10s. + fn returns_within(f: impl FnOnce() -> T + Send + 'static) -> T { + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = tx.send(f()); + }); + rx.recv_timeout(Duration::from_secs(10)).expect( + "a delta-path exec fn must return within the watchdog budget, not block on a hung git", + ) + } + let dl = || Instant::now() + Duration::from_millis(150); + let sha = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"; + + let (p, g, d) = (path.clone(), git_bin.clone(), dl()); + assert!( + returns_within(move || peeled_object_type(&p, sha, &g, d)).is_none(), + "peeled_object_type must time out to None on a hung cat-file" + ); + let (p, g, d) = (path.clone(), git_bin.clone(), dl()); + assert!( + returns_within(move || rev_list_delta(&p, &[sha], &[], &g, d)).is_err(), + "rev_list_delta must error out on a hung rev-list" + ); + let (p, g, d) = (path.clone(), git_bin.clone(), dl()); + assert!( + returns_within(move || list_all_objects_with_type(&p, &g, d)).is_err(), + "list_all_objects_with_type must error out on a hung cat-file" + ); + } } From a030eefb704d68a008e760a01077abc823cbe258 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 16:01:49 -0500 Subject: [PATCH 026/124] test(node): retry the ETXTBSY exec race in build_filtered_pack timeout tests (#174) build_filtered_pack_times_out_a_hung_rev_list and _a_hung_pack_objects spawn a freshly-written fake git directly, bypassing the fake_git_run_with_pids retry the rest of the module uses. Under cargo test fork-storm load a concurrent worker forks while the fake's write fd is still open, so execve returns ETXTBSY and the raw io::Error surfaces where the test asserts GitServiceTimeout, failing test (stable) intermittently. Route both tests through build_filtered_pack_or_exec_race_retry, which retries ONLY errno 26 (ETXTBSY) and keeps the outer 10s watchdog per attempt, so a genuinely-missing async timeout bound still trips the watchdog and a wrong error type still fails at the assertion. Production paths unchanged. --- crates/gitlawb-node/src/git/smart_http.rs | 99 ++++++++++++++++------- 1 file changed, 71 insertions(+), 28 deletions(-) diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index 206e7b27..f885aef3 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -1160,6 +1160,61 @@ mod tests { ); } + /// True when `err` is the transient ETXTBSY exec race a freshly-written fake + /// `git` hits under fork-storm load — a concurrent worker forked while this + /// file's write fd was still open, so `execve` sees it as busy (the same race + /// `fake_git_run_with_pids` retries). Deliberately narrow: only this raw-OS + /// error is retried, so a wrong error *type* (e.g. a missing async bound + /// surfacing as anything other than `GitServiceTimeout`) still fails loudly at + /// the caller's assertion instead of being swallowed. `spawn()?` propagates the + /// `io::Error` with no context, so it survives as the anyhow root. + #[cfg(unix)] + fn is_transient_exec_race(err: &anyhow::Error) -> bool { + // ETXTBSY == 26 on Linux and the BSDs/macOS; std has no stable ErrorKind. + err.downcast_ref::() + .and_then(std::io::Error::raw_os_error) + == Some(26) + } + + /// Drive `build_filtered_pack` under a per-attempt watchdog, retrying ONLY the + /// transient ETXTBSY exec race and returning the terminal error for the caller + /// to classify. Every attempt keeps its own outer `tokio::time::timeout`, so a + /// MISSING async bound — the regression the `build_filtered_pack_times_out_*` + /// tests guard — still trips the watchdog loudly on every attempt: the retry + /// can only absorb a fast exec failure, never a hang (a hang never returns, so + /// it can never reach the retry decision). + #[cfg(unix)] + async fn build_filtered_pack_or_exec_race_retry( + git_bin: &str, + repo_path: &std::path::Path, + withheld: &HashSet, + stage_timeout: Duration, + ) -> anyhow::Error { + for i in 0..FAKE_GIT_RETRY_ATTEMPTS { + let result = tokio::time::timeout( + Duration::from_secs(10), + build_filtered_pack(git_bin, repo_path, withheld, stage_timeout), + ) + .await + .expect( + "build_filtered_pack must return within the watchdog — the git stage \ + must be timeout-bounded, not an uncancellable spawn_blocking", + ); + let err = result.expect_err("a hung git stage must return an error, not hang"); + if is_transient_exec_race(&err) { + // Fresh-fake-git ETXTBSY: back off (growing) so a bursty fork-pressure + // spike subsides before retrying, per fake_git_run_with_pids. + tokio::time::sleep(Duration::from_millis(FAKE_GIT_BACKOFF_STEP_MS * (i + 1))).await; + continue; + } + return err; + } + panic!( + "fake git kept hitting ETXTBSY after {FAKE_GIT_RETRY_ATTEMPTS} attempts \ + (persistent exec failure, not a transient parallel-runner miss)" + ); + } + // Dropping the request future mid-flight (client disconnect) must SIGTERM the // whole group so git AND its pack-objects grandchild die together. Goes RED // if `process_group(0)` or the guard-arming is removed: without its own @@ -1314,21 +1369,15 @@ mod tests { let git_bin = write_fake_git(tmp.path(), body); let withheld = HashSet::new(); - let result = tokio::time::timeout( - Duration::from_secs(10), - build_filtered_pack( - git_bin.to_str().unwrap(), - tmp.path(), - &withheld, - Duration::from_millis(200), - ), + // Retry only the transient ETXTBSY exec race (see the helper); the + // per-attempt watchdog keeps the missing-bound regression loud. + let err = build_filtered_pack_or_exec_race_retry( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_millis(200), ) - .await - .expect( - "build_filtered_pack must return within the watchdog — the pack-objects \ - stage must be timeout-bounded, not an uncancellable spawn_blocking", - ); - let err = result.expect_err("a hung pack-objects must return an error, not hang"); + .await; assert!( err.downcast_ref::().is_some(), "a hung pack-objects must abort with GitServiceTimeout, got: {err}" @@ -1352,21 +1401,15 @@ mod tests { let git_bin = write_fake_git(tmp.path(), body); let withheld = HashSet::new(); - let result = tokio::time::timeout( - Duration::from_secs(10), - build_filtered_pack( - git_bin.to_str().unwrap(), - tmp.path(), - &withheld, - Duration::from_millis(200), - ), + // Retry only the transient ETXTBSY exec race (see the helper); the + // per-attempt watchdog keeps the missing-bound regression loud. + let err = build_filtered_pack_or_exec_race_retry( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_millis(200), ) - .await - .expect( - "build_filtered_pack must return within the watchdog — the rev-list \ - stage must be timeout-bounded, not an uncancellable spawn_blocking", - ); - let err = result.expect_err("a hung rev-list must return an error, not hang"); + .await; assert!( err.downcast_ref::().is_some(), "a hung rev-list must abort with GitServiceTimeout, got: {err}" From 3cc01890fbb3b395a336df951cff0e5b89393530 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 14 Jul 2026 19:27:12 -0500 Subject: [PATCH 027/124] fix(node): keep the visibility-walk watchdog armed until the child is reaped (#174) A group member (or the leader itself) that closes stdout before the deadline made the stdout drain return EOF early, which stood the watchdog down via done_tx before it could ever fire; child.wait() then blocked on the still-live child, pinning the walk thread and its read/write permit past GITLAWB_GIT_SERVICE_TIMEOUT_SECS. This is distinct from the descendant case already covered: there the leader sleeps until the deadline so the watchdog does time out; here the drain-EOF races ahead of it. Stand the watchdog down only once the child has actually terminated, detected without reaping (waitid + WNOWAIT) so the leader's pid stays unreaped and its pgid un-recycled until the watchdog finishes its teardown and is joined. Past the deadline the watchdog still owns the full SIGTERM -> grace -> SIGKILL sequence. Regression: run_bounded_git_reaps_a_leader_that_closes_stdout_then_hangs, RED before (blocks on child.wait past the recv budget), GREEN after; the three existing teardown tests still pass (no recycle-hazard regression). --- .../gitlawb-node/src/git/visibility_pack.rs | 116 ++++++++++++++++-- 1 file changed, 108 insertions(+), 8 deletions(-) diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index 9f40cd4b..9f07020d 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -41,6 +41,31 @@ const WATCHDOG_TERM_GRACE: Duration = Duration::from_secs(1); /// the serve handler maps it to 504. `git_bin` is injectable so a fake `git` can /// drive the teardown in tests without mutating the process-global PATH; /// `stdin_bytes` feeds children that read stdin (empty for the arg-only children). +/// Returns true if `pid` (a process-group leader we spawned) has terminated, WITHOUT +/// reaping it. `waitid(..., WNOWAIT)` reports the exit state but leaves the child +/// waitable, so the caller's later `child.wait()` still collects the status and the +/// pid/pgid stays live until then — which is what keeps the watchdog's `kill(-pgid)` +/// teardown from ever racing a recycled pgid. Used to distinguish "the child actually +/// exited" from "the child merely closed stdout" after the drain returns (#174 P1-a). +#[cfg(unix)] +fn child_terminated_without_reaping(pid: i32) -> bool { + // SAFETY: waitid writes only into the zeroed siginfo and borrows no Rust memory; + // WNOWAIT leaves the child unreaped, WNOHANG makes the probe non-blocking. + let mut info: libc::siginfo_t = unsafe { std::mem::zeroed() }; + let rc = unsafe { + libc::waitid( + libc::P_PID, + pid as libc::id_t, + &mut info, + libc::WEXITED | libc::WNOHANG | libc::WNOWAIT, + ) + }; + // rc == 0 with si_pid == 0 means "no state change yet" (still running); a non-zero + // si_pid means the child has entered a waitable, exited state. EINTR/other errors + // (rc != 0) are treated as "not yet terminated" and the caller re-polls. + rc == 0 && unsafe { info.si_pid() } != 0 +} + #[cfg(unix)] pub(crate) fn run_bounded_git( git_bin: &str, @@ -132,14 +157,28 @@ pub(crate) fn run_bounded_git( // is unreachable in userspace (no signal reaps a D-state process) and matches the // async `reap_group_on_timeout`, which likewise only warns and gives up there. let read_result = stdout.read_to_end(&mut out); - // The drain has returned: either the child exited on its own, or the watchdog's - // teardown closed its pipes. Tell the watchdog the drain is done, then WAIT for it - // to return BEFORE reaping the leader. On the timeout path the watchdog runs the - // full SIGTERM -> grace -> SIGKILL teardown; joining it before `child.wait()` keeps - // the leader's pid unreaped while every `kill(-pgid)` fires (so the pgid can't be - // recycled), and guarantees a SIGTERM-ignoring group member has already been - // SIGKILLed rather than left running past the deadline (finding 3, #174). - let _ = done_tx.send(()); + // The drain has returned, but that only means all stdout write ends are closed — + // NOT that the child has exited. A group member, or the leader itself, can close + // stdout and keep running; standing the watchdog down on the drain alone (as the + // old code did) would then let `child.wait()` block forever on that live child, + // past the deadline, pinning the walk thread and its permit (finding P1-a, #174). + // So stand the watchdog down only once the child has ACTUALLY terminated, detected + // WITHOUT reaping (waitid + WNOWAIT) so the leader's pid stays unreaped and its + // pgid un-recycled until the watchdog finishes and we join it below. Past the + // deadline the watchdog owns the teardown, so we stop polling and let it run the + // full SIGTERM -> grace -> SIGKILL; joining it before `child.wait()` keeps every + // `kill(-pgid)` firing while the pid is still unreaped and guarantees a + // stdout-closing-then-hanging member has been SIGKILLed rather than left running. + loop { + if child_terminated_without_reaping(pgid) { + let _ = done_tx.send(()); + break; + } + if Instant::now() >= deadline { + break; + } + std::thread::sleep(Duration::from_millis(2)); + } let killed = watchdog.join().unwrap_or(false); let status = child.wait().context("git wait failed")?; let err = err_reader.join().unwrap_or_default(); @@ -902,6 +941,67 @@ esac\n"; "a SIGTERM-ignoring descendant (pid {desc}) must be SIGKILLed even after the leader exits cleanly, not orphaned" ); } + + /// #174 U1 (P1-a, RED-before/GREEN-after): the group LEADER closes its own + /// stdout/stderr BEFORE the deadline and then keeps running. On the pre-fix code + /// the stdout drain returns EOF early, `done_tx.send` stands the watchdog down + /// before it ever fires (`recv` gets `Ok` -> `false`, no kill), and `child.wait()` + /// then blocks on the still-alive leader — pinning the walk thread and its read/ + /// write permit past the deadline, bypassing GITLAWB_GIT_SERVICE_TIMEOUT_SECS. + /// This is distinct from the descendant case above: there the leader sleeps until + /// the deadline so the watchdog DOES time out; here the drain-EOF races ahead of + /// the deadline. The fix keeps the watchdog armed until the child is actually + /// reaped, so the deadline SIGTERM still fires and the call returns within budget. + /// A pre-fix build blocks on `child.wait()` past the recv budget (RED). + #[cfg(unix)] + #[test] + fn run_bounded_git_reaps_a_leader_that_closes_stdout_then_hangs() { + use std::time::{Duration, Instant}; + let tmp = TempDir::new().unwrap(); + // rev-list records its (leader) pid, closes stdout+stderr so the drain EOFs + // immediately, then sleeps without trapping TERM. `sleep 30` bounds the worst + // case so a RED run cannot wedge the suite; the recv budget fires first. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) echo $$ > leader.pid; exec 1>&- 2>&-; sleep 30 ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + let (tx, rx) = std::sync::mpsc::channel(); + let path = tmp.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(run_bounded_git( + &git_bin, + &["rev-list"], + &path, + b"", + Instant::now() + Duration::from_millis(100), + )); + }); + let out = rx.recv_timeout(Duration::from_secs(10)).expect( + "run_bounded_git must return within the watchdog budget when the leader closes stdout then hangs, not block on child.wait()", + ); + assert!( + out.is_err(), + "a leader killed at the deadline (no TERM trap) is a timeout, not a success: {out:?}" + ); + let pid: i32 = std::fs::read_to_string(tmp.path().join("leader.pid")) + .unwrap() + .trim() + .parse() + .unwrap(); + let mut gone = false; + for _ in 0..300 { + if unsafe { libc::kill(pid, 0) } != 0 { + gone = true; + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + // Kill it regardless so a RED run leaks no orphan. + unsafe { libc::kill(pid, libc::SIGKILL) }; + assert!( + gone, + "the hung leader (pid {pid}) must be killed and reaped at the deadline, not left running" + ); + } + use crate::db::VisibilityMode; use chrono::Utc; use std::process::Command; From ac59bd7e01bf42b7d42b7adb57ccb927c0309993 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 14 Jul 2026 19:50:41 -0500 Subject: [PATCH 028/124] fix(node): reap the served-git process group on disconnect, not just SIGTERM (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a client disconnect KillGroupOnDrop sent a lone SIGTERM with no escalation or reap, so a group member that ignores SIGTERM (a wedged pack-objects, a malicious helper) survived while the handler released its concurrency permit — letting disconnect-spam accumulate live git processes and defeat the load shed (P1-c). The timeout path already ran the full teardown. Make the disconnect teardown as strong as the timeout path: the guard now owns the child and, on drop, launches a detached reaper that runs the same TERM -> grace -> SIGKILL -> reap-the-group sequence (reap_group_on_timeout). Owning the tokio Child in the reaper keeps it the sole reaper of the leader, so it never races tokio's own orphan reaper (a raw waitpid(-pgid) would). Regression: run_git_service_sigkills_a_sigterm_ignoring_child_on_disconnect, RED before (the descendant survives the lone SIGTERM), GREEN after; all existing teardown/timeout/disarm tests still pass (the three hand-built-guard tests were updated to the owned-child shape). Residual: capacity (the handler's semaphore permit) still releases at disconnect rather than strictly after the reaper confirms the group dead, so accumulation is now bounded to ~one teardown window (<=4s) rather than eliminated; gating the permit on the reap needs threading it through the serve handlers and is left for a follow-up (it also touches the receive-pack permit lifetime U4 changes). --- crates/gitlawb-node/src/git/smart_http.rs | 235 +++++++++++++++++----- 1 file changed, 188 insertions(+), 47 deletions(-) diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index f885aef3..a5649a42 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -108,24 +108,60 @@ pub async fn receive_pack( /// `wait_with_output()` returns, so a request that completed cleanly never signals. #[cfg(unix)] struct KillGroupOnDrop { + // Holds the child + its pgid while armed. The interaction drives the child through + // `child_mut()`; the success/timeout paths call `disarm` once they have reaped it. + // On drop — a client disconnect that drops the whole request future — the guard + // launches a detached reaper that OWNS the child and runs the full + // SIGTERM -> grace -> SIGKILL -> reap sequence (`reap_group_on_timeout`), as strong + // as the timeout path, so a SIGTERM-ignoring group member is SIGKILLed and reaped + // rather than left running until EPIPE to accumulate past the concurrency cap + // (#174 P1-c). Owning the tokio `Child` in the reaper (rather than a raw + // `waitpid(-pgid)`) keeps a single reaper of the leader, so it never races tokio's + // own SIGCHLD-driven orphan reaper. + child: Option, pgid: Option, } #[cfg(unix)] impl KillGroupOnDrop { + /// The child, while armed. The interaction drives stdin/stdout/`wait` through this. + fn child_mut(&mut self) -> &mut tokio::process::Child { + self.child.as_mut().expect("child present while armed") + } + + /// Disarm on the success/timeout path: the body has already reaped the child (its + /// `wait()` returned, or `reap_group_on_timeout` ran), so drop the handle and clear + /// the pgid, leaving the guard's Drop a no-op. fn disarm(&mut self) { self.pgid = None; + self.child = None; } } #[cfg(unix)] impl Drop for KillGroupOnDrop { fn drop(&mut self) { - if let Some(pgid) = self.pgid { - // SAFETY: kill(2) takes only integer arguments and borrows no Rust - // memory. Signalling a stale group just returns ESRCH, which we ignore. - unsafe { - libc::kill(-pgid, libc::SIGTERM); + let (Some(mut child), Some(pgid)) = (self.child.take(), self.pgid) else { + return; + }; + // A sync Drop cannot await, so launch a detached reaper that owns the child and + // runs the same teardown as the timeout path (TERM -> grace -> SIGKILL -> reap + // the whole group). Owning the tokio `Child` means this task is the sole reaper + // of the leader, so it cannot race tokio's orphan reaper. Prefer the current + // runtime handle; if dropped outside a runtime, fall back to a best-effort + // synchronous SIGTERM so the group is at least signalled. + match tokio::runtime::Handle::try_current() { + Ok(handle) => { + handle.spawn(async move { + reap_group_on_timeout(&mut child).await; + }); + } + Err(_) => { + // SAFETY: kill(2) takes only integers and borrows no Rust memory; + // ESRCH on an already-gone group is ignored. + unsafe { + libc::kill(-pgid, libc::SIGTERM); + } } } } @@ -244,20 +280,28 @@ async fn drive_git_child( let mut child = command.spawn()?; - // Arm the group-kill guard for the lifetime of the request. With - // process_group(0) the child is its own group leader, so pgid == its pid. - // This fires on a client disconnect (the whole future is dropped mid-request). + // Own the pipes so the child stays reap-able: wait_with_output would consume it, + // but on a timeout we must actively reap the group first (see + // reap_group_on_timeout), and on a disconnect the guard's detached reaper does. + // Take the pipes before the child moves into the guard below. + let mut stdin = child.stdin.take(); + let mut stdout = child.stdout.take().context("git stdout was not piped")?; + let mut stderr = child.stderr.take().context("git stderr was not piped")?; + + // Arm the group-kill guard for the lifetime of the request. With process_group(0) + // the child is its own group leader, so pgid == its pid. On a client disconnect + // (the whole future is dropped mid-request) the guard's Drop launches a detached + // reaper that OWNS the child and runs the full TERM/grace/SIGKILL/reap — as strong + // as the timeout path below (#174 P1-c). The guard owns the child; the interaction + // drives it through `child_mut()`. + #[cfg(unix)] + let pgid = child.id().map(|id| id as i32); #[cfg(unix)] let mut group_guard = KillGroupOnDrop { - pgid: child.id().map(|id| id as i32), + child: Some(child), + pgid, }; - // Own the pipes so `child` stays reap-able after a timeout: wait_with_output - // would consume it, but on timeout we must actively reap the group before - // returning (see reap_group_on_timeout). - let mut stdin = child.stdin.take(); - let mut stdout = child.stdout.take().context("git stdout was not piped")?; - let mut stderr = child.stderr.take().context("git stderr was not piped")?; let mut out = Vec::new(); let mut err = Vec::new(); @@ -274,11 +318,15 @@ async fn drive_git_child( None => Ok(()), } }; + #[cfg(unix)] + let child_ref = group_guard.child_mut(); + #[cfg(not(unix))] + let child_ref = &mut child; let (write_result, r_out, r_err, status) = tokio::join!( write, stdout.read_to_end(&mut out), stderr.read_to_end(&mut err), - child.wait(), + child_ref.wait(), ); r_out?; r_err?; @@ -290,7 +338,7 @@ async fn drive_git_child( Ok(result) => { // The join runs all arms to completion, so the child is reaped: disarm // before surfacing any interaction error (a read/wait error), else the - // guard's drop would fire SIGTERM on the reaped, possibly-reused pgid. + // guard's drop would reap an already-reaped child / signal a reused pgid. #[cfg(unix)] group_guard.disarm(); result? @@ -301,7 +349,7 @@ async fn drive_git_child( // the (now redundant) guard so its drop can't hit a reused pgid. #[cfg(unix)] { - reap_group_on_timeout(&mut child).await; + reap_group_on_timeout(group_guard.child_mut()).await; group_guard.disarm(); } #[cfg(not(unix))] @@ -960,23 +1008,36 @@ mod tests { #[cfg(unix)] #[tokio::test] async fn kill_group_guard_terminates_child_on_drop() { - let mut child = tokio::process::Command::new("sleep") + let child = tokio::process::Command::new("sleep") .arg("300") .process_group(0) .spawn() .unwrap(); - let pgid = child.id().map(|id| id as i32); + let pid = child.id().unwrap() as i32; { - let _guard = KillGroupOnDrop { pgid }; - } // guard drops here -> SIGTERM to the group - - use std::os::unix::process::ExitStatusExt; - let status = child.wait().await.unwrap(); - assert_eq!( - status.signal(), - Some(libc::SIGTERM), - "child must be terminated by SIGTERM via its process group" + // The guard owns the child; on drop its detached reaper TERM/grace/KILL/ + // reaps the group (a plain sleep dies on the first SIGTERM). + let _guard = KillGroupOnDrop { + child: Some(child), + pgid: Some(pid), + }; + } + + let mut gone = false; + for _ in 0..300 { + if !alive(pid) { + gone = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + unsafe { + libc::kill(pid, libc::SIGKILL); + } + assert!( + gone, + "child must be terminated and reaped via its process group on guard drop" ); } @@ -994,9 +1055,10 @@ mod tests { .process_group(0) .spawn() .unwrap(); - let pgid = child.id().map(|id| id as i32); + let pid = child.id().unwrap() as i32; - // Read the backgrounded grandchild's pid from the first stdout line. + // Read the backgrounded grandchild's pid from the first stdout line, before + // the child moves into the guard. let mut stdout = child.stdout.take().unwrap(); let mut buf = Vec::new(); loop { @@ -1011,20 +1073,25 @@ mod tests { assert!(alive(grandchild), "grandchild should be running"); { - let _guard = KillGroupOnDrop { pgid }; - } // group SIGTERM reaches sh AND the sleep grandchild - - let _ = child.wait().await; // reap sh + // The guard owns the child; on drop the detached reaper group-kills sh AND + // the sleep grandchild. + let _guard = KillGroupOnDrop { + child: Some(child), + pgid: Some(pid), + }; + } - // The grandchild reparents to init and is reaped; poll until it's gone. let mut gone = false; - for _ in 0..200 { + for _ in 0..300 { if !alive(grandchild) { gone = true; break; } tokio::time::sleep(std::time::Duration::from_millis(10)).await; } + unsafe { + libc::kill(grandchild, libc::SIGKILL); + } assert!( gone, "grandchild must be terminated by the group signal (#53)" @@ -1036,27 +1103,31 @@ mod tests { async fn kill_group_guard_disarmed_does_not_kill() { // A request that completed cleanly disarms the guard; dropping it must not // signal anything. - let mut child = tokio::process::Command::new("sleep") + let child = tokio::process::Command::new("sleep") .arg("300") .process_group(0) .spawn() .unwrap(); + let pid = child.id().unwrap() as i32; { let mut guard = KillGroupOnDrop { - pgid: child.id().map(|id| id as i32), + child: Some(child), + pgid: Some(pid), }; guard.disarm(); - } // disarmed -> no kill + } // disarmed -> no reaper, no kill - assert!( - child.try_wait().unwrap().is_none(), - "disarmed guard must not kill the child" - ); + // Give any erroneously-spawned reaper a chance to run, then assert alive. + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + assert!(alive(pid), "disarmed guard must not kill the child"); - // Clean up the still-running child. - let _ = child.kill().await; - let _ = child.wait().await; + // Clean up the still-running child (disarm dropped the handle, so reap by pid). + unsafe { + libc::kill(pid, libc::SIGKILL); + let mut status = 0; + libc::waitpid(pid, &mut status, 0); + } } // ── #62 PR1: end-to-end teardown wiring through run_git_service ───────── @@ -1320,6 +1391,76 @@ mod tests { ); } + /// #174 U2 (P1-c, RED-before/GREEN-after): on a client disconnect the teardown must + /// be as strong as the timeout path — a group member that IGNORES SIGTERM is still + /// SIGKILLed and reaped, not left running to accumulate past the concurrency cap. + /// The old `KillGroupOnDrop::drop` sent a lone SIGTERM with no escalation or reap, + /// so a SIGTERM-ignoring descendant survived the disconnect (RED). The fix launches + /// a detached reaper owning the child that runs the full TERM/grace/SIGKILL/reap. + /// This is distinct from the well-behaved-grandchild test above, whose `sleep` dies + /// on the first SIGTERM. + #[cfg(unix)] + #[tokio::test] + async fn run_git_service_sigkills_a_sigterm_ignoring_child_on_disconnect() { + let tmp = tempfile::TempDir::new().unwrap(); + let descfile = tmp.path().join("desc.pid"); + // Leader (dies on the group SIGTERM) spawns a descendant that traps SIGTERM, + // records its own pid, and loops ~30s (bounded so a RED run leaks no permanent + // orphan; the assertion fires well before then). + let body = format!( + "#!/bin/sh\n\ + sh -c 'trap \"\" TERM; echo $$ > \"{}\"; i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done' &\n\ + wait\n", + descfile.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + + let mut fut = Box::pin(run_git_service( + git_bin.to_str().unwrap(), + "git-upload-pack", + tmp.path(), + Bytes::new(), + Duration::from_secs(60), + )); + // Drive the future a slice at a time until the descendant records its pid. + let mut desc: Option = None; + for _ in 0..500 { + let _ = tokio::time::timeout(Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&descfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + desc = Some(p); + break; + } + } + let desc = desc.expect("the fake leader must have spawned and recorded a descendant"); + let _cleanup = ReapOnPanic(vec![desc]); + assert!(alive(desc), "descendant should be running before the drop"); + + // Client disconnect: drop the request future. The guard's detached reaper must + // escalate SIGTERM -> SIGKILL and reap the SIGTERM-ignoring descendant. + drop(fut); + + let mut gone = false; + for _ in 0..500 { + if !alive(desc) { + gone = true; + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + // Kill regardless so a RED run leaks no orphan. + unsafe { + libc::kill(desc, libc::SIGKILL); + } + assert!( + gone, + "a SIGTERM-ignoring descendant (pid {desc}) must be SIGKILLed and reaped on \ + disconnect, not left running (old KillGroupOnDrop sent SIGTERM only)" + ); + } + // #174 (SC3): the info/refs advertisement is now duration-bounded — previously // it ran a bare `Command::output()` with no deadline, so a hung git pinned its // concurrency slot forever. A hung fake git must abort with GitServiceTimeout From be0cdd6a9eae1b39618f16fdf4e28eae6e83ca51 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 14 Jul 2026 20:09:40 -0500 Subject: [PATCH 029/124] fix(node): hold upload-pack admission through the walk on disconnect (#174) On the path-scoped upload-pack path the per-source and global read permits were handler locals, so a client disconnect dropped the handler future and released both permits while the awaited spawn_blocking withheld-blob walk kept running (spawn_blocking can't be cancelled). A permitless caller could disconnect-spam to exceed the read cap while real git work continued (P1-b). Move both permits into the blocking task and return them out: on success they flow back so the serve phase keeps them; on a dropped future the returned tuple (with the permits) is discarded only when the blocking task completes, so admission tracks the walk's real duration. Regression: upload_pack_permit_held_through_walk_after_disconnect drives the handler to mid-walk (fake git hangs on rev-list), disconnects, and asserts the global read slot stays held (available_permits == 0). RED before (the slot frees to 1 the instant the future drops), GREEN after. The test roots the repo store at its own TempDir so it is isolated per run. --- crates/gitlawb-node/src/api/repos.rs | 166 +++++++++++++++++++++++++-- 1 file changed, 158 insertions(+), 8 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index ea6f126e..8612709c 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -819,9 +819,16 @@ pub async fn git_upload_pack( let resp = if !visibility_pack::has_path_scoped_rule(&rules) { smart_http::upload_pack(&disk_path, body, git_timeout).await } else { - // withheld_blob_oids walks every ref with blocking `git ls-tree`; keep - // that off the async worker thread. - let withheld = { + // withheld_blob_oids walks every ref with blocking `git ls-tree`; keep that + // off the async worker thread. Move BOTH admission permits INTO the blocking + // task so they are held for the walk's real duration: spawn_blocking cannot be + // cancelled, so on a client disconnect the handler future drops but the walk + // keeps running — and now so do its permits, released only when the walk + // finishes rather than the instant the future drops (#174 P1-b). On success the + // task hands the permits back so the serve phase below keeps them; on a + // dropped future the returned tuple (with the permits) is discarded only when + // the blocking task completes, so admission tracks the real git work. + let (withheld, _permit, _caller_permit) = { let path = disk_path.clone(); let rules = rules.clone(); let owner_did = record.owner_did.clone(); @@ -829,7 +836,7 @@ pub async fn git_upload_pack( let is_public = record.is_public; let git_bin = state.git_bin.clone(); tokio::task::spawn_blocking(move || { - visibility_pack::withheld_blob_oids_bounded( + let withheld = visibility_pack::withheld_blob_oids_bounded( &path, &git_bin, git_timeout, @@ -837,14 +844,15 @@ pub async fn git_upload_pack( is_public, &owner_did, caller_owned.as_deref(), - ) + ); + (withheld, _permit, _caller_permit) }) .await .map_err(|e| AppError::Git(e.to_string()))? - // A walk that hit its deadline carries GitServiceTimeout; map it to 504 - // like the smart_http paths, not a generic 500 (#174 U3). - .map_err(|e| git_service_app_error(&e))? }; + // A walk that hit its deadline carries GitServiceTimeout; map it to 504 like + // the smart_http paths, not a generic 500 (#174 U3). + let withheld = withheld.map_err(|e| git_service_app_error(&e))?; if withheld.is_empty() { smart_http::upload_pack(&disk_path, body, git_timeout).await @@ -3456,6 +3464,148 @@ mod tests { ); } + /// #174 U3 (P1-b, RED-before/GREEN-after): a client disconnect during the + /// path-scoped withheld-blob walk must NOT release the read admission while the + /// uncancellable `spawn_blocking` walk is still running. The handler takes the + /// global read permit, enters the walk (a fake git hangs on rev-list), then the + /// request future is dropped mid-walk. With both permits moved into the blocking + /// task the global slot stays occupied until the walk finishes; on the pre-fix code + /// the handler-local permits drop on future-drop and the slot frees instantly (RED), + /// letting disconnect-spam exceed the cap while real git work keeps running. + #[sqlx::test] + async fn upload_pack_permit_held_through_walk_after_disconnect(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + let revlist_pid = tmp.path().join("revlist.pid"); + // Fake git: resolve refs fast, hang on rev-list (recording its pid first). The + // ~6s sleep bounds the walk so a broken fix cannot wedge the suite. + let body = format!( + "#!/bin/sh\ncase \"$1\" in\n rev-list) echo $$ > \"{}\" ; sleep 6 ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n", + revlist_pid.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + // Root the repo store at this test's TempDir so the bare repo is isolated per + // run (the default for_testing store uses a fixed /tmp path that would collide + // across runs). + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.git_read_semaphore = Arc::new(Semaphore::new(1)); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let owner = "z6up3rd"; + let name = "up3"; + state + .db + .upsert_mirror_repo(owner, name, "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + // Real bare repo at the path acquire() computes, so the handler reaches the walk. + state + .repo_store + .init(&rec.owner_did, &rec.name) + .await + .unwrap(); + // A path-scoped rule so has_path_scoped_rule() is true (the walk path) without + // denying the "/" gate for the public repo. + state + .db + .set_visibility_rule( + &rec.id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3ReaderAAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + + let sem = state.git_read_semaphore.clone(); + assert_eq!( + sem.available_permits(), + 1, + "one read slot before the request" + ); + + let router = crate::server::build_router(state); + let peer: SocketAddr = "203.0.113.77:5000".parse().unwrap(); + let mut req = Request::builder() + .method(Method::POST) + .uri(format!("/{owner}/{name}/git-upload-pack")) + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + + let mut fut = Box::pin(router.oneshot(req)); + // Drive until the walk's rev-list starts (its pidfile appears) — i.e. the + // request is inside the spawn_blocking walk, holding the global read permit. + let mut in_walk = false; + for _ in 0..500 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if revlist_pid.exists() { + in_walk = true; + break; + } + } + assert!( + in_walk, + "the walk's rev-list must start (request reached the spawn_blocking walk)" + ); + assert_eq!( + sem.available_permits(), + 0, + "the read slot is held while the walk runs" + ); + + // Client disconnect: drop the request future mid-walk. + drop(fut); + + // Load-bearing: the slot must STAY held while the uncancellable walk runs. On + // the pre-fix code the handler-local permits drop here and the slot frees at + // once (RED). + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + assert_eq!( + sem.available_permits(), + 0, + "on disconnect the read admission must be held until the spawn_blocking walk \ + finishes, not released the instant the future drops (P1-b)" + ); + + // Cleanup: let the walk finish so the slot releases and no blocking task leaks. + for _ in 0..400 { + if sem.available_permits() == 1 { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + if let Some(p) = std::fs::read_to_string(&revlist_pid) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + unsafe { + libc::kill(p, libc::SIGKILL); + } + } + } + /// #174 SC2 (per-source key, U1): the per-caller read sub-cap keys on the /// resolved source IP, NOT the signed DID, so a disposable-DID farm cannot /// multiply its budget. Fill the source IP's single read slot, then drive two From 12643573f6c9f4805bd1876008141edda58f9192 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 14 Jul 2026 20:20:49 -0500 Subject: [PATCH 030/124] fix(node): cap concurrent receive-pack pushes per source IP (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The authenticated git-receive-pack POST acquired only the global write semaphore — no per-source sub-cap, unlike the read and anon-advert pools. Owner enforcement defaults off, so one host minting disposable did:key identities could open max_concurrent_git_pushes slow POSTs from a single source IP and 503 every other source's push; the 600/hour push limiter bounds arrival rate, not in-flight concurrency (P1-d). Add git_write_per_caller (a PerCallerConcurrency sized like the advert cap, max_concurrent_git_pushes/8), and acquire it before the global write permit, keyed on the resolved source IP via read_caller_key — never the signed DID (a DID farm defeats a DID key). This required adding the PeerAddr + HeaderMap extractors to git_receive_pack, which it lacked (without them the key is None and the cap is inert). Neutralized the shared acquire helper's log wording now that it serves both read and write paths. Regression: receive_pack_per_source_write_cap_sheds_capped_source_not_others — a source at its write sub-cap sheds Overloaded/503 (proving the extractors resolve a key), while a different source is not shed. RED before (the capped source proceeds past admission to a git error instead of shedding), GREEN after; the existing receive-pack advert cap test still passes. --- crates/gitlawb-node/src/api/repos.rs | 103 ++++++++++++++++++++++-- crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/main.rs | 7 ++ crates/gitlawb-node/src/state.rs | 7 ++ crates/gitlawb-node/src/test_support.rs | 1 + 5 files changed, 114 insertions(+), 5 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 8612709c..67c6c82a 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -717,7 +717,7 @@ fn acquire_read_caller_permit( Some(k) => match limiter.try_acquire(k) { Some(p) => Ok(Some(p)), None => { - tracing::warn!(repo = %repo, caller = %k, handler, "per-caller read cap reached; shedding with 503"); + tracing::warn!(repo = %repo, caller = %k, handler, "per-caller cap reached; shedding with 503"); Err(AppError::Overloaded( "git service at capacity for this caller, retry shortly".into(), )) @@ -1033,15 +1033,32 @@ pub async fn git_receive_pack( State(state): State, Path((owner, repo)): Path<(String, String)>, Extension(auth): Extension, + crate::rate_limit::PeerAddr(peer): crate::rate_limit::PeerAddr, + headers: axum::http::HeaderMap, body: Bytes, ) -> Result { + let name = smart_http_repo_name(&repo)?; + // Per-source write sub-cap (#174 P1-d): before the global write permit so one + // source IP cannot occupy the whole write pool via many slow authenticated pushes + // and 503 every other source. Owner enforcement defaults off, so any valid did:key + // is accepted (auth != authz), and the 600/hour push limiter bounds arrival RATE, + // not in-flight concurrency — so without this a single host minting disposable DIDs + // saturates the pool. Keyed on the resolved source IP, NEVER the signed DID (a DID + // farm defeats a DID key); no resolvable key -> global write pool only. + let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); + let _caller_permit = acquire_read_caller_permit( + &state.git_write_per_caller, + caller_key.as_deref(), + name, + "receive-pack", + )?; // Shed with a 503 before spawning git when the concurrency cap is saturated. // Pushes draw from the dedicated WRITE pool, separate from reads, so a flood of - // anonymous reads cannot shed an authenticated push (#174). Acquired at the very - // top so it wraps the write-guard below (and precedes the Tigris acquire_write, - // bounding concurrent fresh acquires — INV-10); held for the whole op. + // anonymous reads cannot shed an authenticated push (#174). Taken after the + // per-source cap above so one source cannot occupy global slots it would be + // sub-cap-denied for; still before the Tigris acquire_write, bounding concurrent + // fresh acquires (INV-10); held for the whole op. let _permit = git_permit(&state.git_write_semaphore)?; - let name = smart_http_repo_name(&repo)?; tracing::info!(owner = %owner, repo = %name, "receive-pack request"); let record = state .db @@ -3606,6 +3623,82 @@ mod tests { } } + /// #174 U4 (P1-d, RED-before/GREEN-after): the authenticated receive-pack POST + /// carries a per-source WRITE sub-cap so one source IP cannot monopolize the write + /// pool with many slow pushes (owner enforcement defaults off, so disposable DIDs + /// are free). Global write pool has capacity; the source is pre-held at its single + /// write slot. A push from THAT source sheds (Overloaded/503) — which also proves + /// the PeerAddr+HeaderMap extractors resolve a key (without them the key is None and + /// the cap is inert, never shedding). A push from a DIFFERENT source is NOT shed by + /// the cap. Called directly so the test needs no signed request; the handler is + /// where the cap lives. Remove the `git_write_per_caller` acquire and the capped + /// source no longer sheds (RED). + #[sqlx::test] + async fn receive_pack_per_source_write_cap_sheds_capped_source_not_others(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + let mut state = crate::test_support::test_state(pool).await; + // Global write pool has capacity; the per-source cap is 1. + state.git_write_semaphore = Arc::new(Semaphore::new(4)); + state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6rp4wr", "rp4", "/tmp/rp4-nonexistent", None, false) + .await + .unwrap(); + + let did = "did:key:z6MkReceivePackWriteCapProofDidAAAAAAAAAA"; + let capped: SocketAddr = "203.0.113.44:5000".parse().unwrap(); + let other: SocketAddr = "203.0.113.45:5000".parse().unwrap(); + + // Pin the capped source at its single write slot. + let _slot = state + .git_write_per_caller + .try_acquire(&capped.ip().to_string()) + .expect("first write slot for the capped source IP"); + + // A push from the capped source must shed on the per-source write cap even with + // global write capacity free. The shed also proves the source-IP key resolved + // via the extractors (an inert None key would fall through to Ok(None)). + let capped_result = git_receive_pack( + State(state.clone()), + Path(("z6rp4wr".to_string(), "rp4".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(capped)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await; + assert!( + matches!(capped_result, Err(AppError::Overloaded(_))), + "a source at its per-source write cap must shed (Overloaded/503) with global \ + pool capacity free; got {capped_result:?}" + ); + + // A push from a DIFFERENT source must NOT be shed by the per-source cap — it + // proceeds past admission (and fails later on the nonexistent repo, which is not + // an Overloaded error). + let other_result = git_receive_pack( + State(state.clone()), + Path(("z6rp4wr".to_string(), "rp4".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(other)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await; + assert!( + !matches!(other_result, Err(AppError::Overloaded(_))), + "a different source must not be shed by the per-source write cap while the \ + capped source holds its slot; got {other_result:?}" + ); + } + /// #174 SC2 (per-source key, U1): the per-caller read sub-cap keys on the /// resolved source IP, NOT the signed DID, so a disposable-DID farm cannot /// multiply its budget. Fill the source IP's single read slot, then drive two diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 5d827794..5597cd8d 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -526,6 +526,7 @@ mod tests { git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), + git_write_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), git_bin: "git".to_string(), } } diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 18019395..74504f90 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -399,6 +399,13 @@ async fn main() -> Result<()> { git_push_advert_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( (config.max_concurrent_git_pushes / 8).max(1), ), + // Per-source cap on the authenticated receive-pack POST, sized like the advert + // cap: one source IP can hold at most this many write-pool slots, so + // monopolizing the pool takes ~8 distinct source IPs, each also rate-limited + // (#174 P1-d). + git_write_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( + (config.max_concurrent_git_pushes / 8).max(1), + ), git_bin: "git".to_string(), }; diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index eb07a3e4..6ce158e3 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -123,6 +123,13 @@ pub struct AppState { /// a fraction of `max_concurrent_git_pushes`, so filling the write pool takes many /// distinct source IPs (each also braked by the per-IP push rate limiter). pub git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency, + /// Per-source concurrency sub-cap on the authenticated `git-receive-pack` POST: + /// each source IP may hold at most a small share of `git_write_semaphore`, so one + /// host minting disposable `did:key` identities cannot open enough slow pushes to + /// monopolize the write pool and 503 every other source's push (#174 P1-d). Keyed + /// on the resolved source IP (never the DID — a DID farm defeats a DID key). Sized + /// like `git_push_advert_per_caller`, a fraction of `max_concurrent_git_pushes`. + pub git_write_per_caller: crate::rate_limit::PerCallerConcurrency, /// The `git` executable the served-git withheld-blob walk spawns. Production is /// `"git"` (resolved via PATH); injectable so a fake `git` can drive the walk's /// process-group teardown in handler tests without mutating the process-global diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 0805fec1..9241b518 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -90,6 +90,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( 8, ), + git_write_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), git_bin: "git".to_string(), } } From 2a54c159e6984032205545816e0e44e0881f996c Mon Sep 17 00:00:00 2001 From: t Date: Tue, 14 Jul 2026 20:28:52 -0500 Subject: [PATCH 031/124] fix(node): bound concurrent post-push encryption walks with an admission pool (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After a successful path-scoped push the handler released its write permit and then ran a DETACHED tokio task whose spawn_blocking(withheld_blob_recipients_bounded) performed another full-history git walk under no admission. N fast completed pushes spawned N concurrent full-history walks past GITLAWB_MAX_CONCURRENT_GIT_PUSHES (which bounds only the in-handler phase), and the per-child bounding from 71389e6 caps each walk's duration but not their count (P1-e). Add git_encrypt_semaphore, a pool of its own (sized from max_concurrent_git_pushes, no new knob — Q1) so a long background walk never holds a foreground write slot and a handler holding a write permit can't self-deadlock. Route the walk through a new withheld_recipients_gated helper that acquires the pool before the walk and DEFERS (blocks) when full rather than shedding — dropping the walk would lose the withheld-blob recovery copy, so durability stays fail-closed. Regression: encrypt_walk_defers_when_pool_exhausted drives the gated helper with the pool exhausted and asserts the walk blocks and does not run its rev-list, then runs once a permit frees. RED before (the walk runs regardless of the pool), GREEN after. The detached push task calls this exact helper. --- crates/gitlawb-node/src/api/repos.rs | 138 +++++++++++++++++++++--- crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/main.rs | 6 ++ crates/gitlawb-node/src/state.rs | 11 ++ crates/gitlawb-node/src/test_support.rs | 1 + 5 files changed, 144 insertions(+), 13 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 67c6c82a..430b2f7f 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -727,6 +727,37 @@ fn acquire_read_caller_permit( } } +/// Acquire an encryption-walk admission permit, then run the bounded withheld-blob +/// recipients walk. Blocks (defers) when `git_encrypt_semaphore` is full rather than +/// shedding — the walk is background so added latency is fine, and dropping it would +/// lose the withheld-blob recovery copy (#174 P1-e). Bounds the number of concurrent +/// post-push encryption walks so N fast completed pushes cannot spawn N concurrent +/// full-history git walks. Mirrors the original `spawn_blocking(...).await` return +/// shape so the caller's `Ok(Ok(recipients))` match is unchanged. +async fn withheld_recipients_gated( + encrypt_sem: std::sync::Arc, + repo_path: std::path::PathBuf, + git_bin: String, + timeout: std::time::Duration, + rules: Vec, + is_public: bool, + owner_did: String, +) -> std::result::Result< + anyhow::Result>>, + tokio::task::JoinError, +> { + let _permit = encrypt_sem + .acquire_owned() + .await + .expect("git_encrypt_semaphore is never closed"); + tokio::task::spawn_blocking(move || { + crate::git::visibility_pack::withheld_blob_recipients_bounded( + &repo_path, &git_bin, timeout, &rules, is_public, &owner_did, + ) + }) + .await +} + /// Map an error from a `smart_http` git service call to the right `AppError`: /// [`smart_http::GitServiceTimeout`] to 504, a malformed client request to 400, /// anything else to a 500 git error. Pure (no logging) so it is unit-testable; @@ -1329,6 +1360,7 @@ pub async fn git_receive_pack( let repo_name = record.name.clone(); let enc_git_bin = state.git_bin.clone(); let enc_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let encrypt_sem = state.git_encrypt_semaphore.clone(); tokio::spawn(async move { let pinned = crate::ipfs_pin::pin_new_objects( &ipfs_api, @@ -1351,19 +1383,18 @@ pub async fn git_receive_pack( // the has_path_scoped_rule gate on the other two withheld-walk sites. if let Some(rules) = rules_for_enc.filter(|r| visibility_pack::has_path_scoped_rule(r)) { - let p = repo_path_clone.clone(); - let owner = owner_did.clone(); - let git_bin = enc_git_bin.clone(); - let recip = tokio::task::spawn_blocking(move || { - crate::git::visibility_pack::withheld_blob_recipients_bounded( - &p, - &git_bin, - enc_timeout, - &rules, - is_public, - &owner, - ) - }) + // Bound the number of concurrent post-push encryption walks (#174 P1-e): + // acquire an admission permit before the full-history walk, deferring + // when the pool is full rather than shedding the recovery pin. + let recip = withheld_recipients_gated( + encrypt_sem.clone(), + repo_path_clone.clone(), + enc_git_bin.clone(), + enc_timeout, + rules, + is_public, + owner_did.clone(), + ) .await; if let Ok(Ok(recipients)) = recip { let delta = crate::encrypted_pin::encrypt_and_pin( @@ -3699,6 +3730,87 @@ mod tests { ); } + /// #174 U5 (P1-e, RED-before/GREEN-after): the post-push encryption walk acquires a + /// `git_encrypt_semaphore` permit before running, so completed pushes cannot spawn + /// unbounded concurrent full-history walks. With the pool exhausted the gated walk + /// must DEFER (block on admission) and NOT run its rev-list; on the pre-fix code + /// (no acquire) the walk runs regardless of the pool (RED). It defers rather than + /// sheds — releasing the permit lets the SAME walk run and pin (durability stays + /// fail-closed). Exercises the gating seam directly; the detached push task calls + /// this exact helper. + #[tokio::test] + async fn encrypt_walk_defers_when_pool_exhausted() { + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let marker = tmp.path().join("revlist.ran"); + // Fake git records when rev-list runs (the walk's first git call). + let body = format!( + "#!/bin/sh\ncase \"$1\" in\n rev-list) echo ran > \"{}\" ;;\n *) : ;;\nesac\nexit 0\n", + marker.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + let git_bin = git_path.to_str().unwrap().to_string(); + let owner = "did:key:z6MkEncWalkOwnerAAAAAAAAAAAAAAAAAAAAAAAA".to_string(); + + // Exhaust the pool: hold its only permit so a gated walk must defer. + let sem = Arc::new(Semaphore::new(1)); + let held = sem.clone().acquire_owned().await.unwrap(); + + // Blocked: the gated walk must NOT complete or run rev-list while exhausted. + let blocked = tokio::time::timeout( + Duration::from_millis(500), + withheld_recipients_gated( + sem.clone(), + tmp.path().to_path_buf(), + git_bin.clone(), + Duration::from_secs(5), + Vec::new(), + true, + owner.clone(), + ), + ) + .await; + assert!( + blocked.is_err(), + "the encryption walk must defer (block on admission) when the pool is exhausted" + ); + assert!( + !marker.exists(), + "the walk's rev-list must not run while its admission permit is unavailable (P1-e)" + ); + + // Release admission: the SAME walk now runs (defer, not shed) — rev-list fires. + drop(held); + let ran = withheld_recipients_gated( + sem, + tmp.path().to_path_buf(), + git_bin, + Duration::from_secs(5), + Vec::new(), + true, + owner, + ) + .await; + assert!( + ran.is_ok(), + "with a permit the walk runs and joins: {ran:?}" + ); + assert!( + marker.exists(), + "once admission is available the deferred walk runs its rev-list" + ); + } + /// #174 SC2 (per-source key, U1): the per-caller read sub-cap keys on the /// resolved source IP, NOT the signed DID, so a disposable-DID farm cannot /// multiply its budget. Fill the source IP's single read slot, then drive two diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 5597cd8d..f18b4963 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -523,6 +523,7 @@ mod tests { git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 74504f90..380ab5f4 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -389,6 +389,12 @@ async fn main() -> Result<()> { git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new( config.max_concurrent_git_pushes, )), + // Bounds concurrent detached post-push encryption walks, sized from the push + // pool (no separate knob — Q1): completed pushes cannot outnumber active + // encryption walks past this (#174 P1-e). + git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new( + config.max_concurrent_git_pushes, + )), git_read_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( config.max_concurrent_reads_per_caller, ), diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 6ce158e3..3ff86923 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -110,6 +110,17 @@ pub struct AppState { /// advert pool (each source also capped by `git_push_advert_per_caller` and the /// per-IP push rate limiter), and the reserved POST pool is untouched (#174). pub git_push_advert_semaphore: Arc, + /// Bounds concurrent post-push encrypt-then-pin history walks. Each successful + /// path-scoped push releases its handler write permit and then runs a DETACHED + /// full-history walk (`withheld_blob_recipients_bounded`) to seal withheld blobs; + /// without a cap, N fast pushes spawn N concurrent full-history git walks past + /// `max_concurrent_git_pushes` (which only bounds the in-handler phase). The walk + /// acquires a permit here and DEFERS (blocks) when the pool is full rather than + /// shedding — the work is background and dropping it would lose the recovery copy + /// (#174 P1-e). A pool of its own, not `git_write_semaphore`: a long background + /// walk must not hold a foreground write slot, and a handler already holding a + /// write permit that needed a second would self-deadlock at pool size 1. + pub git_encrypt_semaphore: Arc, /// Per-caller concurrency sub-cap on the read pool: each caller (keyed on the /// resolved source IP, #174 U1) may hold at most `max_concurrent_reads_per_caller` /// in-flight read ops, so one caller cannot monopolize `git_read_semaphore` diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 9241b518..a27be946 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -86,6 +86,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( 8, From 5749df6e67be0d06fd99b69f0a64156ba3df29ca Mon Sep 17 00:00:00 2001 From: t Date: Tue, 14 Jul 2026 20:33:01 -0500 Subject: [PATCH 032/124] test(node): INV-22 completeness guard for the served-git concurrency surface (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A source-scan tripwire that fails when a new site reintroduces the concurrency-cap class PR #174 fixed, or when one of the five gates (U1-U5) is removed. INV-22: a permit held per op recovers only if every path is duration-bounded and reaps the group before releasing admission, and every detached git task carries admission. Lives in tests/ (a separate crate) on purpose — a guard scanning the file it lives in would match its own identifier literals and pass vacuously; scanning src/ from here keeps every check load-bearing. Checks: run_bounded_git confirms child exit via child_terminated_without_reaping (U1); KillGroupOnDrop launches the reaper via Handle::try_current (U2); git_receive_pack acquires state.git_write_per_caller (U4); the encryption walk runs through withheld_recipients_gated over git_encrypt_semaphore (U5); and the bounded recipients walk is invoked exactly once — only inside the gated helper — so a new spawn_blocking bypass (count > 1) fails (P1-e non-bypass). Proven load-bearing: a synthetic second call site turns the count tripwire RED, and each named gate maps to a fix whose own regression already goes RED on revert. --- crates/gitlawb-node/tests/inv22_gates.rs | 80 ++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 crates/gitlawb-node/tests/inv22_gates.rs diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs new file mode 100644 index 00000000..5f576300 --- /dev/null +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -0,0 +1,80 @@ +//! #174 U6 — INV-22 completeness guard (rung-raising). +//! +//! INV-22: a permit held per op recovers only if every path that holds it is also +//! duration-bounded and reaps the process group before releasing admission, and every +//! detached git/blocking task carries its own admission. PR #174 fixed five paths +//! (U1-U5) that violated this. Each fix has a per-unit RED/GREEN regression; together +//! those form the five-revert matrix. This guard adds the missing piece: a source-scan +//! tripwire that fails when a NEW site reintroduces the class, or when one of the five +//! gates is removed. +//! +//! It lives in `tests/` (a separate crate) on purpose: a guard that scanned the same +//! file it lives in would match its own identifier literals and pass vacuously. Here +//! the scanned `src/` files never contain this file's literals, so each check is +//! load-bearing — reverting the named gate turns the assertion red. +//! +//! These are deliberately coarse structural checks, not a parser. They cannot prove a +//! gate is *correct* (the per-unit tests do that); they prove a gate is *present and +//! not bypassed*, which is what stops the class from silently regressing. + +use std::path::Path; + +fn src(rel: &str) -> String { + let p = Path::new(env!("CARGO_MANIFEST_DIR")).join("src").join(rel); + std::fs::read_to_string(&p).unwrap_or_else(|e| panic!("read {}: {e}", p.display())) +} + +#[test] +fn inv22_concurrency_gates_present_and_not_bypassed() { + let repos = src("api/repos.rs"); + let smart_http = src("git/smart_http.rs"); + let vis = src("git/visibility_pack.rs"); + + // U1 / P1-a: run_bounded_git stands the watchdog down only after confirming the + // child actually terminated (WNOWAIT), not on the raw stdout-drain EOF — otherwise + // a child that closes stdout then hangs pins the permit past the deadline. The + // probe is defined and called, so >= 2 occurrences; reverting the fix removes both. + assert!( + vis.matches("child_terminated_without_reaping").count() >= 2, + "U1/P1-a gate missing: run_bounded_git must confirm child exit via \ + child_terminated_without_reaping before signalling the watchdog" + ); + + // U2 / P1-c: on client disconnect KillGroupOnDrop must launch a detached reaper + // that runs the full TERM/grace/KILL/reap, not a lone SIGTERM. The reaper is spawned + // via a runtime handle in Drop; `Handle::try_current` is unique to that launch (the + // timeout path already has an async context and never calls it). + assert!( + smart_http.contains("Handle::try_current"), + "U2/P1-c gate missing: KillGroupOnDrop::drop must launch the full \ + TERM/grace/KILL reaper on disconnect (Handle::try_current), not a lone SIGTERM" + ); + + // U4 / P1-d: git_receive_pack must acquire the per-source write sub-cap before the + // global write permit. The acquire reads `state.git_write_per_caller`; comments name + // the field without the `state.` prefix, so this targets the real acquire site. + assert!( + repos.contains("state.git_write_per_caller"), + "U4/P1-d gate missing: git_receive_pack must acquire the per-source write cap \ + (state.git_write_per_caller) before the global write permit" + ); + + // U5 / P1-e: the detached post-push encryption walk must run through the + // admission-gated helper, which is wired to the shared encrypt pool. + assert!( + repos.contains("fn withheld_recipients_gated") + && repos.contains("state.git_encrypt_semaphore"), + "U5/P1-e gate missing: the encryption walk must run through \ + withheld_recipients_gated, which acquires git_encrypt_semaphore" + ); + + // P1-e non-bypass tripwire: the bounded recipients walk is spawn_blocking'd nowhere + // but inside withheld_recipients_gated. A second call site (count > 1) is a new + // detached git walk that skips the admission gate — exactly the class U5 closed. + assert_eq!( + repos.matches("withheld_blob_recipients_bounded").count(), + 1, + "P1-e bypass: the bounded recipients walk must be invoked only inside \ + withheld_recipients_gated; a new call site bypasses the encrypt-walk admission cap" + ); +} From 72e899dae022dbc757be4198161629dad5245a92 Mon Sep 17 00:00:00 2001 From: t Date: Wed, 15 Jul 2026 11:33:20 -0500 Subject: [PATCH 033/124] fix(node): hold served-git admission until the process group is reaped on the plain spawn paths (#174) On the plain (non-path-scoped) info_refs / upload-pack / receive-pack paths the global + per-source admission permits were handler-locals that dropped the instant the handler future dropped on a client disconnect, while KillGroupOnDrop reaps the git process group in a DETACHED task (SIGTERM -> grace -> SIGKILL -> reap). So a disconnect-spammer admitted replacements while prior groups were still alive, defeating the process cap and the per-source cap during teardown. be0cdd6 already fixed this for the path-scoped upload-pack walk; this closes the residual plain paths (P1-a). Introduce AdmissionGuard, a move-only holder of the permits, threaded through info_refs/upload_pack/receive_pack/run_git_service into drive_git_child's KillGroupOnDrop. On disconnect the guard moves into the detached reaper and drops only after the group is ESRCH-confirmed reaped; on success/timeout disarm() returns it for the earliest provably-free drop. The rev-list/pack-objects visibility-walk callers hold no admission and pass None. Thread git_bin through upload_pack/receive_pack (they hardcoded "git") so the plain POST path is drivable by an injected fake git, matching the existing injectable seam on info_refs/run_git_service/upload_pack_excluding. Handler-layer regression upload_pack_plain_permit_held_through_group_reap_after_disconnect: isolates the global read pool (size 1, per-source + rate limiter permissive), a SIGTERM-ignoring descendant keeps the group alive across the reap window, and asserts available_permits()==0 held on disconnect + a cross-source replacement sheds 503, then frees after ESRCH. Reverting the guard-into-reaper move (release immediately) turns it RED at the held-through-reap assertion. Plus a None-key arm. --- crates/gitlawb-node/src/api/repos.rs | 269 +++++++++++++++++++++- crates/gitlawb-node/src/git/smart_http.rs | 130 +++++++++-- 2 files changed, 382 insertions(+), 17 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 430b2f7f..16a8ed0d 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -653,8 +653,14 @@ pub async fn git_info_refs( AppError::Git(e.to_string()) })?; + // Move the admission permits into the guard so they release only after the spawned + // git process group is confirmed reaped, on complete/timeout/disconnect — not the + // instant a disconnect drops this future while the detached reaper is still tearing + // the group down (#174 P1-a). The handler keeps no copy: `_permit`/`_caller_permit` + // are moved in, so admission tracks the real process lifetime. + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); - smart_http::info_refs("git", &service, &disk_path, git_timeout) + smart_http::info_refs("git", &service, &disk_path, git_timeout, Some(admission)) .await .map_err(|e| { let app = git_service_app_error(&e); @@ -848,7 +854,12 @@ pub async fn git_upload_pack( // withheld walk and serve the pack directly. let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); let resp = if !visibility_pack::has_path_scoped_rule(&rules) { - smart_http::upload_pack(&disk_path, body, git_timeout).await + // Plain (non-path-scoped) serve: move both admission permits into the guard so + // they release only after the spawned git group is reaped, on + // complete/timeout/disconnect — not the instant a disconnect drops this future + // (#174 P1-a). The handler keeps no copy. + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + smart_http::upload_pack(&state.git_bin, &disk_path, body, git_timeout, Some(admission)).await } else { // withheld_blob_oids walks every ref with blocking `git ls-tree`; keep that // off the async worker thread. Move BOTH admission permits INTO the blocking @@ -886,9 +897,17 @@ pub async fn git_upload_pack( let withheld = withheld.map_err(|e| git_service_app_error(&e))?; if withheld.is_empty() { - smart_http::upload_pack(&disk_path, body, git_timeout).await + // No blobs to withhold: serve the plain pack, moving the permits returned by + // the walk into the guard so admission tracks the served git group's reap + // (the walk already held them per be0cdd6; this hands them to the serve). + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + smart_http::upload_pack(&state.git_bin, &disk_path, body, git_timeout, Some(admission)).await } else { tracing::info!(repo = %name, caller = ?caller, withheld = withheld.len(), "serving filtered pack"); + // upload_pack_excluding runs its own rev-list/pack-objects (both pass `None` + // admission internally); the walk's permits stay handler-locals held across + // this serve, as be0cdd6 established, and drop when the handler returns. + let _hold = (_permit, _caller_permit); smart_http::upload_pack_excluding(&disk_path, body, &withheld, git_timeout).await } } @@ -1172,7 +1191,14 @@ pub async fn git_receive_pack( tracing::debug!(repo = %name, path = %disk_path.display(), "running git receive-pack"); let body_len = body.len(); let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); - let receive_result = smart_http::receive_pack(&disk_path, body, git_timeout).await; + // Move both admission permits into the guard so they release only after the spawned + // receive-pack process group is reaped, on complete/timeout/disconnect — not the + // instant a disconnect drops this future while the detached reaper runs (#174 P1-a). + // The handler keeps no copy. This is independent of the write-lock `guard.release` + // below: admission tracks the git process lifetime, the write lock tracks the repo. + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + let receive_result = + smart_http::receive_pack(&state.git_bin, &disk_path, body, git_timeout, Some(admission)).await; // Always release the advisory lock — even on error — to prevent stale locks // from blocking subsequent pushes. Only upload to Tigris when the push @@ -3654,6 +3680,241 @@ mod tests { } } + /// #174 U1 (P1-a, plain-spawn residual, RED-before/GREEN-after): on the PLAIN + /// (non-path-scoped) upload-pack path a client disconnect must NOT release the + /// global read admission while the detached process-group reaper is still tearing + /// down a git group that ignores SIGTERM. The `be0cdd6` fix moved permits into the + /// path-scoped `spawn_blocking` walk; this closes the residual plain path, where the + /// permits were handler-locals that dropped the instant the future was dropped. + /// + /// Isolate the GLOBAL pool: read pool = 1, per-source cap + rate limiter permissive, + /// so the only thing that can shed a replacement is the leaked global permit. Drive + /// the handler until git spawns, disconnect, then assert the global slot stays held + /// (`available_permits() == 0`) AND a replacement sheds 503 while the group is alive; + /// after the reaper SIGKILLs+reaps the group the slot frees and a replacement is no + /// longer shed by the global cap. On the pre-fix code the handler-local permit drops + /// on future-drop and the slot frees at once (RED). + #[cfg(unix)] + #[sqlx::test] + async fn upload_pack_plain_permit_held_through_group_reap_after_disconnect(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + let descfile = tmp.path().join("desc.pid"); + // Fake git for the plain upload-pack path (invoked as `git upload-pack + // --stateless-rpc `). It forks a descendant that TRAPS SIGTERM, records its + // pid, and loops ~20s, then `wait`s — so on disconnect the group leader dies on + // the reaper's SIGTERM but the descendant survives until the reaper escalates to + // SIGKILL, keeping the group alive (ESRCH not reached) across the observation + // window. Bounded so a broken fix leaks no permanent orphan. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + upload-pack)\n\ + sh -c 'trap \"\" TERM; echo $$ > \"{}\"; i=0; while [ $i -lt 20 ]; do sleep 1; i=$((i+1)); done' &\n\ + wait ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + descfile.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + // Isolate the global read pool: size 1; per-source cap + rate limiter permissive + // so only the leaked global permit can shed the replacement. + state.git_read_semaphore = Arc::new(Semaphore::new(1)); + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let owner = "z6up1st"; + let name = "up1"; + state + .db + .upsert_mirror_repo(owner, name, "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + // Real bare repo at the path acquire() computes, so the handler reaches the + // spawn. No path-scoped rule -> the PLAIN serve branch (this test's target). + state + .repo_store + .init(&rec.owner_did, &rec.name) + .await + .unwrap(); + + let sem = state.git_read_semaphore.clone(); + assert_eq!(sem.available_permits(), 1, "one read slot before the request"); + + let router = crate::server::build_router(state); + let make_req = |peer: SocketAddr| { + let mut req = Request::builder() + .method(Method::POST) + .uri(format!("/{owner}/{name}/git-upload-pack")) + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req + }; + + let peer: SocketAddr = "203.0.113.71:5000".parse().unwrap(); + let mut fut = Box::pin(router.clone().oneshot(make_req(peer))); + // Drive until git spawns (the descendant records its pid) — the request is + // inside the plain serve, holding the global read permit. Stop polling the + // instant the future completes (re-polling a completed oneshot panics); read the + // descfile first so a spawn that recorded its pid then returned is still caught. + let mut spawned: Option = None; + let mut early = None; + for _ in 0..500 { + let done = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&descfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + spawned = Some(p); + break; + } + if let Ok(resp) = done { + early = Some(resp.map(|r| r.status())); + break; + } + } + let desc = spawned + .unwrap_or_else(|| panic!("the fake git must have spawned; early finish: {early:?}")); + // Kill the descendant regardless of outcome so a RED run leaks no orphan. + struct ReapOnDrop(i32); + impl Drop for ReapOnDrop { + fn drop(&mut self) { + unsafe { + libc::kill(self.0, libc::SIGKILL); + } + } + } + let _cleanup = ReapOnDrop(desc); + assert!( + unsafe { libc::kill(desc, 0) == 0 }, + "descendant should be running before the disconnect" + ); + assert_eq!( + sem.available_permits(), + 0, + "the read slot is held while the git op runs" + ); + + // Client disconnect: drop the request future. The detached reaper now owns the + // AdmissionGuard and will not drop it until the group is ESRCH-confirmed reaped. + drop(fut); + + // Load-bearing: the slot must STAY held while the SIGTERM-ignoring group is still + // alive. On the pre-fix code the handler-local permit drops here and the slot + // frees at once (RED). Check quickly (before the reaper's ~2s SIGKILL escalation). + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + assert!( + unsafe { libc::kill(desc, 0) == 0 }, + "the SIGTERM-ignoring descendant must still be alive during the hold window" + ); + assert_eq!( + sem.available_permits(), + 0, + "on disconnect the read admission must be HELD until the process group is \ + reaped, not released the instant the future drops (P1-a)" + ); + // A replacement request from a DIFFERENT source must shed 503 — the only pool + // that can shed it is the leaked global permit (per-source cap is permissive). + let peer2: SocketAddr = "203.0.113.72:5000".parse().unwrap(); + let resp = router.clone().oneshot(make_req(peer2)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "while the prior group is still alive the held global permit must shed a \ + replacement with 503" + ); + + // After the reaper SIGKILLs + reaps the group the AdmissionGuard drops and the + // slot frees. Poll for recovery. + let mut freed = false; + for _ in 0..400 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + assert!( + freed, + "once the reaper confirms the group gone the admission guard must drop and \ + free the global slot" + ); + // A replacement is now no longer shed by the global cap (it proceeds past + // admission; it then fails downstream on the fake git, which is not a 503). + let peer3: SocketAddr = "203.0.113.73:5000".parse().unwrap(); + let resp = router.oneshot(make_req(peer3)).await.unwrap(); + assert_ne!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "after the group is reaped the freed slot must admit a replacement" + ); + } + + /// #174 U1 (P1-a): the `None`-key arm — a request with no resolvable source key + /// (no trusted-proxy header, no peer) is bounded by the GLOBAL read pool only, never + /// a per-source cap. With the global read pool exhausted such a request still sheds + /// 503, proving the plain path admits/sheds on the global pool for the `None` arm + /// (the counterpart to the `Some(ip)` arm above). Complements the resolver-arm rule: + /// neither arm is vacuous. + #[tokio::test] + async fn upload_pack_plain_none_key_arm_sheds_on_global_pool() { + use axum::body::Body; + use axum::http::{Method, Request, StatusCode}; + use axum::Router; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state_lazy(); + // Global read pool exhausted; per-source cap permissive so only the global pool + // can shed. No ConnectInfo + no trusted header -> read_caller_key resolves None. + state.git_read_semaphore = Arc::new(Semaphore::new(0)); + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let router = Router::new() + .route( + "/{owner}/{repo}/git-upload-pack", + axum::routing::post(crate::api::repos::git_upload_pack), + ) + .with_state(state); + // No ConnectInfo extension and no XFF header: the caller key is None. + let req = Request::builder() + .method(Method::POST) + .uri("/alice/repo.git/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a None-key request must still shed 503 on the exhausted GLOBAL read pool" + ); + } + /// #174 U4 (P1-d, RED-before/GREEN-after): the authenticated receive-pack POST /// carries a per-source WRITE sub-cap so one source IP cannot monopolize the write /// pool with many slow pushes (owner enforcement defaults off, so disposable DIDs diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index a5649a42..64d3cb7f 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -10,6 +10,40 @@ use std::time::{Duration, Instant}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::process::Command; +/// Owns the served-git admission permits (global + per-source) for the lifetime of +/// the work they admitted, so admission is released only when that work is truly +/// done — not the instant the handler future drops on a client disconnect. +/// +/// A move-only wrapper: no methods beyond construction and `Drop`. The handler +/// MOVEs its permits in and keeps no copy (a retained copy would drop early and +/// release admission the moment the future is dropped, defeating the guard). It is +/// threaded into `drive_git_child`, whose [`KillGroupOnDrop`] moves it into the +/// detached reaper on disconnect, so both permits drop only after the process group +/// is confirmed reaped (`kill(-pgid,0)==ESRCH`) rather than while the group is still +/// alive holding PIDs past the concurrency cap (#174 P1-a, plain-spawn residual). +/// +/// The `be0cdd6` path-scoped upload-pack walk already applies this discipline by +/// moving its permits into the `spawn_blocking`; this generalizes it to the plain +/// (non-path-scoped) `info_refs` / `run_git_service` spawn paths. +pub struct AdmissionGuard { + // Boxed so the concrete permit types stay private to the caller (repos.rs) — the + // guard only needs to hold them and drop them, never inspect them. `Send + + // 'static` so the guard can move into the detached reaper task. + _global: Option>, + _caller: Option>, +} + +impl AdmissionGuard { + /// Take ownership of the global permit and an optional per-caller permit. Both are + /// erased to `Box` — the guard's only job is to hold them until it drops. + pub fn new(global: impl Send + 'static, caller: Option) -> Self { + Self { + _global: Some(Box::new(global)), + _caller: caller.map(|c| Box::new(c) as Box), + } + } +} + /// Handle `GET /:owner/:repo/info/refs?service=git-upload-pack` /// or `?service=git-receive-pack` /// @@ -26,6 +60,7 @@ pub async fn info_refs( service: &str, repo_path: &Path, timeout: Duration, + admission: Option, ) -> Result { validate_service(service)?; @@ -36,7 +71,8 @@ pub async fn info_refs( .arg("--advertise-refs") .arg(repo_path); // No request body — advertise-refs does not read stdin. - let stdout = drive_git_child(command, Bytes::new(), timeout, "advertise-refs").await?; + let stdout = + drive_git_child(command, Bytes::new(), timeout, "advertise-refs", admission).await?; let content_type = format!("application/x-{service}-advertisement"); @@ -61,12 +97,21 @@ pub async fn info_refs( /// Serves pack data for a clone or fetch. This is stateless — the entire /// negotiation happens in a single request/response. pub async fn upload_pack( + git_bin: &str, repo_path: &Path, request_body: Bytes, timeout: Duration, + admission: Option, ) -> Result { - let output = - run_git_service("git", "git-upload-pack", repo_path, request_body, timeout).await?; + let output = run_git_service( + git_bin, + "git-upload-pack", + repo_path, + request_body, + timeout, + admission, + ) + .await?; Ok(Response::builder() .status(StatusCode::OK) @@ -80,12 +125,21 @@ pub async fn upload_pack( /// Accepts a push. The caller MUST verify HTTP Signature auth before /// calling this function. pub async fn receive_pack( + git_bin: &str, repo_path: &Path, request_body: Bytes, timeout: Duration, + admission: Option, ) -> Result { - let output = - run_git_service("git", "git-receive-pack", repo_path, request_body, timeout).await?; + let output = run_git_service( + git_bin, + "git-receive-pack", + repo_path, + request_body, + timeout, + admission, + ) + .await?; Ok(Response::builder() .status(StatusCode::OK) @@ -120,6 +174,12 @@ struct KillGroupOnDrop { // own SIGCHLD-driven orphan reaper. child: Option, pgid: Option, + // The global + per-source admission permits for this op, if any. On the plain + // (non-path-scoped) spawn paths repos.rs moves its permits in here so admission is + // released only when the group is confirmed reaped, on every exit — complete, + // timeout, or client-disconnect (#174 P1-a). The rev-list/pack-objects + // visibility-walk callers hold no admission and pass `None`. + admission: Option, } #[cfg(unix)] @@ -131,17 +191,25 @@ impl KillGroupOnDrop { /// Disarm on the success/timeout path: the body has already reaped the child (its /// `wait()` returned, or `reap_group_on_timeout` ran), so drop the handle and clear - /// the pgid, leaving the guard's Drop a no-op. - fn disarm(&mut self) { + /// the pgid, leaving the guard's Drop a no-op. Returns the admission guard so the + /// caller drops it at the earliest provably-free point (the group is already reaped + /// on both callers of this) rather than at end-of-bookkeeping. + fn disarm(&mut self) -> Option { self.pgid = None; self.child = None; + self.admission.take() } } #[cfg(unix)] impl Drop for KillGroupOnDrop { fn drop(&mut self) { + // Move any admission guard out first so it travels with the reaper (or drops + // here on the no-child / no-runtime fallback), never before the group is gone. + let admission = self.admission.take(); let (Some(mut child), Some(pgid)) = (self.child.take(), self.pgid) else { + // Nothing to reap (already disarmed); dropping `admission` here is correct — + // the group is already gone. return; }; // A sync Drop cannot await, so launch a detached reaper that owns the child and @@ -154,6 +222,13 @@ impl Drop for KillGroupOnDrop { Ok(handle) => { handle.spawn(async move { reap_group_on_timeout(&mut child).await; + // Release admission only now: the group is ESRCH-confirmed gone (or + // hit the ~4s D-state hard cap, past which nothing in userspace can + // free the PIDs anyway). Holding the permits until here is what + // stops disconnect-spam from admitting replacements while the prior + // group is still alive (#174 P1-a). `admission` is moved in and + // dropped when this closure ends. + drop(admission); }); } Err(_) => { @@ -162,6 +237,9 @@ impl Drop for KillGroupOnDrop { unsafe { libc::kill(-pgid, libc::SIGTERM); } + // No runtime to await the reap; drop admission best-effort after the + // synchronous signal (the fallback path is not on the hot request path). + drop(admission); } } } @@ -246,13 +324,14 @@ async fn run_git_service( repo_path: &Path, input: Bytes, timeout: Duration, + admission: Option, ) -> Result> { let mut command = Command::new(git_bin); command .arg(service_to_command(service)) .arg("--stateless-rpc") .arg(repo_path); - drive_git_child(command, input, timeout, service).await + drive_git_child(command, input, timeout, service, admission).await } /// Drive a spawned git child under `timeout` with process-group teardown, returning @@ -267,6 +346,7 @@ async fn drive_git_child( input: Bytes, timeout: Duration, what: &str, + admission: Option, ) -> Result> { command .stdin(Stdio::piped()) @@ -300,7 +380,12 @@ async fn drive_git_child( let mut group_guard = KillGroupOnDrop { child: Some(child), pgid, + admission, }; + // On non-unix there is no process-group teardown, so hold the admission guard here + // for the child's whole interaction; it drops when this function returns. + #[cfg(not(unix))] + let _admission = admission; let mut out = Vec::new(); let mut err = Vec::new(); @@ -339,18 +424,21 @@ async fn drive_git_child( // The join runs all arms to completion, so the child is reaped: disarm // before surfacing any interaction error (a read/wait error), else the // guard's drop would reap an already-reaped child / signal a reused pgid. + // Dropping the returned admission guard here releases the permits at the + // earliest provably-free point on the success path (the op finished). #[cfg(unix)] - group_guard.disarm(); + drop(group_guard.disarm()); result? } Err(_elapsed) => { // Timeout: tear the whole group down and reap it before returning so a // caller releasing a write lock can't race a still-live git. Then disarm - // the (now redundant) guard so its drop can't hit a reused pgid. + // the (now redundant) guard so its drop can't hit a reused pgid. The + // returned admission guard drops here, AFTER the group is confirmed reaped. #[cfg(unix)] { reap_group_on_timeout(group_guard.child_mut()).await; - group_guard.disarm(); + drop(group_guard.disarm()); } #[cfg(not(unix))] { @@ -415,7 +503,9 @@ async fn rev_list_keep( command .args(["rev-list", "--objects", "--all"]) .current_dir(repo_path); - let stdout = drive_git_child(command, Bytes::new(), timeout, "rev-list").await?; + // The visibility-walk callers intentionally hold no admission permit (their + // admission is governed elsewhere), so pass `None` (#174 KTD2). + let stdout = drive_git_child(command, Bytes::new(), timeout, "rev-list", None).await?; let mut keep = Vec::new(); for line in String::from_utf8_lossy(&stdout).lines() { let oid = line.split_whitespace().next().unwrap_or(""); @@ -464,6 +554,8 @@ pub async fn build_filtered_pack( Bytes::from(data), deadline.saturating_duration_since(Instant::now()), "pack-objects", + // Visibility-walk pack build: no admission permit here (#174 KTD2). + None, ) .await } @@ -736,7 +828,7 @@ mod tests { axum::extract::Query(q): axum::extract::Query>, ) -> Response { let service = q.get("service").cloned().unwrap_or_default(); - info_refs("git", &service, &st.repo, Duration::from_secs(30)) + info_refs("git", &service, &st.repo, Duration::from_secs(30), None) .await .unwrap() } @@ -1021,6 +1113,7 @@ mod tests { let _guard = KillGroupOnDrop { child: Some(child), pgid: Some(pid), + admission: None, }; } @@ -1078,6 +1171,7 @@ mod tests { let _guard = KillGroupOnDrop { child: Some(child), pgid: Some(pid), + admission: None, }; } @@ -1114,6 +1208,7 @@ mod tests { let mut guard = KillGroupOnDrop { child: Some(child), pgid: Some(pid), + admission: None, }; guard.disarm(); } // disarmed -> no reaper, no kill @@ -1320,6 +1415,7 @@ mod tests { tmp.path(), Bytes::new(), Duration::from_secs(60), + None, )); // Advance the future a slice at a time until the fake records its @@ -1421,6 +1517,7 @@ mod tests { tmp.path(), Bytes::new(), Duration::from_secs(60), + None, )); // Drive the future a slice at a time until the descendant records its pid. let mut desc: Option = None; @@ -1482,6 +1579,7 @@ mod tests { "git-upload-pack", tmp.path(), Duration::from_millis(200), + None, ), ) .await @@ -1589,6 +1687,7 @@ mod tests { tmp.path(), Bytes::new(), Duration::from_secs(60), + None, ) }) .await; @@ -1633,6 +1732,7 @@ mod tests { tmp.path(), Bytes::new(), Duration::from_secs(60), + None, ) }) .await; @@ -1683,6 +1783,7 @@ mod tests { tmp.path(), Bytes::new(), git_timeout, + None, ), ) }) @@ -1757,6 +1858,7 @@ mod tests { tmp.path(), Bytes::new(), git_timeout, + None, ), ) }) @@ -1794,6 +1896,7 @@ mod tests { tmp.path(), Bytes::new(), Duration::from_millis(200), + None, ), ) .await; @@ -1834,6 +1937,7 @@ mod tests { tmp.path(), big, Duration::from_secs(60), + None, ) .await; From 8d17038e5bc1e37cc76c931288c4d993a17b39c3 Mon Sep 17 00:00:00 2001 From: t Date: Wed, 15 Jul 2026 12:00:02 -0500 Subject: [PATCH 034/124] fix(node): bound the served-git storage-acquisition phase with a release-on-expiry deadline (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The served-git admission permit is acquired, then RepoStore::acquire*/advisory-lock runs before the bounded git runner starts — with no local deadline. acquire/acquire_fresh await Tigris HEAD/GET, and acquire_write's advisory-lock loop awaits a per-iteration pg_try_advisory_lock that can block indefinitely on a hung Postgres pool. Since GITLAWB_GIT_SERVICE_TIMEOUT_SECS only starts once git spawns, a stalled backend pins the permit and drains the pool until every later request 503s (P1-2). Wrap each git-path acquire (git_info_refs read/advert, git_upload_pack, git_receive_pack) and the /ipfs per-repo acquire loop in tokio::time::timeout(git_acquire_timeout_secs, ..). On expiry the git paths return AppError::Overloaded (503 + Retry-After); the permit is a handler-local at that point (moved into the AdmissionGuard only after acquire) so the early return frees the slot. The /ipfs per-repo acquire keeps its fail-closed `continue` on timeout (never serves an un-acquired repo). No repo_store.rs signature change: the outer timeout cancels a mid-sleep/mid-fetch_one future, so no in-loop deadline is needed. New knob GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS (default 30, range 1..), modeled on git_service_timeout_secs, kept separate because acquisition and git execution are distinct cost centers. Handler-layer regression receive_pack_acquire_deadline_sheds_and_releases_permit: holds the same pg advisory lock acquire_write derives on a second connection so the real loop must retry, drives git-receive-pack with the deadline at 2s and write pool size 1, and asserts 503 at ~2s + the permit recovers + a follow-up admits once the lock frees. Removing the timeout wrapper turns it RED (blocks ~31s to the test ceiling on the ~59s advisory loop). --- crates/gitlawb-node/src/api/ipfs.rs | 21 +- crates/gitlawb-node/src/api/repos.rs | 307 ++++++++++++++++++++++++--- crates/gitlawb-node/src/config.rs | 21 ++ 3 files changed, 315 insertions(+), 34 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index e921c39f..284c9398 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -114,9 +114,24 @@ pub async fn get_by_cid( continue; } - let repo_path = match state.repo_store.acquire(&repo.owner_did, &repo.name).await { - Ok(p) => p, - Err(_) => continue, + // Bound the per-repo acquire under `git_acquire_timeout_secs`: this loop shares + // the P1-2 stall vector (a hung Tigris HEAD/GET on one repo would otherwise + // block the whole /ipfs request). On expiry keep the existing fail-closed skip — + // never serve an un-acquired repo; a public copy (if any) still gets its turn. + let acquire_deadline = + std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); + let repo_path = match tokio::time::timeout( + acquire_deadline, + state.repo_store.acquire(&repo.owner_did, &repo.name), + ) + .await + { + Ok(Ok(p)) => p, + Ok(Err(_)) => continue, + Err(_elapsed) => { + tracing::warn!(repo = %repo.name, "repo acquire timed out during /ipfs walk; skipping repo"); + continue; + } }; // Check whether the object exists in this repo before any expensive diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 16a8ed0d..5b11404d 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -637,21 +637,37 @@ pub async fn git_info_refs( // For receive-pack (push), download the latest from Tigris so the client // sees the same refs that acquire_write() will operate on. - let disk_path = if service == "git-receive-pack" { - state - .repo_store - .acquire_fresh(&record.owner_did, &record.name) - .await - } else { - state - .repo_store - .acquire(&record.owner_did, &record.name) - .await - } - .map_err(|e| { - tracing::error!(repo = %name, service = %service, err = %e, "repo acquire failed"); - AppError::Git(e.to_string()) - })?; + // + // Bound the acquire under `git_acquire_timeout_secs`: the concurrency permit is + // already held above, and `git_service_timeout_secs` only starts once git spawns, + // so an un-deadlined acquire (a hung Tigris HEAD/GET here) pins the permit until + // the pool drains (#174 P1-2). On expiry the handler-local `_permit`/`_caller_permit` + // drop on the early return (the AdmissionGuard is not built until after acquire), + // so the shed frees the slot; return a bounded 503. + let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); + let acquire_fut = async { + if service == "git-receive-pack" { + state + .repo_store + .acquire_fresh(&record.owner_did, &record.name) + .await + } else { + state + .repo_store + .acquire(&record.owner_did, &record.name) + .await + } + }; + let disk_path = tokio::time::timeout(acquire_deadline, acquire_fut) + .await + .map_err(|_elapsed| { + tracing::warn!(repo = %name, service = %service, "repo acquire timed out; shedding with 503"); + AppError::Overloaded("git service acquisition timed out, retry shortly".into()) + })? + .map_err(|e| { + tracing::error!(repo = %name, service = %service, err = %e, "repo acquire failed"); + AppError::Git(e.to_string()) + })?; // Move the admission permits into the guard so they release only after the spawned // git process group is confirmed reaped, on complete/timeout/disconnect — not the @@ -842,11 +858,21 @@ pub async fn git_upload_pack( // exec (INV-10). let _permit = git_permit(&state.git_read_semaphore)?; - let disk_path = state - .repo_store - .acquire(&record.owner_did, &record.name) - .await - .map_err(|e| AppError::Git(e.to_string()))?; + // Bound the acquire under `git_acquire_timeout_secs` so a hung Tigris HEAD/GET + // cannot pin the read permit indefinitely (#174 P1-2). The permit is a handler + // local here (moved into the AdmissionGuard only below, once git is spawned), so + // the early return on timeout drops it and frees the slot; shed a bounded 503. + let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); + let disk_path = tokio::time::timeout( + acquire_deadline, + state.repo_store.acquire(&record.owner_did, &record.name), + ) + .await + .map_err(|_elapsed| { + tracing::warn!(repo = %name, "repo acquire timed out; shedding with 503"); + AppError::Overloaded("git service acquisition timed out, retry shortly".into()) + })? + .map_err(|e| AppError::Git(e.to_string()))?; let body_len = body.len(); // No path-scoped rule can withhold an individual blob, and the whole-repo @@ -1179,14 +1205,31 @@ pub async fn git_receive_pack( } tracing::debug!(repo = %name, "acquiring write lock"); - let guard = state - .repo_store - .acquire_write(&record.owner_did, &record.name) - .await - .map_err(|e| { - tracing::error!(repo = %name, err = %e, "acquire_write failed"); - AppError::Git(e.to_string()) - })?; + // Bound the write acquire under `git_acquire_timeout_secs`. acquire_write's + // advisory-lock loop already caps at ~60s, but its per-iteration + // `pg_try_advisory_lock().fetch_one(&pool)` can block indefinitely on a hung / + // exhausted Postgres pool (so the 60-count never advances) — and the write permit + // is held the whole time, draining the pool (#174 P1-2). The outer + // `tokio::time::timeout` cancels a mid-sleep/mid-`fetch_one` future, so it bounds + // both the loop and a hung iteration without any repo_store.rs change (KTD3). The + // permit is a handler local here (moved into the AdmissionGuard only after this), + // so the early return on timeout drops it and frees the slot; shed a bounded 503. + let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); + let guard = tokio::time::timeout( + acquire_deadline, + state + .repo_store + .acquire_write(&record.owner_did, &record.name), + ) + .await + .map_err(|_elapsed| { + tracing::warn!(repo = %name, "acquire_write timed out; shedding with 503"); + AppError::Overloaded("git service acquisition timed out, retry shortly".into()) + })? + .map_err(|e| { + tracing::error!(repo = %name, err = %e, "acquire_write failed"); + AppError::Git(e.to_string()) + })?; let disk_path = guard.path().to_path_buf(); tracing::debug!(repo = %name, path = %disk_path.display(), "running git receive-pack"); let body_len = body.len(); @@ -1197,8 +1240,14 @@ pub async fn git_receive_pack( // The handler keeps no copy. This is independent of the write-lock `guard.release` // below: admission tracks the git process lifetime, the write lock tracks the repo. let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); - let receive_result = - smart_http::receive_pack(&state.git_bin, &disk_path, body, git_timeout, Some(admission)).await; + let receive_result = smart_http::receive_pack( + &state.git_bin, + &disk_path, + body, + git_timeout, + Some(admission), + ) + .await; // Always release the advisory lock — even on error — to prevent stale locks // from blocking subsequent pushes. Only upload to Tigris when the push @@ -3761,7 +3810,11 @@ mod tests { .unwrap(); let sem = state.git_read_semaphore.clone(); - assert_eq!(sem.available_permits(), 1, "one read slot before the request"); + assert_eq!( + sem.available_permits(), + 1, + "one read slot before the request" + ); let router = crate::server::build_router(state); let make_req = |peer: SocketAddr| { @@ -3991,6 +4044,198 @@ mod tests { ); } + /// #174 U2 (P1-2, RED-before/GREEN-after): the storage-acquisition phase is bounded + /// by `git_acquire_timeout_secs`, so a stalled backend releases the admission permit + /// and sheds a 503 instead of pinning the pool. The permit is taken BEFORE + /// `acquire_write`, whose advisory-lock loop can spin ~60s (and whose per-iteration + /// `pg_try_advisory_lock` can block indefinitely on a hung pool), so without the + /// `tokio::time::timeout` wrapper the permit is held far past the deadline. + /// + /// Real stall (no `RepoStore` trait to fake): hold the SAME session-level advisory + /// lock `acquire_write` derives (`advisory_lock_key(owner_slug, repo_name)`, where + /// `owner_slug = owner_did.replace([':','/'], "_")`) on a second pooled connection, + /// so the handler's `pg_try_advisory_lock` returns false every iteration and the loop + /// must retry against the deadline. `git_acquire_timeout_secs = 2`; the request must + /// return 503 (Overloaded) at ~2s (NOT ~59s), and the write permit must be released + /// (`available_permits()` recovers to full once the shed returns). Covers R2. + /// + /// Load-bearing / mutation: remove the `tokio::time::timeout` wrapper on + /// `acquire_write` and the loop runs to ~59s with the permit held the whole time — + /// the `< DEADLINE_CEILING` timing assertion goes RED (observed ~59s) and the permit + /// stays pinned past the deadline. Restore to return GREEN. + #[sqlx::test] + async fn receive_pack_acquire_deadline_sheds_and_releases_permit(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + // Reproduce acquire_write's session-level advisory-lock key exactly so the + // second-connection lock collides with the handler's pg_try_advisory_lock + // (repo_store.rs: advisory_lock_key over owner_slug then repo_name). + fn advisory_lock_key(owner_slug: &str, repo_name: &str) -> i64 { + use std::hash::{Hash, Hasher}; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + owner_slug.hash(&mut hasher); + repo_name.hash(&mut hasher); + hasher.finish() as i64 + } + + let owner = "z6acqdead"; + let name = "acq1"; + // owner_slug as local_path() computes it from the record's owner_did. The + // mirror row stores the short owner as owner_did, so slug == owner (no ':'/'/'). + let owner_slug = owner.replace([':', '/'], "_"); + let lock_key = advisory_lock_key(&owner_slug, name); + + let mut state = crate::test_support::test_state(pool.clone()).await; + // Isolate the write pool at size 1 so available_permits() cleanly reports + // held (0) vs released (1). Per-source cap + trust permissive so only the + // write pool / acquire path can gate. + state.git_write_semaphore = Arc::new(Semaphore::new(1)); + state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Short acquire deadline: the fix must shed here, well before acquire_write's + // ~59s advisory-lock loop would bail on its own. + const ACQUIRE_TIMEOUT_SECS: u64 = 2; + let mut cfg = (*state.config).clone(); + cfg.git_acquire_timeout_secs = ACQUIRE_TIMEOUT_SECS; + // Keep the git-service timeout large so the deadline under test is the acquire + // one, not git execution (which is never reached on the stalled path anyway). + cfg.git_service_timeout_secs = 600; + state.config = std::sync::Arc::new(cfg); + + state + .db + .upsert_mirror_repo(owner, name, "/tmp/z6acqdead-acq1", None, false) + .await + .unwrap(); + + // Hold the advisory lock on a dedicated pooled connection (a distinct session), + // so the handler's pg_try_advisory_lock($lock_key) returns false every iteration + // and acquire_write's real loop must retry against the deadline. Released when + // this connection drops at end of test. + let mut lock_conn = pool + .acquire() + .await + .expect("second connection for the lock"); + sqlx::query("SELECT pg_advisory_lock($1)") + .bind(lock_key) + .execute(&mut *lock_conn) + .await + .expect("hold the advisory lock on the second connection"); + + let did = "did:key:z6MkAcquireDeadlineProofDidAAAAAAAAAAAAAAAA"; + let peer: SocketAddr = "203.0.113.61:5000".parse().unwrap(); + + let sem = state.git_write_semaphore.clone(); + assert_eq!( + sem.available_permits(), + 1, + "one write slot before the request" + ); + + // Drive the authenticated push in the background so we can observe the permit is + // held while acquire_write stalls, then that it is released on the shed. + let state_for_task = state.clone(); + let start = std::time::Instant::now(); + let handle = tokio::spawn(async move { + git_receive_pack( + State(state_for_task), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(peer)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await + }); + + // The handler takes the write permit BEFORE acquire_write, so once it is stalled + // in the advisory-lock loop the pool reports 0 available. Wait for that to prove + // the permit is genuinely held during the stall (and the request really reached + // acquire_write, not an earlier reject). + let mut held = false; + for _ in 0..200 { + if sem.available_permits() == 0 { + held = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert!( + held, + "the write permit must be held while acquire_write stalls on the advisory lock" + ); + + // The bounded acquire deadline must shed with 503 (Overloaded), NOT wait out the + // ~59s advisory-lock loop. Ceiling is comfortably above the 2s deadline + task + // scheduling but far below 59s, so a RED run (no wrapper -> ~59s) fails here. + const DEADLINE_CEILING: std::time::Duration = std::time::Duration::from_secs(20); + let result = tokio::time::timeout( + DEADLINE_CEILING + std::time::Duration::from_secs(10), + handle, + ) + .await + .expect("the handler must return within the ceiling — a hang means the acquire deadline is missing (RED)") + .expect("the receive-pack task must not panic"); + let elapsed = start.elapsed(); + + assert!( + matches!(result, Err(AppError::Overloaded(_))), + "a stalled acquire_write must shed with Overloaded/503 at the acquire deadline; \ + got {result:?}" + ); + assert!( + elapsed < DEADLINE_CEILING, + "the shed must land at ~{ACQUIRE_TIMEOUT_SECS}s (the acquire deadline), not ~59s \ + (the advisory-lock loop). Observed {elapsed:?}; without the timeout wrapper this \ + is ~59s (RED)" + ); + + // Permit release on expiry: the Overloaded return drops the handler-local permit, + // so the isolated write pool must recover to full. A leaked permit here means the + // pool drains under a stalled backend (the #174 P1-2 bug). + let mut freed = false; + for _ in 0..200 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert!( + freed, + "on the acquire-deadline shed the write permit must be released; the pool did \ + not recover to full (permit leaked)" + ); + + // Follow-up admits once the contended lock is released: release the second-conn + // lock, then a fresh push proceeds PAST admission (it fails later on the + // nonexistent on-disk repo, which is NOT an Overloaded/503). Proves the freed + // slot is usable, not merely counted. + sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(lock_key) + .execute(&mut *lock_conn) + .await + .expect("release the advisory lock"); + let followup = git_receive_pack( + State(state.clone()), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some("203.0.113.62:5000".parse().unwrap())), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await; + assert!( + !matches!(followup, Err(AppError::Overloaded(_))), + "once the lock frees, a follow-up push must admit past the (recovered) write \ + pool and acquire; got {followup:?}" + ); + } + /// #174 U5 (P1-e, RED-before/GREEN-after): the post-push encryption walk acquires a /// `git_encrypt_semaphore` permit before running, so completed pushes cannot spawn /// unbounded concurrent full-history walks. With the pool exhausted the gated walk diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index d0a4c319..68125364 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -183,6 +183,27 @@ pub struct Config { )] pub git_service_timeout_secs: u64, + /// Maximum wall-clock time the storage-acquisition phase of a served git + /// operation may run before the request is shed with a 503, in seconds. This + /// bounds `RepoStore::{acquire,acquire_fresh,acquire_write}` — the Tigris + /// HEAD/GET on a read/advert acquire and the advisory-lock retry loop (incl. a + /// per-iteration `pg_try_advisory_lock` that can block on a hung Postgres pool) + /// on a write acquire. A concurrency permit is taken BEFORE this phase, and + /// `git_service_timeout_secs` only starts once git spawns, so without this the + /// acquire phase is unbounded: a stalled backend pins the permit and drains the + /// pool until every later request 503s. On expiry the permit is released and a + /// bounded 503 + Retry-After is returned (fail-closed). Kept separate from + /// `git_service_timeout_secs` because acquisition and git execution are distinct + /// cost centers — one shared budget would let a slow acquire starve git. Must be + /// positive; set it very large to effectively disable the bound. Default: 30s. + #[arg( + long, + env = "GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS", + default_value_t = 30, + value_parser = clap::value_parser!(u64).range(1..) + )] + pub git_acquire_timeout_secs: u64, + /// Maximum connections in the PostgreSQL pool. This is a cap, not a floor /// (connections open lazily). Size against the database server's /// max_connections, remembering admin tooling opens its own pool. From f424ccbbcde49260932d776f3e8b560ec2a674c8 Mon Sep 17 00:00:00 2001 From: t Date: Wed, 15 Jul 2026 12:42:55 -0500 Subject: [PATCH 035/124] fix(node): gate the /ipfs walk with bounded concurrency admission + per-source cap + route rate limit (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GET /ipfs/{cid} (get_by_cid, publicly reachable via optional_signature) ran allowed_blob_set_for_caller_bounded in spawn_blocking inside a per-repo loop with NO global/per-source admission and NO route rate limit — the surface had zero concurrency or rate control. A permissionless caller could fan out concurrent full-history git walks (each up to the walk timeout) exhausting blocking-pool threads and PIDs outside every served-git pool (P1-3). Acquire a global git_ipfs_walk permit (try_acquire_owned, shed 503) plus a per-source sub-cap (with_default_max_keys, reject-before-insert, keyed via client_key/TrustedProxy so XFF can't spoof it) ONCE after CID validation, held as handler locals across the whole request — including every spawn_blocking walk, so the slot reflects real blocking-thread occupancy (a tokio timeout cannot cancel spawn_blocking). None key -> global pool only. Cap repos walked per request so one request can't serialize N full-history walks. Attach rate_limit_by_ip + IpRateLimiter to the /ipfs route (mirrors the git/create/peer routers; the extension is required or the middleware is a silent no-op). Visibility/authz semantics unchanged. New knobs: GITLAWB_MAX_CONCURRENT_IPFS_WALKS (32), GITLAWB_IPFS_WALK_PER_SOURCE (4), GITLAWB_IPFS_MAX_REPOS_WALKED (64), GITLAWB_IPFS_RATE_LIMIT (600/hr, 0 disables) — all range-validated and enforced on-path. Handler-layer regressions (all mutation-verified RED->GREEN): shed-at-capacity 503 (delete acquire -> falls through), per-source cap sheds same source / admits another, None-key arm sheds on the global pool, map self-bounds reject-before-insert, the walk permit is held while the spawn_blocking walk runs (available_permits()==0; dropping it before the loop -> RED), repos-walked cap (remove break -> 2 walks), and the route IP rate limit actually fires 429 (drop the Extension -> RED no-op). --- crates/gitlawb-node/src/api/ipfs.rs | 691 ++++++++++++++++++++++++ crates/gitlawb-node/src/auth/mod.rs | 4 + crates/gitlawb-node/src/config.rs | 120 ++++ crates/gitlawb-node/src/main.rs | 17 + crates/gitlawb-node/src/server.rs | 11 + crates/gitlawb-node/src/state.rs | 27 + crates/gitlawb-node/src/test_support.rs | 6 + 7 files changed, 876 insertions(+) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 284c9398..ac8e615c 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -58,6 +58,11 @@ pub async fn get_by_cid( Path(cid_str): Path, State(state): State, auth: Option>, + // Per-source keying for the walk concurrency sub-cap. Infallible extractors + // (mirror the git handlers in `repos.rs`): `PeerAddr` yields `None` under + // `oneshot` with no `ConnectInfo`, and the header map falls back per `client_key`. + crate::rate_limit::PeerAddr(peer): crate::rate_limit::PeerAddr, + req_headers: HeaderMap, ) -> Result { // 1. Decode the CID and extract the SHA-256 digest let cid = CidGeneric::<64>::from_str(&cid_str) @@ -76,6 +81,36 @@ pub async fn get_by_cid( let caller = auth.as_ref().map(|e| e.0 .0.as_str()); let caller_owned = caller.map(|c| c.to_string()); + // Bounded walk admission (#174 P1-3), taken before any DB/git work so a flood sheds + // cheaply. The per-repo `spawn_blocking` walk below is a full-history git walk with + // no served-git admission of its own; a permissionless caller could otherwise fan + // out concurrent walks past every git pool, exhausting the blocking pool + PIDs. + // Acquire the global permit (and, for a resolvable source, the per-source + // sub-permit) ONCE here and hold BOTH for the whole request — across every + // `spawn_blocking` walk in the loop below — so the slot reflects real blocking-thread + // occupancy (a tokio walk-timeout cannot free it while the blocking work still runs) + // and one request cannot open more than its share of concurrent walks. On + // unavailability shed a clean 503. The per-source key is the resolved source IP + // (`client_key`), never the DID (`/ipfs` admits any `did:key` unthrottled, so a DID + // key would be free to mint around); a `None` key (no trusted header, no peer) is + // bounded by the global pool only, never the per-source sub-cap. + let _ipfs_walk_permit = state + .git_ipfs_walk_semaphore + .clone() + .try_acquire_owned() + .map_err(|_| { + tracing::warn!("/ipfs walk concurrency cap reached; shedding request with 503"); + AppError::Overloaded("ipfs service at capacity, retry shortly".into()) + })?; + let source_key = crate::rate_limit::client_key(&req_headers, peer, state.push_limiter_trust); + let _ipfs_caller_permit = match &source_key { + Some(ip) => Some(state.git_ipfs_walk_per_caller.try_acquire(ip).ok_or_else(|| { + tracing::warn!(key = %ip, "/ipfs per-source walk cap reached; shedding request with 503"); + AppError::Overloaded("ipfs service at capacity for this source, retry shortly".into()) + })?), + None => None, + }; + // 2. Search all repos for an object with this SHA-256 let repos = state .db @@ -104,6 +139,11 @@ pub async fn get_by_cid( // deny entry (#126). let mut allowed_memo: HashMap> = HashMap::new(); + // Cap the number of candidate repos one request walks (it already short-circuits on + // serve): a CID present in — or path-gated out of — many repos must not serialize an + // unbounded number of full-history walks inside the single held admission slot. + let mut repos_walked: usize = 0; + for repo in &repos { // Repo-level read gate against THIS row's own rules (KTD2a). let rules: &[crate::db::VisibilityRule] = rules_by_repo @@ -114,6 +154,19 @@ pub async fn get_by_cid( continue; } + // Loop bound (#174 P1-3): once this request has walked its cap of candidate + // repos (each a git subprocess, up to a full-history walk), stop and fall + // through to the opaque 404 rather than serialize an unbounded number under the + // single held admission slot. + if repos_walked >= state.config.ipfs_max_repos_walked { + tracing::warn!( + cap = state.config.ipfs_max_repos_walked, + "/ipfs request hit the per-request repo-walk cap; stopping the scan" + ); + break; + } + repos_walked += 1; + // Bound the per-repo acquire under `git_acquire_timeout_secs`: this loop shares // the P1-2 stall vector (a hung Tigris HEAD/GET on one repo would otherwise // block the whole /ipfs request). On expiry keep the existing fail-closed skip — @@ -249,3 +302,641 @@ pub async fn list_pins(State(state): State) -> Result Router { + Router::new() + .route( + "/ipfs/{cid}", + axum::routing::get(crate::api::ipfs::get_by_cid), + ) + .layer(axum::middleware::from_fn(crate::auth::optional_signature)) + .with_state(state) + } + + /// A syntactically valid CIDv1(raw, sha2-256) string the handler decodes past its + /// CID/hash-code validation, so the request reaches the walk admission (not a 400). + fn valid_cid() -> String { + gitlawb_core::cid::Cid::from_git_object_bytes(b"blob 5\0hello") + .as_str() + .to_string() + } + + fn get_cid(cid: &str, peer: Option) -> Request { + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + if let Some(p) = peer { + req.extensions_mut().insert(ConnectInfo(p)); + } + req + } + + /// Shed at capacity: an exhausted `git_ipfs_walk_semaphore` sheds a `/ipfs/{cid}` + /// request with 503 BEFORE any DB/git walk (the acquire is the first thing after CID + /// validation), so a lazy DB-free state suffices — exactly like the served-git shed + /// tests. MUTATION (RED): delete the `git_ipfs_walk_semaphore` acquire in + /// `get_by_cid` and the request no longer sheds here (it falls through to the DB / + /// walk and returns something other than 503). + #[tokio::test] + async fn get_by_cid_sheds_with_503_when_walk_pool_exhausted() { + let mut state = crate::test_support::test_state_lazy(); + // Global /ipfs walk pool exhausted; per-source cap permissive so only the global + // pool can shed. Route rate limit is applied as a layer in production, not here. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(0)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let peer: SocketAddr = "203.0.113.9:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted /ipfs walk pool must shed the request with 503" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + + /// Per-source sub-cap, the `Some(ip)` arm: with per-source = 1 and the source pinned + /// at its single slot, a request from THAT source sheds 503 (global pool has room), + /// while a request from a DIFFERENT source is NOT shed by the cap (it proceeds past + /// admission). Pinning proves the `PeerAddr`/`HeaderMap` extractors resolved the key + /// — an inert `None` key would never shed on the per-source cap. MUTATION (RED): + /// delete the `git_ipfs_walk_per_caller` acquire and the capped source no longer + /// sheds. + #[tokio::test] + async fn get_by_cid_per_source_cap_sheds_same_source_admits_other() { + let mut state = crate::test_support::test_state_lazy(); + // Global pool has room; the per-source cap is 1. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(8)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let capped: SocketAddr = "203.0.113.20:5000".parse().unwrap(); + let other: SocketAddr = "203.0.113.21:5000".parse().unwrap(); + + // Pin the capped source at its single walk slot. + let _slot = state + .git_ipfs_walk_per_caller + .try_acquire(&capped.ip().to_string()) + .expect("first walk slot for the capped source IP"); + + let cid = valid_cid(); + // The capped source sheds on the per-source cap even with global capacity free. + let resp = ipfs_router(state.clone()) + .oneshot(get_cid(&cid, Some(capped))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its per-source /ipfs walk cap must shed 503 with global capacity free" + ); + + // A DIFFERENT source is NOT shed by the per-source cap: it clears admission and + // proceeds (then errors on the lazy DB, which is not a 503). + let resp = ipfs_router(state) + .oneshot(get_cid(&cid, Some(other))) + .await + .unwrap(); + assert_ne!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a different source must not be shed by the per-source cap" + ); + } + + /// The `None`-key arm: a request with no resolvable source key (no trusted-proxy + /// header, no `ConnectInfo`) is bounded by the GLOBAL pool only, never the per-source + /// sub-cap. With the global pool exhausted it still sheds 503 (the counterpart to the + /// `Some(ip)` arm above, so neither arm is vacuous). + #[tokio::test] + async fn get_by_cid_none_key_arm_sheds_on_global_pool() { + let mut state = crate::test_support::test_state_lazy(); + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(0)); + // Per-source cap permissive so only the global pool can shed. + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // No ConnectInfo + no trusted header -> client_key resolves None. + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), None)) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a None-key request must still shed 503 on the exhausted GLOBAL /ipfs walk pool" + ); + } + + /// Map self-bound (INV-15): the `/ipfs` per-source map is a `PerCallerConcurrency` + /// built via `with_default_max_keys`, so a distinct-source-key flood cannot grow it + /// past the cap and a rejected key never allocates (reject-before-insert). Mirrors + /// `per_caller_concurrency_map_is_self_bounding_and_reject_before_insert` for the + /// pool U3 adds. + #[tokio::test] + async fn ipfs_walk_per_caller_map_is_self_bounding_and_reject_before_insert() { + let lim = crate::rate_limit::PerCallerConcurrency::new(4, 3); + // Acquire+drop a flood of distinct keys — the map self-empties (a key is removed + // the instant its in-flight count hits zero). + for i in 0..50 { + let _p = lim.try_acquire(&format!("src{i}")); + } + assert_eq!( + lim.tracked_keys(), + 0, + "an acquire+drop flood of distinct sources leaves the /ipfs map empty" + ); + // Reject-before-insert: hold max_keys distinct sources, then a new one sheds + // without growing the map. + let held: Vec<_> = (0..3) + .map(|i| lim.try_acquire(&format!("h{i}")).unwrap()) + .collect(); + assert_eq!( + lim.tracked_keys(), + 3, + "three distinct sources held concurrently" + ); + assert!( + lim.try_acquire("h3").is_none(), + "a new source key at max_keys is rejected" + ); + assert_eq!( + lim.tracked_keys(), + 3, + "the rejected key did not allocate an entry (reject-before-insert)" + ); + drop(held); + } + + /// Retain-through-blocking (R3, the load-bearing async property): the walk + /// admission is held until the `spawn_blocking` walk actually RETURNS, not when a + /// tokio timeout fires. With the global pool at size 1, drive a request until its + /// walk (a fake git that hangs on `rev-list`) is in flight; the slot must stay held + /// (`available_permits() == 0`) and a replacement from a DIFFERENT source must shed + /// 503 for as long as the blocking walk runs — even though the request future is + /// only `.await`ing the blocking join. When the blocking walk ends the permit frees + /// and a replacement is admitted. The permit lives INSIDE the handler across the + /// blocking `.await`; move it out (drop before the walk) and the replacement would + /// be admitted while the walk still burns a blocking thread (the bug this guards). + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_walk_permit_held_through_blocking_walk(pool: sqlx::PgPool) { + use std::process::Command; + + let tmp = tempfile::TempDir::new().unwrap(); + let revlist_pid = tmp.path().join("revlist.pid"); + // Fake git for the /ipfs WALK only (object_type/read_object_content use the real + // `git`, so the object must genuinely exist below). Empty refs (so + // assert_all_refs_are_commits returns Ok without the peel), `rev-parse` resolves, + // and `rev-list` records its pid then sleeps ~6s so the walk BLOCKS + // deterministically. The sleep bounds the walk so a broken fix cannot wedge the + // suite. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + for-each-ref) : ;;\n\ + rev-parse) echo deadbeef ;;\n\ + rev-list) echo $$ > \"{}\"; sleep 6 ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + revlist_pid.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + // Isolate the global walk pool at size 1; per-source cap permissive so only the + // held global permit can shed the replacement. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let owner = "z6ipfs1"; + let name = "ip1"; + state + .db + .upsert_mirror_repo(owner, name, "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + // The exact bare path the handler's `acquire` resolves. Build a REAL SHA-256 bare + // repo there with a committed blob under `src/`, so real `git cat-file -t ` classifies it as a blob (the CID digest IS the sha256 object id in + // object-format=sha256) and the handler reaches the path-scoped walk branch. + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + let _ = std::fs::remove_dir_all(&bare); + std::fs::create_dir_all(&bare).unwrap(); + let run = |args: &[&str], cwd: &std::path::Path| { + let out = Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + }; + let work = tmp.path().join("work"); + std::fs::create_dir_all(work.join("src")).unwrap(); + std::fs::write(work.join("src/secret.txt"), b"ipfs walk retain proof\n").unwrap(); + run( + &["init", "-q", "--object-format=sha256", "-b", "main"], + &work, + ); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "src/secret.txt"], &work); + run(&["commit", "-q", "-m", "seed"], &work); + run( + &[ + "clone", + "--bare", + "-q", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + tmp.path(), + ); + // The blob's SHA-256 object id (= the CID's digest); build the CID from it. + let oid = { + let out = Command::new("git") + .args(["rev-parse", "HEAD:src/secret.txt"]) + .current_dir(&work) + .output() + .expect("git rev-parse runs"); + assert!(out.status.success(), "rev-parse failed"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + let oid_bytes = gitlawb_core::cid::sha256_hex_to_bytes(&oid).unwrap(); + let cid = gitlawb_core::cid::Cid::from_sha256_bytes(&oid_bytes) + .as_str() + .to_string(); + // Precondition: real git classifies the object as a blob (so the handler reaches + // the walk branch, not an early `continue`). + assert_eq!( + crate::git::store::object_type(&bare, &oid) + .unwrap() + .as_deref(), + Some("blob"), + "the seeded sha256 blob must exist so the handler reaches the walk" + ); + // A path-scoped rule so has_path_scoped_rule() is true (the walk branch) without + // denying the "/" gate on the public repo. + state + .db + .set_visibility_rule( + &rec.id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderAAAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + + let sem = state.git_ipfs_walk_semaphore.clone(); + assert_eq!( + sem.available_permits(), + 1, + "one walk slot before the request" + ); + + let router = ipfs_router(state); + let make_req = |peer: SocketAddr| { + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req + }; + + let peer: SocketAddr = "203.0.113.81:5000".parse().unwrap(); + let mut fut = Box::pin(router.clone().oneshot(make_req(peer))); + // Drive until the fake git's rev-list records its pid — the walk is now in the + // blocking pool and the request future is `.await`ing its join, holding the walk + // permit. Stop polling the instant the future completes (re-polling a completed + // oneshot panics). + let mut walk_pid: Option = None; + let mut early = None; + for _ in 0..500 { + let done = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&revlist_pid) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + walk_pid = Some(p); + break; + } + if let Ok(resp) = done { + early = Some(resp.map(|r| r.status())); + break; + } + } + let pid = walk_pid + .unwrap_or_else(|| panic!("the fake git rev-list must have spawned; early: {early:?}")); + // Reap the sleeping child on drop so a RED run leaks no orphan. + struct ReapOnDrop(i32); + impl Drop for ReapOnDrop { + fn drop(&mut self) { + unsafe { + libc::kill(self.0, libc::SIGKILL); + } + } + } + let _cleanup = ReapOnDrop(pid); + + // Load-bearing: while the blocking walk runs, the slot is HELD and a replacement + // from a DIFFERENT source sheds 503 — proving the permit is retained across the + // spawn_blocking join, not freed by a tokio timeout. + assert_eq!( + sem.available_permits(), + 0, + "the walk slot must be held while the spawn_blocking walk runs" + ); + let peer2: SocketAddr = "203.0.113.82:5000".parse().unwrap(); + let resp = router.clone().oneshot(make_req(peer2)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a replacement must shed 503 while the prior request's blocking walk still runs" + ); + + // Drop the in-flight request; the detached blocking walk keeps running (a + // spawn_blocking cannot be cancelled), but on the fix the permit is a handler + // local, so dropping the future releases it once the blocking join is abandoned. + // Either way, kill the sleeping child so the slot frees promptly and poll for + // recovery — the point already proven above is that the slot stayed held for the + // duration of the blocking work. + drop(fut); + unsafe { + libc::kill(pid, libc::SIGKILL); + } + let mut freed = false; + for _ in 0..400 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + assert!( + freed, + "once the blocking walk ends the walk permit must free the global slot" + ); + } + + /// Loop bound (cap N): one `/ipfs/{cid}` request against a CID present in many repos + /// must not serialize an unbounded number of full-history walks. With + /// `ipfs_max_repos_walked = 1` and TWO public, path-scoped repos both carrying the + /// blob at the requested CID, the handler walks only the FIRST candidate then stops + /// (the second is cut by the cap), so the fake git's `rev-list` (one per walk) runs + /// exactly once. MUTATION (RED): remove the `repos_walked >= cap` break and both + /// repos are walked (count 2). + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_caps_repos_walked_per_request(pool: sqlx::PgPool) { + use std::process::Command; + + let tmp = tempfile::TempDir::new().unwrap(); + let walk_log = tmp.path().join("walks.log"); + // Fake git for the WALK: empty refs, `rev-parse` resolves, and each `rev-list` + // appends one line to a log (so the number of walks == the line count) and exits + // with EMPTY output (the allowed-set is empty, so every repo path-gates to a + // `continue` and the request 404s after walking). object_type uses the REAL git, + // so the seeded blob below must genuinely exist. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + for-each-ref) : ;;\n\ + rev-parse) echo deadbeef ;;\n\ + rev-list) echo walk >> \"{}\" ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + walk_log.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // The bound under test: walk at most one candidate repo per request. + let mut cfg = (*state.config).clone(); + cfg.ipfs_max_repos_walked = 1; + state.config = Arc::new(cfg); + + // Seed TWO public repos, each with the SAME blob (same content -> same sha256 OID + // -> same CID) under a path-scoped rule, so both are walk candidates for one CID. + let run = |args: &[&str], cwd: &std::path::Path| { + let out = Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + }; + let mut oid = String::new(); + for (i, name) in ["ipa", "ipb"].iter().enumerate() { + let owner = "z6ipfsN"; + state + .db + .upsert_mirror_repo(owner, name, &format!("/unused-{name}"), None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + let _ = std::fs::remove_dir_all(&bare); + std::fs::create_dir_all(&bare).unwrap(); + let work = tmp.path().join(format!("work{i}")); + std::fs::create_dir_all(work.join("src")).unwrap(); + // Identical content in both repos -> identical sha256 blob OID -> one CID. + std::fs::write(work.join("src/secret.txt"), b"loop bound proof\n").unwrap(); + run( + &["init", "-q", "--object-format=sha256", "-b", "main"], + &work, + ); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "src/secret.txt"], &work); + run(&["commit", "-q", "-m", "seed"], &work); + run( + &[ + "clone", + "--bare", + "-q", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + tmp.path(), + ); + if oid.is_empty() { + let out = Command::new("git") + .args(["rev-parse", "HEAD:src/secret.txt"]) + .current_dir(&work) + .output() + .expect("git rev-parse runs"); + oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); + } + state + .db + .set_visibility_rule( + &rec.id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderBBBBBBBBBBBBBBBBBBBBBBBB".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + } + let oid_bytes = gitlawb_core::cid::sha256_hex_to_bytes(&oid).unwrap(); + let cid = gitlawb_core::cid::Cid::from_sha256_bytes(&oid_bytes) + .as_str() + .to_string(); + + let peer: SocketAddr = "203.0.113.90:5000".parse().unwrap(); + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let resp = ipfs_router(state).oneshot(req).await.unwrap(); + // The empty allowed-set path-gates both repos to a `continue`, so a 404; the + // point is HOW MANY walks ran to get there. + assert_eq!(resp.status(), StatusCode::NOT_FOUND); + + let walks = std::fs::read_to_string(&walk_log) + .map(|s| s.lines().count()) + .unwrap_or(0); + assert_eq!( + walks, 1, + "with the per-request repo-walk cap at 1, only the first candidate repo is \ + walked (the second is cut by the cap), so exactly one walk runs; got {walks}" + ); + } + + /// Route rate limit is WIRED (not a silent no-op): the production `build_router` + /// attaches an `IpRateLimiter` extension to the `/ipfs/{cid}` route, so a per-IP + /// flood is braked with 429. A bare `rate_limit_by_ip` layer with no extension does + /// nothing, so this proves the extension is attached. Drive it through the real + /// router with a tight limiter (1/hr): the second request from the same IP is 429. + /// MUTATION (RED): drop the `axum::Extension(ipfs_limiter)` layer in `server.rs` and + /// the second request is no longer braked (it reaches the handler, 404, not 429). + #[sqlx::test] + async fn ipfs_route_ip_rate_limit_is_attached(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + // Tight per-IP /ipfs bucket so the second request from one IP trips 429. + state.ipfs_rate_limiter = + crate::rate_limit::RateLimiter::new(1, std::time::Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let router = crate::server::build_router(state); + let cid = valid_cid(); + let make = |peer: SocketAddr| { + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req + }; + let peer: SocketAddr = "203.0.113.99:5000".parse().unwrap(); + + // First request from this IP passes the brake and reaches the handler (404 — no + // such object anywhere), debiting the single-slot bucket. + let resp = router.clone().oneshot(make(peer)).await.unwrap(); + assert_ne!( + resp.status(), + StatusCode::TOO_MANY_REQUESTS, + "the first /ipfs request from an IP must pass the rate brake" + ); + // Second request from the SAME IP is braked with 429 — proving the limiter + // extension is attached (a bare no-op layer would let it through to 404). + let resp = router.clone().oneshot(make(peer)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::TOO_MANY_REQUESTS, + "an exhausted per-IP /ipfs bucket must brake with 429 — the IpRateLimiter \ + extension must be attached to the route" + ); + // A DIFFERENT IP still has its own budget (independent bucket). + let other: SocketAddr = "203.0.113.100:5000".parse().unwrap(); + let resp = router.oneshot(make(other)).await.unwrap(); + assert_ne!( + resp.status(), + StatusCode::TOO_MANY_REQUESTS, + "a different IP must not be braked by another IP's exhausted bucket" + ); + } +} diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index f18b4963..eed49253 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -528,6 +528,10 @@ mod tests { git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), git_write_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), + git_ipfs_walk_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_ipfs_walk_per_caller: + crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), + ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), git_bin: "git".to_string(), } } diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 68125364..b3667024 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -335,6 +335,66 @@ pub struct Config { value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) )] pub max_concurrent_reads_per_caller: usize, + + /// Maximum number of concurrent `GET /ipfs/{cid}` requests that may run their + /// visibility walk at once. The publicly-reachable `/ipfs/{cid}` route runs + /// `allowed_blob_set_for_caller_bounded` in `spawn_blocking` — a full-history + /// git walk (up to `git_service_timeout_secs`) — for each candidate repo. It + /// draws from THIS pool, not any served-git pool: a distinct public cost center + /// on a distinct surface, so sharing a git pool would let anonymous /ipfs + /// traffic shed authenticated git ops (the auth-boundary trap). A permit is + /// held for the whole request (across the repo loop) so it reflects real + /// blocking-thread occupancy, not merely the tokio wait. Beyond this the request + /// sheds a clean 503 + Retry-After. Must be between 1 and 1_048_576; the ceiling + /// keeps the value under tokio's `Semaphore` permit limit so an oversized value + /// is a clean CLI error rather than a boot-time panic. Default: 32. + #[arg( + long, + env = "GITLAWB_MAX_CONCURRENT_IPFS_WALKS", + default_value_t = 32, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub max_concurrent_ipfs_walks: usize, + + /// Maximum concurrent `/ipfs/{cid}` walk requests a single source may hold at + /// once, so one source cannot monopolize `max_concurrent_ipfs_walks` (#174). + /// Callers are keyed on the RESOLVED SOURCE IP (`client_key`/`GITLAWB_TRUSTED_PROXY`), + /// never the DID — `/ipfs` accepts any `did:key` via `optional_signature` with no + /// admission step, so keying on the DID would let one host mint disposable DIDs to + /// multiply its budget. A request with no resolvable key (no trusted header, no + /// peer) is bounded by the global pool only, never this sub-cap. Over-cap sheds a + /// clean 503 + Retry-After. Must be between 1 and 1_048_576. Default: 4. + #[arg( + long, + env = "GITLAWB_IPFS_WALK_PER_SOURCE", + default_value_t = 4, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub ipfs_walk_per_source: usize, + + /// Upper bound on the number of candidate repos a single `/ipfs/{cid}` request + /// will walk before giving up (returning the opaque 404). The handler already + /// short-circuits the moment it serves the object, but a CID that is present in + /// (or path-gated out of) many repos could otherwise serialize one full-history + /// walk per repo inside a single held admission slot. Capping the count bounds + /// the worst-case work one request can pin its slot with. Must be between 1 and + /// 1_048_576. Default: 64. + #[arg( + long, + env = "GITLAWB_IPFS_MAX_REPOS_WALKED", + default_value_t = 64, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub ipfs_max_repos_walked: usize, + + /// Per-client-IP rate limit for `GET /ipfs/{cid}`, in requests per hour. The + /// route is publicly reachable (`optional_signature`) and each request can drive + /// a full-history git walk, so it carries a per-IP flood brake in addition to the + /// concurrency cap above (a rate limit bounds request *rate*, the semaphore + /// bounds concurrent slow holds — different axes). Keyed on the resolved client + /// IP via `GITLAWB_TRUSTED_PROXY`. `0` disables. Default: 600. + #[arg(long, env = "GITLAWB_IPFS_RATE_LIMIT", default_value_t = 600)] + pub ipfs_rate_limit: usize, } impl Config { @@ -428,6 +488,66 @@ mod tests { ); } + #[test] + fn max_concurrent_ipfs_walks_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).max_concurrent_ipfs_walks, + 32 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "4"]) + .max_concurrent_ipfs_walks, + 4 + ); + // 0 permits would shed every /ipfs walk with a 503; clap must reject it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "0"]).is_err() + ); + // Above the ceiling would panic tokio's Semaphore::new at boot; clap rejects it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "1048577"]) + .is_err() + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "1048576"]) + .max_concurrent_ipfs_walks, + 1_048_576 + ); + } + + #[test] + fn ipfs_walk_per_source_defaults_and_rejects_out_of_range() { + assert_eq!(Config::parse_from(["gitlawb-node"]).ipfs_walk_per_source, 4); + assert_eq!( + Config::parse_from(["gitlawb-node", "--ipfs-walk-per-source", "2"]) + .ipfs_walk_per_source, + 2 + ); + // 0 would shed every /ipfs walk from a keyed source; clap must reject it. + assert!(Config::try_parse_from(["gitlawb-node", "--ipfs-walk-per-source", "0"]).is_err()); + assert!( + Config::try_parse_from(["gitlawb-node", "--ipfs-walk-per-source", "1048577"]).is_err() + ); + } + + #[test] + fn ipfs_max_repos_walked_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).ipfs_max_repos_walked, + 64 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "8"]) + .ipfs_max_repos_walked, + 8 + ); + // 0 would walk no repos (serve nothing); clap must reject it. + assert!(Config::try_parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "0"]).is_err()); + assert!( + Config::try_parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "1048577"]).is_err() + ); + } + #[test] fn max_concurrent_reads_per_caller_defaults_and_rejects_out_of_range() { assert_eq!( diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 380ab5f4..8727aae1 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -412,8 +412,25 @@ async fn main() -> Result<()> { git_write_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( (config.max_concurrent_git_pushes / 8).max(1), ), + // Bounds concurrent /ipfs visibility walks — a distinct public cost center, so + // its own pool + per-source sub-cap + per-IP rate limiter, never a git pool + // (#174 P1-3). The per-source map is bounded (reject-before-insert, INV-15). + git_ipfs_walk_semaphore: Arc::new(tokio::sync::Semaphore::new( + config.max_concurrent_ipfs_walks, + )), + git_ipfs_walk_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( + config.ipfs_walk_per_source, + ), + ipfs_rate_limiter: rate_limit::RateLimiter::new_bounded( + config.ipfs_rate_limit, + std::time::Duration::from_secs(3600), + 200_000, + ), git_bin: "git".to_string(), }; + if config.ipfs_rate_limit == 0 { + tracing::warn!("GITLAWB_IPFS_RATE_LIMIT=0 — per-IP /ipfs rate limiting disabled"); + } // Periodic peer-count poll for the metrics gauge. If p2p is disabled // we still set the gauge to 0 so dashboards don't show "no data". diff --git a/crates/gitlawb-node/src/server.rs b/crates/gitlawb-node/src/server.rs index f4c0d3e3..de61fcbe 100644 --- a/crates/gitlawb-node/src/server.rs +++ b/crates/gitlawb-node/src/server.rs @@ -214,9 +214,20 @@ pub fn build_router(state: AppState) -> Router { // identity and can apply per-repo visibility (#110); anonymous callers stay // anonymous and still read genuinely public content. `/api/v1/ipfs/pins` // stays unsigned — gating the pin index is tracked separately (#121). + // `/ipfs/{cid}` also carries a per-IP flood brake: it is anon-reachable and each + // request can drive a full-history git walk, so the per-IP rate limiter is the + // outermost layer (rejects a flood before the walk-admission work), mirroring the + // push/create routers. The extension MUST be attached or rate_limit_by_ip is a + // silent no-op. `/api/v1/ipfs/pins` (no walk) is merged in unbraked, as before. + let ipfs_limiter = rate_limit::IpRateLimiter { + limiter: state.ipfs_rate_limiter.clone(), + trust: state.push_limiter_trust, + }; let ipfs_routes = Router::new() .route("/ipfs/{cid}", get(ipfs::get_by_cid)) .layer(middleware::from_fn(auth::optional_signature)) + .layer(middleware::from_fn(rate_limit::rate_limit_by_ip)) + .layer(axum::Extension(ipfs_limiter)) .merge(Router::new().route("/api/v1/ipfs/pins", get(ipfs::list_pins))); // ── Arweave permanent anchors ────────────────────────────────────────── diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 3ff86923..3b8ba250 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -141,6 +141,33 @@ pub struct AppState { /// on the resolved source IP (never the DID — a DID farm defeats a DID key). Sized /// like `git_push_advert_per_caller`, a fraction of `max_concurrent_git_pushes`. pub git_write_per_caller: crate::rate_limit::PerCallerConcurrency, + /// Bounds concurrent `GET /ipfs/{cid}` visibility-walk requests. The public + /// `/ipfs/{cid}` route runs `allowed_blob_set_for_caller_bounded` in + /// `spawn_blocking` (a full-history git walk) with NO served-git admission of its + /// own; without this a permissionless caller fans out concurrent walks past every + /// git pool, exhausting the blocking pool + PIDs (#174 P1-3). A request acquires a + /// permit before the repo loop and holds it for the whole request (across every + /// `spawn_blocking` walk), so the slot reflects real thread occupancy — a tokio + /// walk-timeout cannot free it while the blocking work still runs. A pool of its + /// own (`max_concurrent_ipfs_walks`), NOT a git pool: distinct cost center + public + /// surface, so anonymous /ipfs traffic can never shed an authenticated git op. + pub git_ipfs_walk_semaphore: Arc, + /// Per-source concurrency sub-cap on the `/ipfs/{cid}` walk pool: each source + /// (keyed on the resolved source IP, never the DID — `/ipfs` admits any `did:key` + /// unthrottled, so a DID key would be free to mint around) may hold at most + /// `ipfs_walk_per_source` in-flight walk slots, so one source cannot monopolize + /// `git_ipfs_walk_semaphore` (#174 P1-3). A request with no resolvable key is + /// bounded by the global pool only, never this sub-cap. The key map is bounded + /// (`with_default_max_keys`, reject-before-insert) so a source-key farm cannot grow + /// it (INV-15). + pub git_ipfs_walk_per_caller: crate::rate_limit::PerCallerConcurrency, + /// Per-client-IP rate limiter for `GET /ipfs/{cid}`. The route is publicly + /// reachable and each request can drive a full-history git walk, so it carries a + /// per-IP flood brake in addition to the concurrency cap above — a rate limit + /// bounds request *rate*, the semaphore bounds concurrent slow holds (different + /// axes). Keyed on the resolved client IP via `push_limiter_trust`. Layered on the + /// `/ipfs` route via `rate_limit_by_ip`. + pub ipfs_rate_limiter: RateLimiter, /// The `git` executable the served-git withheld-blob walk spawns. Production is /// `"git"` (resolved via PATH); injectable so a fake `git` can drive the walk's /// process-group teardown in handler tests without mutating the process-global diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index a27be946..cfa676af 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -92,6 +92,12 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { 8, ), git_write_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), + // Generous — a test that drives the /ipfs walk shed overrides these directly. + git_ipfs_walk_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_ipfs_walk_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( + 16, + ), + ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), git_bin: "git".to_string(), } } From 394768711ec6afa834d010734c34cafd1f6063ad Mon Sep 17 00:00:00 2001 From: t Date: Wed, 15 Jul 2026 13:10:04 -0500 Subject: [PATCH 036/124] fix(node): bound the post-push encryption task set by per-repo coalescing without dropping recovery copies (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After a successful path-scoped push the handler spawns a DETACHED task that runs pin_new_objects then withheld_recipients_gated, which acquire_owned().await's on git_encrypt_semaphore — it DEFERS (blocks) when the pool is full. The semaphore caps active walks but nothing capped how many detached tasks were spawned and PARKED on that await: N rapid path-scoped pushes spawned N tasks, each holding cloned object lists/rules/paths/keys — an unbounded parked-waiter set (P2-2). 2a54c15 deliberately chose defer-not-shed because dropping the walk loses the withheld-blob recovery copy and there is no reconciliation sweep to rebuild it. So bound the OUTSTANDING-TASK set by per-repo coalescing rather than shedding: EncryptInflight (a bounded Arc>>) + try_begin — before the spawn, if a task for the repo is already in-flight skip the duplicate (the pending walk covers the newer objects); otherwise spawn and move an RAII EncryptInflightGuard into the task that removes the key on drop (completion, error, or panic-unwind). This bounds the set to <=1 pending task per repo and never drops work; withheld_recipients_gated's defer is unchanged. The second detached post-push spawn (Pinata replication, also runs pin_new_objects) is scoped out with rationale: it parks on no semaphore (no unbounded waiter set) and does per-push per-ref work (branch->CID, gossip, subscription, Arweave, peer-notify) keyed to this push's ref_updates — coalescing it would DROP a later push's announcements, a correctness regression. Regressions (mutation-verified RED->GREEN): the outstanding set is bounded to 1 per repo under saturation (never-coalesce -> 32 tasks RED); a coalesced repo's key is released when its task ends so it is reprocessed, never permanently skipped (guard-drop no-op -> repo locked out, recovery copy lost forever, RED — the exact durability regression the fix prevents); distinct repos each admit; cold-set first push admits; plus an inv22_gates structural tripwire that fails if the handler gate is removed. --- crates/gitlawb-node/src/api/repos.rs | 383 +++++++++++++++++------ crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/main.rs | 5 + crates/gitlawb-node/src/state.rs | 96 ++++++ crates/gitlawb-node/src/test_support.rs | 1 + crates/gitlawb-node/tests/inv22_gates.rs | 10 + 6 files changed, 407 insertions(+), 89 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 5b11404d..6b52273f 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1419,109 +1419,145 @@ pub async fn git_receive_pack( // Pin new git objects to the local IPFS node (no-op if ipfs_api is empty). // Skipped entirely when the public cannot read the repo (withheld == None). + // + // Coalesce per repo (#174 P2-2): this task parks on `git_encrypt_semaphore` + // (which DEFERS when the pool is full rather than dropping the recovery copy). To + // bound the OUTSTANDING parked-task set, only spawn if no encryption task for this + // repo is already in flight; otherwise skip — the pending/next walk over this + // repo's history already covers the newer push's objects, so the recovery copy is + // delayed, never lost. The guard removes the repo key when the task ends (success, + // error, or panic), so a later push is re-admitted (no permanent skip). if withheld.is_some() { - let object_list_ipfs = object_list.clone(); - let ipfs_api = state.config.ipfs_api.clone(); - let repo_path_clone = disk_path.clone(); - let db_clone = state.db.clone(); - let rules_for_enc = rules_opt.clone(); - let repo_id = record.id.clone(); - let owner_did = record.owner_did.clone(); - let is_public = record.is_public; - let irys_url = state.config.irys_url.clone(); - let http_client = std::sync::Arc::clone(&state.http_client); - let node_did_str = state.node_did.to_string(); - let node_seed = state.node_keypair.to_seed(); - let repo_name = record.name.clone(); - let enc_git_bin = state.git_bin.clone(); - let enc_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); - let encrypt_sem = state.git_encrypt_semaphore.clone(); - tokio::spawn(async move { - let pinned = crate::ipfs_pin::pin_new_objects( - &ipfs_api, - &repo_path_clone, - object_list_ipfs, - &db_clone, - ) - .await; - if !pinned.is_empty() { - tracing::info!(count = pinned.len(), "pinned git objects to IPFS"); - for (sha, cid) in &pinned { - tracing::info!(sha = %sha, %cid, "pinned"); - } + match state.encrypt_inflight.try_begin(&record.id) { + None => { + tracing::debug!( + repo = %record.id, + "post-push encryption task already in flight for this repo; coalescing \ + (the pending recovery-copy walk covers this push's objects)" + ); } - - // Option B1: encrypt-then-pin the withheld blobs so authorized - // readers can recover them when the origin cannot serve them. - // No path-scoped rule can withhold a blob, so withheld_blob_recipients - // would return an empty map after a full per-ref walk; skip it. Mirrors - // the has_path_scoped_rule gate on the other two withheld-walk sites. - if let Some(rules) = rules_for_enc.filter(|r| visibility_pack::has_path_scoped_rule(r)) - { - // Bound the number of concurrent post-push encryption walks (#174 P1-e): - // acquire an admission permit before the full-history walk, deferring - // when the pool is full rather than shedding the recovery pin. - let recip = withheld_recipients_gated( - encrypt_sem.clone(), - repo_path_clone.clone(), - enc_git_bin.clone(), - enc_timeout, - rules, - is_public, - owner_did.clone(), - ) - .await; - if let Ok(Ok(recipients)) = recip { - let delta = crate::encrypted_pin::encrypt_and_pin( + Some(inflight_guard) => { + let object_list_ipfs = object_list.clone(); + let ipfs_api = state.config.ipfs_api.clone(); + let repo_path_clone = disk_path.clone(); + let db_clone = state.db.clone(); + let rules_for_enc = rules_opt.clone(); + let repo_id = record.id.clone(); + let owner_did = record.owner_did.clone(); + let is_public = record.is_public; + let irys_url = state.config.irys_url.clone(); + let http_client = std::sync::Arc::clone(&state.http_client); + let node_did_str = state.node_did.to_string(); + let node_seed = state.node_keypair.to_seed(); + let repo_name = record.name.clone(); + let enc_git_bin = state.git_bin.clone(); + let enc_timeout = + std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let encrypt_sem = state.git_encrypt_semaphore.clone(); + tokio::spawn(async move { + // Held for the whole task; drop (on completion/error/panic) releases the + // repo's coalescing key so the next push for this repo can spawn again. + let _inflight_guard = inflight_guard; + let pinned = crate::ipfs_pin::pin_new_objects( &ipfs_api, &repo_path_clone, + object_list_ipfs, &db_clone, - &repo_id, - &node_seed, - &recipients, ) .await; + if !pinned.is_empty() { + tracing::info!(count = pinned.len(), "pinned git objects to IPFS"); + for (sha, cid) in &pinned { + tracing::info!(sha = %sha, %cid, "pinned"); + } + } - // Option B3: anchor a per-push manifest of the blobs sealed - // this push to Arweave, so the oid->cid index survives total - // node loss. Best-effort; never fails the push. - if !delta.is_empty() && !irys_url.is_empty() { - let owner_short = crate::db::normalize_owner_key(&owner_did); - let repo_slug = format!("{owner_short}/{repo_name}"); - let ts = chrono::Utc::now().to_rfc3339(); - let manifest = crate::arweave::EncryptedManifest { - repo: &repo_slug, - owner_did: &owner_did, - node_did: &node_did_str, - timestamp: &ts, - blobs: &delta, - }; - match crate::arweave::anchor_encrypted_manifest( - &http_client, - &irys_url, - &manifest, + // Option B1: encrypt-then-pin the withheld blobs so authorized + // readers can recover them when the origin cannot serve them. + // No path-scoped rule can withhold a blob, so withheld_blob_recipients + // would return an empty map after a full per-ref walk; skip it. Mirrors + // the has_path_scoped_rule gate on the other two withheld-walk sites. + if let Some(rules) = + rules_for_enc.filter(|r| visibility_pack::has_path_scoped_rule(r)) + { + // Bound the number of concurrent post-push encryption walks (#174 P1-e): + // acquire an admission permit before the full-history walk, deferring + // when the pool is full rather than shedding the recovery pin. + let recip = withheld_recipients_gated( + encrypt_sem.clone(), + repo_path_clone.clone(), + enc_git_bin.clone(), + enc_timeout, + rules, + is_public, + owner_did.clone(), ) - .await - { - Ok(tx) if !tx.is_empty() => tracing::info!( - repo = %repo_slug, - tx_id = %tx, - "anchored encrypted manifest to Arweave" - ), - Ok(_) => {} - Err(e) => tracing::warn!( - repo = %repo_slug, - err = %e, - "encrypted manifest anchor failed" - ), + .await; + if let Ok(Ok(recipients)) = recip { + let delta = crate::encrypted_pin::encrypt_and_pin( + &ipfs_api, + &repo_path_clone, + &db_clone, + &repo_id, + &node_seed, + &recipients, + ) + .await; + + // Option B3: anchor a per-push manifest of the blobs sealed + // this push to Arweave, so the oid->cid index survives total + // node loss. Best-effort; never fails the push. + if !delta.is_empty() && !irys_url.is_empty() { + let owner_short = crate::db::normalize_owner_key(&owner_did); + let repo_slug = format!("{owner_short}/{repo_name}"); + let ts = chrono::Utc::now().to_rfc3339(); + let manifest = crate::arweave::EncryptedManifest { + repo: &repo_slug, + owner_did: &owner_did, + node_did: &node_did_str, + timestamp: &ts, + blobs: &delta, + }; + match crate::arweave::anchor_encrypted_manifest( + &http_client, + &irys_url, + &manifest, + ) + .await + { + Ok(tx) if !tx.is_empty() => tracing::info!( + repo = %repo_slug, + tx_id = %tx, + "anchored encrypted manifest to Arweave" + ), + Ok(_) => {} + Err(e) => tracing::warn!( + repo = %repo_slug, + err = %e, + "encrypted manifest anchor failed" + ), + } + } } } - } + }); } - }); + } } - // Pin new git objects to Pinata, then record branch→CID and gossip + // Pin new git objects to Pinata, then record branch→CID and gossip. + // + // #174 P2-2 scope note: this SECOND detached spawn is deliberately NOT brought + // under the per-repo encryption coalescing above. Two reasons: (1) it does not + // park on `git_encrypt_semaphore` (or any semaphore) — the Pinata `pin_new_objects` + // is a bounded reqwest round-trip, so it does not form the unbounded PARKED-waiter + // set that is the P2-2 residual; it runs to completion under the HTTP client's + // network timeouts. (2) Unlike the idempotent recovery-copy walk, this task does + // PER-PUSH, PER-REF work — branch→CID upserts, gossip publish, GraphQL subscription + // broadcast, Arweave anchoring, and peer notify, each keyed to THIS push's + // ref_updates. Coalescing it against an in-flight task for the same repo would DROP + // a later push's ref-update announcements (a correctness regression), not merely + // delay a duplicate. So it is scoped out with rationale, not brought under the bound. { let pinata_jwt = state.config.pinata_jwt.clone(); let pinata_upload_url = state.config.pinata_upload_url.clone(); @@ -4317,6 +4353,175 @@ mod tests { ); } + // ---- #174 U4 (P2-2): post-push encryption task set bounded by per-repo coalescing ---- + // + // The residual jatmn found is not the WALK (bounded by `git_encrypt_semaphore`, + // proven by `encrypt_walk_defers_when_pool_exhausted` above) but the OUTER + // `tokio::spawn` + its parked `acquire_owned().await` waiters: N rapid pushes to a + // repo spawn N tasks that each park holding cloned object lists/rules/keys — an + // unbounded outstanding set. U4 bounds it by coalescing per repo: before spawning, + // if a task for the repo is in flight, skip the duplicate. Crucially this DEFERS a + // duplicate walk (the newer push's objects are covered by the pending one) and does + // NOT shed — there is no reconciliation sweep, so a dropped job would permanently + // lose the withheld-blob recovery copy (`2a54c15`'s fail-closed durability stance). + // + // These drive the coalescing seam (`EncryptInflight`) that the detached spawn at + // `repos.rs` consults directly (the try_begin gate on the in-flight set, guarded by + // `withheld.is_some()`). Observing `encrypt_and_pin`'s IPFS effect end-to-end needs a live IPFS node + // (`pin_git_object` hits the API), so the durability property is proven at this + // layer: a coalesced repo's key is released when its task ends, so a later push for + // that repo is processed once — NOT permanently skipped, which is exactly what a + // coalesce->shed mutation would break by dropping the job with no sweep to recover it. + + /// Bounded outstanding set under saturation (R4). Simulate K rapid path-scoped + /// pushes to the SAME repo while the encrypt pool is saturated (every spawned task + /// would park, so none has finished and removed its key): the first `try_begin` + /// admits (spawns), the rest coalesce (skip). The in-flight set holds at 1, not K. + /// + /// MUTATION (RED): removing the coalescing check makes every push spawn — modeled by + /// `simulate_without_coalescing`, which reaches K. If the coalesced count equaled the + /// un-coalesced one the gate would be a no-op; the strict inequality proves it bites. + #[test] + fn u4_outstanding_encrypt_set_is_bounded_to_one_per_repo_under_saturation() { + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkRepoOwnerAAAAAAAAAAAAAAAAAAAAAAAAAAAA/proj"; + const K: usize = 32; + + // Hold every admitted guard so the tasks are "still in flight" (the saturated + // case: all parked on acquire_owned().await, none finished, none removed a key). + let mut admitted = Vec::new(); + let mut coalesced = 0usize; + for _ in 0..K { + match inflight.try_begin(repo) { + Some(g) => admitted.push(g), + None => coalesced += 1, + } + } + + assert_eq!( + admitted.len(), + 1, + "exactly ONE detached task may spawn per repo while one is in flight — the \ + outstanding set is bounded to 1, not K parked waiters" + ); + assert_eq!( + coalesced, + K - 1, + "the other K-1 rapid pushes to the same repo coalesce (skip spawning)" + ); + assert_eq!( + inflight.len(), + 1, + "the in-flight set holds at most one entry per repo under saturation" + ); + + let no_coalesce = simulate_without_coalescing(K); + assert_eq!( + no_coalesce, K, + "sanity: without the coalescing check all K pushes spawn (the unbounded set \ + the fix prevents) — proves the bound above is not vacuously 1" + ); + assert!( + admitted.len() < no_coalesce, + "coalesced set ({}) must be strictly smaller than the un-coalesced one ({})", + admitted.len(), + no_coalesce + ); + } + + /// Coalescing is PER-REPO: distinct repos are never coalesced against each other, so + /// one repo in flight cannot starve a second repo's recovery copy. + #[test] + fn u4_distinct_repos_each_admit_one_encrypt_task() { + let inflight = crate::state::EncryptInflight::new(); + let a = inflight.try_begin("owner/repo-a"); + let b = inflight.try_begin("owner/repo-b"); + let c = inflight.try_begin("owner/repo-c"); + assert!( + a.is_some() && b.is_some() && c.is_some(), + "three distinct repos each admit their own encryption task" + ); + assert_eq!(inflight.len(), 3, "one in-flight entry per distinct repo"); + } + + /// NO LOST RECOVERY COPY — the security guard (R4/R6). Coalescing must DELAY a + /// duplicate walk, never permanently drop a repo's recovery copy. Observable + /// property: once an in-flight task ENDS (its guard drops — completion, error, or + /// panic-unwind) the repo key is released, so the NEXT push for that repo is admitted + /// and processed again. A coalesce->shed mutation would drop the job AND never + /// re-admit — with no reconciliation sweep the copy is lost forever. Here re-admission + /// survives normal completion AND a panic, so no permanent skip / no leaked key. + #[test] + fn u4_coalesced_repo_is_reprocessed_after_task_ends_not_permanently_skipped() { + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkDurableRepoBBBBBBBBBBBBBBBBBBBBBBBBB/repo"; + + // Push #1 admits and "spawns". A concurrent push #2 (task #1 still in flight) + // coalesces — no duplicate spawn. + let guard1 = inflight.try_begin(repo).expect("first push admits"); + assert!( + inflight.try_begin(repo).is_none(), + "while task #1 is in flight, push #2 to the same repo coalesces" + ); + + // Task #1 finishes (encrypt_and_pin ran or errored): guard drops, key released. + drop(guard1); + assert_eq!( + inflight.len(), + 0, + "when the in-flight task ends its repo key is released — the set does not leak" + ); + + // A LATER push for the SAME repo is admitted again (processed, not skipped + // forever). This is what coalesce->shed breaks: shed drops the job and no sweep + // re-derives the missing copy, so the recovery copy is permanently lost. + let guard2 = inflight.try_begin(repo).expect( + "a later push for a coalesced repo MUST be re-admitted — durability: the \ + deferred recovery copy is produced eventually, never dropped", + ); + drop(guard2); + assert_eq!(inflight.len(), 0); + + // Durability across PANIC: a task that panics mid-walk must still release its key + // (Drop runs on unwind), so one crashed walk never permanently locks a repo out + // of future recovery copies. + let panicked = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _g = inflight.try_begin(repo).expect("admit before the panic"); + assert_eq!(inflight.len(), 1); + panic!("simulate the detached encryption task panicking mid-walk"); + })); + assert!(panicked.is_err(), "the simulated task panicked"); + assert_eq!( + inflight.len(), + 0, + "a panicked encryption task still releases its repo key (Drop on unwind) — no \ + permanent leak that would block every future recovery copy for the repo" + ); + assert!( + inflight.try_begin(repo).is_some(), + "after a panicked task the repo can still be admitted — durability preserved" + ); + } + + /// Degenerate state: the first push on a cold/empty in-flight set always admits + /// (never a false coalesce on an empty set). + #[test] + fn u4_first_push_on_a_cold_set_always_admits() { + let inflight = crate::state::EncryptInflight::new(); + assert!(inflight.is_empty(), "cold set is empty"); + assert!( + inflight.try_begin("owner/first").is_some(), + "the first push on a cold in-flight set must admit (never falsely coalesce)" + ); + } + + /// Model of the pre-fix / mutated code: no coalescing check, so every push spawns. + /// Returns the count of tasks spawned (== the size of the unbounded outstanding set + /// the fix prevents), used as the RED comparison in the bound test above. + fn simulate_without_coalescing(pushes: usize) -> usize { + (0..pushes).count() + } + /// #174 SC2 (per-source key, U1): the per-caller read sub-cap keys on the /// resolved source IP, NOT the signed DID, so a disposable-DID farm cannot /// multiply its budget. Fill the source IP's single read slot, then drive two diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index eed49253..912108a0 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -524,6 +524,7 @@ mod tests { git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + encrypt_inflight: crate::state::EncryptInflight::new(), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 8727aae1..ffac65a2 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -395,6 +395,11 @@ async fn main() -> Result<()> { git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new( config.max_concurrent_git_pushes, )), + // Coalesces the DETACHED post-push encryption tasks per repo so a rapid pusher + // cannot grow the outstanding parked-waiter set past one task per repo (#174 + // P2-2). No knob: it is a natural cap (one entry per distinct repo), not a + // sized pool. + encrypt_inflight: crate::state::EncryptInflight::new(), git_read_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( config.max_concurrent_reads_per_caller, ), diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 3b8ba250..deccf4c7 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -121,6 +121,16 @@ pub struct AppState { /// walk must not hold a foreground write slot, and a handler already holding a /// write permit that needed a second would self-deadlock at pool size 1. pub git_encrypt_semaphore: Arc, + /// Bounds the OUTSTANDING post-push encryption-task set by per-repo coalescing + /// (#174 P2-2). `git_encrypt_semaphore` caps *active* walks; this caps how many + /// detached tasks *spawn and park* on that semaphore's `acquire_owned().await`. + /// Before spawning a per-push encryption task, the receive-pack handler consults + /// this set: if the repo already has a task in flight it coalesces (skips the + /// duplicate spawn) rather than parking a new unbounded waiter. Coalescing only + /// delays a duplicate walk — it never drops the withheld-blob recovery copy, which + /// `2a54c15` deliberately kept fail-closed (there is no reconciliation sweep to + /// re-derive a dropped copy). See [`EncryptInflight`]. + pub encrypt_inflight: EncryptInflight, /// Per-caller concurrency sub-cap on the read pool: each caller (keyed on the /// resolved source IP, #174 U1) may hold at most `max_concurrent_reads_per_caller` /// in-flight read ops, so one caller cannot monopolize `git_read_semaphore` @@ -203,3 +213,89 @@ impl AppState { *self.shutdown_tx.borrow() } } + +/// Bounds the OUTSTANDING post-push encryption-task set by per-repo coalescing +/// (#174 P2-2). Each successful path-scoped push `tokio::spawn`s a DETACHED task that +/// parks on `git_encrypt_semaphore.acquire_owned().await` (which DEFERS when the pool +/// is full rather than shedding — `2a54c15` kept it fail-closed so the withheld-blob +/// recovery copy is never dropped). The semaphore caps *active* walks, but nothing +/// capped how many detached tasks *spawn and park* on that await: N rapid pushes to a +/// repo spawn N parked tasks, each holding cloned object lists/rules/paths/keys — an +/// unbounded outstanding set. +/// +/// This tracks the repo keys with an in-flight encryption task. Before spawning, the +/// handler calls [`try_begin`](Self::try_begin): if a task for the repo is already +/// in-flight it returns `None` and the handler SKIPS spawning a duplicate (coalesce — +/// the newer push's objects are covered by the pending/next walk over the same repo's +/// history). This bounds the outstanding set to <=1 pending task per repo WITHOUT +/// dropping work: coalescing only delays a duplicate walk, it never sheds the recovery +/// copy (there is no reconciliation sweep, so a *dropped* job would be lost forever). +/// +/// The returned [`EncryptInflightGuard`] is moved into the detached task and removes +/// the repo key on drop — on normal completion, error, OR panic (Drop runs on unwind) +/// — so one crashed walk can never permanently lock a repo out of future recovery +/// copies. +#[derive(Clone, Default)] +pub struct EncryptInflight { + // std::sync::Mutex: only ever held for O(1) HashSet insert/remove in a sync + // context (right before `tokio::spawn`, and in the guard's Drop) — never across + // an await, so a std Mutex is correct and cheaper than a tokio one. + repos: Arc>>, +} + +impl EncryptInflight { + pub fn new() -> Self { + Self::default() + } + + /// Try to begin an encryption task for `repo_id`. Returns `Some(guard)` if no task + /// for the repo was in-flight (the caller should spawn), or `None` if one already + /// is (the caller should COALESCE — skip spawning a duplicate). The guard releases + /// the repo key on drop. + pub fn try_begin(&self, repo_id: &str) -> Option { + let mut set = self.repos.lock().expect("encrypt_inflight mutex poisoned"); + if set.insert(repo_id.to_string()) { + Some(EncryptInflightGuard { + repos: Arc::clone(&self.repos), + repo_id: repo_id.to_string(), + }) + } else { + None + } + } + + /// Number of repos with an in-flight encryption task. Test/metrics observability; + /// the bound under saturation is `len() <= number of distinct repos`, i.e. at most + /// one task per repo. + #[allow(dead_code)] + pub fn len(&self) -> usize { + self.repos + .lock() + .expect("encrypt_inflight mutex poisoned") + .len() + } + + #[allow(dead_code)] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +/// RAII guard removing a repo key from [`EncryptInflight`] when the detached +/// encryption task finishes (drop on completion, error, or panic-unwind). Move-only — +/// there is no reason to clone a guard, and cloning would double-remove. +pub struct EncryptInflightGuard { + repos: Arc>>, + repo_id: String, +} + +impl Drop for EncryptInflightGuard { + fn drop(&mut self) { + // A poisoned lock (a prior panic while holding it) still lets us take the inner + // set via into_inner-on-guard; but poisoning here is not expected because the + // only critical sections are the O(1) ops above. Remove best-effort. + if let Ok(mut set) = self.repos.lock() { + set.remove(&self.repo_id); + } + } +} diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index cfa676af..92f41a8d 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -87,6 +87,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + encrypt_inflight: crate::state::EncryptInflight::new(), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( 8, diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index 5f576300..7d3fd1c8 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -77,4 +77,14 @@ fn inv22_concurrency_gates_present_and_not_bypassed() { "P1-e bypass: the bounded recipients walk must be invoked only inside \ withheld_recipients_gated; a new call site bypasses the encrypt-walk admission cap" ); + + // U4 / P2-2: the detached post-push encryption task must be gated by the per-repo + // coalescing set (`encrypt_inflight.try_begin`) so the OUTSTANDING parked-task set is + // bounded to <=1 per repo. Removing the gate lets N rapid pushes spawn N parked + // waiters (the unbounded set U4 closed); the semaphore only caps active walks. + assert!( + repos.contains("state.encrypt_inflight.try_begin"), + "U4/P2-2 gate missing: the detached post-push encryption spawn must consult \ + encrypt_inflight.try_begin to coalesce per repo (bound the outstanding-task set)" + ); } From 413d6cf0bcf1a8c587e01d6c9384de70e2876ecb Mon Sep 17 00:00:00 2001 From: t Date: Wed, 15 Jul 2026 13:16:40 -0500 Subject: [PATCH 037/124] fix(node): reject unsupported git service before the read slot; document the new admission knobs (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P2-1: git_info_refs treated every service other than git-receive-pack as a read op and did the DB/visibility/Tigris work under a read permit, validating the service string only downstream in smart_http. So an unauthenticated `?service=anything` to a public repo consumed a read slot + storage work before rejection. Validate the service is exactly git-upload-pack or git-receive-pack immediately after parsing, returning 400 before the pre-DB shed and the permit. Handler-layer regression: with the read pool exhausted, `?service=git-explode` returns 400 (validated first), not 503 — removing the validation makes it 503 (RED). U5/INV-24 docs: document the new knobs from U2/U3 in .env.example and the README env-var table — GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS, GITLAWB_MAX_CONCURRENT_IPFS_WALKS, GITLAWB_IPFS_WALK_PER_SOURCE, GITLAWB_IPFS_MAX_REPOS_WALKED, GITLAWB_IPFS_RATE_LIMIT. Each is already enforced on the path it names (U2/U3 tests), so the docs match the implemented routing. --- .env.example | 29 ++++++++++++++++ README.md | 5 +++ crates/gitlawb-node/src/api/repos.rs | 51 ++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+) diff --git a/.env.example b/.env.example index 0ef193f8..149ea8b1 100644 --- a/.env.example +++ b/.env.example @@ -115,6 +115,16 @@ GITLAWB_MAX_PACK_BYTES=2147483648 # Default 600. GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600 +# Max seconds the storage-ACQUISITION phase of a served git op may run before the +# request is shed with a 503, separate from the git-run timeout above. A +# concurrency permit is taken before this phase and GITLAWB_GIT_SERVICE_TIMEOUT_SECS +# only starts once git spawns, so without this a stalled backend (a hung Tigris +# HEAD/GET, or a hung pg advisory-lock iteration on push) pins the permit and drains +# the pool until every later request 503s. On expiry the permit is released +# (fail-closed). Kept separate because acquisition and git execution are distinct +# cost centers. Must be positive; set very large to effectively disable. Default 30. +GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS=30 + # Max concurrent git READ ops (upload-pack + the upload-pack info/refs # advertisement) served at once, a global pool separate from the push pool below. # The anon receive-pack info/refs advertisement has its OWN pool (see below), not @@ -146,6 +156,25 @@ GITLAWB_MAX_CONCURRENT_READS_PER_CALLER=16 # IP per hour. 0 disables. Default 600. GITLAWB_PUSH_RATE_LIMIT=600 +# ── /ipfs/{cid} visibility-walk admission (#174) ────────────────────────── +# GET /ipfs/{cid} runs a per-repo full-history git walk in a blocking thread to +# decide whether the caller may read a path-scoped blob. It is publicly reachable, +# so it is bounded to keep a permissionless caller from fanning out unbounded +# concurrent walks and exhausting blocking-pool threads + PIDs. +# Max concurrent /ipfs walks across all callers (a pool of its own, disjoint from +# the served-git pools). Over-cap sheds a 503. Default 32. +GITLAWB_MAX_CONCURRENT_IPFS_WALKS=32 +# Max concurrent /ipfs walks a single SOURCE IP may hold (keyed like the git +# per-caller caps via GITLAWB_TRUSTED_PROXY; reject-before-insert bounded map). +# Default 4. +GITLAWB_IPFS_WALK_PER_SOURCE=4 +# Max repos walked per single /ipfs request, so one request cannot serialize a +# full-history walk over every repo carrying the CID. Default 64. +GITLAWB_IPFS_MAX_REPOS_WALKED=64 +# Max /ipfs/{cid} requests per client IP per hour (route flood brake, distinct +# from the concurrency caps above). 0 disables. Default 600. +GITLAWB_IPFS_RATE_LIMIT=600 + # ── Creation rate limiting (repo/agent/issue/PR flood brake) ────────────── # Max creation requests (POST /api/v1/repos, /api/register, fork, issues, # pulls) per client IP per hour, in addition to the per-DID limit. The per-DID diff --git a/README.md b/README.md index 542a988e..e9765876 100644 --- a/README.md +++ b/README.md @@ -344,6 +344,11 @@ Important node settings: | `GITLAWB_AUTO_SYNC` | Enable automatic sync from known peers. | | `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. | | `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. | +| `GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS` | Max seconds the storage-acquisition phase (Tigris HEAD/GET, push advisory-lock) of a served git op may run before the request is shed with a 503, separate from the git-run timeout. The concurrency permit is released on expiry so a stalled backend cannot pin the pool. Default 30. | +| `GITLAWB_MAX_CONCURRENT_IPFS_WALKS` | Max concurrent `GET /ipfs/{cid}` visibility walks across all callers (own pool, disjoint from the served-git pools); over-cap sheds 503. Default 32. | +| `GITLAWB_IPFS_WALK_PER_SOURCE` | Max concurrent `/ipfs` walks a single source IP may hold. Default 4. | +| `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max repos walked per `/ipfs/{cid}` request, bounding one request's fan-out. Default 64. | +| `GITLAWB_IPFS_RATE_LIMIT` | Max `/ipfs/{cid}` requests per client IP per hour (route flood brake). 0 disables. Default 600. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | | `GITLAWB_IRYS_URL` | Optional Irys/Arweave permanent anchoring. | diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 6b52273f..9b05ece1 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -515,6 +515,16 @@ pub async fn git_info_refs( let service = query .service .ok_or_else(|| AppError::BadRequest("missing ?service= parameter".into()))?; + // Reject an unsupported service BEFORE taking a read slot or doing any DB/Tigris + // work (#174 P2-1). git_info_refs otherwise treats everything that is not + // git-receive-pack as a read op, so an unauthenticated `?service=anything` to a + // public repo would consume a read permit and the visibility/Tigris work before + // validate_service rejected it downstream in smart_http. + if service != "git-upload-pack" && service != "git-receive-pack" { + return Err(AppError::BadRequest(format!( + "unsupported git service: {service}" + ))); + } // #62 cheap pre-DB load shed: if the pool this service draws from is already // saturated, shed with 503 before any DB/disk work. Best-effort (holds no // permit); the authoritative hold is `git_permit` below, after the per-source @@ -3118,6 +3128,47 @@ mod tests { ); } + /// #174 P2-1: an unsupported `?service=` must be rejected with 400 BEFORE taking a + /// read slot or doing DB/Tigris work. Isolate it: exhaust the read pool so a read + /// op WOULD shed 503 at the pre-DB check — a garbage service must still return 400 + /// (validation runs first), proving `?service=anything` cannot consume the read + /// pool. Removing the validation makes this 503 (RED). + #[sqlx::test] + async fn info_refs_rejects_unsupported_service_before_the_read_slot(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state + .db + .upsert_mirror_repo("z6svcowner", "svc", "/tmp/svc", None, false) + .await + .unwrap(); + // Exhaust the read pool: a read op would shed 503 at the pre-DB check. + state.git_read_semaphore = Arc::new(Semaphore::new(0)); + + let router = crate::server::build_router(state); + let peer: SocketAddr = "203.0.113.90:7000".parse().unwrap(); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6svcowner/svc/info/refs?service=git-explode") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + + let status = router.oneshot(req).await.unwrap().status(); + assert_eq!( + status, + StatusCode::BAD_REQUEST, + "an unsupported ?service= must be 400 before the read-pool shed, not 503" + ); + } + /// #174 (jatmn P1): the anon-reachable receive-pack advertisement /// (`GET info/refs?service=git-receive-pack`) draws from a DEDICATED advert pool /// (`git_push_advert_semaphore`), NOT the write pool the authenticated POST uses. From 433c8915bf85a81db469dfead356d3c54a83220a Mon Sep 17 00:00:00 2001 From: t Date: Fri, 17 Jul 2026 10:15:56 -0500 Subject: [PATCH 038/124] feat(node): integrate the #173 IPFS CID tree-gate onto the #174 walk-concurrency base Rebased onto #174 (fix/served-git-concurrency-cap). #173's incremental history cannot replay commit-by-commit because #174 rewrote the same handler files in parallel; this single commit carries the fully-integrated tree (identical to the verified merge result), with the follow-up fixes as separate commits on top. Brings in #173: GET /ipfs/{cid} CID->oid resolution via pinned_cids, per-caller path-scoped blob/tree/commit-tag gating (#135/#173 F1-F6), and the pin-source provenance table. Adapts #173's tree/commit-tag walks to #174's run_bounded_git so the held /ipfs walk-concurrency permit is duration-safe (F5). --- Cargo.lock | 11 +- crates/gitlawb-node/src/api/ipfs.rs | 1131 +++-- crates/gitlawb-node/src/api/mod.rs | 24 +- crates/gitlawb-node/src/api/repos.rs | 3 + crates/gitlawb-node/src/auth/mod.rs | 5 +- crates/gitlawb-node/src/db/mod.rs | 282 +- crates/gitlawb-node/src/error.rs | 12 + crates/gitlawb-node/src/git/store.rs | 20 + .../gitlawb-node/src/git/visibility_pack.rs | 1079 +++++ crates/gitlawb-node/src/ipfs_pin.rs | 42 +- crates/gitlawb-node/src/main.rs | 17 +- crates/gitlawb-node/src/pinata.rs | 31 +- crates/gitlawb-node/src/rate_limit.rs | 29 + crates/gitlawb-node/src/state.rs | 44 +- crates/gitlawb-node/src/test_support.rs | 3764 +++++++++++++++-- crates/gitlawb-node/src/visibility.rs | 25 + crates/gl/src/ipfs_cmd.rs | 202 +- 17 files changed, 5942 insertions(+), 779 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d12daa43..24beb9ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3300,7 +3300,7 @@ dependencies = [ [[package]] name = "git-remote-gitlawb" -version = "0.5.0" +version = "0.5.1" dependencies = [ "anyhow", "gitlawb-core", @@ -3312,7 +3312,7 @@ dependencies = [ [[package]] name = "gitlawb-attest" -version = "0.5.0" +version = "0.5.1" dependencies = [ "base64", "ed25519-dalek", @@ -3329,7 +3329,7 @@ dependencies = [ [[package]] name = "gitlawb-core" -version = "0.5.0" +version = "0.5.1" dependencies = [ "anyhow", "base64", @@ -3356,7 +3356,7 @@ dependencies = [ [[package]] name = "gitlawb-node" -version = "0.5.0" +version = "0.5.1" dependencies = [ "alloy", "anyhow", @@ -3412,7 +3412,7 @@ dependencies = [ [[package]] name = "gl" -version = "0.5.0" +version = "0.5.1" dependencies = [ "alloy", "anyhow", @@ -3824,6 +3824,7 @@ dependencies = [ "reqwest", "serde", "serde_json", + "sha2", "thiserror 2.0.18", "tracing", ] diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index ac8e615c..a764901b 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -1,14 +1,17 @@ //! GET /ipfs/{cid} — content-addressed retrieval of git objects by CIDv1. //! -//! Every git object stored on this node is addressable by its IPFS CIDv1. +//! Every git object pinned on this node is addressable by its IPFS CIDv1. //! The CID is computed as: //! //! CIDv1(codec=raw, multihash=sha2-256(content_bytes)) //! //! where `content_bytes` is the raw object content as returned by -//! `git cat-file ` (i.e. without the git framing header). -//! This is consistent with how `gitlawb_core::cid::Cid::from_git_object_bytes` -//! computes CIDs when objects are pushed. +//! `git cat-file ` (i.e. without the git framing header) — the +//! same bytes `gitlawb_core::cid::Cid::from_git_object_bytes` hashes when the +//! object is pinned. That digest is NOT the object's git oid: git frames the +//! content with a `" \0"` header before hashing, so `sha2-256(content)` +//! and the git oid differ. The handler therefore maps the CID back to its oid via +//! the `pinned_cids` table rather than treating the digest as an oid (#173). //! //! Serving is access-controlled: an object is returned only from a repo row the //! requesting caller is permitted to read (per-caller path-scoped visibility, @@ -27,14 +30,66 @@ use std::str::FromStr; use crate::auth::AuthenticatedDid; use crate::error::{AppError, Result}; use crate::git::store; -use crate::git::visibility_pack::{allowed_blob_set_for_caller_bounded, has_path_scoped_rule}; +use crate::git::visibility_pack::{ + allowed_blob_set_for_caller_bounded, allowed_tree_set_for_caller_bounded, has_path_scoped_rule, + reachable_commit_tag_oids_bounded, +}; use crate::state::AppState; use crate::visibility::{visibility_check, Decision}; +/// Hard ceiling on the number of full-history reachability walks a single +/// `GET /ipfs/{cid}` request may spawn. The per-request `ipfs_rate_limiter` +/// check brakes *repeat* requests, but within one request the object can exist +/// under path-scoped rules in many repos, and each distinct repo pays its own +/// `spawn_blocking` walk (the memo only dedups the same repo). Without a ceiling +/// a single request fans out to O(repos) walks for one rate-limiter token — an +/// amplification sink (INV-10). Once this many walks have run, no further walk is +/// spawned for the rest of the request: any remaining candidate that still needs +/// a walk is skipped (and, with nothing else readable, the request falls through +/// to the opaque 404). The bound is deliberately generous: a legitimate caller +/// serves on the first repo that grants them, so reaching it requires being +/// denied by this many path-scoped repos first, which real traffic effectively +/// never does. Tunable if that assumption stops holding. +pub(crate) const MAX_HISTORY_WALKS_PER_REQUEST: u32 = 16; + +/// Hard per-request ceiling on how many legacy (NULL-provenance) repositories +/// the CID resolver's scan fallback may PROBE (`acquire` + `git cat-file -t`). +/// The provenance path targets one repo; the legacy scan, absent this bound, +/// fans one anonymous request out to O(repos) subprocess spawns and cold-cache +/// Tigris fetches for a CID enumerable from the public pins index (#173 round 3, +/// F1, INV-10). Deliberately generous: a normal node has far fewer repos than +/// this, so a genuine miss still completes the whole scan and returns a truthful +/// 404; only a node larger than the cap truncates, and a truncated search +/// surfaces as a retryable 503 (never a false "absent"). Legacy pins are a +/// shrinking set — each re-pin backfills provenance — so this fallback is a +/// transitional path, not the steady state. Tunable via `AppState`. +pub(crate) const MAX_LEGACY_PROBES_PER_REQUEST: u32 = 256; + +/// Hard ceiling on the byte size of an object `GET /ipfs/{cid}` buffers and serves +/// (#173 round 8, F6, INV-10). The serve reads via a blocking `git cat-file` and +/// buffers the whole object; unbounded, a large public blob (enumerable from the pins +/// index) could exhaust memory or block a runtime worker. A content-addressed serve +/// must verify the whole object hashes to the requested CID before any byte egresses +/// (F2), so it cannot stream — it buffers up to this cap and withholds anything larger +/// (raise the cap if a class of legitimate objects legitimately exceeds it; never +/// stream unverified). 32 MiB is generous for git blobs/trees/commits. Tunable via +/// `AppState` for the test seam, like the sibling caps. +pub(crate) const MAX_SERVED_OBJECT_BYTES: u64 = 32 * 1024 * 1024; + +/// Lazily-loaded context for the legacy (NULL-provenance) scan fallback in +/// `get_by_cid`: all repos, their visibility rules keyed by repo id, and the set of +/// quarantined repo ids. Loaded once per request only if a legacy pin is hit. +type LegacyScanCtx = ( + Vec, + HashMap>, + HashSet, +); + /// GET /ipfs/{cid} /// -/// Search all repos on the node for a git object whose SHA-256 hash matches -/// the given CIDv1, returning its raw content if the caller may read it. +/// Resolve the CIDv1 to its git oid via the `pinned_cids` table, then search all +/// repos on the node for that object, returning its raw content if the caller may +/// read it. /// /// Visibility (#110, #126): the object is served only from a repo row the /// caller passes. For each iterated row we gate against that row's OWN rules @@ -43,13 +98,16 @@ use crate::visibility::{visibility_check, Decision}; /// row than the one read (KTD2a). We check object existence via /// `store::object_type` *before* the expensive reachability walk so random-CID /// spray cannot trigger full-history git walks on repos that don't carry the -/// object. When the row carries path-scoped rules (KTD4) the served object -/// must be either a non-blob (trees/commits are structural; KTD3) OR a blob -/// in the caller's *reachable* allowed-set (`allowed_blob_set_for_caller`). -/// The reachable allowed-set excludes dangling blobs — a blob written via -/// `git hash-object -w` and never committed has no path to gate, so it is -/// fail-closed 404'd under path-scoped rules (#126). Denial and genuine -/// not-found both fall through to an opaque 404. +/// object. When the row carries path-scoped rules (KTD4) the served object is +/// gated by type: a `blob`/`tree` must be in the caller's *reachable* allowed-set +/// (`allowed_blob_set_for_caller` / `allowed_tree_set_for_caller`), and a +/// `commit`/`tag` must be in the repo's *reachable* commit/tag set +/// (`reachable_commit_tag_oids`, #173). A withheld subtree's tree object is denied +/// here exactly as `get_tree` denies its path, so its child names and oids cannot +/// leak by CID (#135). All these sets exclude dangling objects — a blob, tree, +/// commit, or tag written via plumbing and never referenced has no reachable path, +/// so it is fail-closed 404'd under path-scoped rules (#126, #173). Denial and +/// genuine not-found both fall through to an opaque 404. /// /// Scope: this closes the direct unauthenticated scan, including the dangling /// case. A stale-public mirror row still serves withheld content (tracked @@ -57,14 +115,12 @@ use crate::visibility::{visibility_check, Decision}; pub async fn get_by_cid( Path(cid_str): Path, State(state): State, - auth: Option>, - // Per-source keying for the walk concurrency sub-cap. Infallible extractors - // (mirror the git handlers in `repos.rs`): `PeerAddr` yields `None` under - // `oneshot` with no `ConnectInfo`, and the header map falls back per `client_key`. crate::rate_limit::PeerAddr(peer): crate::rate_limit::PeerAddr, - req_headers: HeaderMap, + headers: HeaderMap, + auth: Option>, ) -> Result { - // 1. Decode the CID and extract the SHA-256 digest + // 1. Decode and validate the CID (uniform 400 on a malformed / non-sha2-256 + // CID, before any DB or git work). let cid = CidGeneric::<64>::from_str(&cid_str) .map_err(|e| AppError::BadRequest(format!("invalid CID: {e}")))?; @@ -77,9 +133,13 @@ pub async fn get_by_cid( )); } - let sha256_hex = hex::encode(mh.digest()); - let caller = auth.as_ref().map(|e| e.0 .0.as_str()); - let caller_owned = caller.map(|c| c.to_string()); + // Canonicalize the CID for the pinned_cids lookup. Pins are stored under the + // canonical base32 `cid.to_string()`, but a client may send any equivalent + // multibase spelling (base58/base64) of the same CID; those parse and pass + // the sha2-256 check yet miss the canonical key, so they must be normalized + // before the DB lookup (#173). Response headers and error messages still echo + // the original `cid_str` the client sent. + let canonical_cid = cid.to_string(); // Bounded walk admission (#174 P1-3), taken before any DB/git work so a flood sheds // cheaply. The per-repo `spawn_blocking` walk below is a full-history git walk with @@ -87,13 +147,16 @@ pub async fn get_by_cid( // out concurrent walks past every git pool, exhausting the blocking pool + PIDs. // Acquire the global permit (and, for a resolvable source, the per-source // sub-permit) ONCE here and hold BOTH for the whole request — across every - // `spawn_blocking` walk in the loop below — so the slot reflects real blocking-thread + // `spawn_blocking` walk below — so the slot reflects real blocking-thread // occupancy (a tokio walk-timeout cannot free it while the blocking work still runs) - // and one request cannot open more than its share of concurrent walks. On - // unavailability shed a clean 503. The per-source key is the resolved source IP - // (`client_key`), never the DID (`/ipfs` admits any `did:key` unthrottled, so a DID - // key would be free to mint around); a `None` key (no trusted header, no peer) is - // bounded by the global pool only, never the per-source sub-cap. + // and one request cannot open more than its share of concurrent walks. Holding a + // slot across a walk is only safe because every walk child is duration-bounded + // (`*_bounded` + `run_bounded_git` teardown), so a hung git cannot pin the slot + // past `git_service_timeout_secs`. On unavailability shed a clean 503. The + // per-source key is the resolved source IP (`client_key`), never the DID (`/ipfs` + // admits any `did:key` unthrottled, so a DID key would be free to mint around); a + // `None` key (no trusted header, no peer) is bounded by the global pool only, + // never the per-source sub-cap. let _ipfs_walk_permit = state .git_ipfs_walk_semaphore .clone() @@ -102,7 +165,7 @@ pub async fn get_by_cid( tracing::warn!("/ipfs walk concurrency cap reached; shedding request with 503"); AppError::Overloaded("ipfs service at capacity, retry shortly".into()) })?; - let source_key = crate::rate_limit::client_key(&req_headers, peer, state.push_limiter_trust); + let source_key = crate::rate_limit::client_key(&headers, peer, state.push_limiter_trust); let _ipfs_caller_permit = match &source_key { Some(ip) => Some(state.git_ipfs_walk_per_caller.try_acquire(ip).ok_or_else(|| { tracing::warn!(key = %ip, "/ipfs per-source walk cap reached; shedding request with 503"); @@ -111,178 +174,553 @@ pub async fn get_by_cid( None => None, }; - // 2. Search all repos for an object with this SHA-256 - let repos = state + // Resolve the content-addressed CID to the object's git oid(s). A real pin + // CID digests the raw object content (`Cid::from_git_object_bytes`), NOT the + // git oid (git frames content with a `" \0"` header first), so we + // map it back through `pinned_cids` rather than treating the digest as an oid + // (#173). The cid index is non-unique, so one CID can map to several oids (a + // tree and a blob whose raw bytes collide, or content pinned under two oids); + // we try each candidate below rather than pick one arbitrarily and false-404 + // when the chosen one is withheld or absent while another is readable (#173). + // An empty result is an opaque 404, uniform with a genuine not-found and a + // visibility denial. + let oids = state .db - .list_all_repos() + .oids_for_cid(&canonical_cid) .await .map_err(AppError::Internal)?; + if oids.is_empty() { + return Err(AppError::RepoNotFound(format!( + "no git object found for CID {cid_str}" + ))); + } + let caller = auth.as_ref().map(|e| e.0 .0.as_str()); + let caller_owned = caller.map(|c| c.to_string()); - // Fetch every repo's visibility rules in one query rather than one per row - // (the gate runs each row against its OWN rules — KTD2a). A row absent from - // the map has no rules. - let repo_ids: Vec = repos.iter().map(|r| r.id.clone()).collect(); - let rules_by_repo = state - .db - .list_visibility_rules_for_repos(&repo_ids) - .await - .map_err(AppError::Internal)?; + // Per-request walk budget + memos + throttle flag, shared by the provenance path + // and the legacy scan so both honor the same fan-out ceiling, per-repo memo, and + // IP brake. The caller is constant for one request, so `repo.id` alone keys the memo. + let mut walk = WalkState { + walks: 0, + probes: 0, + truncated: false, + allowed_blob_memo: HashMap::new(), + allowed_tree_memo: HashMap::new(), + reachable_ct_memo: HashMap::new(), + }; + // Set when a walk-requiring candidate is skipped because the source IP's walk quota + // is spent (#173 review, F-C): the scan keeps going so a later walk-free copy still + // serves; only if nothing is servable is it turned into the 429. + let mut throttled = false; + let rctx = ResolveCtx { + caller, + caller_owned: &caller_owned, + headers: &headers, + peer, + cid_str: &cid_str, + canonical_cid: &canonical_cid, + }; - // Request-scoped memo of the per-repo allowed-blob set (KTD1, #126). The - // caller is constant for one request, so `repo.id` alone is a safe, - // sufficient key — never a coarse caller "class", which - // `visibility_check`'s exact full-DID reader match would make unsafe. - // - // We flipped from a deny-set (`withheld_blob_oids`) to an allowed-set - // (`allowed_blob_set_for_caller`) so dangling blobs — never enumerated by - // the reachable walk — fail closed instead of slipping through an empty - // deny entry (#126). - let mut allowed_memo: HashMap> = HashMap::new(); - - // Cap the number of candidate repos one request walks (it already short-circuits on - // serve): a CID present in — or path-gated out of — many repos must not serialize an - // unbounded number of full-history walks inside the single held admission slot. - let mut repos_walked: usize = 0; - - for repo in &repos { - // Repo-level read gate against THIS row's own rules (KTD2a). - let rules: &[crate::db::VisibilityRule] = rules_by_repo - .get(&repo.id) - .map(Vec::as_slice) - .unwrap_or(&[]); - if visibility_check(rules, repo.is_public, &repo.owner_did, caller, "/") == Decision::Deny { - continue; - } + // Legacy scan context (repos + rules + quarantined ids), loaded LAZILY only when a + // legacy NULL-provenance pin is hit — the provenance path must never trigger the + // O(repos) load (that fan-out is exactly what provenance removes, #173 round 2). + let mut scan_ctx: Option = None; - // Loop bound (#174 P1-3): once this request has walked its cap of candidate - // repos (each a git subprocess, up to a full-history walk), stop and fall - // through to the opaque 404 rather than serialize an unbounded number under the - // single held admission slot. - if repos_walked >= state.config.ipfs_max_repos_walked { - tracing::warn!( - cap = state.config.ipfs_max_repos_walked, - "/ipfs request hit the per-request repo-walk cap; stopping the scan" - ); - break; - } - repos_walked += 1; - - // Bound the per-repo acquire under `git_acquire_timeout_secs`: this loop shares - // the P1-2 stall vector (a hung Tigris HEAD/GET on one repo would otherwise - // block the whole /ipfs request). On expiry keep the existing fail-closed skip — - // never serve an un-acquired repo; a public copy (if any) still gets its turn. - let acquire_deadline = - std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); - let repo_path = match tokio::time::timeout( - acquire_deadline, - state.repo_store.acquire(&repo.owner_did, &repo.name), - ) - .await - { - Ok(Ok(p)) => p, - Ok(Err(_)) => continue, - Err(_elapsed) => { - tracing::warn!(repo = %repo.name, "repo acquire timed out during /ipfs walk; skipping repo"); - continue; + for sha256_hex in &oids { + // A pinned object records EVERY repo it was pinned from (#173 round 8, F1). + // Resolve a PROVENANCED pin by trying each source repo (bounded to + // MAX_PIN_SOURCES) through the SAME gate; the first that authorizes serves — no + // scan fan-out. A shared object first pinned from a private/quarantined repo + // still serves from a later PUBLIC source. Deterministic (ORDER BY on the + // union), so no ordering can turn an authorized copy into a 404. + let sources = state + .db + .pin_sources_for_oid(sha256_hex) + .await + .map_err(AppError::Internal)?; + if sources.is_empty() { + // F3 (#173, INV-10/INV-15): the legacy NULL-provenance scan builds an + // O(repos) preload (repos + rules + quarantine) BEFORE gate_and_serve's + // per-probe brake can bite, so a throttled source could still force O(repos) + // DB work on every replay. Peek the per-IP limiter WITHOUT consuming a token: + // an already-throttled source is shed here, before the preload runs. The + // consuming per-probe charge inside gate_and_serve is left UNCHANGED (it is + // load-bearing for the across-request bound), so this adds no double-charge — + // a non-consuming peek plus the existing per-probe charge, never two charges. + if let Some(key) = + crate::rate_limit::client_key(rctx.headers, rctx.peer, state.push_limiter_trust) + { + if state.ipfs_rate_limiter.is_throttled(&key).await { + throttled = true; + continue; + } } - }; - - // Check whether the object exists in this repo before any expensive - // reachability walk. This prevents random-CID spray from triggering - // full-history git walks on repos that don't carry the object. - let obj_type = match store::object_type(&repo_path, &sha256_hex) { - Ok(Some(t)) => t, - Ok(None) => continue, - Err(e) => { - tracing::warn!(repo = %repo.name, err = %e, "error checking git object type"); - continue; + // Legacy pin (recorded before provenance existed): fall back to the repo + // scan, gating each repo through the SAME gate. Load the scan context + // once, lazily. + if scan_ctx.is_none() { + #[cfg(test)] + bump_preload_queries(); + let repos = state + .db + .list_all_repos() + .await + .map_err(AppError::Internal)?; + let repo_ids: Vec = repos.iter().map(|r| r.id.clone()).collect(); + let rules_by_repo = state + .db + .list_visibility_rules_for_repos(&repo_ids) + .await + .map_err(AppError::Internal)?; + let quarantined: HashSet = state + .db + .list_quarantined_repos() + .await + .map_err(AppError::Internal)? + .into_iter() + .map(|r| r.id) + .collect(); + scan_ctx = Some((repos, rules_by_repo, quarantined)); } - }; - - // Per-blob gating only applies when a path-scoped rule exists (KTD4). - // Without any path-scoped rule, the "/" gate above is the whole story. - // Trees/commits are always served under path-scoped rules (KTD3). - let path_scoped = has_path_scoped_rule(rules); - if path_scoped && obj_type == "blob" { - if !allowed_memo.contains_key(&repo.id) { - let rp = repo_path.clone(); - let r = rules.to_vec(); - let is_public = repo.is_public; - let owner = repo.owner_did.clone(); - let caller_for_walk = caller_owned.clone(); - let git_bin = state.git_bin.clone(); - let walk_timeout = - std::time::Duration::from_secs(state.config.git_service_timeout_secs); - // Full-history walk shells out to git — keep it off the async runtime, - // bounded and reaped like the served-git ops (#174). - let walk = tokio::task::spawn_blocking(move || { - allowed_blob_set_for_caller_bounded( - &rp, - &git_bin, - walk_timeout, - &r, - is_public, - &owner, - caller_for_walk.as_deref(), - ) - }) - .await; - // Fail closed on EITHER a task panic (JoinError) or a walk error: - // we cannot prove the caller may read here, so skip this repo and - // let a public copy (if any) serve. Never serve on an unproven gate. - let set = match walk { - Ok(Ok(set)) => set, - Ok(Err(e)) => { - tracing::warn!(repo = %repo.name, err = %e, "allowed-blob walk failed; skipping repo"); - continue; - } - Err(e) => { - tracing::warn!(repo = %repo.name, err = %e, "allowed-blob walk task panicked; skipping repo"); + let (repos, rules_by_repo, quarantined) = scan_ctx.as_ref().unwrap(); + for repo in repos { + let rules = rules_by_repo + .get(&repo.id) + .map(Vec::as_slice) + .unwrap_or(&[]); + let is_quar = quarantined.contains(&repo.id); + match gate_and_serve( + &state, repo, rules, is_quar, sha256_hex, &rctx, &mut walk, true, + ) + .await + { + GateOutcome::Served(resp) => return Ok(resp), + // A throttled walk-requiring candidate is skipped, not fatal: + // keep scanning for a later walk-free copy (#173 review, F-C). + GateOutcome::Throttled => throttled = true, + GateOutcome::Skip => {} + } + } + } else { + for repo_id in &sources { + let repo = match state + .db + .get_repo_by_id(repo_id) + .await + .map_err(AppError::Internal)? + { + Some(r) => r, + // A source repo is gone: skip this source. Do NOT fall back to the + // scan (that would reopen the fan-out); a later source or oid + // candidate may still resolve. + None => continue, + }; + let quarantined = state + .db + .is_repo_quarantined(repo_id) + .await + .map_err(AppError::Internal)?; + let rules_map = state + .db + .list_visibility_rules_for_repos(std::slice::from_ref(repo_id)) + .await + .map_err(AppError::Internal)?; + let rules = rules_map.get(repo_id).map(Vec::as_slice).unwrap_or(&[]); + match gate_and_serve( + &state, + &repo, + rules, + quarantined, + sha256_hex, + &rctx, + &mut walk, + false, // provenance path: bounded source set, no scan fan-out + ) + .await + { + GateOutcome::Served(resp) => return Ok(resp), + GateOutcome::Throttled => { + throttled = true; continue; } - }; - allowed_memo.insert(repo.id.clone(), set); + GateOutcome::Skip => continue, + } } - let in_allowed = allowed_memo - .get(&repo.id) - .is_some_and(|set| set.contains(&sha256_hex)); - if !in_allowed { - continue; + } + } + + // Nothing served — three distinct tails, in precedence order: + // 1. The scan was cut short by a cap (legacy probe ceiling or walk ceiling), so + // the object was NOT proven absent/unreadable everywhere → 503, retryable, and + // explicitly NOT a definitive not-found (#173, F2). This outranks the throttle: + // an incomplete search must not masquerade as a clean rate-limit outcome, and + // it carries only the caller-supplied CID (no object/OID/metadata leak). + // 2. A walk-requiring candidate was skipped for a spent IP quota while the scan + // otherwise completed → 429 (the brake bit; a cheaper copy was sought first). + // 3. A full scan under the caps found nothing readable → opaque 404, uniform with + // a genuine not-found and a visibility denial. + if walk.truncated { + return Err(AppError::SearchIncomplete(format!( + "CID {cid_str} search incomplete — retry" + ))); + } + if throttled { + return Err(AppError::TooManyRequests( + "ipfs retrieval rate limit exceeded — try again later".into(), + )); + } + Err(AppError::RepoNotFound(format!( + "no git object found for CID {cid_str}" + ))) +} + +/// Outcome of gating one repo for one candidate oid. +enum GateOutcome { + /// The object passed the gate; serve this response. + Served(Response), + /// This repo does not serve the object (absent, denied, quarantined, walk-capped, + /// or a walk error) — try the next candidate. + Skip, + /// A walk-requiring candidate hit the per-IP walk quota; skip it but let the caller + /// record the throttle so a later walk-free copy can still serve. + Throttled, +} + +/// Outcome of the bounded, off-worker object read for one gated candidate (F6, #173). +enum ServedRead { + /// Verified: the object's bytes hash to the requested CID; serve them. + Ok(Vec), + /// The bytes do not hash to the requested CID (a legacy provider-CID row); withhold. + Mismatch(String), + /// The object exceeds the served-object size cap; withhold rather than buffer it. + TooLarge(u64), + /// The object is genuinely absent (git reported it does not exist); try the next + /// candidate. Distinct from `ReadErr` so an infra failure is never silently rendered + /// as a clean not-found. + Gone, + /// A git subprocess failed to run (spawn/IO error, not a "no such object"). Logged at + /// the handler layer and skipped — an infra failure must surface as an error, not a + /// silent 404 for an authorized caller (INV-25 spirit, #173). + ReadErr(String), +} + +/// Immutable per-request context threaded into the gate. +struct ResolveCtx<'a> { + caller: Option<&'a str>, + caller_owned: &'a Option, + headers: &'a HeaderMap, + peer: Option, + cid_str: &'a str, + /// Canonical base32 form of the requested CID (`cid.to_string()`), used by the + /// serve-side integrity check to confirm the served bytes actually hash to the + /// requested content address (F2, #173). Compared against the recomputed CID, NOT + /// `cid_str` — a client may send an equivalent non-canonical multibase spelling. + canonical_cid: &'a str, +} + +/// Per-request walk budget + memos, shared across the provenance path and the legacy +/// scan so the fan-out ceiling and per-repo memoization span the whole request. +struct WalkState { + walks: u32, + /// Count of legacy (NULL-provenance) repos actually probed this request, so the + /// scan can stop at `ipfs_max_legacy_probes` instead of fanning out to O(repos) + /// `acquire` + `cat-file` (#173, F1, INV-10). Only the legacy path bumps it. + probes: u32, + /// Set when any cap (the legacy probe ceiling or the walk ceiling) cut the scan + /// short. A truncated scan did NOT prove the object absent/unreadable everywhere, + /// so the tail returns a retryable 503 rather than a definitive 404 (#173, F2). + truncated: bool, + allowed_blob_memo: HashMap>, + allowed_tree_memo: HashMap>, + reachable_ct_memo: HashMap>, +} + +/// Gate ONE repo for ONE candidate oid and, if the caller may read it, serve it. The +/// SINGLE gate both the provenance path and the legacy scan call, so INV-11 (quarantine +/// hard-drops before visibility), INV-2 (the repo's own "/" gate), and the per-object +/// reachability walk hold identically on both paths (KTD5). Never re-resolves via +/// `authorize_repo_read`, whose fuzzy match could authorize a different physical row +/// than the one read (KTD2a). +// The per-repo gate genuinely needs the row, its rules, its quarantine bit, the oid, +// the request context, the shared walk budget, and whether this is the fan-out-bounded +// legacy scan; bundling them buys nothing over the existing threshold. +#[allow(clippy::too_many_arguments)] +async fn gate_and_serve( + state: &AppState, + repo: &crate::db::RepoRecord, + rules: &[crate::db::VisibilityRule], + quarantined: bool, + sha256_hex: &str, + ctx: &ResolveCtx<'_>, + walk: &mut WalkState, + // True only for the legacy NULL-provenance scan, which iterates every repo. The + // provenance path targets one repo (no fan-out) and passes false, so it does not + // consume the per-request probe budget below. + legacy_scan: bool, +) -> GateOutcome { + // Quarantine gate (INV-11): a quarantined mirror is hidden from every reader, owner + // included, BEFORE any visibility check — so an owner whom visibility would Allow + // still 404s. + if quarantined { + return GateOutcome::Skip; + } + // Repo-level "/" read gate against THIS row's own rules (INV-2, KTD2a). + if visibility_check(rules, repo.is_public, &repo.owner_did, ctx.caller, "/") == Decision::Deny { + return GateOutcome::Skip; + } + // Legacy-scan fan-out control (#173, F1/F3, INV-10). The legacy path probes every + // root-visible repo, and the probe below (`acquire` — a possible cold-cache + // Tigris fetch — plus a `git cat-file -t` subprocess) is the expensive part. + // Cap it per request BEFORE that work runs, so an anonymous caller wielding a + // CID from the public pins index cannot amplify one request into O(repos) + // subprocesses. A legacy scan is inherently fan-out (unlike a targeted + // provenance fetch), so EVERY legacy probe is charged to the source IP from the + // first one, not just the ones past a free budget. A per-request-only budget + // reset each request, leaving a NULL-provenance CID open to unbounded ACROSS- + // request amplification: N requests spending N x budget cold `acquire` calls + // against Tigris with zero limiter contact (#173, F3, jatmn). Charging the first + // probe makes those requests accumulate against the per-IP `ipfs_rate_limiter`, + // closing that path. The per-request cap below stays as the second bound (a + // single request's ceiling). A spent quota is the same non-fatal Throttled as the + // walk brake: keep scanning for a walk-free copy, and only a wholly-unservable + // request becomes the 429. No resolvable key (a test oneshot with no peer/header) + // skips the brake, as the walk brake does. The provenance path targets one repo + // (no fan-out) and is exempt (`legacy_scan == false`). + if legacy_scan { + if walk.probes >= state.ipfs_max_legacy_probes { + // Budget spent: stop probing and mark the scan truncated so the tail + // reports an incomplete search (503), not a false 404 (#173, F2). + walk.truncated = true; + return GateOutcome::Skip; + } + if let Some(key) = + crate::rate_limit::client_key(ctx.headers, ctx.peer, state.push_limiter_trust) + { + if !state.ipfs_rate_limiter.check(&key).await { + return GateOutcome::Throttled; } } + walk.probes += 1; + } + let repo_path = match state.repo_store.acquire(&repo.owner_did, &repo.name).await { + Ok(p) => p, + Err(_) => return GateOutcome::Skip, + }; - // Now that we've passed the gate, read the content. - let content = match store::read_object_content(&repo_path, &sha256_hex, &obj_type) { - Ok(c) => c, + // Existence probe before any walk (random-CID spray must not trigger a walk on a + // repo that lacks the object). Off the async runtime — it shells out to + // `git cat-file -t`. Fail closed (skip) on a task panic. + let obj_type = { + let rp = repo_path.clone(); + let sha = sha256_hex.to_string(); + match tokio::task::spawn_blocking(move || store::object_type(&rp, &sha)).await { + Ok(Ok(Some(t))) => t, + Ok(Ok(None)) => return GateOutcome::Skip, + Ok(Err(e)) => { + tracing::warn!(repo = %repo.name, err = %e, "error checking git object type"); + return GateOutcome::Skip; + } Err(e) => { - tracing::warn!(repo = %repo.name, err = %e, "error reading git object content"); - continue; + tracing::warn!(repo = %repo.name, err = %e, "object-type probe task panicked; skipping repo"); + return GateOutcome::Skip; } - }; + } + }; - // 3. Return the content with IPFS-style headers - let mut headers = HeaderMap::new(); - headers.insert( - HeaderName::from_static("content-type"), - HeaderValue::from_static("application/octet-stream"), - ); - headers.insert( - HeaderName::from_static("x-content-cid"), - HeaderValue::from_str(&cid_str).unwrap_or_else(|_| HeaderValue::from_static("invalid")), - ); - headers.insert( - HeaderName::from_static("x-git-hash"), - HeaderValue::from_str(&sha256_hex) - .unwrap_or_else(|_| HeaderValue::from_static("invalid")), - ); + // Per-object gating applies only under a path-scoped rule (KTD4); otherwise the "/" + // gate above is the whole story. A blob is gated on the caller's allowed-blob set, a + // tree on the allowed-tree set (#135), a commit/tag on the repo's reachable + // commit/tag set (#173) — each a full-history walk sharing the per-request cap and + // per-walk IP quota. + let path_scoped = has_path_scoped_rule(rules); + let gated = path_scoped && matches!(obj_type.as_str(), "blob" | "tree" | "commit" | "tag"); + if gated { + let already = match obj_type.as_str() { + "blob" => walk.allowed_blob_memo.contains_key(&repo.id), + "tree" => walk.allowed_tree_memo.contains_key(&repo.id), + "commit" | "tag" => walk.reachable_ct_memo.contains_key(&repo.id), + other => unreachable!("gated admits only blob/tree/commit/tag, got {other}"), + }; + if !already { + // Per-request fan-out ceiling (INV-10): once this many walks have run, skip + // THIS walk-requiring candidate and keep scanning (a later walk-free copy + // must still serve). `walks` is bumped only inside this block, so walk-free + // candidates never consume budget. + if walk.walks >= state.ipfs_max_history_walks { + // The walk ceiling truncated the search: a later repo (possibly one that + // authorizes this caller) is left unwalked, so absence is unproven — + // record it so the tail returns 503, not a false 404 (#173, F2). + walk.truncated = true; + return GateOutcome::Skip; + } + // Brake each spawned walk on the source IP (#173, F3, INV-15), BEFORE + // spending walk budget: a throttled candidate neither walks nor consumes + // budget and must not end the request — skip it and keep scanning + // (#173 review, F-C). No key (a test oneshot with no peer/header) skips the + // brake, as the other IP brakes do. On the LEGACY path the probe brake + // above already charged THIS candidate to the source (#173, F3, jatmn), so + // the walk brake must not double-charge it: only the provenance path + // (`legacy_scan == false`, no probe toll) charges here. + if !legacy_scan { + if let Some(key) = + crate::rate_limit::client_key(ctx.headers, ctx.peer, state.push_limiter_trust) + { + if !state.ipfs_rate_limiter.check(&key).await { + return GateOutcome::Throttled; + } + } + } + walk.walks += 1; - return Ok((StatusCode::OK, headers, content).into_response()); + let rp = repo_path.clone(); + let r = rules.to_vec(); + let is_public = repo.is_public; + let owner = repo.owner_did.clone(); + let caller_for_walk = ctx.caller_owned.clone(); + let kind = obj_type.clone(); + // Every walk is the DURATION-BOUNDED twin (`run_bounded_git` teardown under + // `git_service_timeout_secs`): the handler holds its /ipfs walk permit + // across this spawn_blocking, and a held permit is only safe if no walk + // child can outlive the deadline (#174 F5). + let git_bin = state.git_bin.clone(); + let walk_timeout = + std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let result = tokio::task::spawn_blocking(move || match kind.as_str() { + "blob" => allowed_blob_set_for_caller_bounded( + &rp, + &git_bin, + walk_timeout, + &r, + is_public, + &owner, + caller_for_walk.as_deref(), + ), + "tree" => allowed_tree_set_for_caller_bounded( + &rp, + &git_bin, + walk_timeout, + &r, + is_public, + &owner, + caller_for_walk.as_deref(), + ), + "commit" | "tag" => reachable_commit_tag_oids_bounded(&rp, &git_bin, walk_timeout), + other => unreachable!("gated admits only blob/tree/commit/tag, got {other}"), + }) + .await; + // Fail closed on a walk error or task panic: we cannot prove readability, so + // skip rather than serve on an unproven gate. + let set = match result { + Ok(Ok(set)) => set, + Ok(Err(e)) => { + tracing::warn!(repo = %repo.name, err = %e, "allowed-set walk failed; skipping repo"); + return GateOutcome::Skip; + } + Err(e) => { + tracing::warn!(repo = %repo.name, err = %e, "allowed-set walk task panicked; skipping repo"); + return GateOutcome::Skip; + } + }; + match obj_type.as_str() { + "blob" => walk.allowed_blob_memo.insert(repo.id.clone(), set), + "tree" => walk.allowed_tree_memo.insert(repo.id.clone(), set), + _ => walk.reachable_ct_memo.insert(repo.id.clone(), set), + }; + } + let in_set = match obj_type.as_str() { + "blob" => walk.allowed_blob_memo.get(&repo.id), + "tree" => walk.allowed_tree_memo.get(&repo.id), + _ => walk.reachable_ct_memo.get(&repo.id), + } + .is_some_and(|set| set.contains(sha256_hex)); + if !in_set { + return GateOutcome::Skip; + } } - // Not found in any repo - Err(AppError::RepoNotFound(format!( - "no git object found for CID {cid_str}" - ))) + // Passed the gate — bound the object, read it OFF the async worker, and verify the + // content address, all before any byte egresses. F6 (#173): read_object_content runs a + // blocking `git cat-file` and buffers the whole object; called directly on the Axum + // worker (the type-probe and walk are already off-worker) it blocks a runtime thread, + // and unbounded it can exhaust memory for a large public blob (enumerable from the pins + // index). Precheck the SIZE and run size + read + verify inside spawn_blocking. A + // content-addressed serve cannot verify a STREAMED body (the digest is known only after + // the last byte, by which point the prefix has already egressed), so we never stream: + // buffer-verify-then-serve up to the cap and withhold anything larger. F2's integrity + // check moves in here too, so no unverified bytes are ever assembled into a response. + let max_bytes = state.ipfs_max_served_object_bytes; + let read_repo = repo_path.clone(); + let read_sha = sha256_hex.to_string(); + let read_type = obj_type.clone(); + let want_cid = ctx.canonical_cid.to_string(); + let read = tokio::task::spawn_blocking(move || -> ServedRead { + match store::object_size(&read_repo, &read_sha) { + Ok(Some(size)) if size > max_bytes => return ServedRead::TooLarge(size), + Ok(Some(_)) => {} + // git ran and reported no such object (or an unparseable size): genuine + // not-found for this candidate. + Ok(None) => return ServedRead::Gone, + // git itself failed to run: an infra failure, not a not-found. + Err(e) => return ServedRead::ReadErr(e.to_string()), + } + let content = match store::read_object_content(&read_repo, &read_sha, &read_type) { + Ok(c) => c, + Err(e) => return ServedRead::ReadErr(e.to_string()), + }; + let served = gitlawb_core::cid::Cid::from_git_object_bytes(&content).to_string(); + if served != want_cid { + return ServedRead::Mismatch(served); + } + ServedRead::Ok(content) + }) + .await; + let content = match read { + Ok(ServedRead::Ok(c)) => c, + Ok(ServedRead::TooLarge(size)) => { + tracing::warn!( + repo = %repo.name, size, max = max_bytes, + "withholding object: exceeds the served-object size cap (F6)" + ); + #[cfg(test)] + note_oversize_reject(); + return GateOutcome::Skip; + } + Ok(ServedRead::Mismatch(served)) => { + tracing::warn!( + repo = %repo.name, requested = %ctx.canonical_cid, served = %served, + "withholding object: served bytes do not hash to the requested CID (legacy provider-CID row?)" + ); + return GateOutcome::Skip; + } + Ok(ServedRead::Gone) => return GateOutcome::Skip, + Ok(ServedRead::ReadErr(e)) => { + // Infra failure (git spawn/IO), NOT a not-found: mark the search truncated so + // a wholly-unserved request tails to a retryable 503, never a definitive 404 + // for an authorized caller (INV-25 spirit — logging alone is not surfacing). + tracing::warn!(repo = %repo.name, err = %e, "error reading git object content"); + walk.truncated = true; + return GateOutcome::Skip; + } + Err(e) => { + tracing::warn!(repo = %repo.name, err = %e, "object read task panicked"); + walk.truncated = true; + return GateOutcome::Skip; + } + }; + let mut resp_headers = HeaderMap::new(); + resp_headers.insert( + HeaderName::from_static("content-type"), + HeaderValue::from_static("application/octet-stream"), + ); + resp_headers.insert( + HeaderName::from_static("x-content-cid"), + HeaderValue::from_str(ctx.cid_str).unwrap_or_else(|_| HeaderValue::from_static("invalid")), + ); + resp_headers.insert( + HeaderName::from_static("x-git-hash"), + HeaderValue::from_str(sha256_hex).unwrap_or_else(|_| HeaderValue::from_static("invalid")), + ); + GateOutcome::Served((StatusCode::OK, resp_headers, content).into_response()) } /// GET /api/v1/ipfs/pins @@ -303,6 +741,58 @@ pub async fn list_pins(State(state): State) -> Result = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn reset_preload_queries() { + PRELOAD_QUERIES.with(|c| c.set(0)); +} + +#[cfg(test)] +pub(crate) fn preload_queries() -> usize { + PRELOAD_QUERIES.with(|c| c.get()) +} + +#[cfg(test)] +fn bump_preload_queries() { + PRELOAD_QUERIES.with(|c| c.set(c.get() + 1)); +} + +// Test-only INV-10 cost counter (F6, U6/U7): how many times the serve path withheld an +// object because it exceeded `ipfs_max_served_object_bytes`. The bounded read must reject +// an oversized object rather than buffer it on the worker; the counter is the both-ways +// guard (a removed size precheck stops incrementing it and serves the oversized object). +// Set from the match arm after `spawn_blocking` resolves, i.e. on the test's runtime +// thread, so the thread-local is read on the same thread it is written. +#[cfg(test)] +thread_local! { + static OVERSIZE_REJECTS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn reset_oversize_rejects() { + OVERSIZE_REJECTS.with(|c| c.set(0)); +} + +#[cfg(test)] +pub(crate) fn oversize_rejects() -> usize { + OVERSIZE_REJECTS.with(|c| c.get()) +} + +#[cfg(test)] +fn note_oversize_reject() { + OVERSIZE_REJECTS.with(|c| c.set(c.get() + 1)); +} + #[cfg(test)] mod tests { //! #174 P1-3 (U3): the public `GET /ipfs/{cid}` walk carries bounded CONCURRENCY @@ -310,7 +800,9 @@ mod tests { //! walk, plus a per-IP route rate limit. These are handler-layer proofs: mount the //! real handler/router, drive one request, assert the exact 503 shed, then name the //! mutation that turns each RED. The per-source key resolves an IP only (`Some(ip)` - //! vs `None`), never a DID — both arms are driven so neither is vacuous. + //! vs `None`), never a DID — both arms are driven so neither is vacuous. The + //! CID-resolution / visibility-gate behavior of the handler itself is covered by the + //! `#[sqlx::test]` suite in `test_support.rs`. use axum::body::Body; use axum::extract::ConnectInfo; @@ -500,29 +992,33 @@ mod tests { drop(held); } - /// Retain-through-blocking (R3, the load-bearing async property): the walk - /// admission is held until the `spawn_blocking` walk actually RETURNS, not when a - /// tokio timeout fires. With the global pool at size 1, drive a request until its - /// walk (a fake git that hangs on `rev-list`) is in flight; the slot must stay held - /// (`available_permits() == 0`) and a replacement from a DIFFERENT source must shed - /// 503 for as long as the blocking walk runs — even though the request future is - /// only `.await`ing the blocking join. When the blocking walk ends the permit frees - /// and a replacement is admitted. The permit lives INSIDE the handler across the - /// blocking `.await`; move it out (drop before the walk) and the replacement would - /// be admitted while the walk still burns a blocking thread (the bug this guards). + /// Retain-through-blocking (#174 F5, the load-bearing async property, on the + /// NEWLY-BOUNDED TREE path): the walk admission is held until the `spawn_blocking` + /// walk actually RETURNS, not when a tokio timeout fires. The requested CID + /// resolves to a TREE object under a path-scoped rule, so the gate runs + /// `allowed_tree_set_for_caller_bounded` — the walk this integration converts to + /// `run_bounded_git` — rather than the blob walk #174 already proved. With the + /// global pool at size 1, drive a request until its walk (a fake git that hangs on + /// `rev-list`) is in flight; the slot must stay held (`available_permits() == 0`) + /// and a replacement from a DIFFERENT source must shed 503 for as long as the + /// blocking walk runs — even though the request future is only `.await`ing the + /// blocking join. When the blocking walk ends the permit frees and a replacement + /// is admitted. The permit lives INSIDE the handler across the blocking `.await`; + /// move it out (drop before the walk) and the replacement would be admitted while + /// the walk still burns a blocking thread (the bug this guards). #[cfg(unix)] #[sqlx::test] - async fn get_by_cid_walk_permit_held_through_blocking_walk(pool: sqlx::PgPool) { + async fn get_by_cid_walk_permit_held_through_bounded_tree_walk(pool: sqlx::PgPool) { use std::process::Command; let tmp = tempfile::TempDir::new().unwrap(); let revlist_pid = tmp.path().join("revlist.pid"); - // Fake git for the /ipfs WALK only (object_type/read_object_content use the real - // `git`, so the object must genuinely exist below). Empty refs (so - // assert_all_refs_are_commits returns Ok without the peel), `rev-parse` resolves, - // and `rev-list` records its pid then sleeps ~6s so the walk BLOCKS - // deterministically. The sleep bounds the walk so a broken fix cannot wedge the - // suite. + // Fake git for the /ipfs TREE walk only (object_type/read_object_content use + // the real `git`, so the tree must genuinely exist below). `rev-parse` + // resolves (so the lenient enumeration appends HEAD) and `rev-list` records + // its pid then sleeps ~6s so the walk BLOCKS deterministically inside + // `run_bounded_git`. The sleep bounds the walk so a broken fix cannot wedge + // the suite. let body = format!( "#!/bin/sh\n\ case \"$1\" in\n\ @@ -554,18 +1050,18 @@ mod tests { state.git_bin = git_path.to_str().unwrap().to_string(); state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; - let owner = "z6ipfs1"; - let name = "ip1"; + let owner = "z6ipfstree"; + let name = "iptree"; state .db .upsert_mirror_repo(owner, name, "/unused", None, false) .await .unwrap(); let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); - // The exact bare path the handler's `acquire` resolves. Build a REAL SHA-256 bare - // repo there with a committed blob under `src/`, so real `git cat-file -t ` classifies it as a blob (the CID digest IS the sha256 object id in - // object-format=sha256) and the handler reaches the path-scoped walk branch. + // The exact bare path the handler's `acquire` resolves. Build a REAL SHA-256 + // bare repo there with a committed `src/` directory, so real + // `git cat-file -t ` classifies the requested object as a TREE and the + // handler routes into the tree-walk arm of the gate. let bare = state .repo_store .acquire(&rec.owner_did, &rec.name) @@ -587,7 +1083,11 @@ mod tests { }; let work = tmp.path().join("work"); std::fs::create_dir_all(work.join("src")).unwrap(); - std::fs::write(work.join("src/secret.txt"), b"ipfs walk retain proof\n").unwrap(); + std::fs::write( + work.join("src/secret.txt"), + b"ipfs tree walk retain proof\n", + ) + .unwrap(); run( &["init", "-q", "--object-format=sha256", "-b", "main"], &work, @@ -606,38 +1106,46 @@ mod tests { ], tmp.path(), ); - // The blob's SHA-256 object id (= the CID's digest); build the CID from it. - let oid = { + // The `src` directory's TREE oid — the object the request asks for. + let tree_oid = { let out = Command::new("git") - .args(["rev-parse", "HEAD:src/secret.txt"]) + .args(["rev-parse", "HEAD:src"]) .current_dir(&work) .output() .expect("git rev-parse runs"); assert!(out.status.success(), "rev-parse failed"); String::from_utf8_lossy(&out.stdout).trim().to_string() }; - let oid_bytes = gitlawb_core::cid::sha256_hex_to_bytes(&oid).unwrap(); - let cid = gitlawb_core::cid::Cid::from_sha256_bytes(&oid_bytes) - .as_str() - .to_string(); - // Precondition: real git classifies the object as a blob (so the handler reaches - // the walk branch, not an early `continue`). + // Precondition: real git classifies the object as a TREE (so the handler + // reaches the tree-walk arm, not the blob arm or an early `continue`). assert_eq!( - crate::git::store::object_type(&bare, &oid) + crate::git::store::object_type(&bare, &tree_oid) .unwrap() .as_deref(), - Some("blob"), - "the seeded sha256 blob must exist so the handler reaches the walk" + Some("tree"), + "the seeded sha256 tree must exist so the handler reaches the tree walk" ); - // A path-scoped rule so has_path_scoped_rule() is true (the walk branch) without - // denying the "/" gate on the public repo. + // Pin the tree's content CID WITH provenance so the resolver targets this one + // repo (no legacy scan). A real pin CID digests the raw object content, not + // the git oid, so build it exactly as the pin path does (#173). + let (_ty, raw) = crate::git::store::read_object(&bare, &tree_oid) + .unwrap() + .expect("tree object readable"); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(&raw).to_string(); + state + .db + .record_pinned_cid(&tree_oid, &cid, Some(&rec.id)) + .await + .unwrap(); + // A path-scoped rule so has_path_scoped_rule() is true (the tree-gate branch) + // without denying the "/" gate on the public repo. state .db .set_visibility_rule( &rec.id, - "src/**", + "/src/**", crate::db::VisibilityMode::B, - &["did:key:z6MkU3IpfsReaderAAAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &["did:key:z6MkF5IpfsTreeReaderAAAAAAAAAAAAAAAAAAAA".to_string()], &rec.owner_did, ) .await @@ -663,10 +1171,10 @@ mod tests { let peer: SocketAddr = "203.0.113.81:5000".parse().unwrap(); let mut fut = Box::pin(router.clone().oneshot(make_req(peer))); - // Drive until the fake git's rev-list records its pid — the walk is now in the - // blocking pool and the request future is `.await`ing its join, holding the walk - // permit. Stop polling the instant the future completes (re-polling a completed - // oneshot panics). + // Drive until the fake git's rev-list records its pid — the TREE walk is now in + // the blocking pool and the request future is `.await`ing its join, holding the + // walk permit. Stop polling the instant the future completes (re-polling a + // completed oneshot panics). let mut walk_pid: Option = None; let mut early = None; for _ in 0..500 { @@ -696,28 +1204,28 @@ mod tests { } let _cleanup = ReapOnDrop(pid); - // Load-bearing: while the blocking walk runs, the slot is HELD and a replacement - // from a DIFFERENT source sheds 503 — proving the permit is retained across the - // spawn_blocking join, not freed by a tokio timeout. + // Load-bearing: while the blocking TREE walk runs, the slot is HELD and a + // replacement from a DIFFERENT source sheds 503 — proving the permit is + // retained across the spawn_blocking join, not freed by a tokio timeout. assert_eq!( sem.available_permits(), 0, - "the walk slot must be held while the spawn_blocking walk runs" + "the walk slot must be held while the spawn_blocking tree walk runs" ); let peer2: SocketAddr = "203.0.113.82:5000".parse().unwrap(); let resp = router.clone().oneshot(make_req(peer2)).await.unwrap(); assert_eq!( resp.status(), StatusCode::SERVICE_UNAVAILABLE, - "a replacement must shed 503 while the prior request's blocking walk still runs" + "a replacement must shed 503 while the prior request's blocking tree walk still runs" ); // Drop the in-flight request; the detached blocking walk keeps running (a - // spawn_blocking cannot be cancelled), but on the fix the permit is a handler - // local, so dropping the future releases it once the blocking join is abandoned. - // Either way, kill the sleeping child so the slot frees promptly and poll for - // recovery — the point already proven above is that the slot stayed held for the - // duration of the blocking work. + // spawn_blocking cannot be cancelled), but the permit is a handler local, so + // dropping the future releases it once the blocking join is abandoned. Either + // way, kill the sleeping child so the slot frees promptly and poll for + // recovery — the point already proven above is that the slot stayed held for + // the duration of the blocking work. drop(fut); unsafe { libc::kill(pid, libc::SIGKILL); @@ -736,155 +1244,6 @@ mod tests { ); } - /// Loop bound (cap N): one `/ipfs/{cid}` request against a CID present in many repos - /// must not serialize an unbounded number of full-history walks. With - /// `ipfs_max_repos_walked = 1` and TWO public, path-scoped repos both carrying the - /// blob at the requested CID, the handler walks only the FIRST candidate then stops - /// (the second is cut by the cap), so the fake git's `rev-list` (one per walk) runs - /// exactly once. MUTATION (RED): remove the `repos_walked >= cap` break and both - /// repos are walked (count 2). - #[cfg(unix)] - #[sqlx::test] - async fn get_by_cid_caps_repos_walked_per_request(pool: sqlx::PgPool) { - use std::process::Command; - - let tmp = tempfile::TempDir::new().unwrap(); - let walk_log = tmp.path().join("walks.log"); - // Fake git for the WALK: empty refs, `rev-parse` resolves, and each `rev-list` - // appends one line to a log (so the number of walks == the line count) and exits - // with EMPTY output (the allowed-set is empty, so every repo path-gates to a - // `continue` and the request 404s after walking). object_type uses the REAL git, - // so the seeded blob below must genuinely exist. - let body = format!( - "#!/bin/sh\n\ - case \"$1\" in\n\ - for-each-ref) : ;;\n\ - rev-parse) echo deadbeef ;;\n\ - rev-list) echo walk >> \"{}\" ;;\n\ - *) : ;;\n\ - esac\n\ - exit 0\n", - walk_log.display() - ); - let git_path = tmp.path().join("fakegit"); - std::fs::write(&git_path, &body).unwrap(); - { - use std::os::unix::fs::PermissionsExt; - let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); - perm.set_mode(0o755); - std::fs::set_permissions(&git_path, perm).unwrap(); - } - - let mut state = crate::test_support::test_state(pool.clone()).await; - let repos_dir = tmp.path().join("repos"); - std::fs::create_dir_all(&repos_dir).unwrap(); - state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); - state.git_bin = git_path.to_str().unwrap().to_string(); - state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; - // The bound under test: walk at most one candidate repo per request. - let mut cfg = (*state.config).clone(); - cfg.ipfs_max_repos_walked = 1; - state.config = Arc::new(cfg); - - // Seed TWO public repos, each with the SAME blob (same content -> same sha256 OID - // -> same CID) under a path-scoped rule, so both are walk candidates for one CID. - let run = |args: &[&str], cwd: &std::path::Path| { - let out = Command::new("git") - .args(args) - .current_dir(cwd) - .output() - .expect("git runs"); - assert!( - out.status.success(), - "git {args:?} failed: {}", - String::from_utf8_lossy(&out.stderr) - ); - }; - let mut oid = String::new(); - for (i, name) in ["ipa", "ipb"].iter().enumerate() { - let owner = "z6ipfsN"; - state - .db - .upsert_mirror_repo(owner, name, &format!("/unused-{name}"), None, false) - .await - .unwrap(); - let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); - let bare = state - .repo_store - .acquire(&rec.owner_did, &rec.name) - .await - .unwrap(); - let _ = std::fs::remove_dir_all(&bare); - std::fs::create_dir_all(&bare).unwrap(); - let work = tmp.path().join(format!("work{i}")); - std::fs::create_dir_all(work.join("src")).unwrap(); - // Identical content in both repos -> identical sha256 blob OID -> one CID. - std::fs::write(work.join("src/secret.txt"), b"loop bound proof\n").unwrap(); - run( - &["init", "-q", "--object-format=sha256", "-b", "main"], - &work, - ); - run(&["config", "user.email", "t@t"], &work); - run(&["config", "user.name", "t"], &work); - run(&["add", "src/secret.txt"], &work); - run(&["commit", "-q", "-m", "seed"], &work); - run( - &[ - "clone", - "--bare", - "-q", - work.to_str().unwrap(), - bare.to_str().unwrap(), - ], - tmp.path(), - ); - if oid.is_empty() { - let out = Command::new("git") - .args(["rev-parse", "HEAD:src/secret.txt"]) - .current_dir(&work) - .output() - .expect("git rev-parse runs"); - oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); - } - state - .db - .set_visibility_rule( - &rec.id, - "src/**", - crate::db::VisibilityMode::B, - &["did:key:z6MkU3IpfsReaderBBBBBBBBBBBBBBBBBBBBBBBB".to_string()], - &rec.owner_did, - ) - .await - .unwrap(); - } - let oid_bytes = gitlawb_core::cid::sha256_hex_to_bytes(&oid).unwrap(); - let cid = gitlawb_core::cid::Cid::from_sha256_bytes(&oid_bytes) - .as_str() - .to_string(); - - let peer: SocketAddr = "203.0.113.90:5000".parse().unwrap(); - let mut req = Request::builder() - .method(Method::GET) - .uri(format!("/ipfs/{cid}")) - .body(Body::empty()) - .unwrap(); - req.extensions_mut().insert(ConnectInfo(peer)); - let resp = ipfs_router(state).oneshot(req).await.unwrap(); - // The empty allowed-set path-gates both repos to a `continue`, so a 404; the - // point is HOW MANY walks ran to get there. - assert_eq!(resp.status(), StatusCode::NOT_FOUND); - - let walks = std::fs::read_to_string(&walk_log) - .map(|s| s.lines().count()) - .unwrap_or(0); - assert_eq!( - walks, 1, - "with the per-request repo-walk cap at 1, only the first candidate repo is \ - walked (the second is cut by the cap), so exactly one walk runs; got {walks}" - ); - } - /// Route rate limit is WIRED (not a silent no-op): the production `build_router` /// attaches an `IpRateLimiter` extension to the `/ipfs/{cid}` route, so a per-IP /// flood is braked with 429. A bare `rate_limit_by_ip` layer with no extension does diff --git a/crates/gitlawb-node/src/api/mod.rs b/crates/gitlawb-node/src/api/mod.rs index 10d76daf..86594c03 100644 --- a/crates/gitlawb-node/src/api/mod.rs +++ b/crates/gitlawb-node/src/api/mod.rs @@ -200,9 +200,11 @@ mod authz_guard { (issues, "create_issue", "authorize_repo_read("), (bounties, "create_bounty", "authorize_repo_read("), (repos, "fork_repo", "authorize_repo_read("), - // get_by_cid gates each iterated repo row directly via visibility_check - // (KTD2a: it must NOT route through authorize_repo_read's fuzzy re-resolve). - (ipfs, "get_by_cid", "visibility_check("), + // get_by_cid resolves each candidate (provenance path + legacy scan) through + // the shared `gate_and_serve` (#173 round 2); the gate markers themselves are + // asserted below. This row proves the delegation is real — the gate is + // actually reached, not dead code. + (ipfs, "get_by_cid", "gate_and_serve("), // Bucket C — signer-self: the acting DID is matched/bound to auth.0 (tasks, "create_task", "did_matches("), (tasks, "claim_task", "did_matches("), @@ -236,6 +238,22 @@ mod authz_guard { "visibility::require_owner must use did_matches for DID-safe owner matching" ); + // The CID read surface (#173) enforces its gate inside the shared + // `gate_and_serve`, which BOTH the provenance path and the legacy scan call, so + // the markers must live there (the get_by_cid row above only proves delegation). + // The repo's own "/" visibility check (KTD2a — never authorize_repo_read's fuzzy + // re-resolve) and the quarantine hard-drop BEFORE visibility (INV-11) are both + // load-bearing: removing either re-opens a leak on the provenance path. + let gate_body = fn_body(ipfs, "gate_and_serve"); + assert!( + gate_body.contains("visibility_check("), + "gate_and_serve must gate the CID read surface via visibility_check (KTD2a)" + ); + assert!( + gate_body.contains("if quarantined"), + "gate_and_serve must hard-drop a quarantined repo before the visibility gate (INV-11)" + ); + for (src, func, marker) in rows { let body = fn_body(src, func); assert!( diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 9b05ece1..cb6ed541 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1473,6 +1473,7 @@ pub async fn git_receive_pack( &repo_path_clone, object_list_ipfs, &db_clone, + &repo_id, ) .await; if !pinned.is_empty() { @@ -1573,6 +1574,7 @@ pub async fn git_receive_pack( let pinata_upload_url = state.config.pinata_upload_url.clone(); let repo_path_clone = disk_path.clone(); let db_clone = state.db.clone(); + let repo_id = record.id.clone(); let http_client = Arc::clone(&state.http_client); let node_did_str = state.node_did.to_string(); let repo_slug = format!( @@ -1603,6 +1605,7 @@ pub async fn git_receive_pack( &repo_path_clone, object_list_pinata, &db_clone, + &repo_id, ) .await } else { diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 912108a0..c03eb2e5 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -516,6 +516,10 @@ mod tests { rate_limiter: RateLimiter::new(100, Duration::from_secs(60)), create_ip_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), push_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, + ipfs_max_legacy_probes: crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, + ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, push_limiter_trust: crate::rate_limit::TrustedProxy::None, sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), @@ -532,7 +536,6 @@ mod tests { git_ipfs_walk_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_ipfs_walk_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), - ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), git_bin: "git".to_string(), } } diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 5324a4b9..6b0d394e 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -872,8 +872,63 @@ const MIGRATIONS: &[Migration] = &[ "CREATE UNIQUE INDEX IF NOT EXISTS idx_ref_certs_repo_ref ON ref_certificates(repo_id, ref_name)", ], }, + Migration { + version: 11, + name: "pinned_cids_cid_index", + stmts: &[ + // GET /ipfs/{cid} resolves an incoming CID -> git oid via pinned_cids.cid + // (#173); index it so the per-request lookup is not a table scan. This is + // a NEW versioned migration (not appended to the applied v1 bundle) so a + // node already past v1 actually gets the index. Non-unique on purpose: cid + // is a function of raw content, so a UNIQUE index could reject a legitimate + // record_pinned_cid insert, and colliding rows serve byte-identical content. + "CREATE INDEX IF NOT EXISTS idx_pinned_cids_cid ON pinned_cids(cid)", + ], + }, + Migration { + version: 12, + name: "pinned_cids_repo_provenance", + stmts: &[ + // Record the repository a pin came from so GET /ipfs/{cid} resolves a + // provenanced pin straight to its ONE source repo instead of scanning every + // repo (#173, jatmn round 2 — bounds the anonymous fan-out and removes the + // updated_at-ordering false-404). NEW versioned migration (never appended to + // the applied v1 pinned_cids table) so a node past v1 gets the column. + // Nullable: pins recorded before this migration have no provenance and fall + // back to the legacy repo scan; new pins carry repo_id and resolve to one + // repo. Indexed for the resolver's oid -> repo_id lookup. + "ALTER TABLE pinned_cids ADD COLUMN IF NOT EXISTS repo_id TEXT", + "CREATE INDEX IF NOT EXISTS idx_pinned_cids_repo_id ON pinned_cids(repo_id)", + ], + }, + Migration { + version: 13, + name: "pin_repo_sources", + stmts: &[ + // F1 (#173, jatmn round 8): a shared object (a blob/tree/commit common to + // forks and mirrors) can be pinned from more than one repo. `pinned_cids` + // keeps only the FIRST pinner's `repo_id`, so a shared object first pinned + // from a private/quarantined repo 404s by CID even when a later PUBLIC repo + // also pinned it. Record EVERY pin-path source so `GET /ipfs/{cid}` can try + // each. NEW versioned migration (never appended to an applied block, INV-7). + // Bounded per object at insert time (MAX_PIN_SOURCES) so an adversary pushing + // one object from N repos cannot make resolution O(repos) (R2, INV-10). + "CREATE TABLE IF NOT EXISTS pin_repo_sources ( + sha256_hex TEXT NOT NULL, + repo_id TEXT NOT NULL, + PRIMARY KEY (sha256_hex, repo_id) + )", + "CREATE INDEX IF NOT EXISTS idx_pin_repo_sources_sha ON pin_repo_sources(sha256_hex)", + ], + }, ]; +/// Max distinct source repos recorded per pinned object (F1, #173 jatmn round 8). +/// Bounds both the resolver's per-OID source loop and the `pin_repo_sources` growth, +/// so an adversary re-pushing one object from many repos cannot make resolution +/// O(repos) (R2, INV-10). +pub const MAX_PIN_SOURCES: i64 = 16; + // ── Repos ───────────────────────────────────────────────────────────────────── pub(crate) fn normalize_owner_key(did: &str) -> &str { @@ -1046,6 +1101,22 @@ impl Db { Ok(row.map(row_to_repo)) } + /// Fetch a repo by its stable `id`. Used by the `/ipfs/{cid}` provenance path, + /// which resolves a pin straight to its ONE source repo (#173) instead of + /// scanning `list_all_repos`. `id` is exact, so unlike `get_repo`'s fuzzy + /// owner/name match there is no mirror-vs-canonical disambiguation. + pub async fn get_repo_by_id(&self, id: &str) -> Result> { + let row = sqlx::query( + "SELECT id, name, owner_did, description, is_public, default_branch, + created_at, updated_at, disk_path, forked_from, machine_id + FROM repos WHERE id = $1 LIMIT 1", + ) + .bind(id) + .fetch_optional(&self.pool) + .await?; + Ok(row.map(row_to_repo)) + } + #[allow(dead_code)] pub async fn list_repos(&self, owner_did: &str) -> Result> { let rows = sqlx::query( @@ -2166,20 +2237,148 @@ impl Db { Ok(row.get::("cnt") > 0) } - pub async fn record_pinned_cid(&self, sha256_hex: &str, cid: &str) -> Result<()> { + /// Every git oid a pinned CID maps to (`pinned_cids.cid` -> `sha256_hex`). + /// `GET /ipfs/{cid}` resolves the content-addressed CID a client sends back to + /// the object's git oid this way: a real pin CID digests the raw object + /// content, not the git oid, so the digest cannot be `git cat-file`d directly + /// (#173). The index is unique on the git oid but NON-unique on cid, so two + /// distinct oids can share one content-CID (a tree and a blob whose raw bytes + /// collide, or byte-identical content pinned under two oids). Returning every + /// candidate lets the handler try each rather than pick one arbitrarily and + /// false-404 when the chosen one is withheld or absent while another is + /// readable (#173). Empty when the CID was never pinned on this node. + pub async fn oids_for_cid(&self, cid: &str) -> Result> { + let rows = sqlx::query("SELECT sha256_hex FROM pinned_cids WHERE cid = $1") + .bind(cid) + .fetch_all(&self.pool) + .await?; + Ok(rows + .into_iter() + .map(|r| r.get::("sha256_hex")) + .collect()) + } + + /// Record a pinned object's CID and the repository it was pinned from + /// (`repo_id`, #173). On conflict the `COALESCE` backfills a NULL provenance + /// from a known source while keeping first-pinner-owns: an existing non-NULL + /// `repo_id` is never rewritten by a later push of the same oid, but a legacy + /// pin (or a pin recorded before provenance existed) whose `repo_id` is NULL + /// gets it filled the next time the object is re-pinned with a known source. + /// `cid`/`pinned_at` are left untouched on conflict. `repo_id` is `None` only + /// for a legacy pin with no known source; those fall back to the resolver's scan. + pub async fn record_pinned_cid( + &self, + sha256_hex: &str, + cid: &str, + repo_id: Option<&str>, + ) -> Result<()> { sqlx::query( - "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) - VALUES ($1, $2, $3) - ON CONFLICT(sha256_hex) DO NOTHING", + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) + VALUES ($1, $2, $3, $4) + ON CONFLICT(sha256_hex) DO UPDATE SET + repo_id = COALESCE(pinned_cids.repo_id, EXCLUDED.repo_id)", ) .bind(sha256_hex) .bind(cid) .bind(Utc::now().to_rfc3339()) + .bind(repo_id) + .execute(&self.pool) + .await?; + Ok(()) + } + + /// The repository a pinned object was recorded from (`pinned_cids.repo_id`), + /// or `None` for a legacy pin (recorded before provenance existed) or an + /// unpinned oid. `GET /ipfs/{cid}` uses this to gate+serve the ONE source + /// repo instead of scanning every repo (#173). + pub async fn provenance_for_oid(&self, sha256_hex: &str) -> Result> { + let row = sqlx::query("SELECT repo_id FROM pinned_cids WHERE sha256_hex = $1") + .bind(sha256_hex) + .fetch_optional(&self.pool) + .await?; + Ok(row.and_then(|r| r.get::, _>("repo_id"))) + } + + /// Backfill the source repo on an already-pinned object whose provenance is + /// NULL (a legacy pin recorded before provenance existed, #173, jatmn). The + /// `AND repo_id IS NULL` guard keeps first-pinner-owns: an existing non-NULL + /// provenance is left untouched. Touches only `repo_id` and never re-pins the + /// object's bytes, so it is safe to call on the already-pinned skip path. + pub async fn backfill_pin_provenance(&self, sha256_hex: &str, repo_id: &str) -> Result<()> { + sqlx::query( + "UPDATE pinned_cids SET repo_id = $2 WHERE sha256_hex = $1 AND repo_id IS NULL", + ) + .bind(sha256_hex) + .bind(repo_id) .execute(&self.pool) .await?; Ok(()) } + /// Record a repository as a source for a pinned object (F1, #173 jatmn round 8), + /// bounded to about `MAX_PIN_SOURCES` distinct repos per object. The count guard + /// lives inside the INSERT (a single statement), which suppresses a re-push of the + /// SAME `(oid, repo)` via `ON CONFLICT DO NOTHING`. It does NOT hard-serialize + /// concurrent inserts of DIFFERENT repos for the same object: under Postgres READ + /// COMMITTED each concurrent writer's count subquery reads a snapshot that omits the + /// others' uncommitted rows, so N concurrent pushers can each see `count < cap` and + /// overshoot by up to N-1 rows. The overshoot is a small constant (bounded by + /// concurrent-pusher count, never O(repos)), and the RESOLVER read side + /// (`pin_sources_for_oid`) caps the ADDITIONAL sources at `MAX_PIN_SOURCES` (always + /// keeping the first-pinner), so the INV-10 bound on serve-time work holds at + /// `O(MAX_PIN_SOURCES + 1)` regardless of a table overshoot. + pub async fn record_pin_source(&self, sha256_hex: &str, repo_id: &str) -> Result<()> { + sqlx::query( + "INSERT INTO pin_repo_sources (sha256_hex, repo_id) + SELECT $1, $2 + WHERE (SELECT count(*) FROM pin_repo_sources WHERE sha256_hex = $1) < $3 + ON CONFLICT DO NOTHING", + ) + .bind(sha256_hex) + .bind(repo_id) + .bind(MAX_PIN_SOURCES) + .execute(&self.pool) + .await?; + Ok(()) + } + + /// Every source repository recorded for a pinned object (F1, #173 jatmn round 8): + /// the union of the first-pinner `pinned_cids.repo_id` and the `pin_repo_sources` + /// rows, deduped and ordered for a deterministic resolver walk. + /// + /// The first-pinner (a single row by `pinned_cids`' PK on `sha256_hex`) is ALWAYS + /// included; the `LIMIT MAX_PIN_SOURCES` caps only the ADDITIONAL `pin_repo_sources` + /// rows. This keeps the resolver's per-source work a bounded `O(MAX_PIN_SOURCES + 1)` + /// ceiling (INV-10) while never letting the cap evict the original source. A prior + /// version applied the `LIMIT` to the whole UNION with a lexicographic `ORDER BY`, + /// which let an attacker 404 a legacy public CID (first-pinner in `pinned_cids` but + /// not yet in `pin_repo_sources`) by pushing the same object from `MAX_PIN_SOURCES` + /// repos whose grindable ids sort before it, evicting the public source from the + /// window. Empty for a legacy pin with no known source (it falls back to the repo + /// scan) or an unpinned oid. + pub async fn pin_sources_for_oid(&self, sha256_hex: &str) -> Result> { + let rows = sqlx::query( + "SELECT repo_id FROM pinned_cids + WHERE sha256_hex = $1 AND repo_id IS NOT NULL + UNION + SELECT repo_id FROM ( + SELECT repo_id FROM pin_repo_sources + WHERE sha256_hex = $1 + ORDER BY repo_id + LIMIT $2 + ) capped + ORDER BY repo_id", + ) + .bind(sha256_hex) + .bind(MAX_PIN_SOURCES) + .fetch_all(&self.pool) + .await?; + Ok(rows + .into_iter() + .map(|r| r.get::("repo_id")) + .collect()) + } + pub async fn record_encrypted_blob( &self, repo_id: &str, @@ -2279,18 +2478,34 @@ impl Db { } /// Record the Pinata CID for a git object. - /// Inserts the row if it doesn't exist (objects pinned directly to Pinata - /// without a prior local IPFS pin get cid = pinata_cid). - pub async fn record_pinata_cid(&self, sha256_hex: &str, pinata_cid: &str) -> Result<()> { + /// + /// `raw_cid` is the locally-computed raw-content CID (`Cid::from_git_object_bytes`, + /// CIDv1/raw/sha2-256), the resolver key `GET /ipfs/{cid}` looks up; `pinata_cid` + /// is the provider CID Pinata returned (a dag-pb/UnixFS CID for gateway retrieval). + /// Inserts the row if it doesn't exist (an object pinned directly to Pinata with + /// no prior local IPFS pin gets `cid = raw_cid`, never the provider CID — a dag-pb + /// provider CID must never become an alias that serves raw bytes that do not hash + /// to it, #173). On conflict `cid` is left untouched: a prior local pin already + /// stored the correct raw CID, and the COALESCE backfills a NULL provenance from a + /// known source while keeping first-pinner-owns. + pub async fn record_pinata_cid( + &self, + sha256_hex: &str, + raw_cid: &str, + pinata_cid: &str, + repo_id: Option<&str>, + ) -> Result<()> { sqlx::query( - "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid) - VALUES ($1, $2, $3, $4) - ON CONFLICT(sha256_hex) DO UPDATE SET pinata_cid = EXCLUDED.pinata_cid", + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid, repo_id) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT(sha256_hex) DO UPDATE SET pinata_cid = EXCLUDED.pinata_cid, + repo_id = COALESCE(pinned_cids.repo_id, EXCLUDED.repo_id)", ) .bind(sha256_hex) - .bind(pinata_cid) // fallback local cid if row is new + .bind(raw_cid) // resolver-key cid: locally-computed raw CID, never the provider CID .bind(Utc::now().to_rfc3339()) .bind(pinata_cid) + .bind(repo_id) .execute(&self.pool) .await?; Ok(()) @@ -5037,6 +5252,51 @@ mod ref_certificate_tests { ); } + /// INV-7: upgrade-path test — an existing node already past v1 must still get + /// the `pinned_cids.cid` index. It ships as its OWN v11 migration (not appended + /// to the applied v1 bundle), so dropping the index + its `schema_migrations` + /// row and re-running migrations must recreate it, exercising the real code + /// path rather than hand-copying the SQL. + #[sqlx::test] + async fn v11_pinned_cids_cid_index_applies_on_upgrade(pool: PgPool) { + async fn index_exists(pool: &PgPool) -> bool { + sqlx::query_scalar::<_, bool>( + "SELECT EXISTS(SELECT 1 FROM pg_indexes WHERE indexname = 'idx_pinned_cids_cid')", + ) + .fetch_one(pool) + .await + .unwrap() + } + + let db = Db::for_testing(pool.clone()); + db.run_migrations().await.unwrap(); + assert!( + index_exists(&pool).await, + "fresh migration chain creates the index" + ); + + // Simulate a node at v10 (pre-v11): drop the index and its migration record. + sqlx::query("DROP INDEX IF EXISTS idx_pinned_cids_cid") + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version = 11") + .execute(&pool) + .await + .unwrap(); + assert!( + !index_exists(&pool).await, + "precondition: index and its migration record removed" + ); + + // Re-run migrations: v11 re-applies and recreates the index on the upgrade. + db.run_migrations().await.unwrap(); + assert!( + index_exists(&pool).await, + "v11 must recreate idx_pinned_cids_cid on an upgrading node" + ); + } + /// INV-7: upgrade-path test — seed a database at v9 with duplicate /// ref_certificates, then let the real v10 migration fire via /// run_migrations(). This exercises the migration code path rather than diff --git a/crates/gitlawb-node/src/error.rs b/crates/gitlawb-node/src/error.rs index a07235fa..371f0cf6 100644 --- a/crates/gitlawb-node/src/error.rs +++ b/crates/gitlawb-node/src/error.rs @@ -41,6 +41,9 @@ pub enum AppError { #[error("incomplete: {0}")] Incomplete(String), + #[error("search incomplete: {0}")] + SearchIncomplete(String), + #[error("git error: {0}")] Git(String), @@ -141,6 +144,15 @@ impl IntoResponse for AppError { AppError::Incomplete(msg) => { (StatusCode::UNPROCESSABLE_ENTITY, "incomplete", msg.clone()) } + // A bounded search that could not complete (the CID resolver hit its + // legacy-probe or walk ceiling), distinct from the 404 that asserts a + // definitive not-found: absence was NOT proven, so the caller should + // retry rather than treat it as gone (#173, F2). 503, retryable. + AppError::SearchIncomplete(msg) => ( + StatusCode::SERVICE_UNAVAILABLE, + "search_incomplete", + msg.clone(), + ), AppError::Git(msg) => (StatusCode::INTERNAL_SERVER_ERROR, "git_error", msg.clone()), // 504, distinct from the 500 git_error and from the read-gate's 404 / // the auth 401, so the client can tell a deadline from a failure. diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index 229ee695..25f11246 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -290,6 +290,26 @@ pub fn object_type(repo_path: &Path, sha256_hex: &str) -> Result> )) } +/// Object size in bytes (`git cat-file -s`) WITHOUT reading the content, so an +/// oversized object can be rejected before it is buffered into memory (#173, F6). +/// `None` if the object does not exist or the size is unparseable. +pub fn object_size(repo_path: &Path, sha256_hex: &str) -> Result> { + // allow-unbounded-git: cheap cat-file -s header read (no content), holds no served-git + // permit and cannot hang; exact twin of object_type above. Not an INV-22 lifecycle op. + let out = Command::new("git") + .args(["cat-file", "-s", sha256_hex]) + .current_dir(repo_path) + .output() + .context("failed to run git cat-file -s")?; + if !out.status.success() { + return Ok(None); + } + Ok(String::from_utf8_lossy(&out.stdout) + .trim() + .parse::() + .ok()) +} + /// Read an object's content if its type is already known. pub fn read_object_content(repo_path: &Path, sha256_hex: &str, obj_type: &str) -> Result> { let content_output = Command::new("git") diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index 9f07020d..d463360c 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -648,6 +648,477 @@ pub fn allowed_blob_set_for_caller_bounded( Ok(allowed) } +/// The reachable-commit enumeration for the LENIENT walks (the `/ipfs/{cid}` tree +/// gate and the commit/tag reachability set): bounded `git rev-list --all [HEAD]` +/// under the caller's shared `deadline`, deliberately WITHOUT +/// `assert_all_refs_are_commits`. That guard fail-closes a repo's whole walk when +/// any ref peels to a non-commit (an annotated tag of a tree is pushable through +/// receive-pack), which would 404 every reachable tree/commit/tag CID here for a +/// legitimate reader. `rev-list --all` skips such refs cleanly, so the commit set +/// stays complete; an object reachable only via such a ref is simply excluded — +/// correctly fail-closed. Fails closed on a rev-list error. +/// +/// Safe ONLY for a caller whose output feeds a fail-closed allow-list where absence +/// = withhold: a tolerant walk there over-withholds, never leaks. NOT safe for a +/// serve/replication filter, where a missed reachable object under-withholds — +/// those go through `blob_paths`, which runs the guard first. +fn reachable_commit_oids( + repo_path: &Path, + git_bin: &str, + deadline: Instant, +) -> Result> { + // The HEAD probe is a bounded `git rev-parse --verify HEAD` (a clean exit means + // HEAD resolves), matching `blob_paths`. When HEAD does not resolve (unborn + // branch on an empty repo) `--all` alone yields nothing, which is correct. + let head_resolves = run_bounded_git( + git_bin, + &["rev-parse", "--verify", "HEAD"], + repo_path, + b"", + deadline, + ) + .is_ok(); + let mut rev_args = vec!["rev-list", "--all"]; + if head_resolves { + rev_args.push("HEAD"); + } + let out = run_bounded_git(git_bin, &rev_args, repo_path, b"", deadline)?; + Ok(String::from_utf8_lossy(&out) + .lines() + .map(|l| l.trim().to_string()) + .filter(|l| !l.is_empty()) + .collect()) +} + +/// Every `(oid, "/repo/relative/path", kind)` triple reachable from the given +/// `commits` — the shared ls-tree seam the tree walk filters (`kind == "tree"`). +/// One bounded `git ls-tree -rzt` per commit under the caller's shared `deadline`: +/// `-rzt` is byte-identical to `-rz` for blob records and additionally emits the +/// tree object for each directory at its own path. `kind` is git's object-type +/// string ("blob", "tree", or "commit" for a gitlink). The commit's ROOT tree is +/// not emitted by `ls-tree` (it lists entries *under* a tree); `tree_paths` adds +/// it. Triples are de-duplicated across commits and paths carry a leading "/" to +/// match the glob form of visibility rules ("/secret/**"). +/// +/// Fails closed: if any tree walk fails — or a path is not valid UTF-8 — it +/// returns an error so the caller aborts rather than producing a partial +/// (under-withheld) set. +fn object_paths( + repo_path: &Path, + git_bin: &str, + commits: &[String], + deadline: Instant, +) -> Result> { + let mut out: HashSet<(String, String, String)> = HashSet::new(); + for commit in commits { + let listing_out = run_bounded_git( + git_bin, + &["ls-tree", "-rzt", commit], + repo_path, + b"", + deadline, + )?; + // `-z` NUL-delimits records and emits paths raw; plain `git ls-tree -r` + // C-quotes any path with non-ASCII or special bytes (e.g. café.txt becomes + // "secret/caf\303\251.txt"), and that quoted literal would not match a + // visibility rule like "/secret/**", under-withholding the object. The TAB + // field separator survives `-z`, so the per-record parse is unchanged. + // + // Parse strictly: a lossy decode would replace an invalid byte in a denied + // path (e.g. a non-UTF-8 directory name) with U+FFFD, and the mangled string + // would no longer match its deny rule — the same under-withholding class, one + // layer down. Fail closed instead so the caller aborts rather than leaks. + let Ok(listing_stdout) = std::str::from_utf8(&listing_out) else { + anyhow::bail!( + "git ls-tree -rzt {commit} returned a non-UTF-8 path; \ + refusing to produce a partial (under-withheld) set" + ); + }; + for record in listing_stdout.split('\0') { + // " \t" + let Some((meta, path)) = record.split_once('\t') else { + continue; + }; + let mut parts = meta.split_whitespace(); + let _mode = parts.next(); + let kind = parts.next(); + let oid = parts.next(); + if let (Some(kind), Some(oid)) = (kind, oid) { + out.insert((oid.to_string(), format!("/{path}"), kind.to_string())); + } + } + } + Ok(out) +} + +/// Root tree oid of every reachable commit, at "/". `ls-tree` never emits a commit's +/// own root tree (it lists entries *under* a tree), so it is added explicitly here. +/// Resolved in ONE bounded `git log --no-walk --format=%T --stdin` pass over the +/// shared commit set — not a per-commit `rev-parse` — so a tree-set walk costs the +/// same subprocess order as the blob walk. The commit oids go on STDIN, not argv: a +/// long history has tens of thousands of reachable commits, and passing them all as +/// arguments overflows ARG_MAX so `git log` fails to spawn — which the caller treats +/// as a walk error and fail-closed 404s an authorized reader of a reachable/root +/// tree (#173 P2). `run_bounded_git` drains stdout concurrently with the stdin +/// write, so a large history cannot deadlock the pipes. A commit whose root tree git +/// cannot resolve fails the pass (bail), failing closed. +fn root_tree_pairs( + repo_path: &Path, + git_bin: &str, + commits: &[String], + deadline: Instant, +) -> Result> { + if commits.is_empty() { + return Ok(HashSet::new()); + } + let mut buf = String::with_capacity(commits.len() * 65); + for c in commits { + buf.push_str(c); + buf.push('\n'); + } + let out = run_bounded_git( + git_bin, + &["log", "--no-walk=unsorted", "--format=%T", "--stdin"], + repo_path, + buf.as_bytes(), + deadline, + )?; + let mut set = HashSet::new(); + for line in String::from_utf8_lossy(&out).lines() { + let oid = line.trim(); + if !oid.is_empty() { + set.insert((oid.to_string(), "/".to_string())); + } + } + Ok(set) +} + +/// Every `(tree_oid, "/path")` pair reachable in `repo_path`: the `kind == "tree"` +/// slice of [`object_paths`] (subtree trees at their directory paths) PLUS every +/// reachable commit's root tree at "/" (see [`root_tree_pairs`]). Computes the +/// reachable-commit set ONCE (leniently — see [`reachable_commit_oids`]; the tree +/// allowed-set feeds ONLY the `/ipfs/{cid}` tree gate, where absence = fail-closed +/// 404) and drives both the ls-tree walk and the root-tree pass from it, so the two +/// cannot diverge and neither re-enumerates. The tree analog of [`blob_paths`], +/// bounded by the same shared `deadline`. +fn tree_paths( + repo_path: &Path, + git_bin: &str, + deadline: Instant, +) -> Result> { + let commits = reachable_commit_oids(repo_path, git_bin, deadline)?; + let mut out: HashSet<(String, String)> = object_paths(repo_path, git_bin, &commits, deadline)? + .into_iter() + .filter(|(_, _, kind)| kind == "tree") + .map(|(oid, path, _)| (oid, path)) + .collect(); + out.extend(root_tree_pairs(repo_path, git_bin, &commits, deadline)?); + Ok(out) +} + +/// The OIDs from a `(oid, "/path")` listing that visibility ALLOWS `caller` at some +/// path — the shared inner loop of the blob and tree allowed-sets. An oid reachable +/// at an allowed path is kept even when also reachable at a denied one. +fn allowed_set_from_pairs<'a>( + pairs: impl IntoIterator, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> HashSet { + pairs + .into_iter() + .filter(|(_, path)| { + visibility_check(rules, is_public, owner_did, caller, path) == Decision::Allow + }) + .map(|(oid, _)| oid.clone()) + .collect() +} + +/// Reachable tree OIDs that visibility ALLOWS `caller` at some path — the tree +/// analog of [`allowed_blob_set_for_caller`]. `GET /ipfs/{cid}` gates tree objects +/// with this so the CID surface matches `get_tree`: a tree reachable only at a +/// withheld path is absent from the set and 404'd; the root tree ("/") and any tree +/// on the path to an allowed subtree are present. Fails closed on a +/// dangling/unreachable tree (never enumerated by the reachable walk, so never in +/// the set — the #126 geometry, for trees). A tree reachable at an allowed path is +/// included even when also reachable at a withheld one (its structure is visible to +/// this caller elsewhere). +#[cfg(test)] +pub fn allowed_tree_set_for_caller( + repo_path: &Path, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> Result> { + allowed_tree_set_for_caller_bounded( + repo_path, + "git", + WALK_TIMEOUT, + rules, + is_public, + owner_did, + caller, + ) +} + +/// [`allowed_tree_set_for_caller`] with an injectable `git_bin` and walk `timeout`, +/// for the `GET /ipfs/{cid}` tree gate. One deadline spans the whole walk (the HEAD +/// probe, rev-list, every per-commit ls-tree, and the root-tree pass), matching +/// `blob_paths`, so a slow or hung walk is bounded as a unit while the handler holds +/// its /ipfs walk permit (#174 F5). +pub fn allowed_tree_set_for_caller_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> Result> { + let deadline = Instant::now() + timeout; + Ok(allowed_set_from_pairs( + &tree_paths(repo_path, git_bin, deadline)?, + rules, + is_public, + owner_did, + caller, + )) +} + +/// Object bound for the annotated-tag reachability walk (#173, jatmn tag fan-out). +/// A path-scoped pinned-CID request drives this walk while holding one per-request +/// and one per-IP walk slot, so the total tag work must be finite regardless of how +/// many tag refs the repo has. 8192 is far past any real repo's annotated-tag count +/// (the Linux kernel has a few hundred), yet finite: a repo beyond it fails closed +/// (Err), matching this function's fail-closed-on-any-git-error contract, rather than +/// truncating silently (which would under-withhold a still-reachable tag object). +const MAX_TAG_OBJECTS: usize = 8192; + +/// Walk the annotated-tag chains rooted at `seeds`, inserting every tag object they +/// pass through into `set`. A tag whose target is itself a tag (tag-of-a-tag) +/// discovers the inner tag, which is walked in a later round. +/// +/// #173 (jatmn): the tag inspection is BATCHED, not one process per tag. Each round +/// feeds every not-yet-inspected tag oid to a SINGLE `git cat-file --batch` child on +/// stdin and reads back framed ` \n\n` records, so the +/// number of child processes is bounded by the tag-chain DEPTH (rounds), not the tag +/// COUNT. Oids go on stdin, never argv, so a large tag set cannot overflow ARG_MAX. +/// The child runs through [`run_bounded_git`], which drains stdout concurrently with +/// the stdin write (subsuming #173's F4 writer-thread drain — a round large enough to +/// fill both pipes cannot deadlock) and tears the child down at `deadline`, so a hung +/// cat-file cannot pin the caller's /ipfs walk permit (#174 F5). Total tag objects +/// inspected are capped at `max_tag_objects`; exceeding it is an error (fail closed), +/// not a silent truncation. Takes the bound as a parameter so a test can drive a tiny +/// value while the caller passes the real `MAX_TAG_OBJECTS`. +fn walk_tag_chain( + repo_path: &Path, + git_bin: &str, + seeds: Vec, + set: &mut HashSet, + max_tag_objects: usize, + deadline: Instant, +) -> Result<()> { + // Tag oids known but not yet inspected. Seeds may repeat / already be present; + // the `set.insert` gate below is what actually dedups and terminates cycles. + let mut pending: Vec = seeds; + let mut inspected: usize = 0; + + while !pending.is_empty() { + // Inspect only oids new to `set`; a re-seen oid was already walked. + let round: Vec = pending + .drain(..) + .filter(|oid| set.insert(oid.clone())) + .collect(); + if round.is_empty() { + break; + } + inspected += round.len(); + if inspected > max_tag_objects { + anyhow::bail!( + "annotated-tag walk exceeded the object bound ({max_tag_objects}); refusing to serve" + ); + } + + // One bounded child for the whole round: feed all oids on stdin, read the + // framed records from the returned stdout. + let mut buf = String::with_capacity(round.len() * 65); + for oid in &round { + buf.push_str(oid); + buf.push('\n'); + } + let stdout = run_bounded_git( + git_bin, + &["cat-file", "--batch"], + repo_path, + buf.as_bytes(), + deadline, + )?; + + // Parse one record per requested oid: ` \n\n`. + // A ` missing\n` record has no size/body and is anomalous here (every + // oid came from a ref tip or a prior tag body), so fail closed. + let mut i = 0usize; + for _ in 0..round.len() { + let hdr_end = stdout[i..] + .iter() + .position(|&b| b == b'\n') + .map(|p| i + p) + .context("git cat-file --batch: truncated record header")?; + let header = std::str::from_utf8(&stdout[i..hdr_end]) + .context("git cat-file --batch: non-utf8 record header")?; + i = hdr_end + 1; + let mut fields = header.split(' '); + let _oid = fields.next().unwrap_or(""); + let ty = fields.next().unwrap_or(""); + if ty == "missing" || fields.clone().next().is_none() { + anyhow::bail!("git cat-file --batch: object {header:?} missing or malformed"); + } + let size: usize = fields + .next() + .unwrap_or("") + .parse() + .context("git cat-file --batch: bad record size")?; + let body_end = i + .checked_add(size) + .filter(|&e| e <= stdout.len()) + .context("git cat-file --batch: truncated record body")?; + // Only a tag object can point at an inner tag; walk its header. + if ty == "tag" { + let body = std::str::from_utf8(&stdout[i..body_end]) + .context("git cat-file --batch: non-utf8 tag body")?; + let mut target = None; + let mut is_tag = false; + for line in body.lines() { + if let Some(oid) = line.strip_prefix("object ") { + target = Some(oid.trim().to_string()); + } else if line == "type tag" { + is_tag = true; + } else if line.is_empty() { + break; // end of header + } + } + if is_tag { + if let Some(t) = target { + pending.push(t); + } + } + } + // Skip body plus its trailing newline to the next record. + i = body_end + 1; + } + } + Ok(()) +} + +/// The reachable-commit/tag gate set for the `/ipfs/{cid}` resolver (#173, F2): +/// every reachable commit oid UNION every reachable annotated-tag OBJECT oid. A +/// DANGLING commit/tag (referenced by no ref, directly or via a tag chain) is in +/// neither part, so the resolver denies it under a path-scoped rule instead of +/// leaking its message; a reachable one still serves. +#[cfg(test)] +pub fn reachable_commit_tag_oids(repo_path: &Path) -> Result> { + reachable_commit_tag_oids_bounded(repo_path, "git", WALK_TIMEOUT) +} + +/// [`reachable_commit_tag_oids`] with an injectable `git_bin` and walk `timeout`, +/// for the `GET /ipfs/{cid}` commit/tag gate. One deadline spans the whole walk. +/// +/// Reachable commits come from bounded `git rev-list --all` (+ HEAD for the +/// detached case). Unlike the blob allowed-set, this does NOT run +/// `assert_all_refs_are_commits`: that guard fail-closes a repo's whole walk when +/// any ref peels to a non-commit (an annotated tag of a tree is pushable through +/// receive-pack), which would 404 every reachable commit/tag CID here for a +/// legitimate reader. The guard exists to stop blob/tree UNDER-withholding; it is +/// unnecessary for reachability, since a dangling object is absent from +/// `rev-list --all` and the ref walk below regardless of odd refs — so dropping it +/// recovers availability without admitting any dangling object (no leak). +/// +/// Reachable tag OBJECTS: `rev-list --all` dereferences annotated tags to commits, +/// so the tag objects are absent from it. Collect them by walking every ref tip and +/// peeling each tag's chain, so a nested tag-of-a-tag's INNER tag object (reachable +/// and pinnable, but not itself a ref tip) is included too. Fails closed on any git +/// error. +pub fn reachable_commit_tag_oids_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, +) -> Result> { + let deadline = Instant::now() + timeout; + // Reachable commits — no ref-commit assertion (see docstring). The HEAD probe + // doubles as the seed source for the tag-valued detached HEAD below: + // `rev-parse --verify HEAD` returns the tag oid UNPEELED when HEAD names a tag + // object. Failing to resolve HEAD (unborn/absent) is not fatal — there is + // simply no HEAD to walk or seed. + let head_oid: Option = run_bounded_git( + git_bin, + &["rev-parse", "--verify", "HEAD"], + repo_path, + b"", + deadline, + ) + .ok() + .map(|out| String::from_utf8_lossy(&out).trim().to_string()) + .filter(|s| !s.is_empty()); + let mut rev_args = vec!["rev-list", "--all"]; + if head_oid.is_some() { + rev_args.push("HEAD"); + } + let rev = run_bounded_git(git_bin, &rev_args, repo_path, b"", deadline)?; + let mut set: HashSet = String::from_utf8_lossy(&rev) + .lines() + .map(|l| l.trim().to_string()) + .filter(|l| !l.is_empty()) + .collect(); + + // Ref tips that are annotated tag objects seed the tag-chain walk. + let refs = run_bounded_git( + git_bin, + &["for-each-ref", "--format=%(objectname) %(objecttype)"], + repo_path, + b"", + deadline, + )?; + let mut worklist: Vec = Vec::new(); + for line in String::from_utf8_lossy(&refs).lines() { + let mut it = line.split_whitespace(); + if let (Some(oid), Some("tag")) = (it.next(), it.next()) { + worklist.push(oid.to_string()); + } + } + // A detached/direct HEAD may name an annotated tag object with no ref at that tag + // (#173 review, finding 3): `rev-list --all HEAD` above peels it to its commit and + // `for-each-ref` has no tag row, so the tag OBJECT would be omitted and its pinned + // CID would 404 for an authorized reader. Seed a tag-valued HEAD into the tag-chain + // walk; a `commit` HEAD adds nothing. A cat-file failure here only skips the seed + // (over-withholds that one tag — fail-closed), matching the original's tolerance. + if let Some(head_oid) = head_oid { + if let Ok(ty) = run_bounded_git( + git_bin, + &["cat-file", "-t", &head_oid], + repo_path, + b"", + deadline, + ) { + if String::from_utf8_lossy(&ty).trim() == "tag" { + worklist.push(head_oid); + } + } + } + // Peel every tag object's chain into `set`, adding each tag object it passes + // through. Bounded and batched (#173, jatmn tag fan-out): see `walk_tag_chain`. + walk_tag_chain( + repo_path, + git_bin, + worklist, + &mut set, + MAX_TAG_OBJECTS, + deadline, + )?; + Ok(set) +} + /// Objects safe to replicate, failing closed on blobs (#99). A candidate /// replicates iff it is NOT a blob (`all_blob_oids` — commits and trees are /// structural, never content-withheld) OR it is in `allowed_blobs` (reachable @@ -1068,6 +1539,614 @@ esac\n"; (td, bare, secret, public) } + /// #173 (jatmn round 8, F4 — load-bearing): a repo with enough annotated tags that + /// one `cat-file --batch` round fills BOTH pipes (stdin > 64 KiB of oids while the + /// child blocks on a full stdout) must not deadlock. The old order wrote the whole + /// round to stdin before draining stdout and hung indefinitely, stranding a blocking- + /// pool thread; `run_bounded_git`'s concurrent writer/drain completes. Driven with a + /// completion timeout: GREEN finishes in well under a second, RED (old order) hangs + /// and the recv_timeout fires. ~3000 tags is well past the ~2030-oid deadlock + /// threshold (41 bytes/oid, 64 KiB pipes) and under MAX_TAG_OBJECTS (8192). + /// Bulk-created via one fast-import stream so the fixture cost is one git process, + /// not 3000 `git tag -a` spawns. + #[test] + fn walk_tag_chain_large_batch_does_not_deadlock() { + use std::io::Write; + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + let bare = td.path().join("bare.git"); + let run = |args: &[&str], dir: &Path| { + assert!( + Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(), + "git {args:?} failed" + ); + }; + std::fs::create_dir_all(&work).unwrap(); + std::fs::write(work.join("f.txt"), b"x\n").unwrap(); + run(&["init", "-q"], &work); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "."], &work); + run(&["commit", "-qm", "init"], &work); + let head = { + let out = Command::new("git") + .args(["rev-parse", "HEAD"]) + .current_dir(&work) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + + // Bulk-create ~3000 annotated tags via one fast-import stream. + const N: usize = 3000; + let mut stream = String::new(); + for i in 0..N { + let msg = format!("annotated tag {i}\n"); + stream.push_str(&format!("tag t{i}\n")); + stream.push_str(&format!("from {head}\n")); + stream.push_str("tagger t 1700000000 +0000\n"); + stream.push_str(&format!("data {}\n", msg.len())); + stream.push_str(&msg); + } + let mut fi = Command::new("git") + .args(["fast-import", "--quiet"]) + .current_dir(&work) + .stdin(std::process::Stdio::piped()) + .spawn() + .unwrap(); + fi.stdin + .take() + .unwrap() + .write_all(stream.as_bytes()) + .unwrap(); + assert!(fi.wait().unwrap().success(), "fast-import failed"); + + run( + &[ + "clone", + "-q", + "--bare", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + td.path(), + ); + + // Drive the walk on a worker thread with a completion timeout. The old + // write-all-before-drain order hangs here; the fix completes near-instantly. + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = tx.send(reachable_commit_tag_oids(&bare).map(|s| s.len())); + }); + match rx.recv_timeout(std::time::Duration::from_secs(20)) { + Ok(Ok(n)) => assert!( + n >= N, + "the walk must resolve every annotated tag object (got {n}, expected >= {N})" + ), + Ok(Err(e)) => panic!("walk errored: {e}"), + Err(_) => panic!("walk_tag_chain deadlocked on a large tag batch (F4 regression)"), + } + } + + /// #173 review (finding 3): an annotated tag reachable ONLY through a tag-valued + /// detached HEAD (raw HEAD naming a tag object, with no ref at that tag) must still + /// enter `reachable_commit_tag_oids`. `rev-list --all HEAD` peels such a HEAD to its + /// commit and `for-each-ref` has no tag row, so without a HEAD tag-seed the tag + /// OBJECT is omitted and its pinned CID would 404 for an authorized reader. RED + /// before the HEAD tag-seed (the tag oid is absent); GREEN after. + #[test] + fn reachable_commit_tag_oids_includes_tag_valued_detached_head() { + use std::io::Write; + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + let bare = td.path().join("bare.git"); + let run = |args: &[&str], dir: &Path| -> String { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + std::fs::create_dir_all(&work).unwrap(); + std::fs::write(work.join("a.txt"), b"hi\n").unwrap(); + run(&["init", "-q"], &work); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "."], &work); + run(&["commit", "-qm", "seed"], &work); + run( + &[ + "clone", + "-q", + "--bare", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + td.path(), + ); + let commit = run(&["rev-parse", "HEAD"], &bare); + + // An annotated tag OBJECT in the bare ODB, with NO ref pointing at it. + let tag_body = format!( + "object {commit}\ntype commit\ntag htag\ntagger t 0 +0000\n\nHEAD-only tag\n" + ); + let tag_oid = { + let mut child = Command::new("git") + .args(["hash-object", "-t", "tag", "-w", "--stdin"]) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap(); + child + .stdin + .as_mut() + .unwrap() + .write_all(tag_body.as_bytes()) + .unwrap(); + let out = child.wait_with_output().unwrap(); + assert!(out.status.success()); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + assert_eq!(run(&["cat-file", "-t", &tag_oid], &bare), "tag"); + // Raw-write HEAD directly to the tag object (the only way this state arises; + // update-ref / checkout both refuse a non-commit HEAD). + std::fs::write(bare.join("HEAD"), format!("{tag_oid}\n")).unwrap(); + + let set = reachable_commit_tag_oids(&bare).unwrap(); + assert!( + set.contains(&tag_oid), + "a tag reachable only via a tag-valued detached HEAD must be in the reachable set" + ); + assert!( + set.contains(&commit), + "the commit the HEAD tag peels to stays reachable (no regression)" + ); + } + + /// #173: `reachable_commit_tag_oids` on an empty repo (unborn HEAD) must return an + /// empty set, not error — exercising the `rev-parse HEAD` fail branch of the + /// detached-HEAD tag seed (there is simply no HEAD to seed). + #[test] + fn reachable_commit_tag_oids_handles_unborn_head() { + let td = TempDir::new().unwrap(); + let bare = td.path().join("empty.git"); + let ok = Command::new("git") + .args(["init", "-q", "--bare", bare.to_str().unwrap()]) + .status() + .unwrap() + .success(); + assert!(ok, "git init --bare failed"); + let set = reachable_commit_tag_oids(&bare).unwrap(); + assert!( + set.is_empty(), + "an empty repo (unborn HEAD) yields an empty reachable set with no error" + ); + } + + #[test] + fn object_paths_emits_trees_and_blob_paths_is_the_blob_slice() { + let (_td, bare, secret_oid, public_oid) = fixture(); + let deadline = Instant::now() + WALK_TIMEOUT; + // The lenient enumeration; on this clean fixture it matches the strict one. + let commits = reachable_commit_oids(&bare, "git", deadline).unwrap(); + let objs = object_paths(&bare, "git", &commits, deadline).unwrap(); + + // Blob records survive the `-rzt` change, at their paths (unchanged). + assert!(objs.contains(&(secret_oid.clone(), "/secret/b.txt".into(), "blob".into()))); + assert!(objs.contains(&(public_oid.clone(), "/public/a.txt".into(), "blob".into()))); + + // The #135 addition: subtree tree objects at their directory paths. + assert!( + objs.iter().any(|(_, p, k)| k == "tree" && p == "/secret"), + "the /secret subtree tree must be emitted at its dir path" + ); + assert!( + objs.iter().any(|(_, p, k)| k == "tree" && p == "/public"), + "the /public subtree tree must be emitted at its dir path" + ); + + // blob_paths must equal the blob slice of object_paths exactly — compared as + // SETS (both walks dedup via HashSet; the collected order is nondeterministic). + let bp: HashSet<(String, String)> = blob_paths(&bare, "git", WALK_TIMEOUT) + .unwrap() + .into_iter() + .collect(); + let bp_from_obj: HashSet<(String, String)> = objs + .iter() + .filter(|(_, _, k)| k == "blob") + .map(|(o, p, _)| (o.clone(), p.clone())) + .collect(); + assert_eq!( + bp, bp_from_obj, + "blob_paths output must be byte-identical to object_paths' blob slice" + ); + } + + #[test] + fn allowed_tree_set_gates_withheld_subtree_tree() { + let (_td, bare, _s, _p) = fixture(); + let oid = |rev: &str| { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(&bare) + .output() + .unwrap(); + assert!(out.status.success(), "rev-parse {rev}"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + let secret_tree = oid("HEAD:secret"); + let public_tree = oid("HEAD:public"); + let root_tree = oid("HEAD^{tree}"); + let reader = "did:key:z6MkReader"; + let rules = [rule("/secret/**", &[reader])]; + + // anon: the withheld /secret tree is excluded; root ("/") and /public are in. + let anon = allowed_tree_set_for_caller(&bare, &rules, true, OWNER, None).unwrap(); + assert!( + !anon.contains(&secret_tree), + "withheld /secret subtree tree excluded for anon" + ); + assert!(anon.contains(&root_tree), "root tree included (path /)"); + assert!(anon.contains(&public_tree), "/public subtree tree included"); + + // listed reader: sees the /secret tree (caller-aware, not a blanket deny). + let rd = allowed_tree_set_for_caller(&bare, &rules, true, OWNER, Some(reader)).unwrap(); + assert!( + rd.contains(&secret_tree), + "listed reader sees the /secret tree" + ); + + // owner: sees every reachable tree. + let ow = allowed_tree_set_for_caller(&bare, &rules, true, OWNER, Some(OWNER)).unwrap(); + assert!( + ow.contains(&secret_tree) && ow.contains(&public_tree) && ow.contains(&root_tree), + "owner sees all reachable trees" + ); + } + + #[test] + fn allowed_tree_set_excludes_dangling_tree() { + use std::io::Write; + let (_td, bare, secret_oid, _p) = fixture(); + // A DANGLING tree: written to the ODB but referenced by no commit. Uses a + // UNIQUE entry name so its oid is content-distinct from every reachable tree + // (a content-identical tree would dedup to a reachable oid — that is T2, not + // danglingness). The reachable-only walk never enumerates it -> fail closed. + let mut child = Command::new("git") + .args(["mktree"]) + .current_dir(&bare) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + writeln!( + child.stdin.as_mut().unwrap(), + "100644 blob {secret_oid}\tdangling-only-unreferenced.txt" + ) + .unwrap(); + let out = child.wait_with_output().unwrap(); + assert!(out.status.success(), "git mktree"); + let dangling = String::from_utf8_lossy(&out.stdout).trim().to_string(); + + let rules = [rule("/secret/**", &[])]; + for caller in [None, Some(OWNER)] { + let set = allowed_tree_set_for_caller(&bare, &rules, true, OWNER, caller).unwrap(); + assert!( + !set.contains(&dangling), + "dangling tree must never be in the reachable allowed-set (caller={caller:?})" + ); + } + } + + #[test] + fn allowed_tree_set_includes_tree_shared_across_allowed_and_denied_paths() { + // T2 (content-dedup): the SAME tree oid reachable at both an allowed and a + // withheld path is INCLUDED for anon (allowed-wins) — its structure is + // visible to the caller at the allowed path. Mirrors the blob analog + // `same_blob_at_allowed_and_denied_path_is_not_withheld`. + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + std::fs::create_dir_all(work.join("pub/sub")).unwrap(); + std::fs::create_dir_all(work.join("sec/sub")).unwrap(); + std::fs::write(work.join("pub/sub/f.txt"), b"same bytes\n").unwrap(); + std::fs::write(work.join("sec/sub/f.txt"), b"same bytes\n").unwrap(); + let run = |args: &[&str]| { + assert!( + Command::new("git") + .args(args) + .current_dir(&work) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + run(&["init", "-q"]); + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + run(&["add", "."]); + run(&["commit", "-qm", "seed"]); + let oid = |rev: &str| { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(&work) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + let pub_sub = oid("HEAD:pub/sub"); + let sec_sub = oid("HEAD:sec/sub"); + assert_eq!(pub_sub, sec_sub, "identical content dedups to one tree oid"); + + // Withhold /sec from anon; the shared oid is still reachable at /pub/sub. + let rules = [rule("/sec/**", &[])]; + let anon = allowed_tree_set_for_caller(&work, &rules, true, OWNER, None).unwrap(); + assert!( + anon.contains(&pub_sub), + "a tree reachable at an allowed path is included even when also at a withheld path" + ); + } + + #[test] + fn allowed_tree_set_includes_root_trees_of_all_reachable_commits() { + // The batched root-tree pass (root_tree_pairs) must return EVERY reachable + // commit's root tree, not just HEAD's — two commits with distinct root trees + // both land in the set. Guards the git-log-over-N-commits root derivation. + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + std::fs::create_dir_all(&work).unwrap(); + let run = |args: &[&str]| { + assert!( + Command::new("git") + .args(args) + .current_dir(&work) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + run(&["init", "-q"]); + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + let oid = |rev: &str| { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(&work) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + std::fs::write(work.join("a.txt"), b"one\n").unwrap(); + run(&["add", "."]); + run(&["commit", "-qm", "c1"]); + let root1 = oid("HEAD^{tree}"); + std::fs::write(work.join("b.txt"), b"two\n").unwrap(); + run(&["add", "."]); + run(&["commit", "-qm", "c2"]); + let root2 = oid("HEAD^{tree}"); + assert_ne!(root1, root2, "the two commits have distinct root trees"); + + // Public repo, no rules: every reachable tree is allowed for anon. + let set = allowed_tree_set_for_caller(&work, &[], true, OWNER, None).unwrap(); + assert!( + set.contains(&root1) && set.contains(&root2), + "root trees of BOTH reachable commits are in the set (batched root pass)" + ); + } + + #[test] + fn root_tree_pairs_returns_every_root_tree_at_scale() { + // Parity + liveness at scale for root_tree_pairs (#173 P2): feed every + // reachable commit oid to `git log --format=%T --stdin` and collect each + // commit's root tree. With N commits that is ~N*41 bytes of oids in and + // ~N*41 bytes of %T out — past the ~64 KiB pipe buffer in both directions — + // so this exercises the large-bidirectional-IO path the 2-commit test above + // cannot, and asserts parity: every distinct root tree comes back. + // + // NOTE: this is NOT a deadlock guard. `git log --stdin` reads its whole + // revision list to EOF before emitting any %T, so the naive "write all of + // stdin, then drain stdout" form does not deadlock at any scale for this + // invocation. `run_bounded_git`'s concurrent writer/drain is cheap defensive + // isolation, not load-bearing, and this test does not claim otherwise. The + // 30s watchdog is a general liveness bound so a future regression that + // genuinely hangs fails fast here rather than stalling the suite. + const N: usize = 2500; + let td = TempDir::new().unwrap(); + let bare = td.path().join("many.git"); + assert!(Command::new("git") + .args(["init", "-q", "--bare", bare.to_str().unwrap()]) + .status() + .unwrap() + .success()); + + // fast-import a linear chain of N commits, each adding a distinct file so + // every root tree is distinct (dedup cannot shrink the output). One + // subprocess, ~1s — far cheaper than N `git commit` spawns. + let mut stream = String::new(); + for i in 0..N { + let (b, cm) = (2 * i + 1, 2 * i + 2); + let content = format!("v{i}"); + let msg = format!("c{i}"); + stream.push_str(&format!( + "blob\nmark :{b}\ndata {}\n{content}\n", + content.len() + )); + stream.push_str(&format!( + "commit refs/heads/main\nmark :{cm}\ncommitter t 0 +0000\ndata {}\n{msg}\n", + msg.len() + )); + if i > 0 { + stream.push_str(&format!("from :{}\n", 2 * (i - 1) + 2)); + } + stream.push_str(&format!("M 100644 :{b} f{i}\n\n")); + } + let mut fi = Command::new("git") + .args(["fast-import", "--quiet"]) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .spawn() + .unwrap(); + { + use std::io::Write; + fi.stdin + .take() + .unwrap() + .write_all(stream.as_bytes()) + .unwrap(); + } + assert!(fi.wait().unwrap().success(), "fast-import failed"); + + let commits = reachable_commit_oids(&bare, "git", Instant::now() + WALK_TIMEOUT).unwrap(); + assert_eq!(commits.len(), N, "all {N} commits reachable"); + + // Call root_tree_pairs directly (private, same module) under a liveness + // watchdog, then assert it returned every distinct root tree. + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = tx.send( + root_tree_pairs(&bare, "git", &commits, Instant::now() + WALK_TIMEOUT) + .map(|s| s.len()), + ); + }); + match rx.recv_timeout(std::time::Duration::from_secs(30)) { + Ok(Ok(len)) => assert_eq!(len, N, "every distinct root tree returned"), + Ok(Err(e)) => panic!("root_tree_pairs errored: {e}"), + Err(_) => panic!("root_tree_pairs did not return within 30s"), + } + } + + /// #173 (jatmn tag fan-out): the batched `git cat-file --batch` tag walk must + /// return the SAME reachable set as the old per-tag `cat-file tag` loop — every + /// commit, the outer tag object, AND the inner tag object of a tag-of-a-tag chain + /// (the inner tag is reachable but is not itself a ref tip, so it is only found by + /// peeling the outer tag's target). Behavior-preservation proof for the rewrite. + #[test] + fn reachable_commit_tag_oids_includes_nested_tag_objects() { + let (_td, bare, _secret, _public) = fixture(); + let run = |args: &[&str]| -> String { + let out = Command::new("git") + .args(args) + .current_dir(&bare) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + // v1 -> commit, v2 -> v1 (tag-of-a-tag), plus a couple of sibling tags so the + // round batches more than one oid. Capture v1's oid, then DELETE the v1 ref so + // the inner tag object survives in the ODB but is NOT a ref tip: it is then + // reachable ONLY by peeling v2's target chain. That makes the peel load-bearing + // (breaking the inner-tag enqueue drops v1 from the set), unlike leaving v1 as + // its own ref where `for-each-ref` would seed it directly. + run(&["tag", "-a", "-m", "inner", "v1", "HEAD"]); + run(&["tag", "-a", "-m", "outer", "v2", "v1"]); + run(&["tag", "-a", "-m", "s1", "s1", "HEAD"]); + run(&["tag", "-a", "-m", "s2", "s2", "HEAD"]); + let commit = run(&["rev-parse", "HEAD"]); + let v1 = run(&["rev-parse", "v1"]); + let v2 = run(&["rev-parse", "v2"]); + let s1 = run(&["rev-parse", "s1"]); + let s2 = run(&["rev-parse", "s2"]); + run(&["tag", "-d", "v1"]); + + let set = reachable_commit_tag_oids(&bare).unwrap(); + assert!(set.contains(&commit), "the commit must be reachable"); + assert!( + set.contains(&v2), + "the outer tag object (ref tip) must be present" + ); + assert!( + set.contains(&v1), + "the INNER tag object of a tag-of-a-tag must be present (peeled from v2, no ref)" + ); + assert!(set.contains(&s1), "sibling tag s1 must be present"); + assert!(set.contains(&s2), "sibling tag s2 must be present"); + } + + /// #173 (jatmn tag fan-out): the object bound is load-bearing. A repo whose tag + /// count exceeds the bound must FAIL CLOSED (Err), not return a truncated set that + /// would under-withhold a still-reachable tag. Drives `walk_tag_chain` with a tiny + /// injected bound (the public fn uses the real `MAX_TAG_OBJECTS`); with the bound + /// check removed this would collect all tags and return Ok. + #[test] + fn walk_tag_chain_fails_closed_over_object_bound() { + let (_td, bare, _secret, _public) = fixture(); + let run = |args: &[&str]| { + assert!( + Command::new("git") + .args(args) + .current_dir(&bare) + .status() + .unwrap() + .success(), + "git {args:?} failed" + ); + }; + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + let mut seeds = Vec::new(); + for n in 0..5 { + let name = format!("t{n}"); + run(&["tag", "-a", "-m", &name, &name, "HEAD"]); + let oid = Command::new("git") + .args(["rev-parse", &name]) + .current_dir(&bare) + .output() + .unwrap(); + seeds.push(String::from_utf8_lossy(&oid.stdout).trim().to_string()); + } + + // Within a generous bound: the walk succeeds and collects the tags. + let mut ok_set = HashSet::new(); + walk_tag_chain( + &bare, + "git", + seeds.clone(), + &mut ok_set, + 8192, + Instant::now() + WALK_TIMEOUT, + ) + .unwrap(); + assert!( + seeds.iter().all(|s| ok_set.contains(s)), + "all 5 tags collected under a generous bound" + ); + + // Under a bound of 2 with 5 tags: fail closed (Err), not a partial set. + let mut small_set = HashSet::new(); + let result = walk_tag_chain( + &bare, + "git", + seeds, + &mut small_set, + 2, + Instant::now() + WALK_TIMEOUT, + ); + assert!( + result.is_err(), + "a tag count exceeding the object bound must fail closed (Err), not truncate" + ); + } + #[test] fn anonymous_caller_withholds_only_private_blob() { let (_td, bare, secret_oid, public_oid) = fixture(); diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 3b346190..a4ed8e4d 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -91,7 +91,8 @@ pub async fn cat(ipfs_api: &str, cid: &str) -> Result> { /// `..._fail_closed` filter on the full-scan path before calling, so this /// function never sees a withheld blob. `repo_path` is still needed to read each /// object's bytes. The twin in `pinata.rs` mirrors this shape — change both in -/// lockstep. +/// lockstep. `repo_id` records the pin's provenance so `GET /ipfs/{cid}` resolves +/// straight to this repo instead of scanning every repo (#173). /// /// Returns a list of `(sha256_hex, cid)` pairs for objects pinned this call. pub async fn pin_new_objects( @@ -99,6 +100,7 @@ pub async fn pin_new_objects( repo_path: &std::path::Path, object_list: Vec, db: &crate::db::Db, + repo_id: &str, ) -> Vec<(String, String)> { if ipfs_api.is_empty() { return vec![]; @@ -107,9 +109,36 @@ pub async fn pin_new_objects( let mut pinned = Vec::new(); for sha in object_list { - // Skip if already pinned + // Skip if already pinned — but first backfill provenance if the existing + // pin has none. A legacy pin (recorded before repo_id existed, #173, jatmn) + // is skipped here before record_pinned_cid ever runs, so its NULL provenance + // would never resolve to one repo and known CIDs keep hitting the scan. The + // backfill only sets repo_id (AND repo_id IS NULL guard preserves + // first-pinner-owns) and never re-pins the bytes — the object is already on IPFS. match db.is_pinned(&sha).await { - Ok(true) => continue, + Ok(true) => { + match db.provenance_for_oid(&sha).await { + Ok(None) => { + if let Err(e) = db.backfill_pin_provenance(&sha, repo_id).await { + tracing::warn!(sha = %sha, err = %e, "failed to backfill pin provenance"); + } + } + Ok(Some(_)) => {} + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "DB error reading pin provenance"); + } + } + // F1 (#173 round 8): record this repo as an ADDITIONAL source for the + // already-pinned object. This is the load-bearing skip-branch insert — + // a later repo pushing a shared object hits this path (already pinned), + // and without it `GET /ipfs/{cid}` only ever knows the first pinner, so a + // shared object first pinned from a private/quarantined repo 404s even + // when this repo would serve it. Bounded per object (MAX_PIN_SOURCES). + if let Err(e) = db.record_pin_source(&sha, repo_id).await { + tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); + } + continue; + } Ok(false) => {} Err(e) => { tracing::warn!(sha = %sha, err = %e, "DB error checking pinned status"); @@ -130,9 +159,14 @@ pub async fn pin_new_objects( // Pin to IPFS match pin_git_object(ipfs_api, &sha, &data).await { Ok(cid) if !cid.is_empty() => { - if let Err(e) = db.record_pinned_cid(&sha, &cid).await { + if let Err(e) = db.record_pinned_cid(&sha, &cid, Some(repo_id)).await { tracing::warn!(sha = %sha, err = %e, "failed to record pinned CID in DB"); } + // F1 (#173 round 8): also record the first pinner in pin_repo_sources so + // every source (first and subsequent) is tried uniformly by the resolver. + if let Err(e) = db.record_pin_source(&sha, repo_id).await { + tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); + } pinned.push((sha, cid)); } Ok(_) => {} diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index ffac65a2..37190c76 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -374,6 +374,9 @@ async fn main() -> Result<()> { rate_limiter, create_ip_rate_limiter, push_rate_limiter, + ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, + ipfs_max_legacy_probes: crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, + ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, push_limiter_trust, sync_trigger_rate_limiter, peer_write_rate_limiter, @@ -465,22 +468,16 @@ async fn main() -> Result<()> { // Periodic cleanup of expired rate limit entries + consumed-proof ledger { - let rl = state.rate_limiter.clone(); - let create_ip_rl = state.create_ip_rate_limiter.clone(); - let push_rl = state.push_rate_limiter.clone(); - let sync_trigger_rl = state.sync_trigger_rate_limiter.clone(); - let peer_write_rl = state.peer_write_rate_limiter.clone(); + let cleanup_state = state.clone(); let db = state.db.clone(); let mut shutdown_rx = state.subscribe_shutdown(); tokio::spawn(async move { loop { tokio::select! { _ = tokio::time::sleep(std::time::Duration::from_secs(300)) => { - rl.cleanup().await; - create_ip_rl.cleanup().await; - push_rl.cleanup().await; - sync_trigger_rl.cleanup().await; - peer_write_rl.cleanup().await; + // Sweep every per-IP/DID limiter (incl. the ipfs walk brake) + // so bounded maps shed stale keys instead of sitting at cap. + cleanup_state.sweep_rate_limiters().await; let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|d| d.as_secs() as i64) diff --git a/crates/gitlawb-node/src/pinata.rs b/crates/gitlawb-node/src/pinata.rs index 6c9c0bff..31843bbc 100644 --- a/crates/gitlawb-node/src/pinata.rs +++ b/crates/gitlawb-node/src/pinata.rs @@ -74,8 +74,10 @@ pub async fn pin_object( /// `..._fail_closed` filter on the full-scan path before calling. `repo_path` is /// still needed to read each object's bytes. The twin in `ipfs_pin.rs` mirrors /// this shape — change both in lockstep. Objects already recorded with a -/// `pinata_cid` are skipped. Returns `(sha_hex, cid)` pairs for each newly -/// pinned object. +/// `pinata_cid` are skipped. `repo_id` records the pin's provenance (#173). +/// Returns `(sha_hex, provider_cid)` pairs for each newly pinned object: the +/// provider CID is the Pinata gateway CID (used for branch→CID recording and +/// ref-update gossip), NOT the raw resolver-key CID stored in `pinned_cids.cid`. pub async fn pin_new_objects( client: &reqwest::Client, upload_url: &str, @@ -83,6 +85,7 @@ pub async fn pin_new_objects( repo_path: &std::path::Path, object_list: Vec, db: &crate::db::Db, + repo_id: &str, ) -> Vec<(String, String)> { if jwt.is_empty() { return vec![]; @@ -92,7 +95,15 @@ pub async fn pin_new_objects( for sha in object_list { match db.has_pinata_cid(&sha).await { - Ok(true) => continue, + Ok(true) => { + // F1 (#173 round 8): record this repo as an additional source for the + // already-pinned object (mirrors the ipfs_pin skip-branch insert) so the + // resolver can serve a shared object from any pin-path source. + if let Err(e) = db.record_pin_source(&sha, repo_id).await { + tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); + } + continue; + } Ok(false) => {} Err(e) => { tracing::warn!(sha = %sha, err = %e, "DB error checking pinata_cid"); @@ -111,9 +122,21 @@ pub async fn pin_new_objects( match pin_object(client, upload_url, jwt, &sha, &data).await { Ok(cid) if !cid.is_empty() => { - if let Err(e) = db.record_pinata_cid(&sha, &cid).await { + // The resolver key (`pinned_cids.cid`) must be the locally-computed + // raw-content CID, never the provider CID: Pinata wraps the bytes in + // dag-pb/UnixFS, so its returned CID does not hash the raw content and + // must not become an alias `/ipfs/{cid}` serves raw git bytes for (#173). + let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes(&data).to_string(); + if let Err(e) = db + .record_pinata_cid(&sha, &raw_cid, &cid, Some(repo_id)) + .await + { tracing::warn!(sha = %sha, err = %e, "failed to record pinata_cid in DB"); } + // F1 (#173 round 8): also record the first pinner in pin_repo_sources. + if let Err(e) = db.record_pin_source(&sha, repo_id).await { + tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); + } pinned.push((sha, cid)); } Ok(_) => {} diff --git a/crates/gitlawb-node/src/rate_limit.rs b/crates/gitlawb-node/src/rate_limit.rs index 22281691..14c11a9c 100644 --- a/crates/gitlawb-node/src/rate_limit.rs +++ b/crates/gitlawb-node/src/rate_limit.rs @@ -121,6 +121,28 @@ impl RateLimiter { true } + /// Non-consuming check: is this key ALREADY at its limit for the current window? + /// Unlike [`check`], it records nothing and never inserts a new key — used to shed + /// expensive preparatory work (e.g. the `/ipfs/{cid}` legacy scan's O(repos) DB + /// preload) BEFORE it runs, without perturbing the per-unit budget the consuming + /// `check` maintains (#173, F3). An unknown key or a disabled limiter is not + /// throttled. Prunes the key's expired timestamps as a side effect (keeps state + /// tidy) but adds none, so it cannot itself fill or grow the map. + pub(crate) async fn is_throttled(&self, key: &str) -> bool { + if self.max_requests == 0 { + return false; + } + let now = Instant::now(); + let mut state = self.state.lock().await; + if let Some(window) = state.get_mut(key) { + window + .timestamps + .retain(|t| now.duration_since(*t) < self.window); + return window.timestamps.len() >= self.max_requests; + } + false + } + pub async fn cleanup(&self) { let now = Instant::now(); let mut state = self.state.lock().await; @@ -130,6 +152,13 @@ impl RateLimiter { !w.timestamps.is_empty() }); } + + /// Number of distinct keys currently tracked. Test-only introspection so a + /// cross-module test can assert that a sweep actually evicted expired entries. + #[cfg(test)] + pub(crate) async fn tracked_keys(&self) -> usize { + self.state.lock().await.len() + } } /// A bounded per-caller CONCURRENCY limiter — distinct from [`RateLimiter`], which diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index deccf4c7..10dbbf32 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -65,6 +65,29 @@ pub struct AppState { /// brake a push flood from a DID farm (one throwaway DID per repo), so the /// push path throttles on the resolved client IP instead. pub push_rate_limiter: RateLimiter, + /// Per-client-IP rate limiter for the `GET /ipfs/{cid}` full-history walk. + /// The route is anonymous and a valid tree CID (exposed by the public pins + /// index) makes each repeat request pay a fresh allowed-set walk (rev-list + + /// ls-tree per commit), memoized only per request — unbounded amplification + /// (INV-10). Braking the walk on the non-farmable source IP caps that cost + /// without touching cheap non-walk fetches. Keyed by `push_limiter_trust`. + pub ipfs_rate_limiter: RateLimiter, + /// Per-request ceiling on full-history reachability walks the CID resolver + /// may spawn (default `api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST`). A field, + /// not a bare const, so tests can shrink it to exercise the cap cheaply; + /// production keeps the const default. + pub ipfs_max_history_walks: u32, + /// Per-request ceiling on legacy (NULL-provenance) repo probes in the CID + /// resolver's scan fallback (default `api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST`). + /// Bounds the anonymous `acquire` + `cat-file` fan-out across the node (#173, + /// INV-10); a field for the same test-seam reason as `ipfs_max_history_walks`. + pub ipfs_max_legacy_probes: u32, + /// Hard ceiling on the byte size of an object `GET /ipfs/{cid}` will buffer and + /// serve (default `api::ipfs::MAX_SERVED_OBJECT_BYTES`). The serve reads via a + /// blocking `git cat-file` and buffers the whole object; without a bound a large + /// public blob could exhaust memory or block a runtime worker (#173, F6, INV-10). + /// A field for the same test-seam reason as the sibling caps. + pub ipfs_max_served_object_bytes: u64, /// Which forwarded header (if any) the edge is trusted to set, for /// resolving the push limiter's client-IP key. See `GITLAWB_TRUSTED_PROXY`. /// Node-wide; also keys the two peer-sync limiters below. @@ -171,13 +194,6 @@ pub struct AppState { /// (`with_default_max_keys`, reject-before-insert) so a source-key farm cannot grow /// it (INV-15). pub git_ipfs_walk_per_caller: crate::rate_limit::PerCallerConcurrency, - /// Per-client-IP rate limiter for `GET /ipfs/{cid}`. The route is publicly - /// reachable and each request can drive a full-history git walk, so it carries a - /// per-IP flood brake in addition to the concurrency cap above — a rate limit - /// bounds request *rate*, the semaphore bounds concurrent slow holds (different - /// axes). Keyed on the resolved client IP via `push_limiter_trust`. Layered on the - /// `/ipfs` route via `rate_limit_by_ip`. - pub ipfs_rate_limiter: RateLimiter, /// The `git` executable the served-git withheld-blob walk spawns. Production is /// `"git"` (resolved via PATH); injectable so a fake `git` can drive the walk's /// process-group teardown in handler tests without mutating the process-global @@ -192,6 +208,20 @@ impl AppState { self.shutdown_tx.subscribe() } + /// Sweep expired entries from every per-IP/DID rate limiter. Driven by the + /// periodic cleanup task so a bounded limiter's key map sheds stale entries + /// instead of sitting near its cap until an inline capacity sweep reclaims + /// them. Every limiter on the state is swept here; adding a new limiter means + /// adding it to this list. + pub(crate) async fn sweep_rate_limiters(&self) { + self.rate_limiter.cleanup().await; + self.create_ip_rate_limiter.cleanup().await; + self.push_rate_limiter.cleanup().await; + self.ipfs_rate_limiter.cleanup().await; + self.sync_trigger_rate_limiter.cleanup().await; + self.peer_write_rate_limiter.cleanup().await; + } + /// Trigger graceful shutdown. Idempotent — calling more than once /// has no effect. Returns `true` if this call was the one that /// flipped the signal. diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 92f41a8d..21fa58e4 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -78,6 +78,10 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { rate_limiter: RateLimiter::new(100, Duration::from_secs(60)), create_ip_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), push_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, + ipfs_max_legacy_probes: crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, + ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, push_limiter_trust: crate::rate_limit::TrustedProxy::None, sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), @@ -98,7 +102,6 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { git_ipfs_walk_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( 16, ), - ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), git_bin: "git".to_string(), } } @@ -2020,13 +2023,19 @@ mod tests { /// Seed a SHA-256 source repo (public/a.txt + secret/b.txt), bare-clone it /// into each `/tmp//.git` path, and return guards + oids. - /// SHA-256 object format is required: `get_by_cid` resolves a CID whose - /// multihash digest IS the git object id, which only matches in sha256 repos. + /// SHA-256 object format matches production (`--object-format=sha256`) so the + /// oids are 64-hex. A real CID digests the raw object CONTENT (not the git + /// oid), so tests build the request CID with `pin_cid_for` — mirroring the pin + /// path — and `get_by_cid` maps it back to the oid via `pinned_cids` (#173). struct CidFixture { _guards: Vec, secret_oid: String, public_oid: String, secret_tree_oid: String, + public_tree_oid: String, + root_tree_oid: String, + commit_oid: String, + tag_oid: String, } impl Drop for CidFixture { fn drop(&mut self) { @@ -2060,6 +2069,8 @@ mod tests { run(&["config", "user.name", "t"], &src); run(&["add", "."], &src); run(&["commit", "-qm", "seed"], &src); + // Annotated tag of the commit — exercises the "tags stay served" guard. + run(&["tag", "-a", "-m", "annotated", "v1", "HEAD"], &src); let oid = |rev: &str| { let out = Command::new("git") .args(["rev-parse", rev]) @@ -2072,6 +2083,10 @@ mod tests { let secret_oid = oid("HEAD:secret/b.txt"); let public_oid = oid("HEAD:public/a.txt"); let secret_tree_oid = oid("HEAD:secret"); + let public_tree_oid = oid("HEAD:public"); + let root_tree_oid = oid("HEAD^{tree}"); + let commit_oid = oid("HEAD"); + let tag_oid = oid("refs/tags/v1"); let mut guards = vec![src.clone()]; for name in bare_names { let bare = std::path::PathBuf::from("/tmp") @@ -2089,6 +2104,12 @@ mod tests { ], &src, ); + // `git clone --bare` does NOT copy the source repo's local identity, so + // fixtures that create objects directly in the bare repo (`commit-tree`, + // `git tag -a`) abort with "identity unknown" on a CI runner that has no + // ambient/global git identity. Set it explicitly so the suite is portable. + run(&["config", "user.email", "t@t"], &bare); + run(&["config", "user.name", "t"], &bare); } // One guard for the whole /tmp/ tree covers every bare clone. guards.push(std::path::PathBuf::from("/tmp").join(slug)); @@ -2097,300 +2118,3160 @@ mod tests { secret_oid, public_oid, secret_tree_oid, + public_tree_oid, + root_tree_oid, + commit_oid, + tag_oid, } } - /// CID whose sha2-256 multihash digest equals the given 64-hex git oid, so - /// `get_by_cid` decodes it back to that oid and `git cat-file`s it. - fn cid_for_oid(oid_hex: &str) -> String { - use gitlawb_core::cid::Cid; - let bytes = hex::decode(oid_hex).expect("hex oid"); - let arr: [u8; 32] = bytes.as_slice().try_into().expect("32-byte sha256 oid"); - Cid::from_sha256_bytes(&arr).to_string() + /// Record a pin exactly as the production pin path does — read the object's + /// raw bytes (`git cat-file `, no framing), CID them with + /// `Cid::from_git_object_bytes`, and store the `(oid, cid)` row — then return + /// the CID string the node advertises (`gl ipfs list`) and a client sends to + /// `GET /ipfs/{cid}`. Building the CID from the oid instead (the old + /// `cid_for_oid`) produced an identifier that never occurs in production and + /// made the gate assertions vacuous: a real pin CID digests the raw content, + /// not the git oid, so `get_by_cid` resolves it through `pinned_cids` (#173). + async fn pin_cid_for(bare_repo: &std::path::Path, oid: &str, db: &crate::db::Db) -> String { + let (_ty, raw) = crate::git::store::read_object(bare_repo, oid) + .expect("read object bytes") + .expect("object exists in repo"); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(&raw).to_string(); + // Legacy-style pin (no provenance) so existing CID tests exercise the + // resolver's scan fallback; provenance-path tests pin via `pin_cid_for_repo`. + db.record_pinned_cid(oid, &cid, None) + .await + .expect("record pinned cid"); + cid } - fn cid_router(state: &AppState) -> Router { - Router::new() - .route( - "/ipfs/{cid}", - axum::routing::get(crate::api::ipfs::get_by_cid), - ) - .layer(axum::middleware::from_fn(crate::auth::optional_signature)) - .with_state(state.clone()) + /// Like [`pin_cid_for`] but records the pin's provenance (`repo_id`), so the + /// resolver resolves the CID straight to `repo_id` instead of scanning (#173). + #[allow(dead_code)] // used by the provenance-path resolver tests (P-U3) + async fn pin_cid_for_repo( + bare_repo: &std::path::Path, + oid: &str, + db: &crate::db::Db, + repo_id: &str, + ) -> String { + let (_ty, raw) = crate::git::store::read_object(bare_repo, oid) + .expect("read object bytes") + .expect("object exists in repo"); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(&raw).to_string(); + db.record_pinned_cid(oid, &cid, Some(repo_id)) + .await + .expect("record pinned cid with provenance"); + cid } - async fn cid_parts(resp: axum::response::Response) -> (StatusCode, String) { - let st = resp.status(); - let b = axum::body::to_bytes(resp.into_body(), usize::MAX) + + /// INV-7 upgrade path for the pin-provenance column (#173, jatmn round 2): a node + /// already past v11 gets `pinned_cids.repo_id` from the NEW v12 migration, and a + /// legacy pin recorded before the column existed survives with NULL provenance (so + /// it falls back to the repo scan). Simulate the pre-v12 node by dropping the + /// column and un-applying v12, seed a legacy row, then re-migrate. RED before the + /// v12 migration exists (the column is never re-added → the SELECT errors); GREEN + /// after. + #[sqlx::test] + async fn pinned_cids_repo_provenance_upgrade_path(pool: PgPool) { + let state = test_state(pool.clone()).await; + + // Pre-v12 shape: drop the provenance column and forget v12 was applied. + sqlx::query("ALTER TABLE pinned_cids DROP COLUMN IF EXISTS repo_id") + .execute(&pool) .await .unwrap(); - (st, String::from_utf8_lossy(&b).to_string()) - } - fn cid_anon(cid: &str) -> Request { - Request::builder() - .method(Method::GET) - .uri(format!("/ipfs/{cid}")) - .body(Body::empty()) - .unwrap() - } - fn cid_signed(kp: &gitlawb_core::identity::Keypair, cid: &str) -> Request { - let path = format!("/ipfs/{cid}"); - let s = gitlawb_core::http_sig::sign_request(kp, "GET", &path, b""); - Request::builder() - .method(Method::GET) - .uri(&path) - .header("content-digest", s.content_digest) - .header("signature-input", s.signature_input) - .header("signature", s.signature) - .body(Body::empty()) - .unwrap() + sqlx::query("DELETE FROM schema_migrations WHERE version = 12") + .execute(&pool) + .await + .unwrap(); + + // A legacy pin recorded before provenance existed. + sqlx::query("INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) VALUES ($1, $2, $3)") + .bind("legacyoid") + .bind("legacycid") + .bind("2020-01-01T00:00:00Z") + .execute(&pool) + .await + .unwrap(); + + // Upgrade: re-run migrations → v12 re-adds the column. + state.db.run_migrations().await.expect("migrate to v12"); + + // The legacy pin survives with NULL provenance. + let legacy: Option = + sqlx::query_scalar("SELECT repo_id FROM pinned_cids WHERE sha256_hex = 'legacyoid'") + .fetch_one(&pool) + .await + .expect("legacy pin row survives the upgrade"); + assert!( + legacy.is_none(), + "a pin recorded before v12 must keep NULL provenance (it falls back to the scan)" + ); + + // A new pin can carry provenance. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + ) + .bind("newoid") + .bind("newcid") + .bind("2026-01-01T00:00:00Z") + .bind("repo-abc") + .execute(&pool) + .await + .unwrap(); + let prov: Option = + sqlx::query_scalar("SELECT repo_id FROM pinned_cids WHERE sha256_hex = 'newoid'") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + prov.as_deref(), + Some("repo-abc"), + "a pin recorded after v12 carries its source repo_id" + ); } - /// #110: `GET /ipfs/{cid}` must gate a withheld blob by per-caller visibility. - /// RED before U2 (the current handler serves the secret to anon). + /// #173: a pin records the repository it came from; `provenance_for_oid` reads it + /// back; a legacy pin (no repo) reads back None; and first-pinner-owns holds — a + /// second push of the same oid does NOT rewrite provenance (ON CONFLICT DO + /// NOTHING). This is what lets the resolver gate a CID against its ONE source repo. #[sqlx::test] - async fn ipfs_cid_gate_withholds_blob_from_unauthorized(pool: PgPool) { - use crate::db::VisibilityMode; - use gitlawb_core::identity::Keypair; - - let owner = Keypair::generate(); - let owner_did = owner.did().to_string(); - let reader = Keypair::generate(); - let reader_did = reader.did().to_string(); - let stranger = Keypair::generate(); - let slug = owner_did.replace([':', '/'], "_"); - let short = owner_did.split(':').next_back().unwrap().to_string(); + async fn record_pinned_cid_stores_and_reads_provenance(pool: PgPool) { let state = test_state(pool).await; - let fx = seed_cid_repos(&slug, &short, &["withhold"]); - let secret_cid = cid_for_oid(&fx.secret_oid); - let tree_cid = cid_for_oid(&fx.secret_tree_oid); - let public_cid = cid_for_oid(&fx.public_oid); - state .db - .create_repo(&seed_repo(&owner_did, "withhold")) + .record_pinned_cid("oidA", "cidA", Some("repo-xyz")) .await - .expect("seed repo"); - let rec = state + .unwrap(); + assert_eq!( + state + .db + .provenance_for_oid("oidA") + .await + .unwrap() + .as_deref(), + Some("repo-xyz"), + "a provenanced pin reads back its source repo_id" + ); + + state .db - .get_repo(&owner_did, "withhold") + .record_pinned_cid("oidB", "cidB", None) .await - .unwrap() .unwrap(); + assert_eq!( + state.db.provenance_for_oid("oidB").await.unwrap(), + None, + "a legacy pin (no repo) has NULL provenance" + ); + + // First-pinner-owns: a later push of the same oid must not rewrite provenance. state .db - .set_visibility_rule( - &rec.id, - "/secret/**", - VisibilityMode::B, - std::slice::from_ref(&reader_did), - &owner_did, - ) + .record_pinned_cid("oidA", "cidA", Some("repo-OTHER")) .await - .expect("deny rule"); - - // anon → withheld blob: must 404, must not leak content. (RED on current handler.) - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_anon(&secret_cid)) - .await - .unwrap(), - ) - .await; + .unwrap(); assert_eq!( - st, - StatusCode::NOT_FOUND, - "anon must not read the withheld blob" - ); - assert!( - !body.contains("TOP SECRET"), - "404 body must not leak the secret" + state + .db + .provenance_for_oid("oidA") + .await + .unwrap() + .as_deref(), + Some("repo-xyz"), + "ON CONFLICT DO NOTHING keeps the first repo's provenance" ); - // signed non-reader → 404. - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_signed(&stranger, &secret_cid)) - .await - .unwrap(), - ) - .await; + // An unpinned oid has no provenance. assert_eq!( - st, - StatusCode::NOT_FOUND, - "non-reader must not read the withheld blob" + state.db.provenance_for_oid("never-pinned").await.unwrap(), + None ); - assert!(!body.contains("TOP SECRET")); - - // owner (signed) → 200 + secret bytes. - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_signed(&owner, &secret_cid)) - .await - .unwrap(), - ) - .await; - assert_eq!(st, StatusCode::OK, "owner reads the withheld blob"); - assert!(body.contains("TOP SECRET"), "owner gets the content"); + } - // listed reader (signed) → 200. - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_signed(&reader, &secret_cid)) - .await - .unwrap(), - ) - .await; - assert_eq!(st, StatusCode::OK, "listed reader reads the blob"); - assert!(body.contains("TOP SECRET")); + /// #173 (provenance, happy path): a CID pinned with provenance resolves straight + /// to its ONE source repo and serves an authorized reader — no repo scan. + #[sqlx::test] + async fn ipfs_cid_provenance_serves_from_pinning_repo(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; - // KTD3: anon tree CID under /secret → 200 (trees/commits are not withheld). - let (st, _) = cid_parts( - cid_router(&state) - .oneshot(cid_anon(&tree_cid)) - .await - .unwrap(), - ) - .await; - assert_eq!(st, StatusCode::OK, "tree object is served to anon (KTD3)"); + let _fx = seed_cid_repos(&slug, &short, &["provserve"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("provserve.git"); + let fx = &_fx; - // R3: public blob anon → 200 (non-withheld content not affected). - let (st, _) = cid_parts( - cid_router(&state) - .oneshot(cid_anon(&public_cid)) - .await - .unwrap(), - ) - .await; - assert_eq!(st, StatusCode::OK, "public blob stays served"); + // Build the repo FIRST so the pin can carry its id as provenance. + let repo = seed_repo(&owner_did, "provserve"); // public + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo.id).await; - // R5: a genuine unknown CID also 404, uniform with the withheld 404. - let absent_cid = cid_for_oid(&"ab".repeat(32)); - let (st, _) = cid_parts( - cid_router(&state) - .oneshot(cid_anon(&absent_cid)) - .await - .unwrap(), - ) - .await; + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; assert_eq!( st, - StatusCode::NOT_FOUND, - "absent CID 404 (uniform with withheld)" + StatusCode::OK, + "a provenanced public CID serves its content" + ); + assert!( + body.contains("public bytes"), + "the pinning repo's object is served" ); - - // malformed CID → 400 (unchanged). - let (st, _) = cid_parts( - cid_router(&state) - .oneshot(cid_anon("not-a-cid")) - .await - .unwrap(), - ) - .await; - assert_eq!(st, StatusCode::BAD_REQUEST, "malformed CID still 400"); } - /// R4: the same object withheld in one repo but public in another is still - /// served from the public copy; the withholding repo is iterated first. + /// #173 (provenance, THE load-bearing one — #124 flip + bounded fan-out): a CID + /// pinned from a PRIVATE repo must gate against that pinning repo (404), NOT serve + /// from a byte-identical PUBLIC copy in another repo. Provenance is strictly more + /// restrictive than the old scan (which served the public copy). RED before the + /// rework (the scan serves the public copy → 200 + leaks the secret bytes); GREEN + /// after (provenance → the private repo → 404, no leak). #[sqlx::test] - async fn ipfs_cid_served_from_public_copy_when_withheld_elsewhere(pool: PgPool) { - use crate::db::VisibilityMode; - use chrono::Utc; + async fn ipfs_cid_provenance_private_denies_despite_public_copy(pool: PgPool) { use gitlawb_core::identity::Keypair; - let owner = Keypair::generate(); let owner_did = owner.did().to_string(); let slug = owner_did.replace([':', '/'], "_"); let short = owner_did.split(':').next_back().unwrap().to_string(); let state = test_state(pool).await; - let fx = seed_cid_repos(&slug, &short, &["withhold", "pubcopy"]); - let secret_cid = cid_for_oid(&fx.secret_oid); + let fx = seed_cid_repos(&slug, &short, &["privsrc", "pubcopy"]); + let priv_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("privsrc.git"); - // Withholding repo, iterated FIRST (later updated_at; list_all_repos is DESC). - let mut withhold = seed_repo(&owner_did, "withhold"); - withhold.updated_at = Utc::now(); + // Private source repo, built first so the pin carries its id as provenance. + let mut priv_repo = seed_repo(&owner_did, "privsrc"); + priv_repo.is_public = false; state .db - .create_repo(&withhold) + .create_repo(&priv_repo) .await - .expect("withhold repo"); + .expect("seed private repo"); + let cid = pin_cid_for_repo(&priv_bare, &fx.secret_oid, &state.db, &priv_repo.id).await; + + // A PUBLIC repo holds the SAME object (the old scan would serve it). + let pub_repo = seed_repo(&owner_did, "pubcopy"); // public, no rule state .db - .set_visibility_rule( - &withhold.id, - "/secret/**", - VisibilityMode::B, - &[], - &owner_did, - ) + .create_repo(&pub_repo) .await - .expect("deny rule"); - - // Public copy, no rules, iterated AFTER. - let mut pubcopy = seed_repo(&owner_did, "pubcopy"); - pubcopy.updated_at = Utc::now() - chrono::Duration::seconds(60); - state.db.create_repo(&pubcopy).await.expect("pubcopy repo"); + .expect("seed public copy"); - // anon: denied at the withholding repo (continue), served from the public copy. - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_anon(&secret_cid)) - .await - .unwrap(), - ) - .await; + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; assert_eq!( st, - StatusCode::OK, - "served from the public copy despite the other deny" + StatusCode::NOT_FOUND, + "a provenanced private CID must 404, not serve from a public copy elsewhere (#124 flip)" ); assert!( - body.contains("TOP SECRET"), - "the public copy serves the content" + !body.contains("TOP SECRET"), + "the 404 body must not leak the withheld object" ); } - /// Repo-level "/" gate (KTD2a, first continue branch): a fully private repo - /// (is_public=false, no rules) denies anon before any per-blob check; the - /// owner still reads. The path-scoped tests pass the "/" gate and deny at the - /// per-blob stage, so this exercises the coarser repo-level deny separately. + /// #173 (jatmn round 8, F1 — load-bearing): a shared object first pinned from a + /// PRIVATE repo, then pushed again from a PUBLIC repo through the real pin path, + /// must serve by CID to an anonymous caller from the public source. First-pinner- + /// only provenance 404s it (only the private source is known); recording EVERY + /// pin-path source fixes it. The second push hits the already-pinned skip branch, + /// so this proves the skip-branch source insert fires (and does NOT re-pin: /add + /// expect(0)). RED before U1 (anon 404); GREEN after. #[sqlx::test] - async fn ipfs_cid_private_repo_denies_anon_at_repo_gate(pool: PgPool) { + async fn ipfs_cid_multi_source_serves_from_later_public_pinner(pool: PgPool) { use gitlawb_core::identity::Keypair; - let owner = Keypair::generate(); let owner_did = owner.did().to_string(); let slug = owner_did.replace([':', '/'], "_"); let short = owner_did.split(':').next_back().unwrap().to_string(); let state = test_state(pool).await; - let fx = seed_cid_repos(&slug, &short, &["priv"]); - let blob_cid = cid_for_oid(&fx.public_oid); - - let mut rec = seed_repo(&owner_did, "priv"); - rec.is_public = false; - state.db.create_repo(&rec).await.expect("private repo"); + let fx = seed_cid_repos(&slug, &short, &["privfirst", "pubsecond"]); + let priv_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("privfirst.git"); + let pub_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("pubsecond.git"); + + // Private repo pins the object FIRST — it owns the first-pinner provenance. + let mut priv_repo = seed_repo(&owner_did, "privfirst"); + priv_repo.is_public = false; + state + .db + .create_repo(&priv_repo) + .await + .expect("seed private first-pinner"); + let cid = pin_cid_for_repo(&priv_bare, &fx.public_oid, &state.db, &priv_repo.id).await; + + // A PUBLIC repo pushes the SAME object through the real pin path. The object is + // already pinned, so this hits the already-pinned skip branch, which must record + // the public repo as an additional source without re-pinning (/add expect 0). + let pub_repo = seed_repo(&owner_did, "pubsecond"); // public, no rule + state + .db + .create_repo(&pub_repo) + .await + .expect("seed public second-pinner"); + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyshouldnothappen"}"#) + .expect(0) + .create_async() + .await; + crate::ipfs_pin::pin_new_objects( + &server.url(), + &pub_bare, + vec![fx.public_oid.clone()], + &state.db, + &pub_repo.id, + ) + .await; + m.assert_async().await; // asserts /add was NOT called (already pinned) + + // Anonymous CID fetch: the private first source denies, the public second + // source serves → 200. Before F1 only the private source is known → 404. + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "a shared object must serve by CID from a later public pin-path source (F1)" + ); + assert!( + body.contains("public bytes"), + "the served body is the public object's bytes" + ); + } + + /// #173 (jatmn round 8, F1 — bound, R2): the per-object source set is capped at + /// `MAX_PIN_SOURCES` so an adversary pushing one object from many repos cannot make + /// resolution O(repos). Recording the same oid from `MAX_PIN_SOURCES + 3` distinct + /// repos leaves exactly `MAX_PIN_SOURCES` rows. + #[sqlx::test] + async fn ipfs_cid_pin_sources_capped_at_max(pool: PgPool) { + let state = test_state(pool).await; + let cap = crate::db::MAX_PIN_SOURCES; + for i in 0..(cap + 3) { + state + .db + .record_pin_source("capoid", &format!("repo-{i}")) + .await + .expect("record source"); + } + let sources = state.db.pin_sources_for_oid("capoid").await.unwrap(); + assert_eq!( + sources.len() as i64, + cap, + "the per-object source set is capped at MAX_PIN_SOURCES" + ); + } + + /// #173 (jatmn round 8, F1 — availability, grok-4.5 adversarial catch): the resolver's + /// per-object source cap must NEVER evict the first-pinner. A legacy public pin keeps + /// its source in `pinned_cids.repo_id` but not in `pin_repo_sources` (pre-v13 pins, or + /// a pin whose best-effort `record_pin_source` missed). If the cap `LIMIT` were applied + /// to the whole union with a lexicographic order, an attacker could push the same + /// object from `MAX_PIN_SOURCES` repos whose grindable ids sort before the public + /// source and evict it from the window — turning a public CID that served 200 into a + /// 404. This drives exactly that: a legacy public first-pinner plus `MAX_PIN_SOURCES` + /// lower-sorting attacker sources must STILL serve the public object. RED with a + /// whole-union LIMIT (the first-pinner is dropped → 404); GREEN once the first-pinner + /// is always included and the LIMIT caps only the additional sources. + #[sqlx::test] + async fn ipfs_cid_first_pinner_never_evicted_by_lower_sorting_sources(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["pubfirst"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("pubfirst.git"); + // Public repo whose id sorts AFTER every attacker id below. Legacy shape: the + // source lives in pinned_cids.repo_id only (pin_cid_for_repo records no + // pin_repo_sources row), exactly like a pin from before v13. + let mut pub_repo = seed_repo(&owner_did, "pubfirst"); // public, no rule + pub_repo.id = "zzzzzzzz-pubfirst".to_string(); + state + .db + .create_repo(&pub_repo) + .await + .expect("seed public first-pinner"); + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &pub_repo.id).await; + + // Attacker fills the whole MAX_PIN_SOURCES window with lower-sorting source ids + // (non-existent repos — their mere presence would evict the first-pinner under a + // whole-union LIMIT). + let cap = crate::db::MAX_PIN_SOURCES; + for i in 0..cap { + state + .db + .record_pin_source(&fx.public_oid, &format!("00-attacker-{i:02}")) + .await + .expect("attacker source"); + } + + // The public first-pinner must still serve — never evicted by the cap window. + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "the first-pinner public source must never be evicted by lower-sorting attacker sources (F1 availability)" + ); + assert!( + body.contains("public bytes"), + "the public object is served from the first-pinner" + ); + } + + /// INV-7 upgrade path for the F1 `pin_repo_sources` table (#173, jatmn round 8): a + /// node already past v12 gets the table from the NEW v13 migration. Simulate the + /// pre-v13 node by dropping the table and un-applying v13, then re-migrate and + /// assert a source row round-trips. RED before the v13 migration exists. + #[sqlx::test] + async fn pin_repo_sources_upgrade_path(pool: PgPool) { + let state = test_state(pool.clone()).await; + sqlx::query("DROP TABLE IF EXISTS pin_repo_sources") + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version = 13") + .execute(&pool) + .await + .unwrap(); + state.db.run_migrations().await.expect("re-migrate"); + state + .db + .record_pin_source("upgradeoid", "repo-upg") + .await + .expect("record after re-migrate"); + assert_eq!( + state.db.pin_sources_for_oid("upgradeoid").await.unwrap(), + vec!["repo-upg".to_string()], + "the v13 pin_repo_sources table is present after upgrade" + ); + } + + /// #173 (jatmn round 8, F2 — load-bearing): a legacy `pinned_cids` row keyed on a + /// PROVIDER CID (Pinata/Kubo dag-pb — every release before this branch stored the + /// provider CID as the resolver key, not the raw-content CID) must NOT serve raw git + /// bytes that do not hash to the requested CID. `get_by_cid` recomputes the CID over + /// the served bytes and refuses to serve on mismatch. Seeded with a RAW SQL INSERT + /// because the current helpers store the raw CID, so a helper-seeded row is already + /// correct-shape and the RED assertion would be vacuous (INV-21). RED before U2 + /// (serves the git bytes → 200); GREEN after (not served, no bytes egress). + #[sqlx::test] + async fn ipfs_cid_legacy_provider_cid_row_not_served(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["provsrc"]); + let repo = seed_repo(&owner_did, "provsrc"); // public, no rule + state.db.create_repo(&repo).await.expect("seed repo"); + + // A valid sha2-256 CID whose digest is NOT the object's raw-content digest — + // stands in for a Pinata/Kubo dag-pb provider CID (the legacy resolver key). + let provider_cid = gitlawb_core::cid::Cid::from_git_object_bytes( + b"a decoy object whose CID is not the served object's CID", + ) + .to_string(); + + // Legacy-shape row: cid = the PROVIDER CID (raw SQL — the helpers now store the + // raw CID and cannot reproduce this shape). The object itself is public+servable. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + ) + .bind(&fx.public_oid) + .bind(&provider_cid) + .bind("2020-01-01T00:00:00Z") + .bind(&repo.id) + .execute(&pool) + .await + .unwrap(); + + // Requesting the provider CID resolves the row and passes the repo gate, but the + // served bytes hash to a DIFFERENT CID, so the integrity check must withhold them. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&provider_cid)) + .await + .unwrap(), + ) + .await; + assert_ne!( + st, + StatusCode::OK, + "a provider-CID legacy row must not serve raw git bytes (F2)" + ); + assert!( + !body.contains("public bytes"), + "the mismatched bytes must not egress" + ); + } + + /// #173 (jatmn round 8, F6 — INV-10 cost guard): the serve path buffers the object via + /// a blocking `cat-file`; an object larger than `ipfs_max_served_object_bytes` must be + /// WITHHELD (rejected by the size precheck, never buffered), with zero body bytes + /// egressed. Under the cap it serves unchanged. The oversize-reject counter guards it + /// both ways: a removed size precheck serves the object and leaves the counter at 0. + #[sqlx::test] + async fn ipfs_cid_f6_oversized_object_withheld(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["big"]); + let bare = std::path::PathBuf::from("/tmp").join(&slug).join("big.git"); + let repo = seed_repo(&owner_did, "big"); // public, no rule + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo.id).await; + + // Cap below the object size ("public bytes\n" = 13 bytes) → withheld. + state.ipfs_max_served_object_bytes = 5; + crate::api::ipfs::reset_oversize_rejects(); + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_ne!( + st, + StatusCode::OK, + "an object over the size cap must not serve (F6)" + ); + assert!( + !body.contains("public bytes"), + "no object bytes egress for an over-cap object" + ); + assert_eq!( + crate::api::ipfs::oversize_rejects(), + 1, + "the oversized object was rejected by the size precheck" + ); + + // Control: raise the cap above the object size → serves unchanged. + state.ipfs_max_served_object_bytes = crate::api::ipfs::MAX_SERVED_OBJECT_BYTES; + crate::api::ipfs::reset_oversize_rejects(); + let (st2, body2) = + cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st2, + StatusCode::OK, + "under the cap the object serves normally" + ); + assert!( + body2.contains("public bytes"), + "the served body is the object's bytes" + ); + assert_eq!( + crate::api::ipfs::oversize_rejects(), + 0, + "no oversize reject under the cap" + ); + } + + /// #173 (provenance, INV-11): a quarantined pinning repo must 404 by CID even for + /// its own owner — quarantine hard-drops before the visibility gate on the + /// provenance path too. The owner-signed 404 is the load-bearing negative (a + /// visibility-only gate would Allow the owner). + #[sqlx::test] + async fn ipfs_cid_provenance_quarantined_repo_404_even_owner(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["quarsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("quarsrc.git"); + let repo = seed_repo(&owner_did, "quarsrc"); // public + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo.id).await; + + // Baseline: before quarantine the provenanced CID serves (proves the path works). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&owner, &cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "provenanced CID serves before quarantine" + ); + + state + .db + .set_repo_quarantine(&repo.id, true) + .await + .expect("quarantine"); + + for req in [cid_anon(&cid), cid_signed(&owner, &cid)] { + let (st, body) = cid_parts(cid_router(&state).oneshot(req).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "a quarantined pinning repo must 404 by CID (anon + owner)" + ); + assert!( + !body.contains("public bytes"), + "the 404 body must not leak quarantined content" + ); + } + } + + /// #173 (provenance, bounded — must NOT fall back to the scan): a CID whose + /// provenance points at a repo that no longer exists must 404 rather than scan + /// every repo and serve a byte-identical public copy. Falling back to the scan + /// would reopen the O(repos) anonymous fan-out the provenance rework closes. RED + /// before the rework (the scan serves the public copy → 200); GREEN after. + #[sqlx::test] + async fn ipfs_cid_provenance_missing_repo_404_no_scan_fallback(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["gonesrc", "pubcopy2"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("gonesrc.git"); + + // Pin with provenance = a repo_id that is never created (deleted/absent). + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, "nonexistent-repo-id").await; + + // A public repo holds the SAME object (the old scan would serve it). + let pub_repo = seed_repo(&owner_did, "pubcopy2"); + state + .db + .create_repo(&pub_repo) + .await + .expect("seed public copy"); + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "a provenance pointing at a missing repo must 404, not fall back to the scan" + ); + } + + /// #173 (provenance, path-scoped WALK gate): the #135/#173 per-object gates must + /// run on the NEW provenance path, not only the legacy scan. A provenanced pin from + /// a repo under a `/secret/**` rule runs `allowed_blob_set_for_caller` via the shared + /// gate: a withheld secret blob 404s to anon (no byte leak); the allowed reader gets + /// it. Exercises the walk gate on the provenance path in BOTH directions. + #[sqlx::test] + async fn ipfs_cid_provenance_path_scoped_walk_gates_withheld_blob(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let reader = Keypair::generate(); + let reader_did = reader.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["provwalk"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("provwalk.git"); + let repo = seed_repo(&owner_did, "provwalk"); // public at "/" + state.db.create_repo(&repo).await.expect("seed repo"); + // /secret/** Mode B with the reader allowed → the secret blob walk gates by caller. + state + .db + .set_visibility_rule( + &repo.id, + "/secret/**", + VisibilityMode::B, + std::slice::from_ref(&reader_did), + &owner_did, + ) + .await + .expect("path rule"); + let cid = pin_cid_for_repo(&bare, &fx.secret_oid, &state.db, &repo.id).await; + + // Anon: the walk denies the secret blob → 404, no leak. + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "a withheld secret blob 404s to anon on the provenance path (walk gate runs)" + ); + assert!( + !body.contains("TOP SECRET"), + "the 404 body must not leak the withheld blob" + ); + + // Allowed reader: the walk includes the secret blob → 200 with content. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&reader, &cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "an allowed reader gets the secret blob via the provenance walk gate" + ); + assert!( + body.contains("TOP SECRET"), + "the allowed reader receives the content" + ); + } + + /// #173: the pinata pin path stores the locally-computed raw CID in the + /// resolver-key `cid` column and the provider CID in `pinata_cid`, and its ON + /// CONFLICT COALESCE fills a NULL provenance without overwriting an existing one + /// (first-pinner-owns). On conflict `cid` is left untouched so a prior local pin's + /// raw CID is never clobbered by a provider CID. + #[sqlx::test] + async fn record_pinata_cid_stores_and_coalesces_provenance(pool: PgPool) { + let state = test_state(pool).await; + + // A new row created via the pinata path carries provenance, and stores the + // raw CID in `cid` with the provider CID in `pinata_cid`. + state + .db + .record_pinata_cid("po1", "rawcid1", "pcid1", Some("repoA")) + .await + .unwrap(); + assert_eq!( + state.db.provenance_for_oid("po1").await.unwrap().as_deref(), + Some("repoA") + ); + let po1 = state + .db + .list_pinned_cids() + .await + .unwrap() + .into_iter() + .find(|r| r.sha256_hex == "po1") + .expect("po1 row exists"); + assert_eq!(po1.cid, "rawcid1", "resolver-key cid is the raw CID"); + assert_eq!( + po1.pinata_cid.as_deref(), + Some("pcid1"), + "the provider CID is kept in pinata_cid" + ); + + // An existing NULL-provenance row: the pinata COALESCE fills it, and the + // prior local pin's `cid` is left untouched (not overwritten by the raw arg). + state + .db + .record_pinned_cid("po2", "localcid2", None) + .await + .unwrap(); + state + .db + .record_pinata_cid("po2", "rawcid2", "pcid2", Some("repoB")) + .await + .unwrap(); + assert_eq!( + state.db.provenance_for_oid("po2").await.unwrap().as_deref(), + Some("repoB"), + "pinata fills a NULL provenance" + ); + let po2 = state + .db + .list_pinned_cids() + .await + .unwrap() + .into_iter() + .find(|r| r.sha256_hex == "po2") + .expect("po2 row exists"); + assert_eq!( + po2.cid, "localcid2", + "on conflict the prior local pin's cid is left untouched" + ); + + // An existing provenance: the pinata COALESCE must NOT overwrite it. + state + .db + .record_pinned_cid("po3", "cid3", Some("repoX")) + .await + .unwrap(); + state + .db + .record_pinata_cid("po3", "rawcid3", "pcid3", Some("repoY")) + .await + .unwrap(); + assert_eq!( + state.db.provenance_for_oid("po3").await.unwrap().as_deref(), + Some("repoX"), + "pinata COALESCE keeps the first-pinner's provenance" + ); + } + + /// #173 (jatmn, F4, load-bearing security): a Pinata-first pin (no prior local pin) + /// must make the resolver key (`pinned_cids.cid`) the locally-computed raw CID, NOT + /// the provider CID. Pinata wraps the bytes in dag-pb/UnixFS, so its returned CID + /// does not hash the raw content; if it became the resolver key, `/ipfs/{provider_cid}` + /// would serve raw git bytes that do not hash to it, breaking raw content-addressing. + /// Assert `oids_for_cid(raw_cid)` finds the sha AND `oids_for_cid(provider_cid)` does NOT. + #[sqlx::test] + async fn record_pinata_cid_resolver_key_is_raw_not_provider(pool: PgPool) { + let state = test_state(pool).await; + + let bytes = b"raw git object content for pinata-first pin"; + let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes(bytes).to_string(); + // A distinct provider CID (a dag-pb wrapper CID Pinata would return). + let provider_cid = "QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG"; + assert_ne!( + raw_cid, provider_cid, + "the provider CID must differ from the raw CID for this test to be meaningful" + ); + + // Pinata-first: no prior local pin, so this INSERT creates the row. + state + .db + .record_pinata_cid("pfsha", &raw_cid, provider_cid, Some("repoP")) + .await + .unwrap(); + + // The raw CID resolves to the sha. + assert_eq!( + state.db.oids_for_cid(&raw_cid).await.unwrap(), + vec!["pfsha".to_string()], + "the locally-computed raw CID is the resolver key" + ); + // The provider (dag-pb) CID must NOT resolve raw bytes. + assert!( + state + .db + .oids_for_cid(provider_cid) + .await + .unwrap() + .is_empty(), + "the provider dag-pb CID must never resolve raw git bytes" + ); + } + + /// #173 (end-to-end pin wiring): `pin_new_objects` records the repo_id it is given + /// as the pin's provenance. Drives the real pin path against a mocked IPFS `/add` + /// endpoint (so `pin_git_object` succeeds) and asserts `provenance_for_oid` returns + /// the repo — closing the gap between the push handler's threading and the DB write. + #[sqlx::test] + async fn pin_new_objects_records_provenance(pool: PgPool) { + let state = test_state(pool).await; + + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovtest"}"#) + .expect_at_least(1) + .create_async() + .await; + + let fx = seed_cid_repos("provpin_e2e", "ppe2e", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("provpin_e2e") + .join("pinsrc.git"); + + let pinned = crate::ipfs_pin::pin_new_objects( + &server.url(), + &bare, + vec![fx.public_oid.clone()], + &state.db, + "repoZ", + ) + .await; + assert!( + !pinned.is_empty(), + "the object was pinned via the real pin path" + ); + m.assert_async().await; + assert_eq!( + state + .db + .provenance_for_oid(&fx.public_oid) + .await + .unwrap() + .as_deref(), + Some("repoZ"), + "pin_new_objects records the repo_id it was given as the pin's provenance" + ); + } + + /// #173 (jatmn, F2): a legacy pin with NULL provenance backfills its source + /// via `backfill_pin_provenance`, and the `AND repo_id IS NULL` guard preserves + /// first-pinner-owns (a non-NULL provenance is left untouched). + #[sqlx::test] + async fn backfill_pin_provenance_fills_null_keeps_existing(pool: PgPool) { + let state = test_state(pool).await; + + // A legacy pin: no provenance recorded. + state + .db + .record_pinned_cid("legacy_oid", "legacy_cid", None) + .await + .unwrap(); + assert_eq!( + state.db.provenance_for_oid("legacy_oid").await.unwrap(), + None, + "a legacy pin starts with NULL provenance" + ); + + // Backfill sets the NULL provenance. + state + .db + .backfill_pin_provenance("legacy_oid", "repo-src") + .await + .unwrap(); + assert_eq!( + state + .db + .provenance_for_oid("legacy_oid") + .await + .unwrap() + .as_deref(), + Some("repo-src"), + "backfill fills a NULL provenance from the known source" + ); + + // A pin that already has provenance: backfill must NOT overwrite it. + state + .db + .record_pinned_cid("owned_oid", "owned_cid", Some("repo-first")) + .await + .unwrap(); + state + .db + .backfill_pin_provenance("owned_oid", "repo-second") + .await + .unwrap(); + assert_eq!( + state + .db + .provenance_for_oid("owned_oid") + .await + .unwrap() + .as_deref(), + Some("repo-first"), + "the AND repo_id IS NULL guard keeps the first-pinner's provenance" + ); + } + + /// #173 (jatmn, F2, load-bearing): an object already pinned with NULL provenance + /// (a pre-provenance legacy pin) acquires its source when `pin_new_objects` sees + /// it again. The already-pinned skip path must backfill rather than leave the + /// object stuck on the O(repos) scan fallback — and it must NOT re-pin the bytes + /// (no IPFS `/add` call, the object is already on IPFS). + #[sqlx::test] + async fn pin_new_objects_backfills_legacy_null_provenance(pool: PgPool) { + let state = test_state(pool).await; + + let fx = seed_cid_repos("provpin_backfill", "ppbf", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("provpin_backfill") + .join("pinsrc.git"); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes( + &crate::git::store::read_object(&bare, &fx.public_oid) + .expect("read object bytes") + .expect("object exists") + .1, + ) + .to_string(); + + // Legacy pin: the object is already recorded with NULL provenance. + state + .db + .record_pinned_cid(&fx.public_oid, &cid, None) + .await + .unwrap(); + assert_eq!( + state.db.provenance_for_oid(&fx.public_oid).await.unwrap(), + None, + "the object starts as a legacy pin with NULL provenance" + ); + + // Mock IPFS `/add` and require it is NOT called: the already-pinned object + // must be backfilled, never re-pinned. + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyshouldnothappen"}"#) + .expect(0) + .create_async() + .await; + + let pinned = crate::ipfs_pin::pin_new_objects( + &server.url(), + &bare, + vec![fx.public_oid.clone()], + &state.db, + "repoBF", + ) + .await; + + assert!( + pinned.is_empty(), + "an already-pinned object is not re-pinned (no bytes returned)" + ); + m.assert_async().await; // asserts /add was called 0 times + assert_eq!( + state + .db + .provenance_for_oid(&fx.public_oid) + .await + .unwrap() + .as_deref(), + Some("repoBF"), + "pin_new_objects backfills the legacy pin's NULL provenance" + ); + } + + /// #173 (provenance-path throttle): a walk-requiring provenanced candidate whose + /// per-IP walk quota is spent returns 429 (the provenance arm's Throttled outcome, + /// then the fall-through). quota=1, keyed on XFF. The first reader request runs the + /// walk and spends the token; the second from the same IP is throttled → 429. + #[sqlx::test] + async fn ipfs_cid_provenance_walk_throttle_returns_429(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let reader = Keypair::generate(); + let reader_did = reader.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let fx = seed_cid_repos(&slug, &short, &["provthrottle"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("provthrottle.git"); + let repo = seed_repo(&owner_did, "provthrottle"); + state.db.create_repo(&repo).await.expect("seed repo"); + state + .db + .set_visibility_rule( + &repo.id, + "/secret/**", + VisibilityMode::B, + std::slice::from_ref(&reader_did), + &owner_did, + ) + .await + .expect("path rule"); + let cid = pin_cid_for_repo(&bare, &fx.secret_oid, &state.db, &repo.id).await; + + // 1st reader request runs the walk (reader is allowed) and spends the token. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_signed_xff(&reader, &cid, "1.2.3.4")) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "1st provenance walk from the IP serves"); + + // 2nd request from the same IP: the walk is throttled → 429 (provenance path). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_signed_xff(&reader, &cid, "1.2.3.4")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::TOO_MANY_REQUESTS, + "a throttled provenance walk returns 429" + ); + } + + /// #173 (multi-oid dispatch, mixed provenance + legacy): one CID mapping to a + /// provenanced-then-denied oid AND a legacy (NULL-provenance) oid must still resolve + /// to the legacy-servable copy — the provenance arm's skip does not abort the loop. + #[sqlx::test] + async fn ipfs_cid_mixed_provenance_and_legacy_serves_legacy(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["mixpriv", "mixpub"]); + + // Private repo holds secret_oid, pinned with provenance = itself (denies anon). + let mut priv_repo = seed_repo(&owner_did, "mixpriv"); + priv_repo.is_public = false; + state + .db + .create_repo(&priv_repo) + .await + .expect("seed private"); + // Public repo holds public_oid, legacy pin (NULL provenance -> scan serves it). + let pub_repo = seed_repo(&owner_did, "mixpub"); + state.db.create_repo(&pub_repo).await.expect("seed public"); + + // One REAL CID (the non-unique cid index) maps to BOTH oids: the public oid as a + // legacy (NULL) pin, and the secret oid provenanced to the private repo. + let pub_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("mixpub.git"); + let shared_cid = pin_cid_for(&pub_bare, &fx.public_oid, &state.db).await; + state + .db + .record_pinned_cid(&fx.secret_oid, &shared_cid, Some(&priv_repo.id)) + .await + .unwrap(); + + // Anon: secret_oid (provenance -> private -> denied), public_oid (legacy -> scan + // -> public -> served). Resolves to the public copy regardless of oid order. + let resp = cid_router(&state) + .oneshot(cid_anon(&shared_cid)) + .await + .unwrap(); + let served = resp + .headers() + .get("x-git-hash") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + let (st, body) = cid_parts(resp).await; + assert_eq!( + st, + StatusCode::OK, + "a CID mixing a provenanced-denied oid and a legacy-servable oid resolves" + ); + assert_eq!( + served.as_deref(), + Some(fx.public_oid.as_str()), + "the served object is the legacy public oid" + ); + assert!( + body.contains("public bytes"), + "the public content is served" + ); + } + + // ---- #173 round 3: legacy (NULL-provenance) scan bound + 503-on-truncation ---- + // The provenance path targets one repo and is already bounded. These cover the + // legacy scan fallback, where an anonymous request could otherwise fan out to + // O(repos) `acquire` + `cat-file` probes (F1) and a walk-cap truncation could + // false-404 an object that may be readable (F2). The bound is a per-request probe + // BUDGET, not a per-IP brake: a walk-free public fetch stays un-rate-limited + // (ipfs_walk_rate_limited_per_source), while the expensive walk keeps its IP brake. + + /// T1 (F1): the probe budget gates BEFORE `acquire`/`cat-file`, so it genuinely + /// bounds the fan-out — a repo past the budget is never probed, even one that + /// WOULD serve. With the budget at 0, a PUBLIC legacy copy that would otherwise + /// serve 200 is not probed at all → 503 truncated (absence unproven). RED before + /// the budget check (the repo is probed and serves 200). + #[sqlx::test] + async fn ipfs_cid_legacy_probe_budget_gates_before_serving(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_max_legacy_probes = 0; // probe nothing → any legacy candidate truncates + + let fx = seed_cid_repos(&slug, &short, &["pubprobe"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("pubprobe.git"); + let repo = seed_repo(&owner_did, "pubprobe"); // public, no path rule → would serve + state.db.create_repo(&repo).await.expect("seed repo"); + // Legacy pin (NULL provenance) → resolver takes the scan fallback. + let cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::SERVICE_UNAVAILABLE, + "the probe budget gates before the probe: a servable copy past the budget is not reached → 503" + ); + } + + /// T7 (F1/F3 pre-limit): EVERY legacy probe is braked on the source IP from the + /// FIRST one, so a hostile caller cannot repeatedly force the whole-node `acquire` + /// fan-out across requests (each cold `acquire` is a Tigris round-trip, INV-10). + /// Since #173-F3 (jatmn) there is no free budget: a single-repo legacy scan is + /// itself charged. quota=1 keyed on XFF, one PUBLIC legacy copy that serves + /// walk-free (never touches the walk brake), so the second same-IP request can only + /// be shed by the probe brake: req1 serves and spends the token, req2 → 429. RED + /// before the probe brake (req2 serves 200). The cross-request bound this proves is + /// exactly the amplification F3 closes. + #[sqlx::test] + async fn ipfs_cid_legacy_fanout_braked_on_ip_past_free_budget(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let fx = seed_cid_repos(&slug, &short, &["fanout"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("fanout.git"); + let repo = seed_repo(&owner_did, "fanout"); // public, no path rule → walk-free serve + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; // legacy pin + + let (st1, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&cid, "1.2.3.4")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st1, + StatusCode::OK, + "1st legacy fan-out probe from the IP serves" + ); + + let (st2, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&cid, "1.2.3.4")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st2, + StatusCode::TOO_MANY_REQUESTS, + "with no free budget, a repeat fan-out from the same IP is braked at the first probe" + ); + } + + /// F3 (jatmn, across-request amplification): the pre-fix free-probe budget was + /// PER REQUEST, so a caller could repeat a known NULL-provenance CID and force a + /// fresh batch of `acquire` + `cat-file` probes every request with zero limiter + /// contact, unbounded anonymous amplification against Tigris. Charging every + /// legacy probe from the first one makes those probes accumulate against the + /// per-IP `ipfs_rate_limiter` ACROSS requests. Four repos, none holding the CID, + /// so a full scan probes all four; the per-IP budget is sized to exactly ONE such + /// scan (4 tokens). req1 (a genuine absence) fully scans and 404s, spending the + /// budget; req2 from the SAME IP is shed at the first probe → 429 (it never + /// re-runs the four `acquire` probes). RED with the old free carve-out restored: + /// req2 re-scans un-braked and 404s again (the amplification stays open). This is + /// the load-bearing across-request bound F3 asks for. + #[sqlx::test] + async fn ipfs_cid_legacy_fanout_bounded_across_requests(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + // Budget = one full scan of the four seeded repos. A repeat scan from the same + // IP then finds it spent. Keyed on XFF so `oneshot` can choose the source IP. + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(4, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let names = ["a0", "a1", "a2", "a3"]; + let _fx = seed_cid_repos(&slug, &short, &names); + for n in names { + let repo = seed_repo(&owner_did, n); + state.db.create_repo(&repo).await.expect("seed repo"); + } + // A legacy pin whose oid is absent from every repo → each probed repo misses, + // so req1 scans all four (spending the four-token budget) and 404s cleanly. + let bogus_oid = "0".repeat(64); + let cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"absent-across-requests").to_string(); + state + .db + .record_pinned_cid(&bogus_oid, &cid, None) + .await + .expect("record legacy pin"); + + let (st1, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&cid, "9.9.9.9")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st1, + StatusCode::NOT_FOUND, + "1st scan completes under budget: a genuine absence is a definitive 404" + ); + + let (st2, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&cid, "9.9.9.9")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st2, + StatusCode::TOO_MANY_REQUESTS, + "2nd same-IP scan is shed at the first probe (429), not re-run un-braked: the across-request amplification is closed" + ); + } + + /// #173 (jatmn round 8, F3 — INV-10 cost guard): an already-throttled source's + /// legacy NULL-provenance request must be shed by the non-consuming admission peek + /// BEFORE the O(repos) `scan_ctx` preload runs — not after, where the per-probe + /// brake sits. The preload-query counter proves it both ways: 0 for the throttled + /// replay, 1 for an unthrottled source. RED if the peek is removed (the preload runs + /// while throttled → count 1). The two existing `_fanout_` tests confirm the per- + /// probe consuming charge is untouched (no double-charge, no under-charge). + #[sqlx::test] + async fn ipfs_cid_f3_throttled_source_skips_preload(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + // Budget 1, keyed on XFF so `oneshot` can choose the source IP. + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let _fx = seed_cid_repos(&slug, &short, &["r0"]); + state + .db + .create_repo(&seed_repo(&owner_did, "r0")) + .await + .expect("seed repo"); + // A legacy pin absent from every repo → the scan probes and 404s (spending the + // one token on the first probe). + let bogus_oid = "0".repeat(64); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"f3-absent").to_string(); + state + .db + .record_pinned_cid(&bogus_oid, &cid, None) + .await + .expect("legacy pin"); + + // Req1 from 9.9.9.9 spends the one token (and runs the preload once). + let _ = cid_router(&state) + .oneshot(cid_anon_xff(&cid, "9.9.9.9")) + .await + .unwrap(); + + // Measure the throttled replay: the peek must shed it before the preload runs. + crate::api::ipfs::reset_preload_queries(); + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&cid, "9.9.9.9")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::TOO_MANY_REQUESTS, + "an already-throttled legacy replay is 429" + ); + assert_eq!( + crate::api::ipfs::preload_queries(), + 0, + "a throttled source must NOT run the O(repos) preload (F3): shed before scan_ctx" + ); + + // Control: an unthrottled source (a different IP) still runs the preload once — + // the peek must not over-block. + crate::api::ipfs::reset_preload_queries(); + let _ = cid_router(&state) + .oneshot(cid_anon_xff(&cid, "8.8.8.8")) + .await + .unwrap(); + assert_eq!( + crate::api::ipfs::preload_queries(), + 1, + "an unthrottled source runs the preload once (the peek must not over-block)" + ); + } + + /// T2 (F1): the legacy scan is bounded per request. With the probe ceiling shrunk + /// to 2 and 3 candidate repos none of which hold the object, the 3rd repo is never + /// probed and the search is reported truncated → 503, not an unbounded fan-out. + /// RED before the probe cap (all 3 probe, none serve, definitive 404). + #[sqlx::test] + async fn ipfs_cid_legacy_scan_probe_cap_truncates_to_503(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_max_legacy_probes = 2; + + let _fx = seed_cid_repos(&slug, &short, &["r0", "r1", "r2"]); + for n in ["r0", "r1", "r2"] { + let repo = seed_repo(&owner_did, n); + state.db.create_repo(&repo).await.expect("seed repo"); + } + // A legacy pin whose oid is absent from every repo: each probed repo misses, + // so the cap (not a hit) decides the outcome. + let bogus_oid = "0".repeat(64); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"absent-marker-t2").to_string(); + state + .db + .record_pinned_cid(&bogus_oid, &cid, None) + .await + .expect("record legacy pin"); + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::SERVICE_UNAVAILABLE, + "a scan truncated by the probe cap is a retryable 503, not a definitive 404" + ); + } + + /// T3 (F2): a walk-cap truncation must not false-404. Walk ceiling shrunk to 1; + /// two public repos each carry a path-scoped rule over the object and deny anon. + /// The 1st spends the single walk (deny), the 2nd is skipped at the cap — the + /// resolver did NOT prove the object unreadable everywhere, so 503, not 404. + /// RED before the walk-cap `truncated` flag (returns the opaque 404). + #[sqlx::test] + async fn ipfs_cid_legacy_walk_cap_truncates_to_503(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let reader = Keypair::generate(); + let reader_did = reader.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_max_history_walks = 1; + + let fx = seed_cid_repos(&slug, &short, &["wa", "wb"]); + for n in ["wa", "wb"] { + let repo = seed_repo(&owner_did, n); + state.db.create_repo(&repo).await.expect("seed repo"); + state + .db + .set_visibility_rule( + &repo.id, + "/secret/**", + VisibilityMode::B, + std::slice::from_ref(&reader_did), + &owner_did, + ) + .await + .expect("path rule"); + } + // Legacy pin of the path-scoped secret blob (present in both repos, denies anon). + let bare_wa = std::path::PathBuf::from("/tmp").join(&slug).join("wa.git"); + let cid = pin_cid_for(&bare_wa, &fx.secret_oid, &state.db).await; + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::SERVICE_UNAVAILABLE, + "the walk cap truncated the scan, so absence is unproven → 503, not a false 404" + ); + } + + /// T4 (must-not over-fire): a legacy CID genuinely absent from every repo on a + /// node UNDER the probe cap still returns the definitive 404 — the 503 fires only + /// on real truncation, never as a blanket replacement for not-found. + #[sqlx::test] + async fn ipfs_cid_legacy_true_absence_stays_404(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_max_legacy_probes = 8; // well above the single repo → no truncation + + let _fx = seed_cid_repos(&slug, &short, &["only"]); + let repo = seed_repo(&owner_did, "only"); + state.db.create_repo(&repo).await.expect("seed repo"); + let bogus_oid = "0".repeat(64); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"absent-marker-t4").to_string(); + state + .db + .record_pinned_cid(&bogus_oid, &cid, None) + .await + .expect("record legacy pin"); + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "a fully-scanned genuine absence is a definitive 404, not a 503" + ); + } + + /// T5 (provenance path untouched): the probe cap governs ONLY the legacy scan. + /// With the cap set to 0 (which would truncate any legacy probe immediately) a + /// PROVENANCED pin still resolves to its one repo and serves 200 — proving the + /// `legacy_scan=false` guard exempts the provenance path. RED if the guard were + /// dropped (provenance would truncate to 503). + #[sqlx::test] + async fn ipfs_cid_provenance_serves_despite_zero_probe_cap(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_max_legacy_probes = 0; // would truncate every LEGACY probe + + let fx = seed_cid_repos(&slug, &short, &["provonly"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("provonly.git"); + let repo = seed_repo(&owner_did, "provonly"); // public, no path rule + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo.id).await; + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "the provenance path ignores the legacy probe cap and serves" + ); + } + + fn cid_router(state: &AppState) -> Router { + Router::new() + .route( + "/ipfs/{cid}", + axum::routing::get(crate::api::ipfs::get_by_cid), + ) + .layer(axum::middleware::from_fn(crate::auth::optional_signature)) + .with_state(state.clone()) + } + async fn cid_parts(resp: axum::response::Response) -> (StatusCode, String) { + let st = resp.status(); + let b = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + (st, String::from_utf8_lossy(&b).to_string()) + } + /// Raw body bytes (NOT lossy-decoded). A git tree body stores each child oid + /// as 32 RAW bytes that `from_utf8_lossy` mangles to U+FFFD, so a hex + /// `contains` check on `cid_parts`'s String is vacuous. #135 deny tests must + /// witness the leak on these raw bytes. + async fn cid_bytes(resp: axum::response::Response) -> (StatusCode, Vec) { + let st = resp.status(); + let b = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + (st, b.to_vec()) + } + /// True if `needle` appears as a contiguous byte subsequence of `haystack`. + fn bytes_contain(haystack: &[u8], needle: &[u8]) -> bool { + !needle.is_empty() && haystack.windows(needle.len()).any(|w| w == needle) + } + fn cid_anon(cid: &str) -> Request { + Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap() + } + /// Anonymous CID request carrying `x-forwarded-for: ` — an anon caller with a + /// resolvable source IP, so the per-IP walk brake keys on it (the walk still + /// denies anon at a path rule). + fn cid_anon_xff(cid: &str, xff_ip: &str) -> Request { + Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .header("x-forwarded-for", xff_ip) + .body(Body::empty()) + .unwrap() + } + fn cid_signed(kp: &gitlawb_core::identity::Keypair, cid: &str) -> Request { + let path = format!("/ipfs/{cid}"); + let s = gitlawb_core::http_sig::sign_request(kp, "GET", &path, b""); + Request::builder() + .method(Method::GET) + .uri(&path) + .header("content-digest", s.content_digest) + .header("signature-input", s.signature_input) + .header("signature", s.signature) + .body(Body::empty()) + .unwrap() + } + /// Signed CID request carrying `x-forwarded-for: `. Used by the walk + /// rate-limit test to key the per-IP limiter off a chosen source under + /// `TrustedProxy::XForwardedFor` (the request goes through `oneshot`, which + /// leaves no socket peer, so the header is the only key source). + fn cid_signed_xff( + kp: &gitlawb_core::identity::Keypair, + cid: &str, + xff_ip: &str, + ) -> Request { + let path = format!("/ipfs/{cid}"); + let s = gitlawb_core::http_sig::sign_request(kp, "GET", &path, b""); + Request::builder() + .method(Method::GET) + .uri(&path) + .header("content-digest", s.content_digest) + .header("signature-input", s.signature_input) + .header("signature", s.signature) + .header("x-forwarded-for", xff_ip) + .body(Body::empty()) + .unwrap() + } + + /// #110: `GET /ipfs/{cid}` must gate a withheld blob by per-caller visibility. + /// RED before U2 (the current handler serves the secret to anon). + #[sqlx::test] + async fn ipfs_cid_gate_withholds_blob_from_unauthorized(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let reader = Keypair::generate(); + let reader_did = reader.did().to_string(); + let stranger = Keypair::generate(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["withhold"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("withhold.git"); + // Request CIDs are the production pin CIDs (content-hash), recorded in + // pinned_cids so get_by_cid resolves each back to its oid (#173). + let secret_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; + let tree_cid = pin_cid_for(&bare, &fx.secret_tree_oid, &state.db).await; + let public_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + let root_tree_cid = pin_cid_for(&bare, &fx.root_tree_oid, &state.db).await; + let public_tree_cid = pin_cid_for(&bare, &fx.public_tree_oid, &state.db).await; + let commit_cid = pin_cid_for(&bare, &fx.commit_oid, &state.db).await; + let tag_cid = pin_cid_for(&bare, &fx.tag_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "withhold")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "withhold") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + VisibilityMode::B, + std::slice::from_ref(&reader_did), + &owner_did, + ) + .await + .expect("deny rule"); + + // anon → withheld blob: must 404, must not leak content. (RED on current handler.) + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "anon must not read the withheld blob" + ); + assert!( + !body.contains("TOP SECRET"), + "404 body must not leak the secret" + ); + + // signed non-reader → 404. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&stranger, &secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "non-reader must not read the withheld blob" + ); + assert!(!body.contains("TOP SECRET")); + + // owner (signed) → 200 + secret bytes. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&owner, &secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "owner reads the withheld blob"); + assert!(body.contains("TOP SECRET"), "owner gets the content"); + + // listed reader (signed) → 200. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&reader, &secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "listed reader reads the blob"); + assert!(body.contains("TOP SECRET")); + + // #135: anon tree CID under withheld /secret → 404. The 404 body is an opaque + // error string (never the object), so status is the load-bearing deny check; + // the real leak witness is the CONTRAST with the reader below, who DOES get a + // 200 carrying the child structure that anon is denied. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "withheld subtree tree must not be served to anon (#135)" + ); + + // Over-denial guard + positive leak witness: the listed reader (signed) DOES + // read the withheld subtree's tree, and its body carries the exact child + // structure anon was denied — the child filename plus the child oid as the 32 + // RAW bytes a git tree stores (witnessed on raw bytes, since cid_parts's lossy + // decode would mangle them). This proves b.txt / secret_raw are the real leak + // markers and that the anon 404 above actually withheld them. + let secret_raw = hex::decode(&fx.secret_oid).expect("hex oid"); + let (st, body) = cid_bytes( + cid_router(&state) + .oneshot(cid_signed(&reader, &tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "listed reader reads the withheld subtree tree" + ); + assert!( + bytes_contain(&body, b"b.txt") && bytes_contain(&body, &secret_raw), + "reader's tree body carries the child filename and raw child oid" + ); + + // Root tree (path "/") stays served to anon who passes the "/" gate. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&root_tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "root tree stays served (must-serve)"); + + // /public subtree tree stays served to anon (allowed path). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&public_tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "public subtree tree stays served"); + + // Commit and annotated tag objects stay served (unchanged by #135). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&commit_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "commit object stays served"); + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&tag_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "tag object stays served"); + + // R3: public blob anon → 200 (non-withheld content not affected). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&public_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "public blob stays served"); + + // R5: a genuine unknown CID also 404, uniform with the withheld 404. A + // well-formed pin-style CID that was never recorded in pinned_cids, so the + // oid_for_cid resolve misses (the production not-found path). + let absent_cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"never pinned to this node").to_string(); + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&absent_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "absent CID 404 (uniform with withheld)" + ); + + // malformed CID → 400 (unchanged). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon("not-a-cid")) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::BAD_REQUEST, "malformed CID still 400"); + } + + /// R4: the same object withheld in one repo but public in another is still + /// served from the public copy; the withholding repo is iterated first. + #[sqlx::test] + async fn ipfs_cid_served_from_public_copy_when_withheld_elsewhere(pool: PgPool) { + use crate::db::VisibilityMode; + use chrono::Utc; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["withhold", "pubcopy"]); + // Same content in both clones -> same oid/CID; read from either. + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("withhold.git"); + let secret_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; + + // Withholding repo, iterated FIRST (later updated_at; list_all_repos is DESC). + let mut withhold = seed_repo(&owner_did, "withhold"); + withhold.updated_at = Utc::now(); + state + .db + .create_repo(&withhold) + .await + .expect("withhold repo"); + state + .db + .set_visibility_rule( + &withhold.id, + "/secret/**", + VisibilityMode::B, + &[], + &owner_did, + ) + .await + .expect("deny rule"); + + // Public copy, no rules, iterated AFTER. + let mut pubcopy = seed_repo(&owner_did, "pubcopy"); + pubcopy.updated_at = Utc::now() - chrono::Duration::seconds(60); + state.db.create_repo(&pubcopy).await.expect("pubcopy repo"); + + // anon: denied at the withholding repo (continue), served from the public copy. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "served from the public copy despite the other deny" + ); + assert!( + body.contains("TOP SECRET"), + "the public copy serves the content" + ); + } + + /// Repo-level "/" gate (KTD2a, first continue branch): a fully private repo + /// (is_public=false, no rules) denies anon before any per-blob check; the + /// owner still reads. The path-scoped tests pass the "/" gate and deny at the + /// per-blob stage, so this exercises the coarser repo-level deny separately. + #[sqlx::test] + async fn ipfs_cid_private_repo_denies_anon_at_repo_gate(pool: PgPool) { + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["priv"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("priv.git"); + let blob_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + let mut rec = seed_repo(&owner_did, "priv"); + rec.is_public = false; + state.db.create_repo(&rec).await.expect("private repo"); + + // anon → repo-level deny → 404, no content leaked. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&blob_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "anon denied at a private repo's / gate" + ); + assert!(!body.contains("public bytes"), "404 must not leak content"); + + // owner-signed → 200. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&owner, &blob_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "owner reads their private repo's object" + ); + assert!(body.contains("public bytes"), "owner gets the content"); + } + + /// Fail-closed walk-error arm: if `withheld_blob_oids` errors (here, a ref + /// pointing at a non-tree-ish blob, which `git ls-tree -r` cannot traverse — + /// the same induction as `visibility_pack::fails_closed_when_a_ref_cannot_be_traversed`), + /// the handler skips the whole repo rather than serving. Asserts no leak of the + /// withheld blob AND that even the *public* blob in that repo is withheld — the + /// latter distinguishes fail-closed-skip from normal per-blob withholding and + /// would serve 200 if the error arm wrongly proceeded. + #[sqlx::test] + async fn ipfs_cid_walk_error_fails_closed(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["withhold"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("withhold.git"); + // Recorded pins so get_by_cid resolves each CID to its oid and reaches the + // walk; the 404s below are then the fail-closed skip, not a table miss. + let secret_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; + let public_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + // Force the withheld walk to fail closed: a ref pointing at a blob (not + // tree-ish) makes `git ls-tree -r` error, which `withheld_blob_oids` + // propagates as Err → the handler's `Ok(Err)` arm skips the repo. + std::fs::write( + bare.join("refs/heads/blobref"), + format!("{}\n", fx.secret_oid), + ) + .unwrap(); + + state + .db + .create_repo(&seed_repo(&owner_did, "withhold")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "withhold") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("deny rule"); + + // Withheld secret CID under a walk error → 404, no leak. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "walk error must not serve the withheld blob" + ); + assert!( + !body.contains("TOP SECRET"), + "walk-error 404 must not leak the secret" + ); + + // The PUBLIC blob in the same repo is also 404: the walk error fails closed + // by skipping the whole repo, not by serving. Without the fail-closed arm + // this would serve 200, so this assertion is the load-bearing discriminator. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&public_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "walk error fails closed: repo skipped, even the public blob is not served" + ); + } + + /// #173 review (F2): the commit/tag reachability walk must FAIL CLOSED on a git + /// error, exactly like the blob/tree walk. A ref pointing at a nonexistent object + /// makes `rev-list --all` fail, so `reachable_commit_tag_oids` returns Err, which + /// the handler's shared `Ok(Err) => continue` arm turns into a repo skip. The + /// load-bearing discriminator is that the PUBLIC commit is ALSO 404: if the arm + /// fail-OPENed (served on error) it would 200. Drives the commit/tag branch of + /// the shared fail-closed arm specifically (the sibling test covers blob/tree). + #[sqlx::test] + async fn ipfs_cid_commit_tag_walk_error_fails_closed(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["cterr"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("cterr.git"); + // A reachable commit CID — would serve 200 if the walk succeeded. + let commit_cid = pin_cid_for(&bare, &fx.commit_oid, &state.db).await; + + // A ref to a NONEXISTENT object: `git rev-list --all` fails ("bad object"), + // so reachable_commit_tag_oids bails → the walk arm skips the repo. + std::fs::write( + bare.join("refs/heads/broken"), + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef\n", + ) + .unwrap(); + + state + .db + .create_repo(&seed_repo(&owner_did, "cterr")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "cterr") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + + // Fail-closed: a walk error skips the repo, so even the otherwise-reachable + // public commit is 404 (not served). A fail-OPEN arm would 200 here. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&commit_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "a commit/tag walk error must fail closed (repo skipped), never serve" + ); + } + + /// #126: a dangling blob (written via `git hash-object -w`, never referenced + /// by any commit/tree) must 404 through `GET /ipfs/{cid}` under path-scoped + /// rules — for anon AND the owner. The pre-#126 deny-set was fail-open by + /// construction: dangling oids were absent from the reachable enumeration + /// and thus absent from the deny-set, so the handler served 200. The + /// allowed-set is fail-closed: dangling oids are absent from the reachable + /// allowed-set, so the handler 404s (per team memory: the owner shift to + /// 404 is the accepted fail-closed default — owners can still + /// `git cat-file` directly). + #[sqlx::test] + async fn ipfs_cid_dangling_blob_fails_closed_under_path_rules(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + // Seed a normal repo with `secret/b.txt` reachable from HEAD, so the + // path-scoped rule has something to match — without this the rule has + // no anchor and we'd be testing nothing. + let _fx = seed_cid_repos(&slug, &short, &["dangling"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("dangling.git"); + + // Write a dangling blob: `git hash-object -w --stdin` adds it to the + // object DB but nothing references it, so the reachable walk never + // enumerates it. + let mut cmd = std::process::Command::new("git"); + cmd.args(["hash-object", "-w", "--stdin"]) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()); + let mut child = cmd.spawn().expect("spawn git hash-object"); + { + use std::io::Write; + let stdin = child.stdin.as_mut().expect("stdin"); + stdin.write_all(b"DANGLING SECRET\n").expect("write stdin"); + } + let out = child.wait_with_output().expect("hash-object output"); + assert!( + out.status.success(), + "git hash-object: {}", + String::from_utf8_lossy(&out.stderr) + ); + let dangling_oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); + // Sanity: must be a 64-hex sha256 oid, since the repo is sha256-format. + assert_eq!( + dangling_oid.len(), + 64, + "expected sha256 oid: {dangling_oid}" + ); + // Record the pin so oid_for_cid resolves it — the 404 must then come from + // the allowed-set gate excluding the dangling oid, not from a table miss. + let dangling_cid = pin_cid_for(&bare, &dangling_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "dangling")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "dangling") + .await + .unwrap() + .unwrap(); + // Path-scoped rule triggers the per-blob allowed-set gate (KTD4). + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("deny rule"); + + // anon: the dangling blob is absent from the reachable allowed-set → + // 404, no leak. Pre-#126 (deny-set) would serve 200. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&dangling_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "dangling blob must 404 under path-scoped rules" + ); + assert!( + !body.contains("DANGLING SECRET"), + "404 body must not leak the dangling content" + ); + + // owner (signed): same 404. The dangling blob has no path, so it's + // never visibility-checked → never in the allowed set, even for the + // owner. This is the accepted fail-closed shift documented in the PR. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&owner, &dangling_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "owner also 404s on dangling blobs under path-scoped rules (fail-closed default)" + ); + assert!(!body.contains("DANGLING SECRET")); + } + + /// #135: a DANGLING tree (in the ODB, referenced by no commit) 404s under + /// path-scoped rules for anon AND owner — the reachable-only allowed-tree-set + /// never enumerates it. Handler-level companion to the helper test + /// `allowed_tree_set_excludes_dangling_tree`, proving the `get_by_cid` tree arm + /// (memo insert + `!in_allowed` continue) fails closed on the dangling case. + #[sqlx::test] + async fn ipfs_cid_dangling_tree_fails_closed_under_path_rules(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["dangtree"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("dangtree.git"); + + // Dangling tree via `git mktree`: a UNIQUE entry name so its oid is + // content-distinct from every reachable tree (a content-identical tree would + // dedup to a reachable oid — that is T2, not danglingness). + let mut child = std::process::Command::new("git") + .args(["mktree"]) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .expect("spawn git mktree"); + { + use std::io::Write; + writeln!( + child.stdin.as_mut().unwrap(), + "100644 blob {}\tdangling-only-unreferenced.txt", + fx.secret_oid + ) + .unwrap(); + } + let out = child.wait_with_output().expect("mktree output"); + assert!( + out.status.success(), + "git mktree: {}", + String::from_utf8_lossy(&out.stderr) + ); + let dangling_tree_oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); + assert_eq!(dangling_tree_oid.len(), 64, "expected sha256 oid"); + // Record the pin so the 404 is the allowed-tree-set gate excluding the + // dangling tree, not a table miss. + let dangling_cid = pin_cid_for(&bare, &dangling_tree_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "dangtree")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "dangtree") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("deny rule"); + + for req in [cid_anon(&dangling_cid), cid_signed(&owner, &dangling_cid)] { + let (st, _) = cid_parts(cid_router(&state).oneshot(req).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "dangling tree must 404 under path-scoped rules (anon + owner)" + ); + } + } + + /// #173 (F1): a QUARANTINED repo must not serve a pinned object by CID, to anon + /// OR to the mirror's own owner — quarantine is "hidden from serve/clone/listings, + /// owner included" (authorize_repo_read / feed_quarantined_mirror_withheld_from_owner). + /// The repo is PUBLIC with no path-scoped rule, so the "/" visibility gate ALLOWS + /// it and quarantine is the sole possible denier: RED before the fix (the loop + /// never checks quarantine → serves 200 + bytes), GREEN after the quarantine skip. + /// The owner-signed 404 is the load-bearing negative — a visibility-only gate + /// would Allow the owner and miss this. + #[sqlx::test] + async fn ipfs_cid_quarantined_repo_withheld_from_anon_and_owner(pool: PgPool) { + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["quar"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("quar.git"); + // Pin a ROOT-readable object (public/a.txt) — no path-scoped rule, so only + // quarantine can deny it. + let public_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "quar")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "quar") + .await + .unwrap() + .unwrap(); + + // Baseline: before quarantine the object serves 200 (proves the CID resolves + // and the object is otherwise servable, so the 404 below is quarantine's doing). + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&public_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "public root object serves before quarantine" + ); + assert!(body.contains("public bytes"), "baseline serves the content"); + + // Quarantine it. + state + .db + .set_repo_quarantine(&rec.id, true) + .await + .expect("quarantine"); + + // anon AND owner-signed must both 404 with no content leak. + for req in [cid_anon(&public_cid), cid_signed(&owner, &public_cid)] { + let (st, body) = cid_parts(cid_router(&state).oneshot(req).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "quarantined repo must not serve by CID (anon + owner)" + ); + assert!( + !body.contains("public bytes"), + "404 body must not leak quarantined content" + ); + } + } + + /// #173 (F2): a DANGLING commit or annotated tag (in the ODB, referenced by no + /// ref) must 404 under path-scoped rules for anon AND owner. The resolver proved + /// reachability only for blobs/trees, so a dangling commit/tag fell through to + /// serve, leaking its message/metadata. RED before the fix (serves 200 + + /// sentinel), GREEN after (the reachable commit/tag set excludes them). The + /// reachable-commit/tag serve path is covered by + /// ipfs_cid_gate_withholds_blob_from_unauthorized (commit + annotated tag → 200). + #[sqlx::test] + async fn ipfs_cid_dangling_commit_and_tag_fail_closed_under_path_rules(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["dangct"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("dangct.git"); + + // Run a git plumbing command that reads from stdin and prints an oid. + let oid_from_stdin = |args: &[&str], input: &[u8]| -> String { + use std::io::Write; + let mut child = std::process::Command::new("git") + .args(args) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("spawn git"); + child.stdin.as_mut().unwrap().write_all(input).unwrap(); + let out = child.wait_with_output().expect("git output"); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + + // Dangling commit: commit-tree with a sentinel message, NO ref update. + let dangling_commit_oid = oid_from_stdin( + &["commit-tree", &fx.root_tree_oid], + b"DANGLING COMMIT SECRET\n", + ); + assert_eq!(dangling_commit_oid.len(), 64, "expected sha256 commit oid"); + // Dangling annotated tag: mktag of the dangling commit, NO ref. + let tag_body = format!( + "object {dangling_commit_oid}\ntype commit\ntag dang\ntagger t 0 +0000\n\nDANGLING TAG SECRET\n" + ); + let dangling_tag_oid = oid_from_stdin(&["mktag"], tag_body.as_bytes()); + assert_eq!(dangling_tag_oid.len(), 64, "expected sha256 tag oid"); + + let commit_cid = pin_cid_for(&bare, &dangling_commit_oid, &state.db).await; + let tag_cid = pin_cid_for(&bare, &dangling_tag_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "dangct")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "dangct") + .await + .unwrap() + .unwrap(); + // Path-scoped rule triggers the per-object gate (KTD4). + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("deny rule"); + + for (cid, sentinel) in [ + (&commit_cid, "DANGLING COMMIT SECRET"), + (&tag_cid, "DANGLING TAG SECRET"), + ] { + for req in [cid_anon(cid), cid_signed(&owner, cid)] { + let (st, body) = cid_parts(cid_router(&state).oneshot(req).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "dangling commit/tag must 404 under path-scoped rules (anon + owner)" + ); + assert!( + !body.contains(sentinel), + "404 body must not leak the dangling message: {sentinel}" + ); + } + } + } + + /// #173 review (F2 hardening): a REACHABLE commit must still serve under a + /// path-scoped rule even when the repo carries a pushable non-commit ref (an + /// annotated tag of a tree, accepted by receive-pack). `reachable_commit_tag_oids` + /// must NOT route through `assert_all_refs_are_commits` (which bails on such a + /// ref and would fail-closed 404 every reachable commit/tag CID in the repo). + /// RED before the decoupling (the guard bails → 404), GREEN after. + #[sqlx::test] + async fn ipfs_cid_reachable_commit_served_despite_non_commit_ref(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["weirdref"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("weirdref.git"); + + // A pushable non-commit ref: an annotated tag pointing at a TREE. `git tag -a` + // in the bare repo creates refs/tags/treetag -> tag object -> tree, which + // peels to a non-commit and makes assert_all_refs_are_commits bail. + let out = std::process::Command::new("git") + .args([ + "tag", + "-a", + "treetag", + &fx.root_tree_oid, + "-m", + "tag of a tree", + ]) + .current_dir(&bare) + .output() + .expect("git tag -a"); + assert!( + out.status.success(), + "git tag -a: {}", + String::from_utf8_lossy(&out.stderr) + ); + + // Pin the REACHABLE root commit. + let commit_cid = pin_cid_for(&bare, &fx.commit_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "weirdref")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "weirdref") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + + // The reachable commit must still serve — the non-commit ref must not + // fail-closed the whole repo's commit/tag CID retrieval. + let resp = cid_router(&state) + .oneshot(cid_anon(&commit_cid)) + .await + .unwrap(); + let served_hash = resp + .headers() + .get("x-git-hash") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + let (st, _body) = cid_parts(resp).await; + assert_eq!( + st, + StatusCode::OK, + "a reachable commit must serve despite a pushable non-commit ref in the repo" + ); + assert_eq!( + served_hash.as_deref(), + Some(fx.commit_oid.as_str()), + "the served object is the reachable root commit" + ); + } + + /// #173 review (F-F): an annotated tag pointing at a TREE is pushable through + /// receive-pack, and the TREE allowed-set path + /// (`allowed_tree_set_for_caller` -> `tree_paths` -> `reachable_commits`) runs + /// `assert_all_refs_are_commits`, which bails on that ref and fail-closes the + /// whole repo — 404-ing EVERY tree CID (root + public subtrees) for its owner + /// and readers, not just the offending tag. The tree allowed-set feeds ONLY the + /// CID gate (absence = fail-closed 404), so `tree_paths` uses the lenient + /// reachable-commit enumeration: commit-reachable trees still serve, while a + /// tree reachable only via such a tag stays excluded. `blob_paths` keeps the + /// strict guard (it also feeds serve/replication, where a miss under-withholds). + /// RED before the decoupling (whole-repo bail -> 404 on the root/public tree), + /// GREEN after; the withheld-subtree 404 is the load-bearing must-not. + #[sqlx::test] + async fn ipfs_cid_tree_served_despite_non_commit_ref(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["treeweird"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("treeweird.git"); + + // Pushable non-commit ref: an annotated tag pointing at the ROOT TREE. + let out = std::process::Command::new("git") + .args([ + "tag", + "-a", + "treetag", + &fx.root_tree_oid, + "-m", + "tag of a tree", + ]) + .current_dir(&bare) + .output() + .expect("git tag -a"); + assert!( + out.status.success(), + "git tag -a: {}", + String::from_utf8_lossy(&out.stderr) + ); + + // Pin the reachable root tree and public subtree (both at ALLOWED paths), + // plus the secret subtree (a DENIED path — the fail-closed negative). + let root_tree_cid = pin_cid_for(&bare, &fx.root_tree_oid, &state.db).await; + let public_tree_cid = pin_cid_for(&bare, &fx.public_tree_oid, &state.db).await; + let secret_tree_cid = pin_cid_for(&bare, &fx.secret_tree_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "treeweird")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "treeweird") + .await + .unwrap() + .unwrap(); + // Path-scoped rule triggers the per-object tree gate (KTD4). + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + + // Reachable trees at ALLOWED paths must still serve despite the tag-of-tree. + for (cid, want_oid, label) in [ + (&root_tree_cid, &fx.root_tree_oid, "root tree"), + (&public_tree_cid, &fx.public_tree_oid, "public subtree"), + ] { + let resp = cid_router(&state).oneshot(cid_anon(cid)).await.unwrap(); + let served = resp + .headers() + .get("x-git-hash") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + let (st, _) = cid_parts(resp).await; + assert_eq!( + st, + StatusCode::OK, + "{label} CID must serve despite a pushable tag-of-tree in the repo" + ); + assert_eq!( + served.as_deref(), + Some(want_oid.as_str()), + "{label}: the served object is the reachable tree" + ); + } + + // Fail-closed preserved: the DENIED subtree's CID is still withheld — the + // lenient walk must not under-withhold a path the caller cannot read. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&secret_tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "a withheld subtree's tree CID stays 404 (lenient walk must not under-withhold)" + ); + } + + /// #173 review (F2 hardening): the INNER tag object of a nested tag-of-a-tag is + /// reachable (via the outer ref tag) and pinnable, so its CID must serve under a + /// path rule. `reachable_commit_tag_oids` peels tag chains to include it. RED + /// before the peel loop (the inner tag is not a ref tip and rev-list dereferences + /// to the commit, so it is absent → 404), GREEN after. + #[sqlx::test] + async fn ipfs_cid_nested_tag_inner_object_served(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["nested"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("nested.git"); + + let git_stdin = |args: &[&str], input: &[u8]| -> String { + use std::io::Write; + let mut child = std::process::Command::new("git") + .args(args) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("spawn git"); + child.stdin.as_mut().unwrap().write_all(input).unwrap(); + let out = child.wait_with_output().expect("git output"); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + + // Inner annotated tag of the reachable commit (no ref of its own). + let inner_body = format!( + "object {}\ntype commit\ntag inner\ntagger t 0 +0000\n\ninner\n", + fx.commit_oid + ); + let inner_tag_oid = git_stdin(&["mktag"], inner_body.as_bytes()); + // Outer annotated tag of the inner tag, then a ref to the outer tag. The + // inner tag is reachable only THROUGH the outer, not as a ref tip. + let outer_body = format!( + "object {inner_tag_oid}\ntype tag\ntag outer\ntagger t 0 +0000\n\nouter\n" + ); + let outer_tag_oid = git_stdin(&["mktag"], outer_body.as_bytes()); + let out = std::process::Command::new("git") + .args(["update-ref", "refs/tags/nested", &outer_tag_oid]) + .current_dir(&bare) + .output() + .expect("update-ref"); + assert!( + out.status.success(), + "update-ref: {}", + String::from_utf8_lossy(&out.stderr) + ); + + let inner_cid = pin_cid_for(&bare, &inner_tag_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "nested")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "nested") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&inner_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "the inner tag of a nested tag-of-a-tag is reachable and must serve" + ); + } + + /// #135: with NO path-scoped rule the per-object gate is skipped, so a tree CID + /// is served (the `"/"` gate is the whole story). Guards against over-gating + /// trees — the tree analog of the blob skip-walk branch. + #[sqlx::test] + async fn ipfs_cid_tree_served_when_no_path_scoped_rule(pool: PgPool) { + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["nopathrule"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("nopathrule.git"); + let tree_cid = pin_cid_for(&bare, &fx.secret_tree_oid, &state.db).await; + + // Public repo, no visibility rules → has_path_scoped_rule is false. + state + .db + .create_repo(&seed_repo(&owner_did, "nopathrule")) + .await + .expect("seed repo"); + + let (st, body) = cid_bytes( + cid_router(&state) + .oneshot(cid_anon(&tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "tree served to anon when no path-scoped rule exists" + ); + assert!( + bytes_contain(&body, b"b.txt"), + "served tree carries its child structure" + ); + } + + /// #173 (Fix 1): the pinned_cids lookup must use the canonical base32 CID, not + /// the raw request spelling. A pin is stored under `cid.to_string()` (canonical + /// base32); a request carrying the SAME CID re-encoded to a different multibase + /// (base58btc) parses and passes the sha2-256 check but, on the pre-fix handler, + /// misses the lookup key → false 404. Public repo, no path-scoped rule, so no + /// walk — this isolates the lookup-key canonicalization. + #[sqlx::test] + async fn ipfs_alt_encoding_cid_resolves(pool: PgPool) { + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["altenc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("altenc.git"); + // Canonical base32 CID as stored by the pin path. + let public_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + // Public repo, no visibility rules (no path-scoped walk). + state + .db + .create_repo(&seed_repo(&owner_did, "altenc")) + .await + .expect("seed repo"); + + // Re-encode the SAME CID to base58btc — a different, equally-valid spelling + // that is NOT the stored key. The `cid` crate re-exports `multibase`. + let alt = public_cid + .parse::>() + .unwrap() + .to_string_of_base(cid::multibase::Base::Base58Btc) + .unwrap(); + assert_ne!(alt, public_cid, "alt encoding must differ from canonical"); + + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&alt)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "alt-multibase spelling of a pinned CID must resolve (canonicalized lookup)" + ); + assert!( + body.contains("public bytes"), + "resolved object serves its content" + ); + } + + /// #173 (Fix 2a, db-level): `oids_for_cid` returns EVERY oid recorded under a + /// CID, not an arbitrary one. `record_pinned_cid` is unique on the git oid and + /// non-unique on cid, so two distinct oids can share one content-CID. Old + /// `oid_for_cid` did `LIMIT 1`; the new plural method must surface both. + #[sqlx::test] + async fn oids_for_cid_returns_all_duplicates(pool: PgPool) { + let state = test_state(pool).await; + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"shared content cid").to_string(); + let oid_a = "a".repeat(64); + let oid_b = "b".repeat(64); + state + .db + .record_pinned_cid(&oid_a, &cid, None) + .await + .unwrap(); + state + .db + .record_pinned_cid(&oid_b, &cid, None) + .await + .unwrap(); + + let mut oids = state.db.oids_for_cid(&cid).await.unwrap(); + oids.sort(); + assert_eq!( + oids, + vec![oid_a, oid_b], + "oids_for_cid must return every oid recorded under the shared CID" + ); + } + + /// #173 (Fix 2b, handler-level): when two oids collide on one CID and the + /// first-recorded is absent from every repo while the second is a readable + /// public object, the handler must try both and serve the readable one. The + /// pre-fix handler resolved a single oid (LIMIT 1 → first-inserted for equal + /// keys) and 404'd. Ordering caveat: this relies on `oids_for_cid` returning + /// the absent oid before the readable one (heap/insert order for equal keys); + /// if that ordering ever changes, `oids_for_cid_returns_all_duplicates` remains + /// the load-bearing, deterministic driver for Fix 2. + #[sqlx::test] + async fn ipfs_cid_collision_serves_readable_duplicate(pool: PgPool) { + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["collision"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("collision.git"); + + // A GENUINE content collision: the shared CID is the readable object's REAL + // content CID, and a second (absent) oid is recorded under the SAME cid. The + // handler must try every oid and serve the one whose bytes hash to the CID. + // (F2, #173: the served bytes must match the requested content address, so the + // shared cid has to be the object's real cid — an arbitrary seed would now be + // withheld by the integrity check as an unverifiable provider-CID-style row.) + let (_ty, raw) = crate::git::store::read_object(&bare, &fx.public_oid) + .unwrap() + .unwrap(); + let shared_cid = gitlawb_core::cid::Cid::from_git_object_bytes(&raw).to_string(); + let absent_oid = "c".repeat(64); + state + .db + .record_pinned_cid(&absent_oid, &shared_cid, None) + .await + .expect("record absent oid first"); + state + .db + .record_pinned_cid(&fx.public_oid, &shared_cid, None) + .await + .expect("record readable oid second"); + + // Public repo, no rules → the readable public object is served if reached. + state + .db + .create_repo(&seed_repo(&owner_did, "collision")) + .await + .expect("seed repo"); - // anon → repo-level deny → 404, no content leaked. let (st, body) = cid_parts( cid_router(&state) - .oneshot(cid_anon(&blob_cid)) + .oneshot(cid_anon(&shared_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "handler must try every oid under the CID and serve the readable duplicate" + ); + assert!( + body.contains("public bytes"), + "the readable duplicate's content is served" + ); + } + + /// #173 (Fix 3/F3, INV-10): the expensive legacy fan-out is rate-limited per + /// source IP. A valid tree CID makes the object-type pre-check pass, so each + /// repeat request pays a fresh walk (request-scoped memo only) — unbounded + /// amplification. Since #173-F3 (jatmn) the source charge sits on the LEGACY + /// PROBE (`acquire` + `cat-file`), which precedes the walk, so every legacy + /// candidate is charged to the non-farmable source IP from the first probe; a + /// second identical request from the same IP is shed with 429, but a targeted + /// PROVENANCE fetch (no scan) and a request from a different IP are unaffected. + /// The limiter is sized to admit one full scan of the two seeded repos (2 probes) + /// so the first request serves; the repeat then finds the bucket spent. + #[sqlx::test] + async fn ipfs_walk_rate_limited_per_source(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let reader = Keypair::generate(); + let reader_did = reader.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + + let mut state = test_state(pool).await; + // The scan probes both seeded repos (walklimit + walkpublic) per request, so + // size the per-IP budget to admit exactly one full scan (2 probes). A repeat + // scan from the same IP then finds the bucket spent. Keyed on the rightmost + // X-Forwarded-For hop so the test can choose a source IP under `oneshot`. + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(2, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let fx = seed_cid_repos(&slug, &short, &["walklimit"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("walklimit.git"); + // The tree CID drives a path-scoped walk (the load-bearing amplification + // surface). The reader is allowed under /secret so the walk returns 200. + let secret_tree_cid = pin_cid_for(&bare, &fx.secret_tree_oid, &state.db).await; + + // Oldest `updated_at` → `list_all_repos` (ORDER BY updated_at DESC) probes + // this serving repo LAST, so a scan deterministically charges the walk-free + // `walkpublic` miss first then this serve: exactly 2 probes per scan. + let mut walklimit = seed_repo(&owner_did, "walklimit"); + walklimit.updated_at = chrono::Utc::now() - chrono::Duration::seconds(60); + state.db.create_repo(&walklimit).await.expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "walklimit") + .await + .unwrap() + .unwrap(); + // Mode B path rule over /secret with the reader allowed → the reader's + // secret-tree fetch runs the allowed-tree walk and returns 200. + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + VisibilityMode::B, + std::slice::from_ref(&reader_did), + &owner_did, + ) + .await + .expect("path rule"); + + // The MUST-NOT object must be a genuinely CHEAP fetch: an object served + // from a repo with NO path-scoped rule takes the no-walk path, so the WALK + // brake never rate-limits it. It has to live in a repo that carries no path + // rule AND whose object graph does not overlap `walklimit` (a blob shared + // with the path-scoped repo would still walk there), so we seed a second bare + // repo with UNIQUE content. `acquire(owner, "walkpublic")` resolves to + // `/tmp//walkpublic.git`. This copy is PROVENANCED (`pin_cid_for_repo`) + // so it resolves straight to its repo and skips the legacy probe brake: the + // point here is the WALK brake, and post-#173-F3 a walk-free LEGACY fetch is + // itself source-charged at the probe, so a legacy pin would (correctly) be + // shed from the exhausted IP and no longer isolate the walk brake. + let pub_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("walkpublic.git"); + { + use std::process::Command; + let run = |args: &[&str], cwd: &std::path::Path| { + let out = Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + }; + let src = std::env::temp_dir().join(format!("gl-cid-pub-{short}")); + let _ = std::fs::remove_dir_all(&src); + std::fs::create_dir_all(&src).unwrap(); + std::fs::write(src.join("cheap.txt"), b"cheap public bytes\n").unwrap(); + run(&["init", "-q", "--object-format=sha256"], &src); + run(&["config", "user.email", "t@t"], &src); + run(&["config", "user.name", "t"], &src); + run(&["add", "."], &src); + run(&["commit", "-qm", "cheap"], &src); + let _ = std::fs::remove_dir_all(&pub_bare); + run( + &[ + "clone", + "--bare", + "-q", + src.to_str().unwrap(), + pub_bare.to_str().unwrap(), + ], + &src, + ); + let _ = std::fs::remove_dir_all(&src); + } + let cheap_oid = { + use std::process::Command; + let out = Command::new("git") + .args(["rev-parse", "HEAD:cheap.txt"]) + .current_dir(&pub_bare) + .output() + .unwrap(); + assert!(out.status.success(), "rev-parse cheap.txt"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + // Public repo, NO visibility rules → the cheap object takes the no-walk path. + state + .db + .create_repo(&seed_repo(&owner_did, "walkpublic")) + .await + .expect("seed public repo"); + let pub_rec = state + .db + .get_repo(&owner_did, "walkpublic") + .await + .unwrap() + .unwrap(); + let public_cid = pin_cid_for_repo(&pub_bare, &cheap_oid, &state.db, &pub_rec.id).await; + + // 1st legacy scan from 1.2.3.4 → 200 (its two probes fit the budget; the + // walk ran, reader allowed). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_signed_xff(&reader, &secret_tree_cid, "1.2.3.4")) .await .unwrap(), ) .await; assert_eq!( st, - StatusCode::NOT_FOUND, - "anon denied at a private repo's / gate" + StatusCode::OK, + "1st legacy scan from a source IP is served" ); - assert!(!body.contains("public bytes"), "404 must not leak content"); - // owner-signed → 200. + // 2nd identical scan from the SAME IP → 429 (per-IP probe budget spent). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_signed_xff(&reader, &secret_tree_cid, "1.2.3.4")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::TOO_MANY_REQUESTS, + "2nd legacy scan from the same source IP is shed with 429" + ); + + // MUST-NOT: a targeted PROVENANCE fetch (no scan, no probe brake) from the + // SAME limited IP, even after the 429, is served: the brake is on the legacy + // scan, not the route. let (st, body) = cid_parts( cid_router(&state) - .oneshot(cid_signed(&owner, &blob_cid)) + .oneshot(cid_signed_xff(&reader, &public_cid, "1.2.3.4")) .await .unwrap(), ) @@ -2398,109 +5279,156 @@ mod tests { assert_eq!( st, StatusCode::OK, - "owner reads their private repo's object" + "a provenance (non-scan) fetch is never rate-limited, even from the exhausted IP" + ); + assert!( + body.contains("cheap public bytes"), + "the cheap fetch serves content" + ); + + // PER-SOURCE isolation: the same tree-CID scan from a DIFFERENT IP → 200. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_signed_xff(&reader, &secret_tree_cid, "5.6.7.8")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "one source's exhaustion must not shed another source's walk" ); - assert!(body.contains("public bytes"), "owner gets the content"); } - /// Fail-closed walk-error arm: if `withheld_blob_oids` errors (here, a ref - /// pointing at a non-tree-ish blob, which `git ls-tree -r` cannot traverse — - /// the same induction as `visibility_pack::fails_closed_when_a_ref_cannot_be_traversed`), - /// the handler skips the whole repo rather than serving. Asserts no leak of the - /// withheld blob AND that even the *public* blob in that repo is withheld — the - /// latter distinguishes fail-closed-skip from normal per-blob withholding and - /// would serve 200 if the error arm wrongly proceeded. + /// #173 review (F-C): a SKIPPED legacy candidate (a walk-and-deny denier, OR a + /// probe-throttled repo since #173-F3) must not end the whole request: the scan + /// keeps going so a later walk-free copy still serves, and a spent probe budget is + /// a clean 429, never a false 404/503. Otherwise a public CID would 404/429 solely + /// because a newer path-scoped duplicate sorts ahead of an older no-rule copy under + /// `updated_at DESC`. Two same-oid legacy copies: a NEWER `/secret`-scoped denier + /// and an OLDER no-rule public copy. + /// + /// Two requests from the SAME IP, budget = 2 (one full scan of both copies): + /// req1 probes the denier (charged), its allowed-blob walk denies anon → skip and + /// keep scanning, then probes+serves the walk-free public copy → 200. That proves + /// the denier skip is non-fatal (`continue`, not `break`). req2 from the same IP + /// finds the probe budget spent, so the denier's probe throttles → skip-continue, + /// the public copy's probe throttles too → nothing servable → a clean 429 (not a + /// truncation 503 nor a false 404), proving the throttle is likewise non-fatal but + /// correctly shed. RED before `continue` (a `break` on the skipped denier 404s + /// req1 outright). #[sqlx::test] - async fn ipfs_cid_walk_error_fails_closed(pool: PgPool) { + async fn ipfs_walk_quota_skips_denier_and_serves_public_copy(pool: PgPool) { use crate::db::VisibilityMode; + use chrono::Utc; use gitlawb_core::identity::Keypair; let owner = Keypair::generate(); let owner_did = owner.did().to_string(); let slug = owner_did.replace([':', '/'], "_"); let short = owner_did.split(':').next_back().unwrap().to_string(); - let state = test_state(pool).await; - - let fx = seed_cid_repos(&slug, &short, &["withhold"]); - let secret_cid = cid_for_oid(&fx.secret_oid); - let public_cid = cid_for_oid(&fx.public_oid); - - // Force the withheld walk to fail closed: a ref pointing at a blob (not - // tree-ish) makes `git ls-tree -r` error, which `withheld_blob_oids` - // propagates as Err → the handler's `Ok(Err)` arm skips the repo. - let bare = std::path::PathBuf::from("/tmp") + let mut state = test_state(pool).await; + // Budget = one full two-repo scan (2 probes), keyed on the rightmost XFF hop + // so `oneshot` can choose a source IP (no socket peer). A repeat scan from the + // same IP then finds the budget spent. + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(2, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + // Identical secret-blob content in both bare clones → one CID resolves to + // `secret_oid` in each. A NEWER path-scoped denier (walk-and-deny anon) and an + // OLDER no-rule public copy (walk-free serve). + let fx = seed_cid_repos(&slug, &short, &["scopeddenier", "publiccopy"]); + let denier_bare = std::path::PathBuf::from("/tmp") .join(&slug) - .join("withhold.git"); - std::fs::write( - bare.join("refs/heads/blobref"), - format!("{}\n", fx.secret_oid), - ) - .unwrap(); - + .join("scopeddenier.git"); + let secret_cid = pin_cid_for(&denier_bare, &fx.secret_oid, &state.db).await; + + // Newer denier: public at "/", `/secret/**` Mode B empty readers → an anon + // blob fetch clears "/", runs the allowed-blob walk, is denied → continue. + let mut denier = seed_repo(&owner_did, "scopeddenier"); + denier.updated_at = Utc::now(); + state.db.create_repo(&denier).await.expect("seed denier"); state .db - .create_repo(&seed_repo(&owner_did, "withhold")) - .await - .expect("seed repo"); - let rec = state - .db - .get_repo(&owner_did, "withhold") + .set_visibility_rule(&denier.id, "/secret/**", VisibilityMode::B, &[], &owner_did) .await - .unwrap() - .unwrap(); + .expect("path rule"); + + // Older public copy — NO rule → the secret blob serves via the no-walk path. + let mut public = seed_repo(&owner_did, "publiccopy"); + public.updated_at = Utc::now() - chrono::Duration::seconds(60); state .db - .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .create_repo(&public) .await - .expect("deny rule"); + .expect("seed public copy"); - // Withheld secret CID under a walk error → 404, no leak. - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_anon(&secret_cid)) - .await - .unwrap(), - ) - .await; + // req1 from 1.2.3.4: the denier is skipped (walk denies anon) and the scan + // keeps going to serve the older walk-free public copy. Both probes fit the + // budget, so this leaves the IP bucket spent. + let resp = cid_router(&state) + .oneshot(cid_anon_xff(&secret_cid, "1.2.3.4")) + .await + .unwrap(); + let served_hash = resp + .headers() + .get("x-git-hash") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + let (st, _body) = cid_parts(resp).await; assert_eq!( st, - StatusCode::NOT_FOUND, - "walk error must not serve the withheld blob" + StatusCode::OK, + "a skipped walk-requiring denier must not end the scan: the later walk-free public copy still serves" ); - assert!( - !body.contains("TOP SECRET"), - "walk-error 404 must not leak the secret" + assert_eq!( + served_hash.as_deref(), + Some(fx.secret_oid.as_str()), + "the served object is the secret blob from the no-rule public copy" ); - // The PUBLIC blob in the same repo is also 404: the walk error fails closed - // by skipping the whole repo, not by serving. Without the fail-closed arm - // this would serve 200, so this assertion is the load-bearing discriminator. + // req2 from the SAME exhausted IP: every legacy probe is now throttled. The + // throttle is non-fatal (skip and keep scanning), but nothing is servable, so + // it resolves to a clean 429, not a truncation 503, not a false 404. let (st, _) = cid_parts( cid_router(&state) - .oneshot(cid_anon(&public_cid)) + .oneshot(cid_anon_xff(&secret_cid, "1.2.3.4")) .await .unwrap(), ) .await; assert_eq!( st, - StatusCode::NOT_FOUND, - "walk error fails closed: repo skipped, even the public blob is not served" + StatusCode::TOO_MANY_REQUESTS, + "with the probe budget spent, the repeat legacy scan is shed with a clean 429" ); } - /// #126: a dangling blob (written via `git hash-object -w`, never referenced - /// by any commit/tree) must 404 through `GET /ipfs/{cid}` under path-scoped - /// rules — for anon AND the owner. The pre-#126 deny-set was fail-open by - /// construction: dangling oids were absent from the reachable enumeration - /// and thus absent from the deny-set, so the handler served 200. The - /// allowed-set is fail-closed: dangling oids are absent from the reachable - /// allowed-set, so the handler 404s (per team memory: the owner shift to - /// 404 is the accepted fail-closed default — owners can still - /// `git cat-file` directly). + /// INV-10 amplification bound: a single `GET /ipfs/{cid}` must not fan out an + /// unbounded number of full-history walks. The per-request `ipfs_rate_limiter` + /// check only brakes REPEAT requests (it fires once per request); within one + /// request the same object can exist under path-scoped rules in many repos, + /// each paying its own walk. `MAX_HISTORY_WALKS_PER_REQUEST` caps that fan-out. + /// + /// Load-bearing witness (#173, F4): a readable public copy (no path rule → + /// served via the no-walk path, exactly like + /// `ipfs_cid_served_from_public_copy_when_withheld_elsewhere`) is given the + /// OLDEST `updated_at` so `list_all_repos` (ORDER BY updated_at DESC) iterates it + /// LAST. Ahead of it sit `cap + 1` path-scoped deniers, each forcing an + /// allowed-blob walk that denies anon. The cap bounds SPAWNED walks to `cap`, but + /// hitting it must `continue` (skip only the walk-requiring denier), NOT `break` + /// the whole repo loop: the walk-free public copy needs no walk, so it is still + /// reached and served (200, `x-git-hash` = the blob oid). The old `break` + /// wrongly 404'd this publicly-readable content. Reverting `continue`→`break` + /// turns this 200 back into a 404: the RED proof that the loop keeps scanning for + /// a cheap readable copy after the cap. The `cap` walk ceiling still holds — only + /// `cap` walks are spawned across the deniers regardless (the amplification bound + /// is proven separately by `ipfs_walk_cap_still_serves_walk_free_candidate`). #[sqlx::test] - async fn ipfs_cid_dangling_blob_fails_closed_under_path_rules(pool: PgPool) { + async fn ipfs_walk_fanout_capped_per_request(pool: PgPool) { use crate::db::VisibilityMode; + use chrono::Utc; use gitlawb_core::identity::Keypair; let owner = Keypair::generate(); @@ -2509,96 +5437,246 @@ mod tests { let short = owner_did.split(':').next_back().unwrap().to_string(); let state = test_state(pool).await; - // Seed a normal repo with `secret/b.txt` reachable from HEAD, so the - // path-scoped rule has something to match — without this the rule has - // no anchor and we'd be testing nothing. - let _fx = seed_cid_repos(&slug, &short, &["dangling"]); - let bare = std::path::PathBuf::from("/tmp") - .join(&slug) - .join("dangling.git"); + let cap = crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST as usize; - // Write a dangling blob: `git hash-object -w --stdin` adds it to the - // object DB but nothing references it, so the reachable walk never - // enumerates it. - let mut cmd = std::process::Command::new("git"); - cmd.args(["hash-object", "-w", "--stdin"]) - .current_dir(&bare) - .stdin(std::process::Stdio::piped()) - .stdout(std::process::Stdio::piped()); - let mut child = cmd.spawn().expect("spawn git hash-object"); - { - use std::io::Write; - let stdin = child.stdin.as_mut().expect("stdin"); - stdin.write_all(b"DANGLING SECRET\n").expect("write stdin"); - } - let out = child.wait_with_output().expect("hash-object output"); - assert!( - out.status.success(), - "git hash-object: {}", - String::from_utf8_lossy(&out.stderr) - ); - let dangling_oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); - // Sanity: must be a 64-hex sha256 oid, since the repo is sha256-format. - assert_eq!( - dangling_oid.len(), - 64, - "expected sha256 oid: {dangling_oid}" - ); - let dangling_cid = cid_for_oid(&dangling_oid); + // `cap + 1` deniers guarantee the fan-out crosses the ceiling before the + // readable copy (iterated last) is reached. All bare clones share identical + // content, so the one secret-BLOB CID resolves to `secret_oid` in every repo. + let denier_names: Vec = (0..=cap).map(|i| format!("denier{i}")).collect(); + let mut names: Vec<&str> = vec!["readable"]; + names.extend(denier_names.iter().map(|s| s.as_str())); + let fx = seed_cid_repos(&slug, &short, &names); + let readable_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("readable.git"); + // The secret BLOB CID drives the path-scoped allowed-blob walk in every + // denier (the amplification surface) and is served cheaply from the + // no-rule public copy — the proven serve path. + let secret_cid = pin_cid_for(&readable_bare, &fx.secret_oid, &state.db).await; + + // 1) Readable public copy — OLDEST updated_at → iterated LAST. Public with + // NO visibility rule, so the blob serves via the no-walk path. This is + // the copy an uncapped fan-out would eventually reach and serve. + let mut readable = seed_repo(&owner_did, "readable"); + readable.updated_at = Utc::now() - chrono::Duration::seconds(60); state .db - .create_repo(&seed_repo(&owner_did, "dangling")) - .await - .expect("seed repo"); - let rec = state - .db - .get_repo(&owner_did, "dangling") + .create_repo(&readable) + .await + .expect("seed readable copy"); + + // 2) cap+1 deniers with NEWER updated_at → iterated before the copy. Public + // at "/", but a `/secret/**` Mode B rule with an EMPTY reader list, so an + // anon blob fetch clears the "/" gate, runs the allowed-blob walk, and is + // denied (the secret blob is in no one's set) → continue. Each distinct + // repo.id is its own walk (the memo only dedups the same repo). + for name in &denier_names { + let mut denier = seed_repo(&owner_did, name); + denier.updated_at = Utc::now(); + state.db.create_repo(&denier).await.expect("seed denier"); + state + .db + .set_visibility_rule(&denier.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + } + + // Anon (no peer, no XFF → the IP brake is skipped, so the walk cap is the + // only thing in play). After the cap, `continue` skips only the + // walk-requiring deniers and keeps scanning, reaching the walk-free public + // copy (iterated last) → served 200. The served object is the secret blob + // from the no-rule public copy, which is legitimately public THERE. + let resp = cid_router(&state) + .oneshot(cid_anon(&secret_cid)) .await - .unwrap() .unwrap(); - // Path-scoped rule triggers the per-blob allowed-set gate (KTD4). + let served_hash = resp + .headers() + .get("x-git-hash") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + let (st, _body) = cid_parts(resp).await; + assert_eq!( + st, + StatusCode::OK, + "hitting the walk cap must skip only the walk-requiring candidate, not abandon the walk-free readable copy" + ); + assert_eq!( + served_hash.as_deref(), + Some(fx.secret_oid.as_str()), + "the served object is the blob from the no-rule public copy reached after the cap" + ); + } + + /// Multi-oid companion to `ipfs_walk_fanout_capped_per_request`: exercises the + /// outer oid loop and proves the per-request walk budget PERSISTS across oid + /// candidates, so a commit/tag candidate cannot re-open the fan-out. Since #173 + /// (F2) a `commit`/`tag` under a path-scoped rule is itself walk-gated (its + /// reachability is proven by a `rev-list` walk via `reachable_commit_tag_oids`), + /// so it is NOT walk-free — it draws from the same budget as the blob/tree walks. + /// + /// One CID → TWO oids (the non-unique cid index, #173): a withheld `/secret` + /// blob (walk-triggering, denied to anon in every denier) recorded FIRST so a + /// seq scan tries it first and burns the whole walk budget across the deniers; + /// the reachable root commit is second. Because the budget is already spent, the + /// commit candidate's reachability walk is also capped in every denier, so the + /// request 404s — proving commit/tag walks (F2) respect the fan-out ceiling and + /// cannot be used to bypass it (R6/F3). A reachable commit served with budget to + /// spare is covered by `ipfs_cid_gate_withholds_blob_from_unauthorized`. The + /// withheld blob must not leak. Since #173 F2 a scan the walk cap truncated + /// returns 503 (absence unproven), not the old opaque 404. + #[sqlx::test] + async fn ipfs_walk_commit_tag_candidate_respects_the_walk_cap(pool: PgPool) { + use crate::db::VisibilityMode; + use chrono::Utc; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let cap = crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST as usize; + + // cap+1 path-scoped deniers, all carrying identical content (same oids). + let denier_names: Vec = (0..=cap).map(|i| format!("m{i}")).collect(); + let names: Vec<&str> = denier_names.iter().map(|s| s.as_str()).collect(); + let fx = seed_cid_repos(&slug, &short, &names); + let bare = std::path::PathBuf::from("/tmp").join(&slug).join("m0.git"); + + // ONE cid → TWO oids. The withheld blob is recorded first (seq scan lists it + // first → tried first → burns the budget); the reachable commit is second. + let multi_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; state .db - .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .record_pinned_cid(&fx.commit_oid, &multi_cid, None) .await - .expect("deny rule"); + .expect("co-locate the commit oid under the same cid"); - // anon: the dangling blob is absent from the reachable allowed-set → - // 404, no leak. Pre-#126 (deny-set) would serve 200. + for name in &denier_names { + let mut d = seed_repo(&owner_did, name); + d.updated_at = Utc::now(); + state.db.create_repo(&d).await.expect("seed denier"); + state + .db + .set_visibility_rule(&d.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + } + + // Anon: the blob candidate is denied in every denier (a walk each, spending + // the budget); the commit candidate's reachability walk is then also capped + // in every denier — so no candidate is served AND the walk cap truncated the + // scan, leaving absence unproven → 503 (not the old false 404, #173 F2). + // Either way commit/tag walks respect the ceiling and cannot re-open the + // fan-out (R6/F3). The withheld blob must not leak in the body. let (st, body) = cid_parts( cid_router(&state) - .oneshot(cid_anon(&dangling_cid)) + .oneshot(cid_anon(&multi_cid)) .await .unwrap(), ) .await; assert_eq!( st, - StatusCode::NOT_FOUND, - "dangling blob must 404 under path-scoped rules" + StatusCode::SERVICE_UNAVAILABLE, + "a commit/tag reachability walk respects the per-request cap; a truncated scan is 503, not a false 404" ); assert!( - !body.contains("DANGLING SECRET"), - "404 body must not leak the dangling content" + !body.contains("TOP SECRET"), + "the withheld blob must not leak in the truncation response" ); + } - // owner (signed): same 404. The dangling blob has no path, so it's - // never visibility-checked → never in the allowed set, even for the - // owner. This is the accepted fail-closed shift documented in the PR. - let (st, body) = cid_parts( + /// #173 (F3, INV-15): the per-IP quota debits ONE token per expensive legacy + /// candidate, not once per request, so one IP cannot drive an unbounded fan-out. + /// With quota=1 and two path-scoped deniers holding one CID, a SINGLE request is + /// shed at 429: since #173-F3 (jatmn) each legacy PROBE (`acquire` + `cat-file`, + /// which precedes the walk) debits, so the first denier probes+walks+denies on + /// token 1 and the second denier's probe finds no token → 429. (Before F3 the + /// debit sat on the walk; the outcome is unchanged, the charge point moved earlier + /// to also bound walk-free probes.) Defeating the per-candidate debit let one IP + /// drive up to MAX_HISTORY_WALKS_PER_REQUEST × quota expensive ops/hour. + #[sqlx::test] + async fn ipfs_walk_quota_debited_per_walk(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + // Signed but NOT a reader → cleared at "/", denied at /secret → forces a walk. + let stranger = Keypair::generate(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + + let mut state = test_state(pool).await; + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let fx = seed_cid_repos(&slug, &short, &["w0", "w1"]); + let bare = std::path::PathBuf::from("/tmp").join(&slug).join("w0.git"); + // The secret BLOB CID forces a path-scoped allowed-blob walk in each denier. + let secret_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; + + // Two path-scoped deniers (Mode B /secret, empty readers): each forces a + // walk that denies the signed stranger, so ONE request spawns two walks. + for name in ["w0", "w1"] { + let d = seed_repo(&owner_did, name); + state.db.create_repo(&d).await.expect("seed denier"); + state + .db + .set_visibility_rule(&d.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + } + + // ONE request, quota 1: walk 1 debits the token, walk 2 has none → 429. + let (st, _) = cid_parts( cid_router(&state) - .oneshot(cid_signed(&owner, &dangling_cid)) + .oneshot(cid_signed_xff(&stranger, &secret_cid, "1.2.3.4")) .await .unwrap(), ) .await; assert_eq!( st, - StatusCode::NOT_FOUND, - "owner also 404s on dangling blobs under path-scoped rules (fail-closed default)" + StatusCode::TOO_MANY_REQUESTS, + "the second full-history walk in one request must be shed with 429 (per-walk debit)" + ); + } + + /// The periodic cleanup task must sweep the ipfs walk limiter, not only its + /// five siblings. Drives `AppState::sweep_rate_limiters` — the exact method the + /// 300s loop calls — and asserts the ipfs limiter's expired entry is evicted. + /// Dropping `ipfs_rate_limiter.cleanup()` from that method leaves the entry in + /// place (`tracked_keys` stays 1): the RED proof that the sweep covers it. + #[sqlx::test] + async fn sweep_rate_limiters_includes_ipfs_limiter(pool: PgPool) { + let mut state = test_state(pool).await; + // Short window so a single recorded hit is already expired at sweep time. + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(5, Duration::from_millis(50)); + + assert!( + state.ipfs_rate_limiter.check("1.2.3.4").await, + "record a hit on the ipfs limiter" + ); + assert_eq!( + state.ipfs_rate_limiter.tracked_keys().await, + 1, + "the source-IP key is tracked before the sweep" + ); + + // Expire the entry (still mapped — cleanup hasn't run), then sweep. + tokio::time::sleep(Duration::from_millis(60)).await; + state.sweep_rate_limiters().await; + + assert_eq!( + state.ipfs_rate_limiter.tracked_keys().await, + 0, + "the periodic sweep must evict the ipfs limiter's expired entries" ); - assert!(!body.contains("DANGLING SECRET")); } // --------------------------------------------------------------------------- diff --git a/crates/gitlawb-node/src/visibility.rs b/crates/gitlawb-node/src/visibility.rs index 56616872..a8d7ddba 100644 --- a/crates/gitlawb-node/src/visibility.rs +++ b/crates/gitlawb-node/src/visibility.rs @@ -437,6 +437,31 @@ mod tests { ); } + // #135 T1: a Mode-B rule on `/secret/**` must DENY the withheld directory's + // OWN path `/secret` (the `path == prefix` arm), not just strict descendants — + // otherwise get_by_cid's tree gate would serve the /secret tree object and leak + // its children. Pins parity with get_tree, which denies the /secret path. + #[test] + fn subtree_rule_denies_the_withheld_directory_itself() { + let reader = "did:key:z6MkReader"; + let rules = [rule("/secret/**", VisibilityMode::B, &[reader])]; + assert_eq!( + visibility_check(&rules, true, OWNER, None, "/secret"), + Decision::Deny, + "anon denied at the withheld directory's OWN path /secret" + ); + assert_eq!( + visibility_check(&rules, true, OWNER, None, "/public"), + Decision::Allow, + "anon allowed at a sibling path outside the withheld subtree" + ); + assert_eq!( + visibility_check(&rules, true, OWNER, Some(reader), "/secret"), + Decision::Allow, + "listed reader allowed at the withheld directory (caller-aware)" + ); + } + // #153 regression: cross-method DID must still be denied even when the // trailing segment collides with a bare owner key. #[test] diff --git a/crates/gl/src/ipfs_cmd.rs b/crates/gl/src/ipfs_cmd.rs index d4cb64fc..fa7a3f3f 100644 --- a/crates/gl/src/ipfs_cmd.rs +++ b/crates/gl/src/ipfs_cmd.rs @@ -33,13 +33,16 @@ pub enum IpfsCmd { cid: String, #[arg(long, default_value = "https://node.gitlawb.com", env = "GITLAWB_NODE")] node: String, + /// Identity directory (default: ~/.gitlawb) + #[arg(long)] + dir: Option, }, } pub async fn run(args: IpfsArgs) -> Result<()> { match args.cmd { IpfsCmd::List { node, dir } => cmd_list(node, dir).await, - IpfsCmd::Get { cid, node } => cmd_get(cid, node).await, + IpfsCmd::Get { cid, node, dir } => cmd_get(cid, node, dir).await, } } @@ -86,11 +89,32 @@ async fn cmd_list(node: String, dir: Option) -> Result<()> { Ok(()) } -async fn cmd_get(cid: String, node: String) -> Result<()> { - let client = NodeClient::new(&node, None); - let path = format!("/ipfs/{cid}"); +async fn cmd_get(cid: String, node: String, dir: Option) -> Result<()> { + // #173 (F5): the resolver now serves path-scoped objects to authorized readers, + // so sign with an available identity like `gl ipfs list` — otherwise an owner or + // listed reader gets the opaque anonymous 404 for content they can read. + // `get_authed` signs when a keypair is present and falls back to unsigned. + // + // An explicit `--dir` is a request to use THAT identity: propagate a + // missing/corrupt-keystore error (like `list`) instead of silently sending an + // anonymous request the authorized reader would see as the node's opaque 404 + // (#173 review). Only the default (no `--dir`) keeps the best-effort unsigned + // fallback, so `get` stays usable for genuinely public content. + let keypair = match dir.as_deref() { + Some(dir) => Some(crate::identity::load_keypair_from_dir(Some(dir))?), + None => crate::identity::load_keypair_from_dir(None).ok(), + }; + let client = NodeClient::new(&node, keypair); + // #173 review (F1): the node now accepts equivalent multibase spellings, + // including base64 CIDs (prefix 'm'), whose alphabet contains '/', '+', '='. + // Interpolating the CID raw would make the client request (and sign) + // `/ipfs//`, which neither matches the single-segment Axum + // route nor points at the intended target. Percent-encode the CID as exactly + // one path segment so the signed and sent target agree and the server's + // `Path` extractor decodes it back to the original CID. + let path = format!("/ipfs/{}", encode_cid_segment(&cid)); let resp = client - .get(&path) + .get_authed(&path) .await .with_context(|| format!("failed to fetch CID {cid} from {node}"))?; @@ -119,6 +143,16 @@ async fn cmd_get(cid: String, node: String) -> Result<()> { Ok(()) } +/// Percent-encode a CID so it occupies exactly one path segment of `/ipfs/`. +/// `urlencoding::encode` escapes every byte outside the RFC 3986 unreserved set +/// (ALPHA / DIGIT / `-._~`), so the base64-CID characters that would otherwise +/// break the single-segment route — `/`, `+`, `=` — are all escaped, and the +/// server's `Path` extractor decodes the result back to the original CID (#173 +/// review, F1). +fn encode_cid_segment(cid: &str) -> String { + urlencoding::encode(cid).into_owned() +} + #[cfg(test)] mod tests { use super::*; @@ -235,4 +269,162 @@ mod tests { m.assert_async().await; } + + /// #173 (F5): `gl ipfs get` must SIGN with an available identity, like + /// `gl ipfs list`, so an owner/reader can retrieve a path-scoped object the node + /// now resolves by CID. RED before the fix: cmd_get ignores the identity dir and + /// sends an unsigned request, so the signature-matching mock is never hit + /// (cmd_get errors on the unmatched 501, and m.assert fails). GREEN after: the + /// signed request carries the RFC 9421 headers and is served 200. + #[tokio::test] + async fn test_cmd_get_signs_when_identity_present() { + let mut server = mockito::Server::new_async().await; + let keystore = seed_keystore(); + + let m = server + .mock("GET", "/ipfs/bafkreitestcid") + .match_header("signature", mockito::Matcher::Any) + .match_header("signature-input", mockito::Matcher::Any) + .with_status(200) + .with_header("content-type", "application/octet-stream") + .with_header("x-git-hash", "abc123") + .with_body("object bytes") + .create_async() + .await; + + cmd_get( + "bafkreitestcid".to_string(), + server.url(), + Some(keystore.path().to_path_buf()), + ) + .await + .expect("signed get of a resolvable object should succeed"); + + m.assert_async().await; + } + + /// #173 (F5) must-not: a genuine anonymous denial must surface as an error, not + /// be masked as success. With no identity dir the request is unsigned; a 404 + /// from the node must produce an Err mentioning the status. + #[tokio::test] + async fn test_cmd_get_anonymous_denial_is_error() { + let mut server = mockito::Server::new_async().await; + + let m = server + .mock("GET", "/ipfs/bafkreidenied") + .with_status(404) + .with_header("content-type", "text/plain") + .with_body("no git object found") + .create_async() + .await; + + let err = cmd_get("bafkreidenied".to_string(), server.url(), None) + .await + .expect_err("a 404 denial must be an error, not masked success"); + assert!( + err.to_string().contains("404"), + "error should mention the status, got: {err}" + ); + + m.assert_async().await; + } + + /// #173 (INV-8) must-not: the node's new 503 "search incomplete" (the legacy CID + /// scan hit its bound and could not prove absence) must surface as an actionable + /// Err naming the status, NOT be rendered as an empty/"not found" success — a + /// retryable outcome the caller has to see. Mirrors the 404 denial case for the + /// bounded-search response the resolver now emits. + #[tokio::test] + async fn test_cmd_get_search_incomplete_503_is_error() { + let mut server = mockito::Server::new_async().await; + + let m = server + .mock("GET", "/ipfs/bafkreiincomplete") + .with_status(503) + .with_header("content-type", "application/json") + .with_body(r#"{"error":"search_incomplete","message":"CID search incomplete — retry"}"#) + .create_async() + .await; + + let err = cmd_get("bafkreiincomplete".to_string(), server.url(), None) + .await + .expect_err("a 503 incomplete-search must be an error, not masked as not-found"); + assert!( + err.to_string().contains("503"), + "error should mention the status, got: {err}" + ); + + m.assert_async().await; + } + + /// #173 review (F1): a base64 CID (multibase prefix 'm') can contain '/', '+', + /// and '='. The client must percent-encode it into ONE path segment before + /// building and signing `/ipfs/`; otherwise the '/' splits the target so + /// it misses the single-segment Axum route and the signature covers the wrong + /// path. Assert the encoded segment carries no raw '/', '+', or '=', and that + /// it decodes back to the original CID (the server's `Path` extractor performs + /// that same decode). RED with the old raw `format!("/ipfs/{cid}")`: the + /// segment still contains '/'. + #[test] + fn test_encode_cid_segment_escapes_base64_alphabet() { + let cid = "mFoo/Bar+baz=="; + let encoded = encode_cid_segment(cid); + + assert!( + !encoded.contains('/'), + "encoded CID must be a single path segment (no raw '/'), got: {encoded}" + ); + assert!( + !encoded.contains('+'), + "encoded CID must escape '+', got: {encoded}" + ); + assert!( + !encoded.contains('='), + "encoded CID must escape '=', got: {encoded}" + ); + + let decoded = urlencoding::decode(&encoded).expect("encoded CID must decode"); + assert_eq!( + decoded, cid, + "encoding must round-trip back to the original CID" + ); + } + + /// #173 review: `gl ipfs get --dir ` must PROPAGATE a missing/corrupt + /// identity-load error like `gl ipfs list`, not silently fall back to an anonymous + /// request — otherwise an authorized reader pointing `--dir` at a broken keystore + /// gets the node's opaque 404 instead of the actionable key-load error. The + /// unsigned fallback is preserved only when NO `--dir` is given (covered by + /// `test_cmd_get_anonymous_denial_is_error`). RED before the fix (`.ok()` swallows + /// the error, an anonymous request is sent, and the `.expect(0)` mock is hit), + /// GREEN after. + #[tokio::test] + async fn test_cmd_get_explicit_dir_no_identity_errors_without_request() { + let mut server = mockito::Server::new_async().await; + // Empty keystore dir passed explicitly via --dir: no identity.pem present. + let empty = tempfile::TempDir::new().unwrap(); + + // The endpoint must never be hit when an explicit --dir fails to load. + let m = server + .mock("GET", "/ipfs/bafkreitestcid") + .expect(0) + .create_async() + .await; + + let err = cmd_get( + "bafkreitestcid".to_string(), + server.url(), + Some(empty.path().to_path_buf()), + ) + .await + .expect_err("an explicit --dir that fails to load must be an error"); + assert!( + err.to_string().contains("gl identity new") + || err.to_string().contains("no identity found") + || err.to_string().contains("failed to load keypair"), + "error should name the key-load failure, got: {err}" + ); + + m.assert_async().await; + } } From 8afc9eaf97988d9e677a8cc256d2307531d277b5 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 17 Jul 2026 09:06:21 -0500 Subject: [PATCH 039/124] fix(node): close grok P1 findings on the #173/#174 IPFS integration P1-A: gate_and_serve held the /ipfs walk permit across a bare repo_store.acquire; wrap it in git_acquire_timeout_secs (mirroring #174's own handler) so a cold/hung Tigris acquire cannot pin the global walk slot. On expiry skip the repo and mark the search truncated (retryable 503, not a false 404). P1-B: the local-IPFS pin path recorded Kubo's provider Hash as pinned_cids.cid; for objects above the block size that is a dag-pb root that does not hash the raw content, so GET /ipfs/{cid} listed then 404'd them under the F2 integrity check. Record the locally-computed raw-content CID, mirroring the pinata twin. --- crates/gitlawb-node/src/api/ipfs.rs | 23 ++++++++++++++++++++--- crates/gitlawb-node/src/ipfs_pin.rs | 12 ++++++++++-- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index a764901b..fe16582b 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -502,9 +502,26 @@ async fn gate_and_serve( } walk.probes += 1; } - let repo_path = match state.repo_store.acquire(&repo.owner_did, &repo.name).await { - Ok(p) => p, - Err(_) => return GateOutcome::Skip, + // Bound the per-repo acquire under `git_acquire_timeout_secs`: this gate runs while + // the /ipfs walk permit is held (F5), so a hung or cold-Tigris acquire would otherwise + // pin the global walk slot for the whole request. On expiry skip the repo (a public + // copy may still serve) and mark the search truncated so a wholly-unserved request + // tails to a retryable 503, never a false 404 (reopened the #174 P1-2 stall vector on + // this path otherwise). + let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); + let repo_path = match tokio::time::timeout( + acquire_deadline, + state.repo_store.acquire(&repo.owner_did, &repo.name), + ) + .await + { + Ok(Ok(p)) => p, + Ok(Err(_)) => return GateOutcome::Skip, + Err(_elapsed) => { + tracing::warn!(repo = %repo.name, "repo acquire timed out during /ipfs gate; skipping repo"); + walk.truncated = true; + return GateOutcome::Skip; + } }; // Existence probe before any walk (random-CID spray must not trigger a walk on a diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index a4ed8e4d..bdc6e090 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -159,7 +159,15 @@ pub async fn pin_new_objects( // Pin to IPFS match pin_git_object(ipfs_api, &sha, &data).await { Ok(cid) if !cid.is_empty() => { - if let Err(e) = db.record_pinned_cid(&sha, &cid, Some(repo_id)).await { + // The resolver key (`pinned_cids.cid`) must be the locally-computed + // raw-content CID, never the provider Hash: Kubo returns a dag-pb/UnixFS + // root for objects above its block size, which does not hash the raw + // content, so `GET /ipfs/{provider_cid}` would resolve then fail the F2 + // integrity check (list-then-404). The serve path reads bytes from git and + // verifies them against the requested CID, so the raw CID is the correct + // key. Mirrors the pinata twin, which already records the raw CID. + let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes(&data).to_string(); + if let Err(e) = db.record_pinned_cid(&sha, &raw_cid, Some(repo_id)).await { tracing::warn!(sha = %sha, err = %e, "failed to record pinned CID in DB"); } // F1 (#173 round 8): also record the first pinner in pin_repo_sources so @@ -167,7 +175,7 @@ pub async fn pin_new_objects( if let Err(e) = db.record_pin_source(&sha, repo_id).await { tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); } - pinned.push((sha, cid)); + pinned.push((sha, raw_cid)); } Ok(_) => {} Err(e) => { From c767658b7b731daf00dff7572ddc287f28df911c Mon Sep 17 00:00:00 2001 From: t Date: Fri, 17 Jul 2026 09:18:16 -0500 Subject: [PATCH 040/124] fix(node): bound the /ipfs cat-file probes under the walk permit; backfill pinata provenance P2-E: the object-type probe and the F6 size+read ran bare git cat-file inside spawn_blocking while the /ipfs walk permit was held, so a wedged cat-file could pin the global walk slot for the request's life. Bound both under git_service_timeout_secs; on timeout free the slot (truncated -> retryable 503) and skip the repo. P2-D: the pinata already-pinned branch now backfills NULL first-pinner provenance in lockstep with the ipfs_pin skip branch (consistency; the object was already resolvable via the pin_repo_sources union). --- crates/gitlawb-node/src/api/ipfs.rs | 101 ++++++++++++++++++---------- crates/gitlawb-node/src/pinata.rs | 16 +++++ 2 files changed, 82 insertions(+), 35 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index fe16582b..cb18d52a 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -530,17 +530,34 @@ async fn gate_and_serve( let obj_type = { let rp = repo_path.clone(); let sha = sha256_hex.to_string(); - match tokio::task::spawn_blocking(move || store::object_type(&rp, &sha)).await { - Ok(Ok(Some(t))) => t, - Ok(Ok(None)) => return GateOutcome::Skip, - Ok(Err(e)) => { + // Bound the blocking `git cat-file -t` under `git_service_timeout_secs`: this probe + // runs while the /ipfs walk permit is held, so a wedged cat-file (corrupt pack, NFS + // stall) would otherwise pin the global walk slot for the request's life. On timeout + // free the slot (mark truncated -> retryable 503) and skip the repo. spawn_blocking + // cannot be cancelled, so the child may linger on a blocking-pool thread, but it no + // longer holds the walk permit. + let probe_deadline = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + match tokio::time::timeout( + probe_deadline, + tokio::task::spawn_blocking(move || store::object_type(&rp, &sha)), + ) + .await + { + Ok(Ok(Ok(Some(t)))) => t, + Ok(Ok(Ok(None))) => return GateOutcome::Skip, + Ok(Ok(Err(e))) => { tracing::warn!(repo = %repo.name, err = %e, "error checking git object type"); return GateOutcome::Skip; } - Err(e) => { + Ok(Err(e)) => { tracing::warn!(repo = %repo.name, err = %e, "object-type probe task panicked; skipping repo"); return GateOutcome::Skip; } + Err(_elapsed) => { + tracing::warn!(repo = %repo.name, "object-type probe timed out under the /ipfs walk permit; skipping repo"); + walk.truncated = true; + return GateOutcome::Skip; + } } }; @@ -670,30 +687,49 @@ async fn gate_and_serve( let read_sha = sha256_hex.to_string(); let read_type = obj_type.clone(); let want_cid = ctx.canonical_cid.to_string(); - let read = tokio::task::spawn_blocking(move || -> ServedRead { - match store::object_size(&read_repo, &read_sha) { - Ok(Some(size)) if size > max_bytes => return ServedRead::TooLarge(size), - Ok(Some(_)) => {} - // git ran and reported no such object (or an unparseable size): genuine - // not-found for this candidate. - Ok(None) => return ServedRead::Gone, - // git itself failed to run: an infra failure, not a not-found. - Err(e) => return ServedRead::ReadErr(e.to_string()), + // Bound the blocking size+read+verify under `git_service_timeout_secs` (same rationale + // as the object-type probe): a hung cat-file must not pin the held /ipfs walk permit. + let read_deadline = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let read = tokio::time::timeout( + read_deadline, + tokio::task::spawn_blocking(move || -> ServedRead { + match store::object_size(&read_repo, &read_sha) { + Ok(Some(size)) if size > max_bytes => return ServedRead::TooLarge(size), + Ok(Some(_)) => {} + // git ran and reported no such object (or an unparseable size): genuine + // not-found for this candidate. + Ok(None) => return ServedRead::Gone, + // git itself failed to run: an infra failure, not a not-found. + Err(e) => return ServedRead::ReadErr(e.to_string()), + } + let content = match store::read_object_content(&read_repo, &read_sha, &read_type) { + Ok(c) => c, + Err(e) => return ServedRead::ReadErr(e.to_string()), + }; + let served = gitlawb_core::cid::Cid::from_git_object_bytes(&content).to_string(); + if served != want_cid { + return ServedRead::Mismatch(served); + } + ServedRead::Ok(content) + }), + ) + .await; + let served_read = match read { + Ok(Ok(sr)) => sr, + Ok(Err(e)) => { + tracing::warn!(repo = %repo.name, err = %e, "object read task panicked"); + walk.truncated = true; + return GateOutcome::Skip; } - let content = match store::read_object_content(&read_repo, &read_sha, &read_type) { - Ok(c) => c, - Err(e) => return ServedRead::ReadErr(e.to_string()), - }; - let served = gitlawb_core::cid::Cid::from_git_object_bytes(&content).to_string(); - if served != want_cid { - return ServedRead::Mismatch(served); + Err(_elapsed) => { + tracing::warn!(repo = %repo.name, "object read timed out under the /ipfs walk permit; skipping repo"); + walk.truncated = true; + return GateOutcome::Skip; } - ServedRead::Ok(content) - }) - .await; - let content = match read { - Ok(ServedRead::Ok(c)) => c, - Ok(ServedRead::TooLarge(size)) => { + }; + let content = match served_read { + ServedRead::Ok(c) => c, + ServedRead::TooLarge(size) => { tracing::warn!( repo = %repo.name, size, max = max_bytes, "withholding object: exceeds the served-object size cap (F6)" @@ -702,15 +738,15 @@ async fn gate_and_serve( note_oversize_reject(); return GateOutcome::Skip; } - Ok(ServedRead::Mismatch(served)) => { + ServedRead::Mismatch(served) => { tracing::warn!( repo = %repo.name, requested = %ctx.canonical_cid, served = %served, "withholding object: served bytes do not hash to the requested CID (legacy provider-CID row?)" ); return GateOutcome::Skip; } - Ok(ServedRead::Gone) => return GateOutcome::Skip, - Ok(ServedRead::ReadErr(e)) => { + ServedRead::Gone => return GateOutcome::Skip, + ServedRead::ReadErr(e) => { // Infra failure (git spawn/IO), NOT a not-found: mark the search truncated so // a wholly-unserved request tails to a retryable 503, never a definitive 404 // for an authorized caller (INV-25 spirit — logging alone is not surfacing). @@ -718,11 +754,6 @@ async fn gate_and_serve( walk.truncated = true; return GateOutcome::Skip; } - Err(e) => { - tracing::warn!(repo = %repo.name, err = %e, "object read task panicked"); - walk.truncated = true; - return GateOutcome::Skip; - } }; let mut resp_headers = HeaderMap::new(); resp_headers.insert( diff --git a/crates/gitlawb-node/src/pinata.rs b/crates/gitlawb-node/src/pinata.rs index 31843bbc..4d6b9704 100644 --- a/crates/gitlawb-node/src/pinata.rs +++ b/crates/gitlawb-node/src/pinata.rs @@ -96,6 +96,22 @@ pub async fn pin_new_objects( for sha in object_list { match db.has_pinata_cid(&sha).await { Ok(true) => { + // Backfill NULL first-pinner provenance from a known source, in lockstep + // with the ipfs_pin skip branch: a pinata-only node otherwise leaves + // pre-provenance rows' `pinned_cids.repo_id` NULL forever (grok P2-D). The + // resolver still finds the object via the pin_repo_sources union below, so + // this is a consistency backfill, not a correctness fix. + match db.provenance_for_oid(&sha).await { + Ok(None) => { + if let Err(e) = db.backfill_pin_provenance(&sha, repo_id).await { + tracing::warn!(sha = %sha, err = %e, "failed to backfill pin provenance"); + } + } + Ok(Some(_)) => {} + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "DB error reading pin provenance"); + } + } // F1 (#173 round 8): record this repo as an additional source for the // already-pinned object (mirrors the ipfs_pin skip-branch insert) so the // resolver can serve a shared object from any pin-path source. From 2eea354188cbeaf1e1298fe41457e2c39e656f05 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 17 Jul 2026 09:27:56 -0500 Subject: [PATCH 041/124] fix(node): tie the /ipfs walk ceiling to MAX_PIN_SOURCES + 1 A provenanced object has a bounded source set (first-pinner + up to MAX_PIN_SOURCES additional). With the walk ceiling at 16 == MAX_PIN_SOURCES, an authorizing public source sorting after 16 path-scoped denials was never reached: the ceiling truncated the search and returned a false 503 for a readable object. Raise the ceiling to MAX_PIN_SOURCES + 1 so the whole bounded provenance set is always tried; the legacy scan stays bounded by MAX_LEGACY_PROBES_PER_REQUEST. --- crates/gitlawb-node/src/api/ipfs.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index cb18d52a..9cb663b2 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -50,7 +50,15 @@ use crate::visibility::{visibility_check, Decision}; /// serves on the first repo that grants them, so reaching it requires being /// denied by this many path-scoped repos first, which real traffic effectively /// never does. Tunable if that assumption stops holding. -pub(crate) const MAX_HISTORY_WALKS_PER_REQUEST: u32 = 16; +/// +/// Kept at `MAX_PIN_SOURCES + 1` so the ceiling can never truncate a request +/// BEFORE its whole bounded provenance source set (first-pinner + up to +/// `MAX_PIN_SOURCES` additional) has been tried: an authorizing public source that +/// sorts after `MAX_PIN_SOURCES` path-scoped denials must still be reached and +/// served, not falsely 503'd as a truncated search. The legacy scan's fan-out is +/// separately bounded by `MAX_LEGACY_PROBES_PER_REQUEST`, so widening this by one +/// does not loosen that path. +pub(crate) const MAX_HISTORY_WALKS_PER_REQUEST: u32 = crate::db::MAX_PIN_SOURCES as u32 + 1; /// Hard per-request ceiling on how many legacy (NULL-provenance) repositories /// the CID resolver's scan fallback may PROBE (`acquire` + `git cat-file -t`). From 94ca74426965f9bb1d72eade791fa08b3526e207 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 17 Jul 2026 09:37:43 -0500 Subject: [PATCH 042/124] fix(node): align ipfs_pin return with pinata; add Retry-After to SearchIncomplete - ipfs_pin::pin_new_objects returns the provider Hash (not the raw resolver key), matching the pinata twin's contract; the DB cid stays the raw content CID. The return is logging-only here, so this is drift-avoidance, not a functional fix. - AppError::SearchIncomplete now carries Retry-After: 1 like Overloaded, so both retryable 503s from the /ipfs handler advertise the retry hint consistently. --- crates/gitlawb-node/src/error.rs | 12 ++++++++---- crates/gitlawb-node/src/ipfs_pin.rs | 6 +++++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/crates/gitlawb-node/src/error.rs b/crates/gitlawb-node/src/error.rs index 371f0cf6..c13b92b4 100644 --- a/crates/gitlawb-node/src/error.rs +++ b/crates/gitlawb-node/src/error.rs @@ -182,10 +182,14 @@ impl IntoResponse for AppError { })); let mut resp = (status, body).into_response(); - // Overloaded advertises when to retry. It rides the shared tail above for - // its body/status, so the header is attached here rather than in a bespoke - // early return — keeping the variant handled in exactly one place. - if matches!(self, AppError::Overloaded(_)) { + // Both retryable 503s advertise when to retry: Overloaded (capacity shed) and + // SearchIncomplete (a bounded CID search cut short by a cap — retry may complete + // it). They ride the shared tail above for body/status, so the header is attached + // here rather than in bespoke early returns, keeping each variant handled once. + if matches!( + self, + AppError::Overloaded(_) | AppError::SearchIncomplete(_) + ) { resp.headers_mut().insert( axum::http::header::RETRY_AFTER, axum::http::HeaderValue::from_static("1"), diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index bdc6e090..e70d8f7b 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -175,7 +175,11 @@ pub async fn pin_new_objects( if let Err(e) = db.record_pin_source(&sha, repo_id).await { tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); } - pinned.push((sha, raw_cid)); + // Return the provider Hash (not the resolver key), mirroring the pinata + // twin's contract: the DB `cid` is the raw resolver key (recorded above), + // the returned value is the provider CID. Here the return is consumed only + // for logging, but keeping the twins structurally identical avoids drift. + pinned.push((sha, cid)); } Ok(_) => {} Err(e) => { From d26cf6dd1d6150dad03c7c9b0e2fdfaf01914cc8 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 17 Jul 2026 10:04:04 -0500 Subject: [PATCH 043/124] fix(node): close the /ipfs pin-source griefing hole via a bounded scan fallback An attacker who pins a public object from MAX_PIN_SOURCES repos before the legitimate public source registers fills pin_repo_sources (record_pin_source is first-N-wins and drops later sources silently); the buried public source is then never reached because a non-empty provenance set suppressed the legacy scan, so anon GET /ipfs/{cid} 404s forever for a public object (re-breaks F1). U1: db::pin_sources_at_cap reports whether pin_repo_sources is at MAX_PIN_SOURCES for an oid (the only observable signal that a servable source may have been dropped, since the write cap never overshoots). U2: get_by_cid falls back to the bounded legacy scan on a provenance miss when the set is empty OR at_cap. The scan gates every repo through the real per-caller gate, so it finds the buried public copy. A complete (non-full) set still fast-404s, so ordinary denials never fan out to O(repos) (INV-10/F3); the fallback honors the is_throttled peek. Regression ipfs_cid_buried_public_source_still_serves_via_scan_fallback: 404 before, 200 after; pin_sources_at_cap_flips_at_max covers the boundary. --- crates/gitlawb-node/src/api/ipfs.rs | 127 +++++++++++++----------- crates/gitlawb-node/src/db/mod.rs | 19 ++++ crates/gitlawb-node/src/test_support.rs | 123 +++++++++++++++++++++++ 3 files changed, 213 insertions(+), 56 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 9cb663b2..c8911bac 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -246,15 +246,77 @@ pub async fn get_by_cid( .pin_sources_for_oid(sha256_hex) .await .map_err(AppError::Internal)?; - if sources.is_empty() { - // F3 (#173, INV-10/INV-15): the legacy NULL-provenance scan builds an - // O(repos) preload (repos + rules + quarantine) BEFORE gate_and_serve's - // per-probe brake can bite, so a throttled source could still force O(repos) - // DB work on every replay. Peek the per-IP limiter WITHOUT consuming a token: - // an already-throttled source is shed here, before the preload runs. The + // Provenance fast-path: try each recorded source repo through the SAME gate + // (bounded to first-pinner + MAX_PIN_SOURCES). Empty for a legacy NULL-provenance + // pin. The first source that authorizes serves — no scan fan-out on the common + // path. + for repo_id in &sources { + let repo = match state + .db + .get_repo_by_id(repo_id) + .await + .map_err(AppError::Internal)? + { + Some(r) => r, + // A source repo is gone: skip it; a later source or the scan fallback + // below may still resolve. + None => continue, + }; + let quarantined = state + .db + .is_repo_quarantined(repo_id) + .await + .map_err(AppError::Internal)?; + let rules_map = state + .db + .list_visibility_rules_for_repos(std::slice::from_ref(repo_id)) + .await + .map_err(AppError::Internal)?; + let rules = rules_map.get(repo_id).map(Vec::as_slice).unwrap_or(&[]); + match gate_and_serve( + &state, + &repo, + rules, + quarantined, + sha256_hex, + &rctx, + &mut walk, + false, + ) + .await + { + GateOutcome::Served(resp) => return Ok(resp), + GateOutcome::Throttled => { + throttled = true; + continue; + } + GateOutcome::Skip => continue, + } + } + + // Bounded legacy-scan fallback. Run it when the provenance set could not have + // served the caller AND may be INCOMPLETE: + // - empty -> a legacy NULL-provenance pin (recorded before provenance existed), or + // - at_cap -> `record_pin_source` stops inserting at MAX_PIN_SOURCES and drops + // later sources SILENTLY, so a full table may hide a servable source + // (e.g. a later PUBLIC pinner buried by 16 attacker sources — the + // pin-source griefing hole). The scan gates every repo through the + // real per-caller gate, so it finds that copy. + // A non-empty, non-full set is COMPLETE (every recorded source was just tried), so + // skip the scan and let the tail 404 — ordinary denials never fan out to O(repos) + // (INV-10 / F3). The at_cap query runs only on a provenance MISS (we return above + // on Served), so it never costs the serve path. + let needs_scan = sources.is_empty() + || state + .db + .pin_sources_at_cap(sha256_hex) + .await + .map_err(AppError::Internal)?; + if needs_scan { + // F3 (#173, INV-10/INV-15): peek the per-IP limiter WITHOUT consuming a token + // so an already-throttled source is shed BEFORE the O(repos) preload; the // consuming per-probe charge inside gate_and_serve is left UNCHANGED (it is - // load-bearing for the across-request bound), so this adds no double-charge — - // a non-consuming peek plus the existing per-probe charge, never two charges. + // load-bearing for the across-request bound), so this adds no double-charge. if let Some(key) = crate::rate_limit::client_key(rctx.headers, rctx.peer, state.push_limiter_trust) { @@ -263,9 +325,7 @@ pub async fn get_by_cid( continue; } } - // Legacy pin (recorded before provenance existed): fall back to the repo - // scan, gating each repo through the SAME gate. Load the scan context - // once, lazily. + // Load the scan context once, lazily (shared across oid candidates). if scan_ctx.is_none() { #[cfg(test)] bump_preload_queries(); @@ -309,51 +369,6 @@ pub async fn get_by_cid( GateOutcome::Skip => {} } } - } else { - for repo_id in &sources { - let repo = match state - .db - .get_repo_by_id(repo_id) - .await - .map_err(AppError::Internal)? - { - Some(r) => r, - // A source repo is gone: skip this source. Do NOT fall back to the - // scan (that would reopen the fan-out); a later source or oid - // candidate may still resolve. - None => continue, - }; - let quarantined = state - .db - .is_repo_quarantined(repo_id) - .await - .map_err(AppError::Internal)?; - let rules_map = state - .db - .list_visibility_rules_for_repos(std::slice::from_ref(repo_id)) - .await - .map_err(AppError::Internal)?; - let rules = rules_map.get(repo_id).map(Vec::as_slice).unwrap_or(&[]); - match gate_and_serve( - &state, - &repo, - rules, - quarantined, - sha256_hex, - &rctx, - &mut walk, - false, // provenance path: bounded source set, no scan fan-out - ) - .await - { - GateOutcome::Served(resp) => return Ok(resp), - GateOutcome::Throttled => { - throttled = true; - continue; - } - GateOutcome::Skip => continue, - } - } } } diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 6b0d394e..8d723ffd 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -2379,6 +2379,25 @@ impl Db { .collect()) } + /// Whether `pin_repo_sources` is at the `MAX_PIN_SOURCES` cap for this oid, i.e. + /// the provenance source set returned by [`Self::pin_sources_for_oid`] may be + /// INCOMPLETE. `record_pin_source` stops inserting at exactly `MAX_PIN_SOURCES` + /// rows and drops later sources silently, so a full table is the only observable + /// signal that a servable source (e.g. a later public pinner) may have been + /// dropped. `get_by_cid` uses this to decide whether a provenance miss should fall + /// back to the bounded legacy scan (which gates every repo through the real + /// visibility gate and so finds a dropped public source) rather than 404 — closing + /// the pin-source griefing hole where 16 attacker sources bury a public one. `>=` + /// (not `==`) is defensive against any future overshoot. + pub async fn pin_sources_at_cap(&self, sha256_hex: &str) -> Result { + let count: i64 = + sqlx::query_scalar("SELECT count(*) FROM pin_repo_sources WHERE sha256_hex = $1") + .bind(sha256_hex) + .fetch_one(&self.pool) + .await?; + Ok(count >= MAX_PIN_SOURCES) + } + pub async fn record_encrypted_blob( &self, repo_id: &str, diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 21fa58e4..45b2b807 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -2450,6 +2450,129 @@ mod tests { ); } + /// U1 (grok round-4 P1): `pin_sources_at_cap` flips exactly at `MAX_PIN_SOURCES`. + /// It is the signal `get_by_cid` uses to decide a provenance miss may be hiding a + /// dropped servable source and must fall back to the bounded scan. + #[sqlx::test] + async fn pin_sources_at_cap_flips_at_max(pool: PgPool) { + let state = test_state(pool).await; + let cap = crate::db::MAX_PIN_SOURCES; + assert!( + !state.db.pin_sources_at_cap("atcapoid").await.unwrap(), + "an oid with no pin_repo_sources rows is not at cap" + ); + for i in 0..(cap - 1) { + state + .db + .record_pin_source("atcapoid", &format!("r-{i:02}")) + .await + .unwrap(); + } + assert!( + !state.db.pin_sources_at_cap("atcapoid").await.unwrap(), + "one below MAX_PIN_SOURCES is not at cap" + ); + state + .db + .record_pin_source("atcapoid", "r-last") + .await + .unwrap(); + assert!( + state.db.pin_sources_at_cap("atcapoid").await.unwrap(), + "exactly MAX_PIN_SOURCES rows is at cap" + ); + } + + /// U2 (grok round-4 P1, load-bearing): the pin-source GRIEFING hole. A private + /// first-pinner denies anon; an attacker fills the whole `MAX_PIN_SOURCES` source + /// window with deny-anon sources BEFORE a legitimate public repo pins the same + /// object, so the public repo's `record_pin_source` no-ops (cap full) and it is + /// buried — present in NO provenance record. The resolver's provenance set is then + /// {private + 16 attacker}, all deny anon. Because the set is at_cap (may hide a + /// dropped source), the handler falls back to the bounded legacy scan, which gates + /// every repo through the real gate and finds the buried PUBLIC copy → 200. + /// MUTATION (RED): remove the `at_cap` fallback edge in `get_by_cid` and the buried + /// public object 404s forever. + #[sqlx::test] + async fn ipfs_cid_buried_public_source_still_serves_via_scan_fallback(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["privfirst", "pubburied"]); + let priv_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("privfirst.git"); + let pub_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("pubburied.git"); + + // Private repo pins FIRST — owns the first-pinner provenance, denies anon. + let mut priv_repo = seed_repo(&owner_did, "privfirst"); + priv_repo.is_public = false; + state + .db + .create_repo(&priv_repo) + .await + .expect("seed private first-pinner"); + let cid = pin_cid_for_repo(&priv_bare, &fx.public_oid, &state.db, &priv_repo.id).await; + + // Attacker fills the ENTIRE MAX_PIN_SOURCES window with deny-anon (non-existent) + // sources BEFORE the public repo registers, so the cap is full. + let cap = crate::db::MAX_PIN_SOURCES; + for i in 0..cap { + state + .db + .record_pin_source(&fx.public_oid, &format!("00-attacker-{i:02}")) + .await + .expect("attacker source"); + } + + // A PUBLIC repo pushes the SAME object through the real pin path. Already pinned + // (skip branch), so it only tries record_pin_source — which NO-OPS because the + // cap is full. The public repo is thus buried: not the first-pinner, not in + // pin_repo_sources. + let pub_repo = seed_repo(&owner_did, "pubburied"); // public, no rule + state + .db + .create_repo(&pub_repo) + .await + .expect("seed public buried source"); + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyshouldnothappen"}"#) + .expect(0) + .create_async() + .await; + crate::ipfs_pin::pin_new_objects( + &server.url(), + &pub_bare, + vec![fx.public_oid.clone()], + &state.db, + &pub_repo.id, + ) + .await; + m.assert_async().await; // /add NOT called (already pinned) + + // The buried public object must STILL serve: the provenance set is at_cap and + // all-deny, so the handler falls back to the bounded scan, which finds pubburied. + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "a public source buried by a full attacker source window must still serve via the bounded scan fallback (F1)" + ); + assert!( + body.contains("public bytes"), + "the served body is the buried public object's bytes" + ); + } + /// #173 (jatmn round 8, F1 — bound, R2): the per-object source set is capped at /// `MAX_PIN_SOURCES` so an adversary pushing one object from many repos cannot make /// resolution O(repos). Recording the same oid from `MAX_PIN_SOURCES + 3` distinct From 3af50a53040d7eef18444ecbedf1f663a31e992e Mon Sep 17 00:00:00 2001 From: t Date: Fri, 17 Jul 2026 17:05:28 -0500 Subject: [PATCH 044/124] fix(node): hold filtered-pack admission through disconnect teardown (#174) drive_git_child hands the AdmissionGuard back on success so upload_pack_excluding can thread it across the rev-list and pack-objects stages; on timeout or error it is dropped post-reap, and on future-drop it rides the detached reaper as before. The upload-pack handler builds the guard once ahead of the withheld/plain branch, so a client disconnect mid-filtered-serve no longer frees the read-pool and per-source permits while the git process group is still being reaped. --- crates/gitlawb-node/src/api/repos.rs | 249 +++++++++++- crates/gitlawb-node/src/git/smart_http.rs | 446 +++++++++++++++++++--- 2 files changed, 637 insertions(+), 58 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 9b05ece1..bb9e743b 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -932,19 +932,30 @@ pub async fn git_upload_pack( // the smart_http paths, not a generic 500 (#174 U3). let withheld = withheld.map_err(|e| git_service_app_error(&e))?; + // Move the permits returned by the walk into the guard, ONE construction + // site for both serve arms below, so admission tracks the served git group's + // reap (complete/timeout/disconnect) whether the pack is plain or filtered. + // The handler keeps no copy (F1: handler-local permits would drop the + // instant a disconnect drops this future, mid-reap). + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); if withheld.is_empty() { - // No blobs to withhold: serve the plain pack, moving the permits returned by - // the walk into the guard so admission tracks the served git group's reap - // (the walk already held them per be0cdd6; this hands them to the serve). - let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + // No blobs to withhold: serve the plain pack (the walk already held the + // permits per be0cdd6; the guard hands them to the serve). smart_http::upload_pack(&state.git_bin, &disk_path, body, git_timeout, Some(admission)).await } else { tracing::info!(repo = %name, caller = ?caller, withheld = withheld.len(), "serving filtered pack"); - // upload_pack_excluding runs its own rev-list/pack-objects (both pass `None` - // admission internally); the walk's permits stay handler-locals held across - // this serve, as be0cdd6 established, and drop when the handler returns. - let _hold = (_permit, _caller_permit); - smart_http::upload_pack_excluding(&disk_path, body, &withheld, git_timeout).await + // The guard threads through both filtered-pack stages (rev-list, then + // pack-objects), so a disconnect mid-stage keeps the permits held until + // that stage's process group is reaped (F1). + smart_http::upload_pack_excluding( + &state.git_bin, + &disk_path, + body, + &withheld, + git_timeout, + Some(admission), + ) + .await } } .map_err(|e| { @@ -4013,6 +4024,226 @@ mod tests { ); } + /// F1 (filtered-serve residual of #174 P1-a, RED-before/GREEN-after): on the + /// FILTERED (path-scoped, non-empty withheld) upload-pack branch a client + /// disconnect mid-pack-objects must NOT release the read admission while the + /// detached reaper is still tearing down the git group. Pre-fix the handler held + /// the permits as locals (`_hold`) across `upload_pack_excluding`, so dropping + /// the future released them instantly and disconnect-spam could exceed the read + /// caps during each reap window (RED). The fix threads the AdmissionGuard through + /// both filtered-pack stages so it rides `KillGroupOnDrop` into the reaper. + /// + /// Same isolation as the plain-path test above: read pool = 1, per-source cap + /// permissive, so only the global permit can shed a replacement. The fake git + /// serves the withheld walk (for-each-ref/rev-parse/rev-list/ls-tree) with a blob + /// under the denied `/src/**` subtree so the filtered branch is taken, answers + /// the pack build's rev-list fast, and hangs pack-objects in a SIGTERM-trapping + /// descendant. The descendant hang is first-invocation-only (keyed on the + /// pidfile's existence) so the post-reap replacement request completes fast. + #[cfg(unix)] + #[sqlx::test] + async fn upload_pack_filtered_permit_held_through_group_reap_after_disconnect( + pool: sqlx::PgPool, + ) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + let descfile = tmp.path().join("desc.pid"); + let commit = "1111111111111111111111111111111111111111"; + let blob = "2222222222222222222222222222222222222222"; + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + rev-parse) echo {commit} ;;\n\ + rev-list) echo {commit} ;;\n\ + ls-tree) printf '100644 blob {blob}\\tsrc/x.txt' ;;\n\ + pack-objects)\n\ + if [ ! -e \"{desc}\" ]; then\n\ + sh -c 'trap \"\" TERM; echo $$ > \"{desc}\"; i=0; while [ $i -lt 20 ]; do sleep 1; i=$((i+1)); done' &\n\ + wait\n\ + fi ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + desc = descfile.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + // Isolate the global read pool: size 1; per-source cap + rate limiter permissive + // so only the leaked global permit can shed the replacement. + state.git_read_semaphore = Arc::new(Semaphore::new(1)); + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let owner = "z6upf1"; + let name = "upf"; + state + .db + .upsert_mirror_repo(owner, name, "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + // Real bare repo at the path acquire() computes, so the handler reaches the + // walk and the filtered serve. + state + .repo_store + .init(&rec.owner_did, &rec.name) + .await + .unwrap(); + // Path-scoped rule denying the anonymous caller under /src, matching the + // fake ls-tree's blob path, so the withheld set is NON-EMPTY and the + // filtered (upload_pack_excluding) branch is taken. + state + .db + .set_visibility_rule( + &rec.id, + "/src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkUF1ReaderAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + + let sem = state.git_read_semaphore.clone(); + assert_eq!( + sem.available_permits(), + 1, + "one read slot before the request" + ); + + let router = crate::server::build_router(state); + let make_req = |peer: SocketAddr| { + let mut req = Request::builder() + .method(Method::POST) + .uri(format!("/{owner}/{name}/git-upload-pack")) + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req + }; + + let peer: SocketAddr = "203.0.113.81:5000".parse().unwrap(); + let mut fut = Box::pin(router.clone().oneshot(make_req(peer))); + // Drive until the pack-objects descendant records its pid: the request is + // past the walk, inside the filtered serve's stage 2, holding the read permit. + // Stop polling the instant the future completes (re-polling a completed + // oneshot panics); read the descfile first so a spawn that recorded its pid + // then returned is still caught. + let mut spawned: Option = None; + let mut early = None; + for _ in 0..500 { + let done = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&descfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + spawned = Some(p); + break; + } + if let Ok(resp) = done { + early = Some(resp.map(|r| r.status())); + break; + } + } + let desc = spawned.unwrap_or_else(|| { + panic!("the fake pack-objects must have spawned; early finish: {early:?}") + }); + // Kill the descendant regardless of outcome so a RED run leaks no orphan. + struct ReapOnDrop(i32); + impl Drop for ReapOnDrop { + fn drop(&mut self) { + unsafe { + libc::kill(self.0, libc::SIGKILL); + } + } + } + let _cleanup = ReapOnDrop(desc); + assert!( + unsafe { libc::kill(desc, 0) == 0 }, + "descendant should be running before the disconnect" + ); + assert_eq!( + sem.available_permits(), + 0, + "the read slot is held while the filtered serve runs" + ); + + // Client disconnect: drop the request future mid-pack-objects. The detached + // reaper must now own the AdmissionGuard and hold it until ESRCH. + drop(fut); + + // Load-bearing: the slot must STAY held while the SIGTERM-ignoring group is + // still alive. On the pre-fix code the handler-local `_hold` drops here and + // the slot frees at once (RED). Check quickly (before the reaper's ~2s + // SIGKILL escalation). + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + assert!( + unsafe { libc::kill(desc, 0) == 0 }, + "the SIGTERM-ignoring descendant must still be alive during the hold window" + ); + assert_eq!( + sem.available_permits(), + 0, + "on disconnect the read admission must be HELD until the filtered serve's \ + process group is reaped, not released the instant the future drops (F1)" + ); + // A replacement request from a DIFFERENT source must shed 503: the only pool + // that can shed it is the held global permit (per-source cap is permissive). + let peer2: SocketAddr = "203.0.113.82:5000".parse().unwrap(); + let resp = router.clone().oneshot(make_req(peer2)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "while the prior group is still alive the held global permit must shed a \ + replacement with 503" + ); + + // After the reaper SIGKILLs + reaps the group the AdmissionGuard drops and + // the slot frees. Poll for recovery. + let mut freed = false; + for _ in 0..400 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + assert!( + freed, + "once the reaper confirms the group gone the admission guard must drop and \ + free the global slot" + ); + // A replacement is now admitted and completes: the fake pack-objects takes + // its fast path (the descfile exists), so the filtered serve returns instead + // of hanging. + let peer3: SocketAddr = "203.0.113.83:5000".parse().unwrap(); + let resp = router.oneshot(make_req(peer3)).await.unwrap(); + assert_ne!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "after the group is reaped the freed slot must admit a replacement" + ); + } + /// #174 U1 (P1-a): the `None`-key arm — a request with no resolvable source key /// (no trusted-proxy header, no peer) is bounded by the GLOBAL read pool only, never /// a per-source cap. With the global read pool exhausted such a request still sheds diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index 64d3cb7f..f8b46ce8 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -24,7 +24,9 @@ use tokio::process::Command; /// /// The `be0cdd6` path-scoped upload-pack walk already applies this discipline by /// moving its permits into the `spawn_blocking`; this generalizes it to the plain -/// (non-path-scoped) `info_refs` / `run_git_service` spawn paths. +/// (non-path-scoped) `info_refs` / `run_git_service` spawn paths and, via the +/// hand-back contract on `drive_git_child`, across both stages of the filtered +/// (`upload_pack_excluding`) pack build (F1). pub struct AdmissionGuard { // Boxed so the concrete permit types stay private to the caller (repos.rs) — the // guard only needs to hold them and drop them, never inspect them. `Send + @@ -71,8 +73,11 @@ pub async fn info_refs( .arg("--advertise-refs") .arg(repo_path); // No request body — advertise-refs does not read stdin. - let stdout = + let (stdout, admission) = drive_git_child(command, Bytes::new(), timeout, "advertise-refs", admission).await?; + // Single-stage op: the advertisement's group is reaped by now; release admission + // rather than holding it across the pure response framing below. + drop(admission); let content_type = format!("application/x-{service}-advertisement"); @@ -174,11 +179,12 @@ struct KillGroupOnDrop { // own SIGCHLD-driven orphan reaper. child: Option, pgid: Option, - // The global + per-source admission permits for this op, if any. On the plain - // (non-path-scoped) spawn paths repos.rs moves its permits in here so admission is - // released only when the group is confirmed reaped, on every exit — complete, - // timeout, or client-disconnect (#174 P1-a). The rev-list/pack-objects - // visibility-walk callers hold no admission and pass `None`. + // The global + per-source admission permits for this op, if any. repos.rs moves + // its permits in here (directly on the plain spawn paths, via the stage threading + // on the filtered rev-list/pack-objects ones) so admission is released only when + // the group is confirmed reaped, on every exit: complete, timeout, or + // client-disconnect (#174 P1-a, F1). `None` when the op carries no admission + // (e.g. the smart_http test harness). admission: Option, } @@ -191,9 +197,10 @@ impl KillGroupOnDrop { /// Disarm on the success/timeout path: the body has already reaped the child (its /// `wait()` returned, or `reap_group_on_timeout` ran), so drop the handle and clear - /// the pgid, leaving the guard's Drop a no-op. Returns the admission guard so the - /// caller drops it at the earliest provably-free point (the group is already reaped - /// on both callers of this) rather than at end-of-bookkeeping. + /// the pgid, leaving the guard's Drop a no-op. Returns the admission guard: the + /// group is already reaped on both callers of this, so the success path hands it + /// back to drive_git_child's caller (a multi-stage build carries it to the next + /// stage) and the timeout path drops it there and then. fn disarm(&mut self) -> Option { self.pgid = None; self.child = None; @@ -331,23 +338,36 @@ async fn run_git_service( .arg(service_to_command(service)) .arg("--stateless-rpc") .arg(repo_path); - drive_git_child(command, input, timeout, service, admission).await + let (out, admission) = drive_git_child(command, input, timeout, service, admission).await?; + // Single-stage op: the child group is already reaped when drive_git_child + // returns, so releasing admission here keeps the permits held exactly for the + // process lifetime, no longer. + drop(admission); + Ok(out) } /// Drive a spawned git child under `timeout` with process-group teardown, returning -/// its stdout. Shared core for [`run_git_service`] and [`info_refs`]: the caller +/// its stdout together with the admission guard. Shared core for +/// [`run_git_service`], [`info_refs`], and the filtered-pack stages: the caller /// passes a `Command` with its args set; this adds piped stdio and `process_group(0)`. /// On the deadline the whole group is torn down and reaped before returning /// [`GitServiceTimeout`]; on a dropped future (client disconnect) the /// [`KillGroupOnDrop`] guard fires. `input` is written to the child's stdin (empty /// for the advertise-refs path, which has no request body); `what` labels errors. +/// +/// Admission contract: on success the guard is handed BACK so a multi-stage caller +/// (the filtered-pack build) can carry one admission across consecutive children +/// instead of releasing it between stages. On every Err return (timeout, spawn +/// failure, interaction error, non-zero exit) the guard is dropped internally, and +/// only after the child is reaped: an Err cannot carry the guard, and by then the +/// group is confirmed gone, so releasing admission there is correct. async fn drive_git_child( mut command: Command, input: Bytes, timeout: Duration, what: &str, admission: Option, -) -> Result> { +) -> Result<(Vec, Option)> { command .stdin(Stdio::piped()) .stdout(Stdio::piped()) @@ -382,10 +402,10 @@ async fn drive_git_child( pgid, admission, }; - // On non-unix there is no process-group teardown, so hold the admission guard here - // for the child's whole interaction; it drops when this function returns. - #[cfg(not(unix))] - let _admission = admission; + // On non-unix there is no process-group teardown; the `admission` parameter + // itself stays live for the child's whole interaction (nothing moves it until + // the match below), so the success path hands it back like the unix path and + // the timeout path drops it after the child is waited on. let mut out = Vec::new(); let mut err = Vec::new(); @@ -419,16 +439,18 @@ async fn drive_git_child( }; let timed = tokio::time::timeout(timeout, interact).await; - let (write_result, status) = match timed { + let (write_result, status, admission) = match timed { Ok(result) => { // The join runs all arms to completion, so the child is reaped: disarm // before surfacing any interaction error (a read/wait error), else the // guard's drop would reap an already-reaped child / signal a reused pgid. - // Dropping the returned admission guard here releases the permits at the - // earliest provably-free point on the success path (the op finished). + // Disarming hands the admission guard back; it rides to the Ok return + // below, and every error return between here and there drops it, always + // AFTER the reap, the earliest provably-free point. #[cfg(unix)] - drop(group_guard.disarm()); - result? + let admission = group_guard.disarm(); + let (write_result, status) = result?; + (write_result, status, admission) } Err(_elapsed) => { // Timeout: tear the whole group down and reap it before returning so a @@ -444,6 +466,7 @@ async fn drive_git_child( { let _ = child.start_kill(); let _ = tokio::time::timeout(Duration::from_secs(2), child.wait()).await; + drop(admission); } return Err(GitServiceTimeout.into()); } @@ -452,7 +475,8 @@ async fn drive_git_child( // Surface git's own failure (its stderr, which the handler may classify as a // 400) before any stdin-write error: when git rejects a malformed body it // exits non-zero and closes stdin, so the write's EPIPE would otherwise mask - // the real cause. + // the real cause. `admission` drops on these error returns; the child is + // already reaped by the completed join above. if !status.success() { let stderr = String::from_utf8_lossy(&err); bail!("{what} failed: {stderr}"); @@ -460,7 +484,7 @@ async fn drive_git_child( write_result.context("failed to write to git stdin")?; - Ok(out) + Ok((out, admission)) } fn service_to_command(service: &str) -> &str { @@ -498,14 +522,17 @@ async fn rev_list_keep( repo_path: &Path, withheld: &HashSet, timeout: Duration, -) -> Result> { + admission: Option, +) -> Result<(Vec, Option)> { let mut command = Command::new(git_bin); command .args(["rev-list", "--objects", "--all"]) .current_dir(repo_path); - // The visibility-walk callers intentionally hold no admission permit (their - // admission is governed elsewhere), so pass `None` (#174 KTD2). - let stdout = drive_git_child(command, Bytes::new(), timeout, "rev-list", None).await?; + // The filtered-serve caller's admission rides through drive_git_child so a + // disconnect mid-enumeration keeps the permits held until the rev-list group is + // reaped; on success the guard comes back for the pack-objects stage (F1). + let (stdout, admission) = + drive_git_child(command, Bytes::new(), timeout, "rev-list", admission).await?; let mut keep = Vec::new(); for line in String::from_utf8_lossy(&stdout).lines() { let oid = line.split_whitespace().next().unwrap_or(""); @@ -514,7 +541,7 @@ async fn rev_list_keep( } keep.push(oid.to_string()); } - Ok(keep) + Ok((keep, admission)) } /// Build a packfile containing every object reachable from all refs EXCEPT the @@ -527,20 +554,30 @@ async fn rev_list_keep( /// disconnect. An outer `tokio::time::timeout` around a `spawn_blocking` cannot /// cancel the blocking thread, so neither stage may live off the async side /// (#174, KTD5). +/// +/// `admission` carries the filtered-serve permits across BOTH stages: stage 1 hands +/// it back on success, stage 2 takes it, and the caller receives it after stage 2 so +/// release happens only once the last child group is reaped. On a disconnect the +/// guard rides the active stage's detached reaper instead of dropping with the +/// future (F1). A stage that fails or times out (including a stage 2 whose +/// remaining budget has saturated to zero) drops the guard inside drive_git_child, +/// after that stage's reap. pub async fn build_filtered_pack( git_bin: &str, repo_path: &Path, withheld: &HashSet, timeout: Duration, -) -> Result> { + admission: Option, +) -> Result<(Vec, Option)> { // One deadline spans both git stages so a slow rev-list eats into the pack // budget rather than granting each stage a fresh `timeout` (2x the permit hold). let deadline = Instant::now() + timeout; - let keep = rev_list_keep( + let (keep, admission) = rev_list_keep( git_bin, repo_path, withheld, deadline.saturating_duration_since(Instant::now()), + admission, ) .await?; let mut data = keep.join("\n").into_bytes(); @@ -554,8 +591,7 @@ pub async fn build_filtered_pack( Bytes::from(data), deadline.saturating_duration_since(Instant::now()), "pack-objects", - // Visibility-walk pack build: no admission permit here (#174 KTD2). - None, + admission, ) .await } @@ -584,16 +620,24 @@ pub async fn build_filtered_pack( /// instead of a thin delta. Honoring negotiation for smaller fetch packs is an /// optimization follow-up, not a correctness requirement. pub async fn upload_pack_excluding( + git_bin: &str, repo_path: &Path, request_body: Bytes, withheld: &HashSet, timeout: Duration, + admission: Option, ) -> Result { - // The rev-list enumeration runs blocking off the runtime; the streaming - // pack-objects stage is duration-bounded and its process group is reaped on - // disconnect via drive_git_child (#174), so a hung build no longer pins its - // concurrency slot and a client disconnect no longer orphans the git child. - let pack = build_filtered_pack("git", repo_path, withheld, timeout).await?; + // Both filtered-pack stages run async under drive_git_child (duration-bounded, + // process group reaped on disconnect, #174), and `admission` threads through + // them so the caller's permits release only after the active stage's group is + // reaped, never the instant a disconnect drops this future (F1). `git_bin` is + // injectable for the same fake-git testing reason as `run_git_service` + // (production passes the configured git binary). + let (pack, admission) = + build_filtered_pack(git_bin, repo_path, withheld, timeout, admission).await?; + // Both git stages are done and their groups reaped; release admission before the + // pure in-memory response framing below. + drop(admission); // The client lists its capabilities on the first `want` line. Honor // side-band-64k when offered (every modern smart-HTTP client offers it); @@ -712,9 +756,10 @@ mod tests { let mut withheld = std::collections::HashSet::new(); withheld.insert(secret.clone()); - let pack = build_filtered_pack("git", &bare, &withheld, Duration::from_secs(30)) - .await - .unwrap(); + let (pack, _admission) = + build_filtered_pack("git", &bare, &withheld, Duration::from_secs(30), None) + .await + .unwrap(); let ids = pack_object_ids(&pack); assert!(ids.contains(&public), "public blob must be in the pack"); assert!( @@ -776,9 +821,10 @@ mod tests { b"0098want 0000000000000000000000000000000000000000 \ side-band-64k ofs-delta agent=git/2\n00000009done\n", ); - let resp = upload_pack_excluding(&bare, req, &withheld, Duration::from_secs(30)) - .await - .unwrap(); + let resp = + upload_pack_excluding("git", &bare, req, &withheld, Duration::from_secs(30), None) + .await + .unwrap(); let body = to_bytes(resp.into_body(), usize::MAX).await.unwrap(); let ids = pack_object_ids(&extract_pack(&body)); assert!( @@ -837,9 +883,16 @@ mod tests { axum::extract::State(st): axum::extract::State>, body: Bytes, ) -> Response { - upload_pack_excluding(&st.repo, body, &st.withheld, Duration::from_secs(30)) - .await - .unwrap() + upload_pack_excluding( + "git", + &st.repo, + body, + &st.withheld, + Duration::from_secs(30), + None, + ) + .await + .unwrap() } /// Spawn the server for `bare`, withholding `withheld`. Returns the clone URL @@ -1359,14 +1412,19 @@ mod tests { for i in 0..FAKE_GIT_RETRY_ATTEMPTS { let result = tokio::time::timeout( Duration::from_secs(10), - build_filtered_pack(git_bin, repo_path, withheld, stage_timeout), + build_filtered_pack(git_bin, repo_path, withheld, stage_timeout, None), ) .await .expect( "build_filtered_pack must return within the watchdog — the git stage \ must be timeout-bounded, not an uncancellable spawn_blocking", ); - let err = result.expect_err("a hung git stage must return an error, not hang"); + // `match` rather than `expect_err`: the Ok arm carries an + // AdmissionGuard, which has no Debug impl for expect_err to print. + let err = match result { + Ok(_) => panic!("a hung git stage must return an error, not hang"), + Err(e) => e, + }; if is_transient_exec_race(&err) { // Fresh-fake-git ETXTBSY: back off (growing) so a bursty fork-pressure // spike subsides before retrying, per fake_git_run_with_pids. @@ -1655,6 +1713,296 @@ mod tests { ); } + // ── F1: filtered-serve admission threaded through both pack stages ────── + // + // The handler-level disconnect regression lives in api/repos.rs + // (upload_pack_filtered_permit_held_through_group_reap_after_disconnect); these + // exercise the guard contract at the smart_http seam: held through a + // mid-rev-list disconnect, handed back on success, and recovered (post-reap) + // on every error return. + + /// F1: a client disconnect mid-REV-LIST (stage 1 of the filtered build) must + /// keep the threaded admission held until the rev-list group is reaped, and + /// stage 2 (pack-objects) must never spawn, since the dropped future cannot + /// advance to it. + #[cfg(unix)] + #[tokio::test] + async fn build_filtered_pack_holds_admission_through_rev_list_reap_on_disconnect() { + let tmp = tempfile::TempDir::new().unwrap(); + let descfile = tmp.path().join("desc.pid"); + let packfile = tmp.path().join("pack.ran"); + // rev-list: SIGTERM-trapping descendant records its pid and loops (bounded + // so a broken fix leaks no permanent orphan); the leader waits, keeping the + // interaction pending. pack-objects: records that it ran (it must never run). + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + rev-list)\n\ + sh -c 'trap \"\" TERM; echo $$ > \"{desc}\"; i=0; while [ $i -lt 20 ]; do sleep 1; i=$((i+1)); done' &\n\ + wait ;;\n\ + pack-objects) : > \"{pack}\" ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + desc = descfile.display(), + pack = packfile.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + let guard = AdmissionGuard::new(sem.clone().try_acquire_owned().unwrap(), None::<()>); + let withheld = HashSet::new(); + + let mut fut = Box::pin(build_filtered_pack( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_secs(60), + Some(guard), + )); + // Drive until the rev-list descendant records its pid. + let mut desc: Option = None; + for _ in 0..500 { + let _ = tokio::time::timeout(Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&descfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + desc = Some(p); + break; + } + } + let desc = desc.expect("the fake rev-list must have spawned its descendant"); + let _cleanup = ReapOnPanic(vec![desc]); + assert_eq!( + sem.available_permits(), + 0, + "admission held while rev-list runs" + ); + + // Client disconnect mid-enumeration. + drop(fut); + + // Load-bearing: the permit must stay held while the SIGTERM-ignoring group + // member is still alive (check before the reaper's ~2s SIGKILL escalation). + tokio::time::sleep(Duration::from_millis(100)).await; + assert!( + alive(desc), + "the SIGTERM-ignoring descendant must still be alive during the hold window" + ); + assert_eq!( + sem.available_permits(), + 0, + "on disconnect mid-rev-list the admission must be held until the group is \ + reaped, not released the instant the future drops (F1)" + ); + + let mut freed = false; + for _ in 0..400 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + assert!( + freed, + "the reaper must release admission once the rev-list group is gone" + ); + assert!( + !packfile.exists(), + "pack-objects must never spawn after a disconnect mid-rev-list" + ); + } + + /// F1 success path: a filtered serve that completes hands the guard back + /// through both stages and releases admission when the serve returns. No leak, + /// no double-release. + #[tokio::test] + async fn upload_pack_excluding_releases_admission_after_success() { + let td = TempDir::new().unwrap(); + let (_work, bare, secret_oid, _public_oid) = fixture_with_secret(&td); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + let guard = AdmissionGuard::new(sem.clone().try_acquire_owned().unwrap(), None::<()>); + let withheld = HashSet::from([secret_oid]); + let resp = upload_pack_excluding( + "git", + &bare, + Bytes::from_static(b"0000"), + &withheld, + Duration::from_secs(30), + Some(guard), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!( + sem.available_permits(), + 1, + "admission must be released once the filtered serve returns" + ); + } + + /// F1 error path: a rev-list that exits non-zero fails the build AND recovers + /// the threaded admission: drive_git_child drops the guard only after the + /// completed join has reaped the child, and the error return must not leak it. + #[cfg(unix)] + #[tokio::test] + async fn build_filtered_pack_releases_admission_after_rev_list_failure() { + let tmp = tempfile::TempDir::new().unwrap(); + let body = + "#!/bin/sh\ncase \"$1\" in\n rev-list) echo boom >&2; exit 2 ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + let withheld = HashSet::new(); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + + for i in 0..FAKE_GIT_RETRY_ATTEMPTS { + let guard = AdmissionGuard::new(sem.clone().try_acquire_owned().unwrap(), None::<()>); + let result = build_filtered_pack( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_secs(30), + Some(guard), + ) + .await; + let err = match result { + Ok(_) => panic!("a non-zero rev-list exit must fail the build"), + Err(e) => e, + }; + // Load-bearing on EVERY attempt: even a transient spawn-failure return + // must recover the permit (the guard drops before the Err surfaces). + assert_eq!( + sem.available_permits(), + 1, + "the admission permit must be recovered on the error return, not leaked" + ); + if is_transient_exec_race(&err) { + tokio::time::sleep(Duration::from_millis(FAKE_GIT_BACKOFF_STEP_MS * (i + 1))).await; + continue; + } + assert!( + err.to_string().contains("rev-list failed"), + "the failure must surface rev-list's own error, got: {err}" + ); + return; + } + panic!( + "fake git kept hitting ETXTBSY after {FAKE_GIT_RETRY_ATTEMPTS} attempts \ + (persistent exec failure, not a transient parallel-runner miss)" + ); + } + + /// F1: a pack-objects stage that times out (stage 2 of the filtered build, + /// guard threaded through) recovers the admission after the reap: the timeout + /// Err cannot carry the guard, so drive_git_child must drop it post-reap. + #[cfg(unix)] + #[tokio::test] + async fn build_filtered_pack_recovers_admission_after_pack_objects_timeout() { + let tmp = tempfile::TempDir::new().unwrap(); + // rev-list returns one oid fast (guard handed back); pack-objects hangs. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) echo deadbeefdeadbeefdeadbeefdeadbeefdeadbeef ;;\n pack-objects) sleep 300 ;;\n *) exit 1 ;;\nesac\n"; + let git_bin = write_fake_git(tmp.path(), body); + let withheld = HashSet::new(); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + + for i in 0..FAKE_GIT_RETRY_ATTEMPTS { + let guard = AdmissionGuard::new(sem.clone().try_acquire_owned().unwrap(), None::<()>); + let result = tokio::time::timeout( + Duration::from_secs(10), + build_filtered_pack( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_millis(200), + Some(guard), + ), + ) + .await + .expect( + "build_filtered_pack must return within the watchdog; the git stage \ + must be timeout-bounded", + ); + let err = match result { + Ok(_) => panic!("a hung pack-objects must fail the build"), + Err(e) => e, + }; + assert_eq!( + sem.available_permits(), + 1, + "admission must be recovered after the stage-2 timeout reap, not leaked" + ); + if is_transient_exec_race(&err) { + tokio::time::sleep(Duration::from_millis(FAKE_GIT_BACKOFF_STEP_MS * (i + 1))).await; + continue; + } + assert!( + err.downcast_ref::().is_some(), + "a hung pack-objects must abort with GitServiceTimeout, got: {err}" + ); + return; + } + panic!( + "fake git kept hitting ETXTBSY after {FAKE_GIT_RETRY_ATTEMPTS} attempts \ + (persistent exec failure, not a transient parallel-runner miss)" + ); + } + + /// F1 zero-budget contract: build_filtered_pack hands pack-objects + /// `deadline.saturating_duration_since(now)`, which saturates to ZERO once + /// rev-list has consumed the whole budget. A zero-budget stage must still route + /// the guard through the timeout's reap-then-drop rather than leak it: the + /// child spawns, the deadline fires immediately, the group is reaped, and only + /// then does the guard drop. + #[cfg(unix)] + #[tokio::test] + async fn drive_git_child_zero_budget_drops_admission_after_reap() { + let tmp = tempfile::TempDir::new().unwrap(); + let git_bin = write_fake_git(tmp.path(), "#!/bin/sh\nsleep 300\n"); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + + for i in 0..FAKE_GIT_RETRY_ATTEMPTS { + let guard = AdmissionGuard::new(sem.clone().try_acquire_owned().unwrap(), None::<()>); + let mut command = tokio::process::Command::new(git_bin.to_str().unwrap()); + command + .args(["pack-objects", "--stdout"]) + .current_dir(tmp.path()); + let result = tokio::time::timeout( + Duration::from_secs(10), + drive_git_child( + command, + Bytes::new(), + Duration::ZERO, + "pack-objects", + Some(guard), + ), + ) + .await + .expect("a zero-budget stage must abort via its own deadline, not hang"); + let err = match result { + Ok(_) => panic!("a zero-budget stage must return an error"), + Err(e) => e, + }; + assert_eq!( + sem.available_permits(), + 1, + "the guard must drop after the zero-budget timeout reap, not leak" + ); + if is_transient_exec_race(&err) { + tokio::time::sleep(Duration::from_millis(FAKE_GIT_BACKOFF_STEP_MS * (i + 1))).await; + continue; + } + assert!( + err.downcast_ref::().is_some(), + "a zero-budget stage must abort with GitServiceTimeout, got: {err}" + ); + return; + } + panic!( + "fake git kept hitting ETXTBSY after {FAKE_GIT_RETRY_ATTEMPTS} attempts \ + (persistent exec failure, not a transient parallel-runner miss)" + ); + } + // A request that runs to completion must DISARM the guard after reaping, so // no stray group SIGTERM fires. The fake exits non-zero (surfacing as Err) // but leaves a grandchild alive; the grandchild must survive. Goes RED if the From 138b48210a1ae949edc798a93afd1ee38412cfb9 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 17 Jul 2026 17:35:01 -0500 Subject: [PATCH 045/124] fix(node): make /ipfs scan verdicts honest: walk-scoped cap, visit ceiling, 503 on truncation (#174) A request-scoped taint list replaces the silent folds: any repo skipped without a verdict (acquire timeout, probe or read failure, walk failure, cap or ceiling exhaustion) turns the terminal into a retryable 503 with Retry-After, and 404 is reserved for a scan in which every candidate reached a verdict. The walk cap now counts only expensive allowed-set walks and skip-continues on exhaustion so a plain public copy past the cap still serves; a new GITLAWB_IPFS_MAX_REPO_VISITS knob (default 1024) bounds the acquire+probe cost class and stops the scan, documented with its worst-case object-store fetch count. --- .env.example | 12 +- README.md | 3 +- crates/gitlawb-node/src/api/ipfs.rs | 706 ++++++++++++++++++++++-- crates/gitlawb-node/src/config.rs | 54 +- crates/gitlawb-node/src/git/tigris.rs | 19 + crates/gitlawb-node/src/test_support.rs | 25 +- 6 files changed, 765 insertions(+), 54 deletions(-) diff --git a/.env.example b/.env.example index 149ea8b1..144108ba 100644 --- a/.env.example +++ b/.env.example @@ -168,9 +168,17 @@ GITLAWB_MAX_CONCURRENT_IPFS_WALKS=32 # per-caller caps via GITLAWB_TRUSTED_PROXY; reject-before-insert bounded map). # Default 4. GITLAWB_IPFS_WALK_PER_SOURCE=4 -# Max repos walked per single /ipfs request, so one request cannot serialize a -# full-history walk over every repo carrying the CID. Default 64. +# Max EXPENSIVE path-scope visibility walks per single /ipfs request (only a +# blob in a path-scoped repo costs a full-history walk). Over-cap repos are +# skipped without a verdict and the scan continues; if the object is then found +# nowhere the request sheds a retryable 503 instead of a false 404. Default 64. GITLAWB_IPFS_MAX_REPOS_WALKED=64 +# Ceiling on repos one /ipfs request may VISIT past the visibility gate. Each +# visit costs a repo acquire — on a Tigris cache miss a full archive download, +# so this is also the worst-case object-store fetch count per request — plus a +# cat-file probe. On exhaustion the scan stops and sheds a retryable 503. +# Default 1024. +GITLAWB_IPFS_MAX_REPO_VISITS=1024 # Max /ipfs/{cid} requests per client IP per hour (route flood brake, distinct # from the concurrency caps above). 0 disables. Default 600. GITLAWB_IPFS_RATE_LIMIT=600 diff --git a/README.md b/README.md index e9765876..f3fa361b 100644 --- a/README.md +++ b/README.md @@ -347,7 +347,8 @@ Important node settings: | `GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS` | Max seconds the storage-acquisition phase (Tigris HEAD/GET, push advisory-lock) of a served git op may run before the request is shed with a 503, separate from the git-run timeout. The concurrency permit is released on expiry so a stalled backend cannot pin the pool. Default 30. | | `GITLAWB_MAX_CONCURRENT_IPFS_WALKS` | Max concurrent `GET /ipfs/{cid}` visibility walks across all callers (own pool, disjoint from the served-git pools); over-cap sheds 503. Default 32. | | `GITLAWB_IPFS_WALK_PER_SOURCE` | Max concurrent `/ipfs` walks a single source IP may hold. Default 4. | -| `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max repos walked per `/ipfs/{cid}` request, bounding one request's fan-out. Default 64. | +| `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max expensive path-scope visibility walks per `/ipfs/{cid}` request; over-cap repos are skipped and the scan continues, shedding a retryable 503 (not a false 404) if the object is then found nowhere. Default 64. | +| `GITLAWB_IPFS_MAX_REPO_VISITS` | Ceiling on repos one `/ipfs/{cid}` request may visit (acquire + probe) past the visibility gate — also the worst-case per-request Tigris fetch count. On exhaustion the scan stops with a retryable 503. Default 1024. | | `GITLAWB_IPFS_RATE_LIMIT` | Max `/ipfs/{cid}` requests per client IP per hour (route flood brake). 0 disables. Default 600. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index ac8e615c..d4a6c7a8 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -51,6 +51,15 @@ use crate::visibility::{visibility_check, Decision}; /// fail-closed 404'd under path-scoped rules (#126). Denial and genuine /// not-found both fall through to an opaque 404. /// +/// Scan completeness (F2): the 404 above is returned ONLY when every candidate +/// repo reached a VERDICT — visibility deny, probe-says-absent, walk-gate deny, +/// or served. A candidate skipped WITHOUT a verdict (acquire failure/timeout, +/// probe error, walk failure/panic, content-read error, or truncation by +/// `ipfs_max_repos_walked` / `ipfs_max_repo_visits`) taints the scan, and a +/// tainted scan that found nothing sheds a retryable 503 + Retry-After naming +/// the truncation sources — existing content is never misreported absent +/// because of unrelated repos or transient faults. +/// /// Scope: this closes the direct unauthenticated scan, including the dangling /// case. A stale-public mirror row still serves withheld content (tracked /// separately, #124). @@ -139,13 +148,35 @@ pub async fn get_by_cid( // deny entry (#126). let mut allowed_memo: HashMap> = HashMap::new(); - // Cap the number of candidate repos one request walks (it already short-circuits on - // serve): a CID present in — or path-gated out of — many repos must not serialize an - // unbounded number of full-history walks inside the single held admission slot. + // Verdict-or-taint bookkeeping (F2): a candidate repo the loop cannot bring to a + // VERDICT (visibility deny / probe-says-absent / walk-gate deny / served) marks + // the scan truncated with its source. A truncated scan that finds nothing must + // NOT report 404 — the object may sit in a repo we skipped — so the terminal arm + // sheds a retryable 503 naming the sources, keyed so the operator can tell which + // knob (or backend) to look at. + let mut truncated_by: Vec<&'static str> = Vec::new(); + fn taint(truncated_by: &mut Vec<&'static str>, source: &'static str) { + if !truncated_by.contains(&source) { + truncated_by.push(source); + } + } + + // Cap on EXPENSIVE walks only (F2): counts the repos that actually require the + // full-history `allowed_blob_set_for_caller_bounded` walk (a path-scoped blob), + // checked immediately before the spawn_blocking below. Cheap probe-only visits + // are bounded by `repos_visited` — counting them here starved later-ordered + // repos out of a plain 200 on nodes with more readable repos than the cap. let mut repos_walked: usize = 0; + // Ceiling on VISITS (F2): every repo past the visibility gate costs an acquire + // (worst case a full Tigris archive download on a cache miss) plus a cat-file + // probe, so one request can trigger at most `ipfs_max_repo_visits` object-store + // fetches. On exhaustion the scan STOPS — there is no cheaper way to continue. + let mut repos_visited: usize = 0; for repo in &repos { - // Repo-level read gate against THIS row's own rules (KTD2a). + // Repo-level read gate against THIS row's own rules (KTD2a). Deny is a + // VERDICT: this repo would never serve the caller, so skipping it cannot + // hide content from them. let rules: &[crate::db::VisibilityRule] = rules_by_repo .get(&repo.id) .map(Vec::as_slice) @@ -154,23 +185,24 @@ pub async fn get_by_cid( continue; } - // Loop bound (#174 P1-3): once this request has walked its cap of candidate - // repos (each a git subprocess, up to a full-history walk), stop and fall - // through to the opaque 404 rather than serialize an unbounded number under the - // single held admission slot. - if repos_walked >= state.config.ipfs_max_repos_walked { + // Visit ceiling (F2): bound the acquire+probe cost class. Stopping here + // leaves the remaining candidates unproven, so the scan is truncated. + if repos_visited >= state.config.ipfs_max_repo_visits { tracing::warn!( - cap = state.config.ipfs_max_repos_walked, - "/ipfs request hit the per-request repo-walk cap; stopping the scan" + ceiling = state.config.ipfs_max_repo_visits, + "/ipfs request hit the per-request repo-visit ceiling \ + (GITLAWB_IPFS_MAX_REPO_VISITS); stopping the scan without a verdict" ); + taint(&mut truncated_by, "visit-ceiling"); break; } - repos_walked += 1; + repos_visited += 1; // Bound the per-repo acquire under `git_acquire_timeout_secs`: this loop shares // the P1-2 stall vector (a hung Tigris HEAD/GET on one repo would otherwise - // block the whole /ipfs request). On expiry keep the existing fail-closed skip — - // never serve an un-acquired repo; a public copy (if any) still gets its turn. + // block the whole /ipfs request). On expiry keep the fail-closed skip — never + // serve an un-acquired repo; a public copy (if any) still gets its turn — but + // the repo got no verdict, so the skip taints the scan. let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); let repo_path = match tokio::time::timeout( @@ -180,21 +212,28 @@ pub async fn get_by_cid( .await { Ok(Ok(p)) => p, - Ok(Err(_)) => continue, + Ok(Err(e)) => { + tracing::warn!(repo = %repo.name, err = %e, "repo acquire failed during /ipfs scan; skipping repo without a verdict"); + taint(&mut truncated_by, "acquire"); + continue; + } Err(_elapsed) => { - tracing::warn!(repo = %repo.name, "repo acquire timed out during /ipfs walk; skipping repo"); + tracing::warn!(repo = %repo.name, "repo acquire timed out during /ipfs scan; skipping repo without a verdict"); + taint(&mut truncated_by, "acquire"); continue; } }; // Check whether the object exists in this repo before any expensive // reachability walk. This prevents random-CID spray from triggering - // full-history git walks on repos that don't carry the object. + // full-history git walks on repos that don't carry the object. Absent + // (`Ok(None)`) is a VERDICT; a probe that could not run is not. let obj_type = match store::object_type(&repo_path, &sha256_hex) { Ok(Some(t)) => t, Ok(None) => continue, Err(e) => { - tracing::warn!(repo = %repo.name, err = %e, "error checking git object type"); + tracing::warn!(repo = %repo.name, err = %e, "object-type probe failed during /ipfs scan; skipping repo without a verdict"); + taint(&mut truncated_by, "probe"); continue; } }; @@ -205,6 +244,21 @@ pub async fn get_by_cid( let path_scoped = has_path_scoped_rule(rules); if path_scoped && obj_type == "blob" { if !allowed_memo.contains_key(&repo.id) { + // Walk cap (F2), checked at the one site that actually spends a walk: + // on exhaustion skip THIS repo without a verdict and KEEP scanning — + // later candidates may still reach a cheap probe-only verdict (a plain + // public copy serves its 200 with no walk at all). + if repos_walked >= state.config.ipfs_max_repos_walked { + tracing::warn!( + cap = state.config.ipfs_max_repos_walked, + repo = %repo.name, + "/ipfs request hit the per-request walk cap \ + (GITLAWB_IPFS_MAX_REPOS_WALKED); skipping repo without a verdict" + ); + taint(&mut truncated_by, "walk-cap"); + continue; + } + repos_walked += 1; let rp = repo_path.clone(); let r = rules.to_vec(); let is_public = repo.is_public; @@ -229,20 +283,25 @@ pub async fn get_by_cid( .await; // Fail closed on EITHER a task panic (JoinError) or a walk error: // we cannot prove the caller may read here, so skip this repo and - // let a public copy (if any) serve. Never serve on an unproven gate. + // let a public copy (if any) serve. Never serve on an unproven gate + // — and never report absent on one either (no verdict, taint). let set = match walk { Ok(Ok(set)) => set, Ok(Err(e)) => { - tracing::warn!(repo = %repo.name, err = %e, "allowed-blob walk failed; skipping repo"); + tracing::warn!(repo = %repo.name, err = %e, "allowed-blob walk failed during /ipfs scan; skipping repo without a verdict"); + taint(&mut truncated_by, "walk-failure"); continue; } Err(e) => { - tracing::warn!(repo = %repo.name, err = %e, "allowed-blob walk task panicked; skipping repo"); + tracing::warn!(repo = %repo.name, err = %e, "allowed-blob walk task panicked during /ipfs scan; skipping repo without a verdict"); + taint(&mut truncated_by, "walk-failure"); continue; } }; allowed_memo.insert(repo.id.clone(), set); } + // Not in the caller's reachable allowed-set: a VERDICT (deny), the walk + // proved this repo would never serve the blob to this caller. let in_allowed = allowed_memo .get(&repo.id) .is_some_and(|set| set.contains(&sha256_hex)); @@ -251,11 +310,14 @@ pub async fn get_by_cid( } } - // Now that we've passed the gate, read the content. + // Now that we've passed the gate, read the content. A failed read after a + // passed gate is not an absence verdict — the probe just said the object + // exists here — so the skip taints the scan. let content = match store::read_object_content(&repo_path, &sha256_hex, &obj_type) { Ok(c) => c, Err(e) => { - tracing::warn!(repo = %repo.name, err = %e, "error reading git object content"); + tracing::warn!(repo = %repo.name, err = %e, "object content read failed during /ipfs scan; skipping repo without a verdict"); + taint(&mut truncated_by, "read"); continue; } }; @@ -279,7 +341,20 @@ pub async fn get_by_cid( return Ok((StatusCode::OK, headers, content).into_response()); } - // Not found in any repo + // Truncated scan (F2): at least one candidate repo yielded no verdict, so the + // object is not proven absent. A 404 here would misreport existing content, so + // shed retryable instead — Overloaded is the single 503 + Retry-After site in + // error.rs, and the message names the truncation sources so the operator can + // map the shed to the right knob or backend. + if !truncated_by.is_empty() { + return Err(AppError::Overloaded(format!( + "ipfs scan incomplete ({}) for CID {cid_str}; retry shortly", + truncated_by.join("+") + ))); + } + + // Complete scan: every candidate reached a verdict and none served, so the + // object is definitively absent (or denied) for this caller. Err(AppError::RepoNotFound(format!( "no git object found for CID {cid_str}" ))) @@ -353,6 +428,554 @@ mod tests { req } + /// Run real git, asserting success. Shared by the F2 scan-verdict tests. + fn run_git(args: &[&str], cwd: &std::path::Path) { + let out = std::process::Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + } + + /// Seed a repo row plus a REAL sha256 bare repo at its acquired path holding one + /// committed blob (`src/secret.txt` = `content`). Returns `(repo_id, blob_oid)`. + /// Same recipe as `get_by_cid_walk_permit_held_through_blocking_walk`: the CID + /// digest IS the sha256 object id under `--object-format=sha256`, so the real + /// `cat-file` probe finds the blob. + async fn seed_repo_with_blob( + state: &crate::state::AppState, + tmp: &std::path::Path, + owner: &str, + name: &str, + content: &[u8], + ) -> (String, String) { + state + .db + .upsert_mirror_repo(owner, name, &format!("/unused-{name}"), None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + let _ = std::fs::remove_dir_all(&bare); + std::fs::create_dir_all(&bare).unwrap(); + let work = tmp.join(format!("work-{owner}-{name}")); + std::fs::create_dir_all(work.join("src")).unwrap(); + std::fs::write(work.join("src/secret.txt"), content).unwrap(); + run_git( + &["init", "-q", "--object-format=sha256", "-b", "main"], + &work, + ); + run_git(&["config", "user.email", "t@t"], &work); + run_git(&["config", "user.name", "t"], &work); + run_git(&["add", "src/secret.txt"], &work); + run_git(&["commit", "-q", "-m", "seed"], &work); + run_git( + &[ + "clone", + "--bare", + "-q", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + tmp, + ); + let out = std::process::Command::new("git") + .args(["rev-parse", "HEAD:src/secret.txt"]) + .current_dir(&work) + .output() + .expect("git rev-parse runs"); + assert!(out.status.success(), "rev-parse failed"); + let oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); + (rec.id, oid) + } + + /// CIDv1(raw, sha2-256) for a sha256 object id, as the handler resolves it. + fn cid_for_oid(oid: &str) -> String { + let oid_bytes = gitlawb_core::cid::sha256_hex_to_bytes(oid).unwrap(); + gitlawb_core::cid::Cid::from_sha256_bytes(&oid_bytes) + .as_str() + .to_string() + } + + /// Fake git for the WALK only (`state.git_bin`): empty refs, `rev-parse` + /// resolves, and each `rev-list` appends one line to `log` and prints nothing — + /// every walked repo yields an EMPTY allowed-set (path-gate deny verdict) and + /// the log's line count == the number of expensive walks run. The probe and the + /// content read shell to the real `git`, so seeded objects must genuinely exist. + #[cfg(unix)] + fn walk_logging_fake_git(dir: &std::path::Path, log: &std::path::Path) -> String { + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + for-each-ref) : ;;\n\ + rev-parse) echo deadbeef ;;\n\ + rev-list) echo walk >> \"{}\" ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + log.display() + ); + let git_path = dir.join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + git_path.to_str().unwrap().to_string() + } + + /// F2 buried-row repro: with more readable repos than `ipfs_max_repos_walked`, + /// existing PUBLIC content past the cap must still serve. The cap counts + /// EXPENSIVE walks only — this request has no path-scoped rules anywhere, so it + /// runs ZERO walks (the fake-git walk log stays empty) and the cap can never cut + /// the scan: the blob buried in the OLDER-updated repo (iterated last under + /// `list_all_repos`' updated_at DESC) serves its 200. Before F2 the cap counted + /// visibility-passing VISITS and broke the loop into the opaque 404 — existing + /// content misreported absent because of unrelated repos. MUTATION (RED): count + /// visits against the cap again (re-add the check+increment at the visibility + /// gate) and the buried row 503s instead of serving. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_buried_public_row_past_walk_cap_still_serves(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let walk_log = tmp.path().join("walks.log"); + state.git_bin = walk_logging_fake_git(tmp.path(), &walk_log); + // Tighter than the repo count: the old visit-counting cap cut the scan here. + let mut cfg = (*state.config).clone(); + cfg.ipfs_max_repos_walked = 1; + state.config = Arc::new(cfg); + + // Seed the blob-carrying repo FIRST so its updated_at is OLDER: the empty + // repo is iterated first and the blob row sits past the old visit budget. + let (_, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6f2buried", + "buried", + b"buried row proof\n", + ) + .await; + seed_repo_with_blob( + &state, + tmp.path(), + "z6f2buried", + "fresh", + b"unrelated content\n", + ) + .await; + + let peer: SocketAddr = "203.0.113.60:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&cid_for_oid(&oid), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::OK, + "a public blob in a repo past the walk cap must still serve — the cap \ + counts expensive walks and this scan needs none" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + assert_eq!(&body[..], b"buried row proof\n"); + let walks = std::fs::read_to_string(&walk_log) + .map(|s| s.lines().count()) + .unwrap_or(0); + assert_eq!( + walks, 0, + "a request with no path-scoped rules anywhere must run zero expensive walks" + ); + } + + /// F2 walk-cap skip-and-continue: exhausting `ipfs_max_repos_walked` skips the + /// walk-NEEDING repo without a verdict but keeps the scan alive. Three public + /// repos carry the same blob, newest first: the first (path-scoped) consumes the + /// cap-of-1 walk and denies (empty allowed-set — a verdict); the second + /// (path-scoped) needs a walk the cap forbids and is skipped WITHOUT one (taint); + /// the third is plain public and serves the 200 from a cheap probe — found beats + /// taint, and exactly one expensive walk ran. Before F2 the cap broke the loop at + /// the second repo and the request 404'd despite the public copy. MUTATION (RED): + /// turn the walk-cap skip back into a `break` and the public copy never serves. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_walk_cap_skip_continues_to_later_public_copy(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let walk_log = tmp.path().join("walks.log"); + state.git_bin = walk_logging_fake_git(tmp.path(), &walk_log); + let mut cfg = (*state.config).clone(); + cfg.ipfs_max_repos_walked = 1; + state.config = Arc::new(cfg); + + // Insert order = oldest first, so iteration (updated_at DESC) is reversed: + // gatedwalk, then gatedskip, then pubcopy. Identical content -> one CID. + let content = b"skip and continue proof\n"; + let (_, oid) = + seed_repo_with_blob(&state, tmp.path(), "z6f2skip", "pubcopy", content).await; + let (skip_id, _) = + seed_repo_with_blob(&state, tmp.path(), "z6f2skip", "gatedskip", content).await; + let (walk_id, _) = + seed_repo_with_blob(&state, tmp.path(), "z6f2skip", "gatedwalk", content).await; + for id in [&walk_id, &skip_id] { + state + .db + .set_visibility_rule( + id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderCCCCCCCCCCCCCCCCCCCCCCCC".to_string()], + "z6f2skip", + ) + .await + .unwrap(); + } + + let peer: SocketAddr = "203.0.113.61:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&cid_for_oid(&oid), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::OK, + "the walk-cap skip must continue the scan so the plain public copy serves" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + assert_eq!(&body[..], content.as_slice()); + let walks = std::fs::read_to_string(&walk_log) + .map(|s| s.lines().count()) + .unwrap_or(0); + assert_eq!( + walks, 1, + "cap honored exactly: the first path-scoped repo walks, the second is cut" + ); + } + + /// F2 visit ceiling: `ipfs_max_repo_visits` bounds the acquire+probe cost class + /// (each visit can be a full Tigris archive fetch on a cache miss). Unlike the + /// walk cap there is no cheap way to keep scanning, so exhaustion STOPS the scan + /// — and the stop is a truncation, not an absence: with ceiling 1 the newer + /// empty repo consumes the only visit and the blob-carrying older repo is never + /// probed, so the request sheds a retryable 503 + Retry-After, never a false + /// 404. MUTATION (RED): drop the ceiling check and the blob serves (200); drop + /// only the taint on the break and the 503 decays to a 404. + #[sqlx::test] + async fn get_by_cid_visit_ceiling_stops_scan_with_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let mut cfg = (*state.config).clone(); + cfg.ipfs_max_repo_visits = 1; + state.config = Arc::new(cfg); + + // Blob repo first (older, iterated second); empty repo second (newer, + // consumes the single visit). + let (_, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6f2visit", + "buried", + b"visit ceiling proof\n", + ) + .await; + seed_repo_with_blob(&state, tmp.path(), "z6f2visit", "fresh", b"unrelated\n").await; + + let peer: SocketAddr = "203.0.113.62:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&cid_for_oid(&oid), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a visit-ceiling truncation must shed a retryable 503, not report absent" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + } + + /// F2 negative arm: a COMPLETE scan that finds nothing keeps its definitive 404 + /// — the truncation 503 must never fire when every candidate reached a verdict. + /// Two public repos both probe clean (the requested CID is nowhere), no rules, + /// no cap or ceiling hit: 404 with no Retry-After. MUTATION (RED): taint the + /// scan unconditionally and this decays into a 503. + #[sqlx::test] + async fn get_by_cid_complete_scan_keeps_definitive_404(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + seed_repo_with_blob(&state, tmp.path(), "z6f2clean", "one", b"content one\n").await; + seed_repo_with_blob(&state, tmp.path(), "z6f2clean", "two", b"content two\n").await; + + // valid_cid() is the "hello" blob — present in neither repo. + let peer: SocketAddr = "203.0.113.63:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::NOT_FOUND, + "a complete clean scan is a definitive absence — 404, never the 503 shed" + ); + assert!( + resp.headers().get("retry-after").is_none(), + "a definitive 404 must not advertise a retry" + ); + } + + /// F2 acquire taint: a repo row with NO local copy over a Tigris backend that + /// stalls (non-routable endpoint — the connect just hangs) hits the 1s acquire + /// timeout at the read-acquire site. The skip carries no verdict, so the scan is + /// truncated: retryable 503 + Retry-After, never the old silent-skip 404. + /// MUTATION (RED): drop the taint on the acquire-timeout arm and this decays to + /// a 404. + #[sqlx::test] + async fn get_by_cid_acquire_timeout_taints_scan_to_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + // Endpoint-pinned test client (no AWS_* env reads — env is racy under a + // parallel test run); 10.255.255.1 is non-routable so the HEAD hangs. + let tigris = crate::git::tigris::TigrisClient::for_testing_with_endpoint( + "test-bucket", + "http://10.255.255.1:9000", + ) + .await; + state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let mut cfg = (*state.config).clone(); + cfg.git_acquire_timeout_secs = 1; + state.config = Arc::new(cfg); + + // Row exists in the DB but has no local copy, so the read acquire must + // consult Tigris (local-miss path) and stall until the timeout. + state + .db + .upsert_mirror_repo("z6f2acq", "ghost", "/unused-ghost", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.64:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an acquire timeout leaves the repo unproven — the scan must shed 503, not 404" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + } + + /// F2 probe taint: a repo row whose local dir does not exist (no Tigris) — + /// `RepoStore::acquire` returns the path anyway (local passthrough), and the + /// `cat-file -t` probe cannot even spawn (missing working dir), so + /// `object_type` is Err. That is not an absence verdict, so the scan is + /// truncated: 503, never 404. A second, real repo probes clean (absent verdict) + /// — the one bad row is what taints. NOTE: the probe shells to the real `git` + /// (not `state.git_bin`) and maps a NONZERO cat-file exit to `Ok(None)` (an + /// absent verdict), so a fake git exiting nonzero cannot reach this arm; only a + /// spawn failure is Err, hence the missing-dir recipe. MUTATION (RED): drop the + /// taint on the probe-error arm and this decays to a 404. + #[sqlx::test] + async fn get_by_cid_probe_error_taints_scan_to_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // Older row: a real repo that probes clean. Newer row: no dir on disk. + seed_repo_with_blob(&state, tmp.path(), "z6f2probe", "real", b"probe clean\n").await; + state + .db + .upsert_mirror_repo("z6f2probe", "ghost", "/unused-ghost", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.65:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a failed probe leaves the repo unproven — the scan must shed 503, not 404" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + } + + /// F2 read taint: the gate passes (the probe reads the truncated loose object's + /// intact "blob 64" header) but the content read fails (`cat-file blob` dies on + /// the deflate stream cut mid-content) — the probe just said the object EXISTS + /// here, so the failed read is no absence verdict: 503, never 404. The loose + /// object is hand-rolled: zlib header + one stored deflate block declaring 72 + /// bytes ("blob 64\0" + 64), truncated after the header NUL + 4 content bytes, + /// no adler trailer. MUTATION (RED): drop the taint on the read-error arm and + /// this decays to a 404. + #[sqlx::test] + async fn get_by_cid_read_error_taints_scan_to_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + state + .db + .upsert_mirror_repo("z6f2read", "corrupt", "/unused-corrupt", None, false) + .await + .unwrap(); + let rec = state + .db + .get_repo("z6f2read", "corrupt") + .await + .unwrap() + .unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + std::fs::create_dir_all(&bare).unwrap(); + run_git(&["init", "-q", "--bare", "--object-format=sha256"], &bare); + // Hand-rolled truncated loose object (dangling is fine: no path-scoped rules, + // so the "/" gate is the whole story and the read follows the probe). + let oid = "6bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459c"; + let mut corrupt: Vec = vec![0x78, 0x01, 0x01, 0x48, 0x00, 0xb7, 0xff]; + corrupt.extend_from_slice(b"blob 64\0AAAA"); + let obj_dir = bare.join("objects").join(&oid[..2]); + std::fs::create_dir_all(&obj_dir).unwrap(); + std::fs::write(obj_dir.join(&oid[2..]), &corrupt).unwrap(); + // Preconditions: the probe classifies it as a blob, the full read fails — + // otherwise the test would pass vacuously via some other arm. + assert_eq!( + crate::git::store::object_type(&bare, oid) + .unwrap() + .as_deref(), + Some("blob"), + "the truncated loose object's header must still probe as a blob" + ); + assert!( + crate::git::store::read_object_content(&bare, oid, "blob").is_err(), + "the truncated loose object's content read must fail" + ); + + let peer: SocketAddr = "203.0.113.66:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&cid_for_oid(oid), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a failed read after a passed gate leaves the repo unproven — 503, not 404" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + } + + /// F2 denied-is-a-verdict: repos that DENY the caller at the visibility gate + /// are settled, not skipped — an all-denied scan is COMPLETE: 404, zero visits. + /// The private rows deliberately have no local dirs: if the deny didn't + /// short-circuit before the visit, the missing-dir probe would taint the scan + /// into a 503, which the 404 assertion rules out — so the 404 also proves zero + /// acquires, probes, or walks ran for denied rows. + #[sqlx::test] + async fn get_by_cid_all_denied_is_complete_scan_404(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + for name in ["priv-a", "priv-b"] { + let now = chrono::Utc::now(); + state + .db + .create_repo(&crate::db::RepoRecord { + id: uuid::Uuid::new_v4().to_string(), + name: name.to_string(), + owner_did: "did:key:z6MkF2DenyOwnerAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + description: None, + is_public: false, + default_branch: "main".to_string(), + created_at: now, + updated_at: now, + disk_path: format!("/nonexistent/{name}"), + forked_from: None, + machine_id: None, + }) + .await + .unwrap(); + } + + let peer: SocketAddr = "203.0.113.67:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::NOT_FOUND, + "an anonymous caller denied by every repo gets a complete-scan 404 — a deny \ + is a verdict and must not visit, taint, or 503" + ); + } + /// Shed at capacity: an exhausted `git_ipfs_walk_semaphore` sheds a `/ipfs/{cid}` /// request with 503 BEFORE any DB/git walk (the acquire is the first thing after CID /// validation), so a lazy DB-free state suffices — exactly like the served-git shed @@ -736,13 +1359,16 @@ mod tests { ); } - /// Loop bound (cap N): one `/ipfs/{cid}` request against a CID present in many repos - /// must not serialize an unbounded number of full-history walks. With - /// `ipfs_max_repos_walked = 1` and TWO public, path-scoped repos both carrying the - /// blob at the requested CID, the handler walks only the FIRST candidate then stops - /// (the second is cut by the cap), so the fake git's `rev-list` (one per walk) runs - /// exactly once. MUTATION (RED): remove the `repos_walked >= cap` break and both - /// repos are walked (count 2). + /// Loop bound (cap N) + F2 truncation verdict: one `/ipfs/{cid}` request against a + /// CID present in many path-scoped repos must not serialize an unbounded number of + /// full-history walks — and cutting a candidate WITHOUT a verdict must not report + /// the object absent. With `ipfs_max_repos_walked = 1` and TWO public, path-scoped + /// repos both carrying the blob, the first candidate is walked (empty allowed-set → + /// a deny VERDICT) and the second is cut by the cap (no verdict), so the fake git's + /// `rev-list` runs exactly once and the request sheds a retryable 503 + Retry-After + /// — never the old false 404 (the blob genuinely sits in the second repo). + /// MUTATION (RED): remove the `repos_walked >= cap` skip and both repos are walked + /// (count 2); drop the truncation taint on the skip and the 503 decays to a 404. #[cfg(unix)] #[sqlx::test] async fn get_by_cid_caps_repos_walked_per_request(pool: sqlx::PgPool) { @@ -871,9 +1497,21 @@ mod tests { .unwrap(); req.extensions_mut().insert(ConnectInfo(peer)); let resp = ipfs_router(state).oneshot(req).await.unwrap(); - // The empty allowed-set path-gates both repos to a `continue`, so a 404; the - // point is HOW MANY walks ran to get there. - assert_eq!(resp.status(), StatusCode::NOT_FOUND); + // The first repo's walk yields the empty allowed-set (deny verdict); the second + // repo NEEDS a walk the cap forbids, so the scan is truncated without a verdict + // on it: retryable 503, never a false 404 for the blob it genuinely carries. + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a walk-cap truncation must shed a retryable 503, not report the object absent" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); let walks = std::fs::read_to_string(&walk_log) .map(|s| s.lines().count()) diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index b3667024..3e0a8e50 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -372,13 +372,17 @@ pub struct Config { )] pub ipfs_walk_per_source: usize, - /// Upper bound on the number of candidate repos a single `/ipfs/{cid}` request - /// will walk before giving up (returning the opaque 404). The handler already - /// short-circuits the moment it serves the object, but a CID that is present in - /// (or path-gated out of) many repos could otherwise serialize one full-history - /// walk per repo inside a single held admission slot. Capping the count bounds - /// the worst-case work one request can pin its slot with. Must be between 1 and - /// 1_048_576. Default: 64. + /// Upper bound on the number of EXPENSIVE visibility walks + /// (`allowed_blob_set_for_caller_bounded`, a full-history git walk in a + /// blocking thread) a single `/ipfs/{cid}` request may run. Only a blob in a + /// path-scoped repo costs a walk, so the cap counts exactly those candidates + /// — cheap probe-only visits are bounded by `ipfs_max_repo_visits` instead + /// (counting them here would starve a plain public copy past the cap out of + /// its 200). On exhaustion the walk-needing repo is skipped WITHOUT a verdict + /// and the scan continues; if the request then finds the object nowhere it + /// sheds a retryable 503 + Retry-After rather than misreport existing content + /// absent with a 404. The handler still short-circuits the moment it serves. + /// Must be between 1 and 1_048_576. Default: 64. #[arg( long, env = "GITLAWB_IPFS_MAX_REPOS_WALKED", @@ -387,6 +391,24 @@ pub struct Config { )] pub ipfs_max_repos_walked: usize, + /// Ceiling on the number of repos a single `/ipfs/{cid}` request may VISIT — + /// pass the repo-level visibility gate into the acquire + `cat-file` probe. + /// Each visit costs a `RepoStore::acquire` (on a Tigris cache miss that is a + /// full repo-archive download from object storage, so the worst-case + /// object-store fetch count for one request equals this ceiling) plus a git + /// probe subprocess. On exhaustion the scan STOPS — unlike + /// `ipfs_max_repos_walked`, which skips just the walk-needing repo, there is + /// no cheaper way to keep scanning — and the request sheds a retryable 503 + + /// Retry-After rather than a false 404. Must be between 1 and 1_048_576. + /// Default: 1024. + #[arg( + long, + env = "GITLAWB_IPFS_MAX_REPO_VISITS", + default_value_t = 1024, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub ipfs_max_repo_visits: usize, + /// Per-client-IP rate limit for `GET /ipfs/{cid}`, in requests per hour. The /// route is publicly reachable (`optional_signature`) and each request can drive /// a full-history git walk, so it carries a per-IP flood brake in addition to the @@ -548,6 +570,24 @@ mod tests { ); } + #[test] + fn ipfs_max_repo_visits_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).ipfs_max_repo_visits, + 1024 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--ipfs-max-repo-visits", "8"]) + .ipfs_max_repo_visits, + 8 + ); + // 0 would visit no repos (serve nothing); clap must reject it. + assert!(Config::try_parse_from(["gitlawb-node", "--ipfs-max-repo-visits", "0"]).is_err()); + assert!( + Config::try_parse_from(["gitlawb-node", "--ipfs-max-repo-visits", "1048577"]).is_err() + ); + } + #[test] fn max_concurrent_reads_per_caller_defaults_and_rejects_out_of_range() { assert_eq!( diff --git a/crates/gitlawb-node/src/git/tigris.rs b/crates/gitlawb-node/src/git/tigris.rs index ad26ddc5..cf7abfd5 100644 --- a/crates/gitlawb-node/src/git/tigris.rs +++ b/crates/gitlawb-node/src/git/tigris.rs @@ -31,6 +31,25 @@ impl TigrisClient { }) } + /// Test-only constructor with an explicit S3 endpoint, region, and static + /// credentials — no env-var reads, so parallel tests cannot race each other's + /// `AWS_*` environment the way the env-based `new` would. Lets a test point + /// the client at a non-routable endpoint to exercise acquire-stall paths. + #[cfg(test)] + pub(crate) async fn for_testing_with_endpoint(bucket: &str, endpoint_url: &str) -> Self { + let creds = aws_sdk_s3::config::Credentials::new("test", "test", None, None, "test"); + let config = aws_config::defaults(aws_config::BehaviorVersion::latest()) + .endpoint_url(endpoint_url) + .region(aws_config::Region::new("auto")) + .credentials_provider(creds) + .load() + .await; + Self { + s3: S3Client::new(&config), + bucket: bucket.to_string(), + } + } + /// S3 key for a given repo: `repos/v1/{owner_slug}/{repo_name}.tar.zst` fn repo_key(owner_slug: &str, repo_name: &str) -> String { format!("repos/v1/{owner_slug}/{repo_name}.tar.zst") diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 92f41a8d..df095b44 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -2409,7 +2409,10 @@ mod tests { /// the handler skips the whole repo rather than serving. Asserts no leak of the /// withheld blob AND that even the *public* blob in that repo is withheld — the /// latter distinguishes fail-closed-skip from normal per-blob withholding and - /// would serve 200 if the error arm wrongly proceeded. + /// would serve 200 if the error arm wrongly proceeded. The skip carries no + /// VERDICT (F2), so the response is the retryable truncation 503, not a 404 + /// claiming the object is absent — never-serve-unproven and never-404-unproven + /// hold together. #[sqlx::test] async fn ipfs_cid_walk_error_fails_closed(pool: PgPool) { use crate::db::VisibilityMode; @@ -2454,7 +2457,8 @@ mod tests { .await .expect("deny rule"); - // Withheld secret CID under a walk error → 404, no leak. + // Withheld secret CID under a walk error → the repo is skipped without a + // verdict, so the scan is truncated (503), and nothing leaks. let (st, body) = cid_parts( cid_router(&state) .oneshot(cid_anon(&secret_cid)) @@ -2464,17 +2468,17 @@ mod tests { .await; assert_eq!( st, - StatusCode::NOT_FOUND, - "walk error must not serve the withheld blob" + StatusCode::SERVICE_UNAVAILABLE, + "walk error must not serve the withheld blob — the unproven skip sheds 503" ); assert!( !body.contains("TOP SECRET"), - "walk-error 404 must not leak the secret" + "walk-error 503 must not leak the secret" ); - // The PUBLIC blob in the same repo is also 404: the walk error fails closed - // by skipping the whole repo, not by serving. Without the fail-closed arm - // this would serve 200, so this assertion is the load-bearing discriminator. + // The PUBLIC blob in the same repo is also not served: the walk error fails + // closed by skipping the whole repo. Without the fail-closed arm this would + // serve 200, so this assertion is the load-bearing discriminator. let (st, _) = cid_parts( cid_router(&state) .oneshot(cid_anon(&public_cid)) @@ -2484,8 +2488,9 @@ mod tests { .await; assert_eq!( st, - StatusCode::NOT_FOUND, - "walk error fails closed: repo skipped, even the public blob is not served" + StatusCode::SERVICE_UNAVAILABLE, + "walk error fails closed: repo skipped without a verdict, even the public \ + blob is not served and the scan sheds 503" ); } From 39d30dc3c11d08b1c816b38c108eb56f8111d11d Mon Sep 17 00:00:00 2001 From: t Date: Fri, 17 Jul 2026 18:03:49 -0500 Subject: [PATCH 046/124] fix(node): bound an admitted /ipfs request's total lifetime with a shared budget (#174) GITLAWB_IPFS_REQUEST_BUDGET_SECS (default 600) is captured as one deadline at handler entry: every acquire and expensive walk is clamped to the remaining budget, every stage checks remaining before starting, and exhaustion taints the scan into the retryable 503. A walk in flight is never aborted at the tokio layer (that would free the walk permit while the blocking thread still runs); its clamped git deadline plus group teardown ends it. Docs state the residual overshoot: the in-flight stage's kill/reap slack plus the unclamped probe subprocesses. --- .env.example | 7 + README.md | 1 + crates/gitlawb-node/src/api/ipfs.rs | 500 +++++++++++++++++++++++++++- crates/gitlawb-node/src/config.rs | 44 +++ 4 files changed, 546 insertions(+), 6 deletions(-) diff --git a/.env.example b/.env.example index 144108ba..94685069 100644 --- a/.env.example +++ b/.env.example @@ -179,6 +179,13 @@ GITLAWB_IPFS_MAX_REPOS_WALKED=64 # cat-file probe. On exhaustion the scan stops and sheds a retryable 503. # Default 1024. GITLAWB_IPFS_MAX_REPO_VISITS=1024 +# Absolute wall-clock budget for one admitted /ipfs request's acquire+walk +# lifetime (all stages of the whole scan). Bounds how LONG one request may hold +# its walk slot: per-stage timeouts are clamped to the remaining budget and no +# stage starts once it is exhausted (the scan then sheds a retryable 503). +# Residual overshoot: the in-flight clamped stage's kill/reap slack plus one +# unclamped cat-file probe subprocess. Default 600. +GITLAWB_IPFS_REQUEST_BUDGET_SECS=600 # Max /ipfs/{cid} requests per client IP per hour (route flood brake, distinct # from the concurrency caps above). 0 disables. Default 600. GITLAWB_IPFS_RATE_LIMIT=600 diff --git a/README.md b/README.md index f3fa361b..bc338339 100644 --- a/README.md +++ b/README.md @@ -349,6 +349,7 @@ Important node settings: | `GITLAWB_IPFS_WALK_PER_SOURCE` | Max concurrent `/ipfs` walks a single source IP may hold. Default 4. | | `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max expensive path-scope visibility walks per `/ipfs/{cid}` request; over-cap repos are skipped and the scan continues, shedding a retryable 503 (not a false 404) if the object is then found nowhere. Default 64. | | `GITLAWB_IPFS_MAX_REPO_VISITS` | Ceiling on repos one `/ipfs/{cid}` request may visit (acquire + probe) past the visibility gate — also the worst-case per-request Tigris fetch count. On exhaustion the scan stops with a retryable 503. Default 1024. | +| `GITLAWB_IPFS_REQUEST_BUDGET_SECS` | Absolute wall-clock budget for one admitted `/ipfs/{cid}` request's acquire+walk lifetime. Per-stage timeouts are clamped to the remaining budget and no stage starts once it is exhausted; the scan then stops with a retryable 503. Overshoot is bounded by the in-flight clamped stage's kill/reap slack plus one unclamped `cat-file` probe. Default 600. | | `GITLAWB_IPFS_RATE_LIMIT` | Max `/ipfs/{cid}` requests per client IP per hour (route flood brake). 0 disables. Default 600. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index d4a6c7a8..28b9cc67 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -55,11 +55,19 @@ use crate::visibility::{visibility_check, Decision}; /// repo reached a VERDICT — visibility deny, probe-says-absent, walk-gate deny, /// or served. A candidate skipped WITHOUT a verdict (acquire failure/timeout, /// probe error, walk failure/panic, content-read error, or truncation by -/// `ipfs_max_repos_walked` / `ipfs_max_repo_visits`) taints the scan, and a +/// `ipfs_max_repos_walked` / `ipfs_max_repo_visits` / +/// `ipfs_request_budget_secs`) taints the scan, and a /// tainted scan that found nothing sheds a retryable 503 + Retry-After naming /// the truncation sources — existing content is never misreported absent /// because of unrelated repos or transient faults. /// +/// Request budget (F3): one absolute clock (`ipfs_request_budget_secs`) spans +/// the whole admitted request. No stage (acquire, probe, walk, content read) +/// starts once it is exhausted, and the acquire wait and walk deadline are +/// clamped to the remainder, so an admitted request cannot hold its scarce walk +/// slot past the budget plus the in-flight stage's kill/reap slack and one +/// unclamped probe subprocess. +/// /// Scope: this closes the direct unauthenticated scan, including the dangling /// case. A stale-public mirror row still serves withheld content (tracked /// separately, #124). @@ -90,6 +98,19 @@ pub async fn get_by_cid( let caller = auth.as_ref().map(|e| e.0 .0.as_str()); let caller_owned = caller.map(|c| c.to_string()); + // One absolute budget bounds this request's whole acquire+walk lifetime (F3), + // captured before admission so the clock covers everything the walk permit + // holds. Each stage below (acquire, probe, walk, read) starts only while + // budget remains, and the acquire wait + walk deadline run clamped to the + // remainder, so an admitted request cannot hold its scarce walk slot for + // hours by drawing a fresh per-stage timeout every iteration. The budget + // NEVER aborts a running spawn_blocking walk: the clamped git deadline + // inside the walk is what ends it (a tokio timeout around the walk future + // would free the walk permit while the blocking thread still runs, the + // exact hole the held permit closes). + let request_deadline = std::time::Instant::now() + + std::time::Duration::from_secs(state.config.ipfs_request_budget_secs); + // Bounded walk admission (#174 P1-3), taken before any DB/git work so a flood sheds // cheaply. The per-repo `spawn_blocking` walk below is a full-history git walk with // no served-git admission of its own; a permissionless caller could otherwise fan @@ -161,6 +182,16 @@ pub async fn get_by_cid( } } + // Remaining request budget (F3), or None once exhausted. A stage is never + // started with zero remaining; the probe and read subprocesses carry no + // internal duration clamp, so the check before them is their entire bound + // (documented overshoot: at most one unclamped cat-file plus the in-flight + // clamped stage's kill/reap slack). + fn budget_remaining(deadline: std::time::Instant) -> Option { + let left = deadline.saturating_duration_since(std::time::Instant::now()); + (!left.is_zero()).then_some(left) + } + // Cap on EXPENSIVE walks only (F2): counts the repos that actually require the // full-history `allowed_blob_set_for_caller_bounded` walk (a path-scoped blob), // checked immediately before the spawn_blocking below. Cheap probe-only visits @@ -185,6 +216,21 @@ pub async fn get_by_cid( continue; } + // Budget gate for the acquire stage (F3): once the request budget is + // exhausted the scan STOPS, leaving this and every later candidate + // unproven (taint, never a false 404). Checked ahead of the visit + // bookkeeping so an unstarted acquire is not counted as a visit. + let Some(budget_left) = budget_remaining(request_deadline) else { + tracing::warn!( + repo = %repo.name, + budget_secs = state.config.ipfs_request_budget_secs, + "/ipfs request budget exhausted before the repo acquire \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); stopping the scan without a verdict" + ); + taint(&mut truncated_by, "budget"); + break; + }; + // Visit ceiling (F2): bound the acquire+probe cost class. Stopping here // leaves the remaining candidates unproven, so the scan is truncated. if repos_visited >= state.config.ipfs_max_repo_visits { @@ -202,9 +248,13 @@ pub async fn get_by_cid( // the P1-2 stall vector (a hung Tigris HEAD/GET on one repo would otherwise // block the whole /ipfs request). On expiry keep the fail-closed skip — never // serve an un-acquired repo; a public copy (if any) still gets its turn — but - // the repo got no verdict, so the skip taints the scan. - let acquire_deadline = - std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); + // the repo got no verdict, so the skip taints the scan. Clamped to the + // remaining request budget (F3) so per-repo acquires cannot each draw a + // fresh full timeout past it. + let acquire_deadline = std::cmp::min( + std::time::Duration::from_secs(state.config.git_acquire_timeout_secs), + budget_left, + ); let repo_path = match tokio::time::timeout( acquire_deadline, state.repo_store.acquire(&repo.owner_did, &repo.name), @@ -224,6 +274,18 @@ pub async fn get_by_cid( } }; + // Budget gate for the probe stage (F3). The probe subprocess has no + // internal duration clamp, so this pre-start check is its entire bound. + if budget_remaining(request_deadline).is_none() { + tracing::warn!( + repo = %repo.name, + "/ipfs request budget exhausted before the object-type probe; \ + stopping the scan without a verdict" + ); + taint(&mut truncated_by, "budget"); + break; + } + // Check whether the object exists in this repo before any expensive // reachability walk. This prevents random-CID spray from triggering // full-history git walks on repos that don't carry the object. Absent @@ -244,6 +306,21 @@ pub async fn get_by_cid( let path_scoped = has_path_scoped_rule(rules); if path_scoped && obj_type == "blob" { if !allowed_memo.contains_key(&repo.id) { + // Budget gate for the walk stage (F3): a walk is never STARTED + // with zero remaining (probed-present is not a serve), and a + // started walk runs its git children under a deadline clamped + // to the remainder (the min below), so a walk can never + // complete past the budget. + let Some(budget_left) = budget_remaining(request_deadline) else { + tracing::warn!( + repo = %repo.name, + budget_secs = state.config.ipfs_request_budget_secs, + "/ipfs request budget exhausted before the visibility walk \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); stopping the scan without a verdict" + ); + taint(&mut truncated_by, "budget"); + break; + }; // Walk cap (F2), checked at the one site that actually spends a walk: // on exhaustion skip THIS repo without a verdict and KEEP scanning — // later candidates may still reach a cheap probe-only verdict (a plain @@ -265,8 +342,10 @@ pub async fn get_by_cid( let owner = repo.owner_did.clone(); let caller_for_walk = caller_owned.clone(); let git_bin = state.git_bin.clone(); - let walk_timeout = - std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let walk_timeout = std::cmp::min( + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + budget_left, + ); // Full-history walk shells out to git — keep it off the async runtime, // bounded and reaped like the served-git ops (#174). let walk = tokio::task::spawn_blocking(move || { @@ -310,6 +389,21 @@ pub async fn get_by_cid( } } + // Budget gate for the content-read stage (F3): the read subprocess is + // unclamped, so it never starts past the budget. Tainting instead of + // serving keeps the terminal arm honest and the stop unconditional; the + // retryable 503 tells the caller to come back rather than letting an + // over-budget request keep spending. + if budget_remaining(request_deadline).is_none() { + tracing::warn!( + repo = %repo.name, + "/ipfs request budget exhausted before the content read; \ + stopping the scan without a verdict" + ); + taint(&mut truncated_by, "budget"); + break; + } + // Now that we've passed the gate, read the content. A failed read after a // passed gate is not an absence verdict — the probe just said the object // exists here — so the skip taints the scan. @@ -976,6 +1070,400 @@ mod tests { ); } + /// F3 budget expiry mid-loop: one absolute request budget + /// (`ipfs_request_budget_secs`) bounds the whole admitted scan; per-repo + /// stages may not each draw a fresh timeout past it. Budget 1s, per-iteration + /// acquire timeout 2s; the NEWER row is a Tigris-backed ghost (no local copy, + /// non-routable endpoint) whose acquire stalls, the OLDER row is a plain + /// public repo carrying the blob. The ghost's acquire runs clamped to the ~1s + /// remainder and times out; at the next repo the budget gate sees zero + /// remaining, taints "budget", and STOPS the scan, so the blob repo is never + /// visited (a visit would probe the healthy public copy and serve 200, which + /// the 503 assertion rules out) and the shed names the budget. Without the + /// budget the acquire would time out at its own 2s, the scan would continue, + /// and the buried blob would serve 200 (the recorded RED). MUTATION (RED): + /// remove the `request_deadline` capture (or make the remaining budget + /// infinite) and this serves 200 again. + #[sqlx::test] + async fn get_by_cid_request_budget_expiry_stops_scan_with_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Seed the blob repo through a LOCAL-ONLY store first, so seeding never + // consults the (deliberately unreachable) Tigris endpoint. + state.repo_store = + crate::git::repo_store::RepoStore::for_testing(repos_dir.clone(), pool.clone()); + let (_, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6f3budget", + "buried", + b"budget expiry proof\n", + ) + .await; + // Swap in a Tigris-backed store over the SAME repos_dir (the seeded bare + // repo stays a fast local hit) and add a NEWER ghost row with no local + // copy: its acquire consults the non-routable endpoint and stalls + // (endpoint-pinned test client, no AWS_* env reads; 10.255.255.1 hangs). + let tigris = crate::git::tigris::TigrisClient::for_testing_with_endpoint( + "test-bucket", + "http://10.255.255.1:9000", + ) + .await; + state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); + state + .db + .upsert_mirror_repo("z6f3budget", "ghost", "/unused-ghost", None, false) + .await + .unwrap(); + let mut cfg = (*state.config).clone(); + cfg.ipfs_request_budget_secs = 1; + cfg.git_acquire_timeout_secs = 2; + state.config = Arc::new(cfg); + + let peer: SocketAddr = "203.0.113.70:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&cid_for_oid(&oid), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted request budget must stop the scan with a retryable 503; \ + scanning on into the later public blob repo would have served 200" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the budget-truncation 503 must carry Retry-After" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("budget"), + "the truncation body must name the budget taint so the operator can \ + map the shed to GITLAWB_IPFS_REQUEST_BUDGET_SECS; got: {body}" + ); + } + + /// F3 clamped walk at expiry: a walk that starts with little budget left runs + /// its git children under `min(git_service_timeout_secs, remaining)`, so the + /// clamp (not any tokio-level abort) is what ends it and a walk can never + /// complete past the budget. Budget 2s, service timeout at its 600s default, + /// fake walk git that sleeps 8s: the walk STARTS (pid file), the walk permit + /// stays held while the blocking walk runs (`available_permits == 0`), the + /// clamped deadline SIGTERM/SIGKILLs the child group at ~2s remaining (the + /// response lands after the ~1s watchdog grace, far before the 8s sleep, and + /// the recorded pid is already dead: a tokio abort would have left it + /// running), the log shows the walk started but never completed, and the + /// request sheds the terminal budget-truncated 503 without ever reaching the + /// OLDER public copy of the same blob (which would have served 200). After + /// the response the permit is free: the spawn_blocking closure genuinely + /// returned. MUTATION (RED): drop the `min` clamp on `walk_timeout` and the + /// walk runs its full 8s sleep (elapsed and log-completion assertions fail). + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_budget_clamps_walk_deadline_and_holds_permit(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let walk_log = tmp.path().join("walks.log"); + let revlist_pid = tmp.path().join("revlist.pid"); + // Fake git for the WALK only: `rev-list` records its pid and a start + // marker, sleeps far past the budget, then records a done marker. Under + // the clamped walk deadline the whole process group is torn down mid + // sleep, so "done" never appears. The 8s sleep also bounds a RED run. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + for-each-ref) : ;;\n\ + rev-parse) echo deadbeef ;;\n\ + rev-list) echo $$ > \"{pid}\"; echo start >> \"{log}\"; sleep 8; echo done >> \"{log}\" ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + pid = revlist_pid.display(), + log = walk_log.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Global walk pool of 1 so the held permit is observable; per-source cap + // permissive so only the global pool matters. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + let mut cfg = (*state.config).clone(); + // The budget is the ONLY thing that can end this walk early: the service + // timeout stays at its generous 600s default. + cfg.ipfs_request_budget_secs = 2; + state.config = Arc::new(cfg); + + // Older row: a plain public copy of the same blob, which must never be + // reached. Newer row: path-scoped, so its blob costs the clamped walk. + let content = b"budget walk clamp proof\n"; + let (_, oid) = + seed_repo_with_blob(&state, tmp.path(), "z6f3clamp", "pubcopy", content).await; + let (walk_id, _) = + seed_repo_with_blob(&state, tmp.path(), "z6f3clamp", "gated", content).await; + state + .db + .set_visibility_rule( + &walk_id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderDDDDDDDDDDDDDDDDDDDDDDDD".to_string()], + "z6f3clamp", + ) + .await + .unwrap(); + + let sem = state.git_ipfs_walk_semaphore.clone(); + let router = ipfs_router(state); + let started = std::time::Instant::now(); + let peer: SocketAddr = "203.0.113.71:5000".parse().unwrap(); + let mut fut = Box::pin(router.oneshot(get_cid(&cid_for_oid(&oid), Some(peer)))); + + // Drive until the fake git's rev-list records its pid: the walk is now in + // the blocking pool and the request future is `.await`ing its join. Stop + // polling the instant the future completes (re-polling would panic). + let mut walk_pid: Option = None; + let mut early = None; + for _ in 0..500 { + let done = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&revlist_pid) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + walk_pid = Some(p); + break; + } + if let Ok(resp) = done { + early = Some(resp.map(|r| r.status())); + break; + } + } + let pid = walk_pid.unwrap_or_else(|| { + panic!( + "the budget-clamped walk must have STARTED (nonzero remaining); early: {early:?}" + ) + }); + // Reap the sleeping child on drop so a RED run leaks no orphan. + struct ReapOnDrop(i32); + impl Drop for ReapOnDrop { + fn drop(&mut self) { + unsafe { + libc::kill(self.0, libc::SIGKILL); + } + } + } + let _cleanup = ReapOnDrop(pid); + + // While the blocking walk runs the permit is HELD: the budget never frees + // a slot whose blocking thread is still burning. + assert_eq!( + sem.available_permits(), + 0, + "the walk permit must stay held while the budget-clamped walk runs" + ); + + let resp = tokio::time::timeout(std::time::Duration::from_secs(20), &mut fut) + .await + .expect("the clamped walk deadline must end the request; it never hung") + .unwrap(); + let elapsed = started.elapsed(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a budget-clamped walk that could not finish leaves no verdict: 503, not 404/200" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("budget"), + "the terminal shed must name the budget taint; got: {body}" + ); + // Deadline-killed at ~remaining, not run to completion: the response + // lands at ~budget + the watchdog's kill/reap slack, well before the 8s + // sleep could have finished. + assert!( + elapsed < std::time::Duration::from_secs(7), + "the clamped git deadline must end the walk at ~remaining; got {elapsed:?}" + ); + // The child group is already dead AT response time: the clamp killed it. + // (A tokio-level abort of the walk future would have answered while the + // blocking thread and its child still ran.) + assert_eq!( + unsafe { libc::kill(pid, 0) }, + -1, + "the walk's git child must be reaped by the clamped deadline before the response" + ); + let log = std::fs::read_to_string(&walk_log).unwrap_or_default(); + assert!( + log.contains("start"), + "the walk must have started (the budget gate passed with remaining > 0)" + ); + assert!( + !log.contains("done"), + "the walk must never complete past the budget; the clamp kills it mid-run" + ); + // The spawn_blocking closure returned and the handler finished: the + // permit is free again (held through the blocking run, no longer). + assert_eq!( + sem.available_permits(), + 1, + "the walk permit must free once the blocking walk genuinely returns" + ); + } + + /// F3 expiry between probe and walk: an object PROBED PRESENT in a + /// path-scoped repo is still not served once the budget is gone; the + /// walk-stage gate taints and stops before any walk starts. The probe is + /// stalled past the 1s budget while still SUCCEEDING via a FIFO at + /// `objects/info/alternates`: real `git cat-file -t` blocks opening it at + /// odb setup until a plain OS thread feeds it an empty alternates list at + /// ~2s, after which the probe finds the genuine loose object and reports + /// "blob". The walk-needing repo thus reaches the walk gate with zero + /// remaining: 503 naming the budget, never a 404 (the walk-gate deny of the + /// fake git's empty allowed-set would have been a verdict) and never a + /// serve, and the fake-git walk log stays EMPTY (no walk ever started). + /// MUTATION (RED): drop the walk-stage budget gate and this decays to the + /// walked 404 with one logged walk. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_budget_expiry_between_probe_and_walk_sheds_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let walk_log = tmp.path().join("walks.log"); + state.git_bin = walk_logging_fake_git(tmp.path(), &walk_log); + let mut cfg = (*state.config).clone(); + cfg.ipfs_request_budget_secs = 1; + state.config = Arc::new(cfg); + + let (repo_id, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6f3probe", + "gated", + b"probe-then-expire proof\n", + ) + .await; + state + .db + .set_visibility_rule( + &repo_id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderEEEEEEEEEEEEEEEEEEEEEEEE".to_string()], + "z6f3probe", + ) + .await + .unwrap(); + + // Stall the REAL-git probe past the budget while still letting it + // succeed: `objects/info/alternates` as a FIFO blocks cat-file's odb + // setup until a writer appears. The feeder thread supplies an EMPTY + // alternates list (open + close, no bytes) after 2s, so the probe then + // resolves the genuine loose object. Non-blocking open with retries so + // a handler that never probes cannot wedge the feeder forever. + let rec = state + .db + .get_repo("z6f3probe", "gated") + .await + .unwrap() + .unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + let fifo = bare.join("objects").join("info").join("alternates"); + let c_path = std::ffi::CString::new(fifo.to_str().unwrap()).unwrap(); + assert_eq!( + unsafe { libc::mkfifo(c_path.as_ptr(), 0o644) }, + 0, + "mkfifo(objects/info/alternates) must succeed" + ); + let feeder = std::thread::spawn(move || { + use std::os::unix::fs::OpenOptionsExt; + std::thread::sleep(std::time::Duration::from_secs(2)); + for _ in 0..300 { + if std::fs::OpenOptions::new() + .write(true) + .custom_flags(libc::O_NONBLOCK) + .open(&fifo) + .is_ok() + { + return; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + } + }); + + let peer: SocketAddr = "203.0.113.72:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&cid_for_oid(&oid), Some(peer))) + .await + .unwrap(); + feeder.join().unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "probed-present with the budget gone must shed the truncation 503: \ + never the walked 404, never a serve" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("budget"), + "the shed must name the budget taint; got: {body}" + ); + let walks = std::fs::read_to_string(&walk_log) + .map(|s| s.lines().count()) + .unwrap_or(0); + assert_eq!( + walks, 0, + "no walk may START once the budget is exhausted, even for a probed-present object" + ); + } + /// Shed at capacity: an exhausted `git_ipfs_walk_semaphore` sheds a `/ipfs/{cid}` /// request with 503 BEFORE any DB/git walk (the acquire is the first thing after CID /// validation), so a lazy DB-free state suffices — exactly like the served-git shed diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 3e0a8e50..d564a554 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -409,6 +409,32 @@ pub struct Config { )] pub ipfs_max_repo_visits: usize, + /// Absolute wall-clock budget for one admitted `GET /ipfs/{cid}` request's + /// acquire+walk lifetime, in seconds. `max_concurrent_ipfs_walks` bounds how + /// MANY requests hold walk slots; this bounds how LONG one admitted request + /// may keep its slot. Without it, each repo iteration draws a fresh + /// `git_acquire_timeout_secs` and each expensive walk a fresh + /// `git_service_timeout_secs`, so one request scanning many repos could hold + /// a scarce walk slot for hours. Every stage (acquire, `cat-file` probe, + /// visibility walk, content read) starts only while budget remains, and the + /// acquire wait and walk deadline are clamped to `min(their own timeout, + /// remaining budget)`; a stage is never started with zero remaining. On + /// exhaustion the scan stops without a verdict and the request sheds a + /// retryable 503 + Retry-After rather than a false 404. Residual overshoot + /// past the budget is bounded by the kill/reap slack of the one in-flight + /// clamped stage (the walk watchdog's SIGTERM grace + SIGKILL settle) plus + /// the `object_type` / `read_object_content` probe subprocesses, which are + /// budget-checked before they start but carry no internal duration clamp. + /// Must be positive. Default: 600s (10 min), matching + /// `git_service_timeout_secs` so a single full-length walk still fits. + #[arg( + long, + env = "GITLAWB_IPFS_REQUEST_BUDGET_SECS", + default_value_t = 600, + value_parser = clap::value_parser!(u64).range(1..) + )] + pub ipfs_request_budget_secs: u64, + /// Per-client-IP rate limit for `GET /ipfs/{cid}`, in requests per hour. The /// route is publicly reachable (`optional_signature`) and each request can drive /// a full-history git walk, so it carries a per-IP flood brake in addition to the @@ -588,6 +614,24 @@ mod tests { ); } + #[test] + fn ipfs_request_budget_secs_defaults_to_600_and_rejects_zero() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).ipfs_request_budget_secs, + 600 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--ipfs-request-budget-secs", "30"]) + .ipfs_request_budget_secs, + 30 + ); + // 0 would expire every /ipfs request at its first stage (unconditional + // 503); clap must reject it. + assert!( + Config::try_parse_from(["gitlawb-node", "--ipfs-request-budget-secs", "0"]).is_err() + ); + } + #[test] fn max_concurrent_reads_per_caller_defaults_and_rejects_out_of_range() { assert_eq!( From daf9195f29192b7dba610238529091e4402e7edd Mon Sep 17 00:00:00 2001 From: t Date: Fri, 17 Jul 2026 18:37:40 -0500 Subject: [PATCH 047/124] fix(node): admit the post-receive git scans to the encrypt pool, defer-not-shed (#174) replication_withheld_set's walk arm, resolve_candidates_for_push's git stages, and fail_closed_full_scan_objects each take a git_encrypt_semaphore permit before their spawn_blocking and carry it inside the closure, so a push burst holds at most pool-many concurrent scans and a started scan finishes with its permit. Deletion-only pushes and repos without path-scoped rules never park. Contention defers the landed push's tail (logged with queue_wait_ms), never errors it; the acquire sites document the accepted disconnect-during-park residual. --- crates/gitlawb-node/src/api/repos.rs | 523 ++++++++++++++++++++++ crates/gitlawb-node/src/git/push_delta.rs | 156 +++++++ crates/gitlawb-node/src/state.rs | 22 +- crates/gitlawb-node/tests/inv22_gates.rs | 33 ++ 4 files changed, 726 insertions(+), 8 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index bb9e743b..f25293b8 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -39,7 +39,15 @@ const ZERO_SHA: &str = "0000000000000000000000000000000000000000"; /// `withheld` is `None`, so an unvetted push neither replicates blobs nor /// announces. Returning both keeps the gate's announce decision a single /// source rather than recomputing it at each call site. +/// +/// The walk arm runs under a `git_encrypt_semaphore` admission permit (#174 F4): +/// by the time the receive-pack tail calls this, the handler's write permit has +/// already been released (receive_pack's AdmissionGuard drops when the git group +/// is reaped), so without the gate a burst of completed pushes accumulates +/// unbounded concurrent full-history walks. `encrypt_sem` is threaded in so the +/// no-walk fast paths (not announceable; no path-scoped rule) never touch it. async fn replication_withheld_set( + encrypt_sem: std::sync::Arc, rules: Option>, owner_did: &str, is_public: bool, @@ -66,7 +74,31 @@ async fn replication_withheld_set( // that off the async worker thread. Some(rules) => { let owner_did = owner_did.to_string(); + // Scan admission (#174 F4): DEFER (await), never shed — dropping the + // walk would skip the vetting and fail the push's replication closed + // for no reason. Accepted residuals, stated honestly: (1) the park + // wait is queue-depth multiplied — post-receive tails are no longer + // admission-bounded once the write permit is released, so N landed + // pushes can queue N walks and the last waits N walk-durations; + // (2) a client-timeout disconnect while parked HERE drops the + // handler future and silently loses this push's replication work — + // the encrypt_inflight coalescing requeue does NOT cover it, because + // this park precedes the `try_begin` spawn gate. + let parked = std::time::Instant::now(); + let permit = encrypt_sem + .acquire_owned() + .await + .expect("git_encrypt_semaphore is never closed"); + tracing::debug!( + repo = %disk_path.display(), + queue_wait_ms = parked.elapsed().as_millis() as u64, + "post-push withheld walk admitted to the scan pool" + ); tokio::task::spawn_blocking(move || { + // The permit lives inside the blocking closure: a started walk + // always completes holding it (a disconnect cannot cancel + // spawn_blocking or leak the permit mid-walk). + let _permit = permit; crate::git::visibility_pack::withheld_blob_oids_bounded( &disk_path, &git_bin, timeout, &rules, is_public, &owner_did, None, ) @@ -102,7 +134,15 @@ async fn replication_withheld_set( /// non-blobs plus allowed blobs. Any error in either walk (or a task panic) /// pins nothing this push, mirroring the degraded-path shape of /// `replication_withheld_set`. +/// +/// Always walks (there is no no-git arm), so the whole blocking scan runs under +/// one `git_encrypt_semaphore` admission permit (#174 F4) — see +/// `replication_withheld_set`'s acquire for the defer rationale and the honest +/// residuals (queue-depth-multiplied park wait; a disconnect while parked loses +/// this push's replication work, uncovered by the coalescing requeue). +#[allow(clippy::too_many_arguments)] async fn fail_closed_full_scan_objects( + encrypt_sem: std::sync::Arc, disk_path: std::path::PathBuf, rules: Vec, is_public: bool, @@ -111,7 +151,20 @@ async fn fail_closed_full_scan_objects( git_bin: String, timeout: std::time::Duration, ) -> Vec { + // Scan admission (#174 F4): DEFER, never shed; the permit moves into the + // closure so a started scan always completes holding it. + let parked = std::time::Instant::now(); + let permit = encrypt_sem + .acquire_owned() + .await + .expect("git_encrypt_semaphore is never closed"); + tracing::debug!( + repo = %disk_path.display(), + queue_wait_ms = parked.elapsed().as_millis() as u64, + "post-push fail-closed full scan admitted to the scan pool" + ); tokio::task::spawn_blocking(move || -> anyhow::Result> { + let _permit = permit; let allowed = crate::git::visibility_pack::replicable_blob_set_bounded( &disk_path, &git_bin, timeout, &rules, is_public, &owner_did, )?; @@ -1385,6 +1438,7 @@ pub async fn git_receive_pack( // never leaks. let rules_opt = state.db.list_visibility_rules(&record.id).await.ok(); let (announce, withheld) = replication_withheld_set( + state.git_encrypt_semaphore.clone(), rules_opt.clone(), &record.owner_did, record.is_public, @@ -1413,6 +1467,7 @@ pub async fn git_receive_pack( .filter(|s| s != ZERO_SHA) .collect(); let pin_set = crate::git::push_delta::resolve_candidates_for_push( + state.git_encrypt_semaphore.clone(), disk_path.clone(), new_tips, old_tips, @@ -1422,6 +1477,7 @@ pub async fn git_receive_pack( .await; if pin_set.full_scan { fail_closed_full_scan_objects( + state.git_encrypt_semaphore.clone(), disk_path.clone(), rules_opt.clone().unwrap_or_default(), record.is_public, @@ -2286,6 +2342,7 @@ mod tests { // Private: no rules at all. let (announce, _) = replication_withheld_set( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), None, OWNER_DID, false, @@ -2298,6 +2355,7 @@ mod tests { // Private: empty rule set, is_public=false → still not listable at root. let (announce, _) = replication_withheld_set( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), Some(vec![]), OWNER_DID, false, @@ -2310,6 +2368,7 @@ mod tests { // Public: empty rule set, is_public=true → listable at root, announces. let (announce, _) = replication_withheld_set( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), Some(vec![]), OWNER_DID, true, @@ -2441,6 +2500,7 @@ mod tests { let result = tokio::time::timeout( Duration::from_secs(10), replication_withheld_set( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), rules, OWNER_DID, true, @@ -4635,6 +4695,469 @@ mod tests { ); } + /// F4 defer proof 1: `replication_withheld_set`'s WALK arm acquires a + /// `git_encrypt_semaphore` permit before its spawn_blocking git walk, deferring + /// (never shedding) when the pool is exhausted — while its no-walk fast paths + /// (no path-scoped rule; not announceable) complete WITHOUT touching the pool. + /// On ungated code the walk runs regardless of a zero-permit pool (RED). + #[cfg(unix)] + #[tokio::test] + async fn replication_walk_defers_when_scan_pool_exhausted() { + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let marker = tmp.path().join("git.ran"); + // Fake git records ANY invocation (the walk's first call is rev-parse), then + // behaves well enough for a successful empty walk: HEAD probe succeeds, + // rev-list lists no commits. + let body = format!( + "#!/bin/sh\necho ran >> \"{}\"\ncase \"$1\" in\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n", + marker.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + let scoped_rules = || Some(vec![vis_rule("/secret/**", &[])]); + + // Zero-permit pool: every gated walk must park forever. + let sem: Arc = Arc::new(Semaphore::new(0)); + + // Fast path A (negative arm): announceable, NO path-scoped rule -> zero git + // work, must complete immediately without acquiring from the empty pool. + let fast = tokio::time::timeout( + Duration::from_millis(500), + replication_withheld_set( + sem.clone(), + Some(vec![]), + OWNER_DID, + true, + tmp.path().to_path_buf(), + git_bin.clone(), + Duration::from_secs(5), + ), + ) + .await + .expect("the no-path-scoped-rule fast path must not park on the scan pool"); + assert_eq!(fast, (true, Some(std::collections::HashSet::new()))); + + // Fast path B (negative arm): not announceable (no rules) -> zero git work, + // must complete immediately without acquiring. + let fast = tokio::time::timeout( + Duration::from_millis(500), + replication_withheld_set( + sem.clone(), + None, + OWNER_DID, + false, + tmp.path().to_path_buf(), + git_bin.clone(), + Duration::from_secs(5), + ), + ) + .await + .expect("the not-announceable fast path must not park on the scan pool"); + assert_eq!(fast, (false, None)); + assert!(!marker.exists(), "the fast paths must spawn no git at all"); + + // Walk arm with the pool exhausted: must DEFER (park), spawning no git. + let blocked = tokio::time::timeout( + Duration::from_millis(500), + replication_withheld_set( + sem.clone(), + scoped_rules(), + OWNER_DID, + true, + tmp.path().to_path_buf(), + git_bin.clone(), + Duration::from_secs(5), + ), + ) + .await; + assert!( + blocked.is_err(), + "the withheld walk must defer (park on admission) when the pool is exhausted" + ); + assert!( + !marker.exists(), + "the withheld walk's git must not spawn while its admission permit is unavailable (F4)" + ); + + // Release admission: the SAME walk now runs (defer, not shed) and succeeds. + sem.add_permits(1); + let ran = replication_withheld_set( + sem, + scoped_rules(), + OWNER_DID, + true, + tmp.path().to_path_buf(), + git_bin, + Duration::from_secs(5), + ) + .await; + assert!( + marker.exists(), + "once admission is available the deferred withheld walk runs its git" + ); + assert_eq!( + ran, + (true, Some(std::collections::HashSet::new())), + "the released walk completes and vets the (empty) withheld set" + ); + } + + /// F4 defer proof 3: `fail_closed_full_scan_objects` ALWAYS walks, so its + /// spawn_blocking is always admission-gated: with the pool exhausted it defers + /// and spawns no git; with a permit the same call runs. Ungated it runs + /// regardless (RED). + #[cfg(unix)] + #[tokio::test] + async fn full_scan_pin_walk_defers_when_scan_pool_exhausted() { + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let marker = tmp.path().join("git.ran"); + let body = format!( + "#!/bin/sh\necho ran >> \"{}\"\ncase \"$1\" in\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n", + marker.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + let candidates = vec!["3333333333333333333333333333333333333333".to_string()]; + + let sem: Arc = Arc::new(Semaphore::new(0)); + let blocked = tokio::time::timeout( + Duration::from_millis(500), + fail_closed_full_scan_objects( + sem.clone(), + tmp.path().to_path_buf(), + vec![vis_rule("/secret/**", &[])], + true, + OWNER_DID.to_string(), + candidates.clone(), + git_bin.clone(), + Duration::from_secs(5), + ), + ) + .await; + assert!( + blocked.is_err(), + "the fail-closed full scan must defer (park on admission) when the pool is exhausted" + ); + assert!( + !marker.exists(), + "the full scan's git must not spawn while its admission permit is unavailable (F4)" + ); + + // Release admission: the SAME scan now runs (defer, not shed). + sem.add_permits(1); + let _objs = fail_closed_full_scan_objects( + sem, + tmp.path().to_path_buf(), + vec![vis_rule("/secret/**", &[])], + true, + OWNER_DID.to_string(), + candidates, + git_bin, + Duration::from_secs(5), + ) + .await; + assert!( + marker.exists(), + "once admission is available the deferred full scan runs its git" + ); + } + + /// Shared fixture for the F4 handler-layer tests: a state whose repo_store and + /// git_bin point at the given tempdir/fake-git, plus a seeded on-disk repo, + /// optionally with a path-scoped rule (so the post-receive walks actually run). + #[cfg(unix)] + async fn f4_state_with_repo( + pool: sqlx::PgPool, + tmp: &std::path::Path, + git_bin: &str, + owner: &str, + name: &str, + path_scoped: bool, + ) -> AppState { + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.git_bin = git_bin.to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo(owner, name, &format!("/unused-{owner}-{name}"), None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + state + .repo_store + .init(&rec.owner_did, &rec.name) + .await + .unwrap(); + if path_scoped { + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkF4ReaderAAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + } + state + } + + /// A pkt-line receive-pack body carrying one branch-create ref update, so the + /// handler's post-receive tail resolves a non-empty new-tip set (the delta + /// scan's git stages run). + fn ref_update_body(new_sha: &str) -> axum::body::Bytes { + let line = format!("{ZERO_SHA} {new_sha} refs/heads/main"); + axum::body::Bytes::from(format!("{:04x}{}0000", line.len() + 4, line)) + } + + /// F4 scenario 2 — push-burst bound at the handler layer: with a scan pool of + /// ONE, two concurrent pushes to two path-scoped repos never have more than one + /// scan's git alive at a time (an atomic mkdir lock in the fake git detects any + /// overlap), and BOTH pushes still succeed 200 — defer, not shed. Two distinct + /// repos on purpose: the per-repo advisory write lock must not be what + /// serializes the scans. + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_burst_scans_serialized_and_both_pushes_succeed(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let lockdir = tmp.path().join("scan.lock"); + let ranfile = tmp.path().join("scan.ran"); + let overlap = tmp.path().join("scan.overlap"); + // receive-pack succeeds instantly; every candidate-scan git op (cat-file / + // rev-list / ls-tree) holds an atomic mkdir lock for 150ms — a second scan + // process alive at the same instant records an overlap. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + receive-pack) cat > /dev/null 2>/dev/null ;;\n\ + rev-parse) echo deadbeef ;;\n\ + cat-file|rev-list|ls-tree)\n\ + if mkdir \"{lock}\" 2>/dev/null; then\n\ + echo 1 >> \"{ran}\"\n\ + sleep 0.15\n\ + rmdir \"{lock}\"\n\ + else\n\ + echo 1 >> \"{over}\"\n\ + fi\n\ + if [ \"$1\" = cat-file ]; then echo commit; fi ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + lock = lockdir.display(), + ran = ranfile.display(), + over = overlap.display(), + ); + let git_bin = write_fake_git(tmp.path(), &body); + + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f4burst1", "b1", true).await; + // Second path-scoped repo on the same state/store. + state + .db + .upsert_mirror_repo("z6f4burst2", "b2", "/unused-z6f4burst2-b2", None, false) + .await + .unwrap(); + let rec2 = state + .db + .get_repo("z6f4burst2", "b2") + .await + .unwrap() + .unwrap(); + state + .repo_store + .init(&rec2.owner_did, &rec2.name) + .await + .unwrap(); + state + .db + .set_visibility_rule( + &rec2.id, + "/secret/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkF4ReaderAAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec2.owner_did, + ) + .await + .unwrap(); + // Scan pool of ONE: at most one post-receive walk may run at a time. + state.git_encrypt_semaphore = Arc::new(Semaphore::new(1)); + + let did = "did:key:z6MkF4BurstPusherAAAAAAAAAAAAAAAAAAAAAAAA"; + let new_sha = "1111111111111111111111111111111111111111"; + let push = |owner: &'static str, name: &'static str, peer: &'static str| { + let state = state.clone(); + tokio::spawn(async move { + git_receive_pack( + State(state), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(peer.parse::().unwrap())), + axum::http::HeaderMap::new(), + ref_update_body(new_sha), + ) + .await + }) + }; + + let (a, b) = ( + push("z6f4burst1", "b1", "203.0.113.71:5000"), + push("z6f4burst2", "b2", "203.0.113.72:5000"), + ); + let a = tokio::time::timeout(std::time::Duration::from_secs(60), a) + .await + .expect("push A must complete — a scan gate must defer, never wedge") + .expect("push A task must not panic"); + let b = tokio::time::timeout(std::time::Duration::from_secs(60), b) + .await + .expect("push B must complete — a scan gate must defer, never wedge") + .expect("push B task must not panic"); + let a = a.expect("push A must succeed"); + let b = b.expect("push B must succeed"); + assert_eq!(a.status(), 200, "push A lands 200 despite scan contention"); + assert_eq!(b.status(), 200, "push B lands 200 despite scan contention"); + + // Let the detached recipients walks drain through the same pool before + // reading the detector files. + tokio::time::sleep(std::time::Duration::from_millis(800)).await; + assert!( + !overlap.exists(), + "with a scan pool of 1, no two scans' git may ever be alive at once \ + (found overlap records: {:?})", + std::fs::read_to_string(&overlap).unwrap_or_default() + ); + let ran = std::fs::read_to_string(&ranfile).unwrap_or_default(); + assert!( + ran.lines().count() >= 6, + "both pushes' scans must actually have run (withheld walk + delta probe + \ + delta rev-list each); got {} runs", + ran.lines().count() + ); + } + + /// F4 scenario 3 — fast-path non-acquisition at the handler layer: a push to a + /// public repo with NO path-scoped rules does zero post-receive git scanning + /// (the withheld short-circuit; a deletion-free flush-only body resolves no new + /// tips), so it must complete 200 even with the scan pool at ZERO permits. + /// A gate that wrongly captured a no-walk path would park this push forever. + /// Note: `resolve_candidates_for_push` spawns git for ANY non-empty new-tip set + /// (the per-tip cat-file probe), so the genuinely git-free negative arm is the + /// no-ref-update body, not a branch-create push. + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_no_scan_fast_path_completes_with_zero_scan_permits(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let marker = tmp.path().join("scan.ran"); + let body = format!( + "#!/bin/sh\ncase \"$1\" in\n receive-pack) cat > /dev/null 2>/dev/null ;;\n cat-file|rev-list|ls-tree) echo 1 >> \"{}\" ;;\n *) : ;;\nesac\nexit 0\n", + marker.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f4fast", "f1", false).await; + state.git_encrypt_semaphore = Arc::new(Semaphore::new(0)); + + let peer: SocketAddr = "203.0.113.73:5000".parse().unwrap(); + let resp = tokio::time::timeout( + std::time::Duration::from_secs(30), + git_receive_pack( + State(state), + Path(("z6f4fast".to_string(), "f1".to_string())), + Extension(crate::auth::AuthenticatedDid( + "did:key:z6MkF4FastPusherAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + )), + crate::rate_limit::PeerAddr(Some(peer)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ), + ) + .await + .expect("a no-scan push must not park on the (empty) scan pool") + .expect("the push must succeed"); + assert_eq!(resp.status(), 200); + assert!( + !marker.exists(), + "the no-walk fast paths must spawn no scan git at all" + ); + } + + /// F4 scenario 4 — landed-push-never-fails: a push whose post-receive walk must + /// park (pool held elsewhere) DEFERS and then returns the receive-pack success + /// once admission frees; contention never converts the landed push into a 5xx. + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_landed_push_defers_on_scan_contention_never_errors(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let body = "#!/bin/sh\ncase \"$1\" in\n receive-pack) cat > /dev/null 2>/dev/null ;;\n rev-parse) echo deadbeef ;;\n cat-file) echo commit ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f4park", "p1", true).await; + let sem = Arc::new(Semaphore::new(1)); + state.git_encrypt_semaphore = sem.clone(); + // The test holds the pool's only permit: the push's withheld walk must park. + let held = sem.clone().acquire_owned().await.unwrap(); + + let peer: SocketAddr = "203.0.113.74:5000".parse().unwrap(); + let mut handle = tokio::spawn(git_receive_pack( + State(state), + Path(("z6f4park".to_string(), "p1".to_string())), + Extension(crate::auth::AuthenticatedDid( + "did:key:z6MkF4ParkPusherAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + )), + crate::rate_limit::PeerAddr(Some(peer)), + axum::http::HeaderMap::new(), + ref_update_body("2222222222222222222222222222222222222222"), + )); + + // Parked, not errored: the handler must NOT complete while the pool is held. + let parked = tokio::time::timeout(std::time::Duration::from_millis(700), &mut handle).await; + assert!( + parked.is_err(), + "the post-receive walk must defer while the scan pool is held; got {parked:?}" + ); + + // Release admission: the SAME push now finishes with the receive-pack 200. + drop(held); + let resp = tokio::time::timeout(std::time::Duration::from_secs(30), handle) + .await + .expect("the deferred push must complete once admission frees") + .expect("the push task must not panic") + .expect("contention must never convert a landed push into an error"); + assert_eq!( + resp.status(), + 200, + "the response is the receive-pack success, not a contention 5xx" + ); + } + // ---- #174 U4 (P2-2): post-push encryption task set bounded by per-repo coalescing ---- // // The residual jatmn found is not the WALK (bounded by `git_encrypt_semaphore`, diff --git a/crates/gitlawb-node/src/git/push_delta.rs b/crates/gitlawb-node/src/git/push_delta.rs index 962c9c65..18f2c4da 100644 --- a/crates/gitlawb-node/src/git/push_delta.rs +++ b/crates/gitlawb-node/src/git/push_delta.rs @@ -282,14 +282,47 @@ pub struct PinCandidateSet { /// it can never leak because the withheld/fail-closed filter still runs on /// whatever set is returned. `full_scan` rides on the returned set so the caller /// knows when the dangling-inclusive filter is required. +/// +/// `scan_sem` is the post-receive scan admission pool (`git_encrypt_semaphore`, +/// #174 F4): both git-spawning stages — the per-tip `cat-file` probe + delta +/// rev-list, and the full-scan fallback — run under ONE permit held for the +/// whole blocking scan. The deletion-only fast path (no new tips) spawns no git +/// and never parks. pub async fn resolve_candidates_for_push( + scan_sem: std::sync::Arc, repo_path: PathBuf, new_tips: Vec, old_tips: Vec, git_bin: String, timeout: Duration, ) -> PinCandidateSet { + // No-git fast path: a deletion-only push resolves to an empty delta without + // spawning any git child, so it must not park on the scan pool. The + // kill-switch disqualifies it — forcing the full scan spawns git. + if new_tips.is_empty() && !force_full_scan() { + tracing::info!(delta = 0usize, repo = %repo_path.display(), "pin candidate set from push delta"); + return PinCandidateSet { + candidates: Vec::new(), + full_scan: false, + }; + } + // Scan admission (#174 F4): DEFER (await), never shed — a dropped scan would + // silently under-pin this push. The permit moves into the blocking closure so + // a started scan always completes holding it. Residuals as documented at + // `replication_withheld_set`'s acquire: park wait is queue-depth multiplied, + // and a client disconnect while parked loses this push's replication work. + let parked = Instant::now(); + let permit = scan_sem + .acquire_owned() + .await + .expect("git_encrypt_semaphore is never closed"); + tracing::debug!( + repo = %repo_path.display(), + queue_wait_ms = parked.elapsed().as_millis() as u64, + "pin-candidate scan admitted to the scan pool" + ); tokio::task::spawn_blocking(move || { + let _permit = permit; // ONE shared deadline for the whole scan, per jatmn ("the same deadline"). let deadline = Instant::now() + timeout; let new_refs: Vec<&str> = new_tips.iter().map(String::as_str).collect(); @@ -547,6 +580,7 @@ mod tests { let c1 = repo.commit_file("a.txt", "one\n"); let c2 = repo.commit_file("b.txt", "two\n"); let set = resolve_candidates_for_push( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), repo.path.clone(), vec![c2.clone()], vec![c1.clone()], @@ -576,6 +610,7 @@ mod tests { .into_iter() .collect(); let set = resolve_candidates_for_push( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), repo.path.clone(), vec![blob], vec![], @@ -588,6 +623,127 @@ mod tests { assert_eq!(got, all, "non-commit tip falls back to full repo scan"); } + /// F4 defer proof 2: `resolve_candidates_for_push`'s git stages (the per-tip + /// cat-file type probe + delta rev-list, and the full-scan fallback) run under a + /// scan-admission permit: with a zero-permit pool the call parks and spawns no + /// git; once a permit is available the SAME call runs (defer, not shed). On + /// ungated code the git runs regardless of the pool (RED). + #[cfg(unix)] + #[tokio::test] + async fn resolve_candidates_defers_when_scan_pool_exhausted() { + use std::os::unix::fs::PermissionsExt; + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + + let dir = tempfile::TempDir::new().unwrap(); + let marker = dir.path().join("git.ran"); + // Fake git records ANY invocation; cat-file reports a commit tip so the + // delta stage proceeds, rev-list yields an empty delta. + let fake = dir.path().join("fakegit"); + std::fs::write( + &fake, + format!( + "#!/bin/sh\necho ran >> \"{}\"\ncase \"$1\" in\n cat-file) echo commit ;;\n *) : ;;\nesac\nexit 0\n", + marker.display() + ), + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + let git_bin = fake.to_str().unwrap().to_string(); + let tip = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef".to_string(); + + let sem: Arc = Arc::new(Semaphore::new(0)); + let blocked = tokio::time::timeout( + Duration::from_millis(500), + resolve_candidates_for_push( + sem.clone(), + dir.path().to_path_buf(), + vec![tip.clone()], + vec![], + git_bin.clone(), + Duration::from_secs(5), + ), + ) + .await; + assert!( + blocked.is_err(), + "the pin-candidate scan must defer (park on admission) when the pool is exhausted" + ); + assert!( + !marker.exists(), + "the scan's git must not spawn while its admission permit is unavailable (F4)" + ); + + // Release admission: the SAME scan now runs (defer, not shed). + sem.add_permits(1); + let set = resolve_candidates_for_push( + sem, + dir.path().to_path_buf(), + vec![tip], + vec![], + git_bin, + Duration::from_secs(5), + ) + .await; + assert!( + marker.exists(), + "once admission is available the deferred scan runs its git" + ); + assert!(!set.full_scan, "commit tip + empty rev-list is a delta"); + assert!(set.candidates.is_empty()); + } + + /// F4 fast-path negative arm: a deletion-only push (no new tips, kill-switch + /// off) computes its empty delta without spawning ANY git child, so it must + /// complete without acquiring from the scan pool — even at zero permits. The + /// per-tip cat-file probe means every push with a non-empty new tip DOES spawn + /// git, so this is the only genuinely git-free stage. + #[cfg(unix)] + #[tokio::test] + async fn resolve_candidates_no_git_fast_path_skips_admission() { + use std::os::unix::fs::PermissionsExt; + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + + let dir = tempfile::TempDir::new().unwrap(); + let marker = dir.path().join("git.ran"); + let fake = dir.path().join("fakegit"); + std::fs::write( + &fake, + format!("#!/bin/sh\necho ran >> \"{}\"\nexit 0\n", marker.display()), + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + + let set = tokio::time::timeout( + Duration::from_millis(500), + resolve_candidates_for_push( + Arc::new(Semaphore::new(0)), + dir.path().to_path_buf(), + vec![], + vec!["deadbeefdeadbeefdeadbeefdeadbeefdeadbeef".to_string()], + fake.to_str().unwrap().to_string(), + Duration::from_secs(5), + ), + ) + .await + .expect("the no-new-tips fast path must not park on the scan pool"); + assert_eq!( + set, + PinCandidateSet { + candidates: Vec::new(), + full_scan: false + } + ); + assert!(!marker.exists(), "the fast path must spawn no git at all"); + } + #[test] fn missing_oid_tip_forces_full_scan() { let repo = Repo::new(); diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index deccf4c7..0badf802 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -110,14 +110,20 @@ pub struct AppState { /// advert pool (each source also capped by `git_push_advert_per_caller` and the /// per-IP push rate limiter), and the reserved POST pool is untouched (#174). pub git_push_advert_semaphore: Arc, - /// Bounds concurrent post-push encrypt-then-pin history walks. Each successful - /// path-scoped push releases its handler write permit and then runs a DETACHED - /// full-history walk (`withheld_blob_recipients_bounded`) to seal withheld blobs; - /// without a cap, N fast pushes spawn N concurrent full-history git walks past - /// `max_concurrent_git_pushes` (which only bounds the in-handler phase). The walk - /// acquires a permit here and DEFERS (blocks) when the pool is full rather than - /// shedding — the work is background and dropping it would lose the recovery copy - /// (#174 P1-e). A pool of its own, not `git_write_semaphore`: a long background + /// Bounds concurrent post-receive git scans. Each successful push releases its + /// handler write permit the moment receive-pack's git group is reaped, then runs + /// up to four scans over the repo: the anonymous withheld walk + /// (`replication_withheld_set`), the pin-candidate scan + /// (`resolve_candidates_for_push`), the fail-closed full scan + /// (`fail_closed_full_scan_objects`), and the DETACHED encrypt-then-pin walk + /// (`withheld_blob_recipients_bounded`). Without a cap, N fast pushes spawn N + /// concurrent full-history git walks past `max_concurrent_git_pushes` (which only + /// bounds the in-handler receive-pack phase) — #174 P1-e closed the detached walk, + /// F4 closed the other three. Each scan acquires ONE permit here per walk and + /// DEFERS (blocks) when the pool is full rather than shedding — dropping the work + /// would lose the recovery copy or silently under-pin the push. No-walk fast + /// paths (not announceable, no path-scoped rule, deletion-only push) never touch + /// the pool. A pool of its own, not `git_write_semaphore`: a long background /// walk must not hold a foreground write slot, and a handler already holding a /// write permit that needed a second would self-deadlock at pool size 1. pub git_encrypt_semaphore: Arc, diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index 7d3fd1c8..bcd0758e 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -87,4 +87,37 @@ fn inv22_concurrency_gates_present_and_not_bypassed() { "U4/P2-2 gate missing: the detached post-push encryption spawn must consult \ encrypt_inflight.try_begin to coalesce per repo (bound the outstanding-task set)" ); + + // F4: every post-receive scan helper acquires a `git_encrypt_semaphore` permit + // BEFORE its spawn_blocking git walk, so a push burst cannot accumulate unbounded + // concurrent scans once the write permit is released. Structural check: within each + // helper the first `acquire_owned` precedes the first `spawn_blocking`. Removing a + // gate pushes the next `acquire_owned` occurrence past the helper's own + // `spawn_blocking` (or off the end of the file), turning the assertion red. + let push_delta = src("git/push_delta.rs"); + for (file_src, file, helper) in [ + (&repos, "api/repos.rs", "fn replication_withheld_set"), + (&repos, "api/repos.rs", "fn fail_closed_full_scan_objects"), + ( + &push_delta, + "git/push_delta.rs", + "fn resolve_candidates_for_push", + ), + ] { + let start = file_src + .find(helper) + .unwrap_or_else(|| panic!("{file}: `{helper}` not found")); + let tail = &file_src[start..]; + let acquire = tail.find("acquire_owned").unwrap_or_else(|| { + panic!("F4 gate missing: {file} `{helper}` no longer acquires a scan permit") + }); + let spawn = tail.find("spawn_blocking").unwrap_or_else(|| { + panic!("{file}: `{helper}` lost its spawn_blocking walk — update this guard") + }); + assert!( + acquire < spawn, + "F4 gate bypassed: {file} `{helper}` must acquire its git_encrypt_semaphore \ + permit BEFORE dispatching the blocking git scan" + ); + } } From 8c95d05661fe5c570ccc60e029297c98fd3b58a9 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 17 Jul 2026 19:17:17 -0500 Subject: [PATCH 048/124] fix(node): requeue coalesced post-push work via an in-lock pending slot and loop-drain (#174) try_begin now merges a losing push's tip pairs into the in-flight key's pending slot in the same critical section as the presence check, bounded at 1024 pairs with a full-scan marker on overflow (force_full_scan threads to the resolver; empty tips never encode the marker). The task loop-drains: finish_or_take_pending either hands back the batch with the key retained, or removes the key and disarms the guard atomically, so a successor task's key survives the guard drop. Drains re-fetch rules fresh and replay the tail pipeline under the U4 helper gates only; the loop holds no task-level pool permit, so a hot repo cannot deadlock the pool at size 1. --- crates/gitlawb-node/src/api/repos.rs | 1035 ++++++++++++++++++--- crates/gitlawb-node/src/git/push_delta.rs | 81 +- crates/gitlawb-node/src/state.rs | 214 ++++- crates/gitlawb-node/tests/inv22_gates.rs | 17 + 4 files changed, 1166 insertions(+), 181 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index f25293b8..2b9cf1d4 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -843,6 +843,244 @@ async fn withheld_recipients_gated( .await } +/// Everything the detached post-push pin/encrypt task needs, cloned once at spawn +/// and shared by the snapshot iteration and every coalesced-drain iteration +/// (#174 F5). +struct EncryptTaskCtx { + ipfs_api: String, + repo_path: std::path::PathBuf, + db: Arc, + repo_id: String, + owner_did: String, + repo_name: String, + irys_url: String, + http_client: Arc, + node_did: String, + node_keypair: Arc, + git_bin: String, + git_timeout: std::time::Duration, + encrypt_sem: Arc, +} + +/// The detached post-push pin/encrypt task (#174 P2-2 + F5): run this push's own +/// pre-resolved snapshot through the pipeline, then loop-drain every push that +/// coalesced against the in-flight key until a finish attempt finds nothing +/// pending (which releases the key in the same critical section). +/// +/// The loop holds NO encrypt-pool permit at the task level: each helper it calls +/// (`replication_withheld_set`, `resolve_candidates_for_push`, +/// `fail_closed_full_scan_objects`, `withheld_recipients_gated`) acquires and +/// releases its own walk permit, so the drain makes progress at pool size 1 — a +/// task-level permit would nest over those same-semaphore acquires and deadlock. +async fn run_encrypt_pin_task( + ctx: EncryptTaskCtx, + guard: crate::state::EncryptInflightGuard, + snapshot_objects: Vec, + snapshot_rules: Option>, + snapshot_is_public: bool, +) { + pin_and_encrypt_objects(&ctx, snapshot_objects, snapshot_rules, snapshot_is_public).await; + let mut guard = guard; + loop { + match guard.finish_or_take_pending() { + crate::state::FinishOutcome::Finished(_) => break, + crate::state::FinishOutcome::Pending(g, pending) => { + guard = g; + if let Some((object_list, rules, is_public)) = + resolve_drain_object_list(&ctx, pending).await + { + pin_and_encrypt_objects(&ctx, object_list, rules, is_public).await; + } + } + } + } +} + +/// Resolve a coalesced-drain iteration's replicable object list. Re-fetches the +/// repo record and visibility rules FRESH — rules tightened between the coalesced +/// push and its drain must be honored, fail closed: a newly-withheld blob is not +/// pinned, and a repo that is no longer announceable (or whose record cannot be +/// re-read) pins nothing at all (`None`). Returns the filtered object list plus +/// the fresh rules/is_public snapshot for the encrypt stage — the same +/// resolution → withheld-filter pipeline the receive-pack tail runs. +async fn resolve_drain_object_list( + ctx: &EncryptTaskCtx, + pending: crate::state::PendingWork, +) -> Option<(Vec, Option>, bool)> { + let record = match ctx.db.get_repo(&ctx.owner_did, &ctx.repo_name).await { + Ok(Some(r)) => r, + Ok(None) => { + tracing::warn!( + repo = %ctx.repo_id, + "coalesced drain: repo record is gone; dropping the pending work" + ); + return None; + } + Err(e) => { + tracing::warn!( + repo = %ctx.repo_id, + err = %e, + "coalesced drain: repo re-fetch failed; pinning nothing (fail closed)" + ); + return None; + } + }; + let rules_opt = ctx.db.list_visibility_rules(&ctx.repo_id).await.ok(); + let (_announce, withheld) = replication_withheld_set( + ctx.encrypt_sem.clone(), + rules_opt.clone(), + &record.owner_did, + record.is_public, + ctx.repo_path.clone(), + ctx.git_bin.clone(), + ctx.git_timeout, + ) + .await; + let withheld_set = match withheld { + Some(w) => w, + None => { + tracing::info!( + repo = %ctx.repo_id, + "coalesced drain: repo is not announceable under current rules; \ + pinning nothing (fail closed)" + ); + return None; + } + }; + let (new_tips, old_tips, force_full_scan) = match pending { + crate::state::PendingWork::Tips(pairs) => { + let new_tips: Vec = pairs + .iter() + .map(|(_, n)| n.clone()) + .filter(|s| s != ZERO_SHA) + .collect(); + let old_tips: Vec = pairs + .into_iter() + .map(|(o, _)| o) + .filter(|s| s != ZERO_SHA) + .collect(); + (new_tips, old_tips, false) + } + // The overflow marker forces the full scan via the explicit flag. It must + // never be encoded as a plain empty-tips call: empty tips resolve to an + // empty delta and would pin nothing (the F5 silent loss again). + crate::state::PendingWork::FullScan => (Vec::new(), Vec::new(), true), + }; + let pin_set = crate::git::push_delta::resolve_candidates_for_push( + ctx.encrypt_sem.clone(), + ctx.repo_path.clone(), + new_tips, + old_tips, + ctx.git_bin.clone(), + ctx.git_timeout, + force_full_scan, + ) + .await; + let object_list = if pin_set.full_scan { + fail_closed_full_scan_objects( + ctx.encrypt_sem.clone(), + ctx.repo_path.clone(), + rules_opt.clone().unwrap_or_default(), + record.is_public, + record.owner_did.clone(), + pin_set.candidates, + ctx.git_bin.clone(), + ctx.git_timeout, + ) + .await + } else { + crate::git::visibility_pack::replicable_objects(pin_set.candidates, &withheld_set) + }; + Some((object_list, rules_opt, record.is_public)) +} + +/// The pin/encrypt pipeline shared by the snapshot iteration and the +/// coalesced-drain iterations: local IPFS pin, then (path-scoped rules only) the +/// admission-gated recipients walk → encrypt-then-pin → Arweave manifest anchor. +async fn pin_and_encrypt_objects( + ctx: &EncryptTaskCtx, + object_list: Vec, + rules: Option>, + is_public: bool, +) { + let pinned = + crate::ipfs_pin::pin_new_objects(&ctx.ipfs_api, &ctx.repo_path, object_list, &ctx.db).await; + if !pinned.is_empty() { + tracing::info!(count = pinned.len(), "pinned git objects to IPFS"); + for (sha, cid) in &pinned { + tracing::info!(sha = %sha, %cid, "pinned"); + } + } + + // Option B1: encrypt-then-pin the withheld blobs so authorized readers can + // recover them when the origin cannot serve them. No path-scoped rule can + // withhold a blob, so withheld_blob_recipients would return an empty map + // after a full per-ref walk; skip it. Mirrors the has_path_scoped_rule gate + // on the other two withheld-walk sites. + if let Some(rules) = rules.filter(|r| visibility_pack::has_path_scoped_rule(r)) { + // Bound the number of concurrent post-push encryption walks (#174 P1-e): + // acquire an admission permit before the full-history walk, deferring + // when the pool is full rather than shedding the recovery pin. + let recip = withheld_recipients_gated( + ctx.encrypt_sem.clone(), + ctx.repo_path.clone(), + ctx.git_bin.clone(), + ctx.git_timeout, + rules, + is_public, + ctx.owner_did.clone(), + ) + .await; + if let Ok(Ok(recipients)) = recip { + let node_seed = ctx.node_keypair.to_seed(); + let delta = crate::encrypted_pin::encrypt_and_pin( + &ctx.ipfs_api, + &ctx.repo_path, + &ctx.db, + &ctx.repo_id, + &node_seed, + &recipients, + ) + .await; + + // Option B3: anchor a per-push manifest of the blobs sealed this + // push to Arweave, so the oid->cid index survives total node loss. + // Best-effort; never fails the push. + if !delta.is_empty() && !ctx.irys_url.is_empty() { + let owner_short = crate::db::normalize_owner_key(&ctx.owner_did); + let repo_slug = format!("{owner_short}/{}", ctx.repo_name); + let ts = chrono::Utc::now().to_rfc3339(); + let manifest = crate::arweave::EncryptedManifest { + repo: &repo_slug, + owner_did: &ctx.owner_did, + node_did: &ctx.node_did, + timestamp: &ts, + blobs: &delta, + }; + match crate::arweave::anchor_encrypted_manifest( + &ctx.http_client, + &ctx.irys_url, + &manifest, + ) + .await + { + Ok(tx) if !tx.is_empty() => tracing::info!( + repo = %repo_slug, + tx_id = %tx, + "anchored encrypted manifest to Arweave" + ), + Ok(_) => {} + Err(e) => tracing::warn!( + repo = %repo_slug, + err = %e, + "encrypted manifest anchor failed" + ), + } + } + } + } +} + /// Map an error from a `smart_http` git service call to the right `AppError`: /// [`smart_http::GitServiceTimeout`] to 504, a malformed client request to 400, /// anything else to a 500 git error. Pure (no logging) so it is unit-testable; @@ -1473,6 +1711,7 @@ pub async fn git_receive_pack( old_tips, state.git_bin.clone(), std::time::Duration::from_secs(state.config.git_service_timeout_secs), + false, ) .await; if pin_set.full_scan { @@ -1497,127 +1736,56 @@ pub async fn git_receive_pack( // Pin new git objects to the local IPFS node (no-op if ipfs_api is empty). // Skipped entirely when the public cannot read the repo (withheld == None). // - // Coalesce per repo (#174 P2-2): this task parks on `git_encrypt_semaphore` - // (which DEFERS when the pool is full rather than dropping the recovery copy). To - // bound the OUTSTANDING parked-task set, only spawn if no encryption task for this - // repo is already in flight; otherwise skip — the pending/next walk over this - // repo's history already covers the newer push's objects, so the recovery copy is - // delayed, never lost. The guard removes the repo key when the task ends (success, - // error, or panic), so a later push is re-admitted (no permanent skip). + // Coalesce-and-requeue per repo (#174 P2-2 + F5): the spawned task's walks park + // on `git_encrypt_semaphore` (which DEFERS when the pool is full rather than + // dropping the recovery copy). To bound the OUTSTANDING task set, at most one + // task per repo is in flight; a push arriving while one is in flight does NOT + // spawn a duplicate — and is NOT dropped either. The in-flight task pins only + // its own pre-spawn object-list snapshot, so this push's (old, new) tip pairs + // are merged into the in-flight key's pending slot in the same critical section + // as the presence check, and the task loop-drains them (fresh rules, fail + // closed) before releasing the key. Without the requeue a coalesced push's pins + // and recovery copies would be silently absent until an unrelated later push + // (the F5 loss). The guard still releases the key on panic (Drop on unwind), so + // a crashed walk never permanently locks the repo out. if withheld.is_some() { - match state.encrypt_inflight.try_begin(&record.id) { - None => { + let tip_pairs: Vec<(String, String)> = ref_updates + .iter() + .map(|u| (u.old_sha.clone(), u.new_sha.clone())) + .collect(); + match state.encrypt_inflight.try_begin(&record.id, tip_pairs) { + crate::state::BeginOutcome::Coalesced => { tracing::debug!( repo = %record.id, - "post-push encryption task already in flight for this repo; coalescing \ - (the pending recovery-copy walk covers this push's objects)" + "post-push encryption task already in flight for this repo; coalesced \ + — this push's tip pairs are queued for that task's drain" ); } - Some(inflight_guard) => { - let object_list_ipfs = object_list.clone(); - let ipfs_api = state.config.ipfs_api.clone(); - let repo_path_clone = disk_path.clone(); - let db_clone = state.db.clone(); - let rules_for_enc = rules_opt.clone(); - let repo_id = record.id.clone(); - let owner_did = record.owner_did.clone(); - let is_public = record.is_public; - let irys_url = state.config.irys_url.clone(); - let http_client = std::sync::Arc::clone(&state.http_client); - let node_did_str = state.node_did.to_string(); - let node_seed = state.node_keypair.to_seed(); - let repo_name = record.name.clone(); - let enc_git_bin = state.git_bin.clone(); - let enc_timeout = - std::time::Duration::from_secs(state.config.git_service_timeout_secs); - let encrypt_sem = state.git_encrypt_semaphore.clone(); - tokio::spawn(async move { - // Held for the whole task; drop (on completion/error/panic) releases the - // repo's coalescing key so the next push for this repo can spawn again. - let _inflight_guard = inflight_guard; - let pinned = crate::ipfs_pin::pin_new_objects( - &ipfs_api, - &repo_path_clone, - object_list_ipfs, - &db_clone, - ) - .await; - if !pinned.is_empty() { - tracing::info!(count = pinned.len(), "pinned git objects to IPFS"); - for (sha, cid) in &pinned { - tracing::info!(sha = %sha, %cid, "pinned"); - } - } - - // Option B1: encrypt-then-pin the withheld blobs so authorized - // readers can recover them when the origin cannot serve them. - // No path-scoped rule can withhold a blob, so withheld_blob_recipients - // would return an empty map after a full per-ref walk; skip it. Mirrors - // the has_path_scoped_rule gate on the other two withheld-walk sites. - if let Some(rules) = - rules_for_enc.filter(|r| visibility_pack::has_path_scoped_rule(r)) - { - // Bound the number of concurrent post-push encryption walks (#174 P1-e): - // acquire an admission permit before the full-history walk, deferring - // when the pool is full rather than shedding the recovery pin. - let recip = withheld_recipients_gated( - encrypt_sem.clone(), - repo_path_clone.clone(), - enc_git_bin.clone(), - enc_timeout, - rules, - is_public, - owner_did.clone(), - ) - .await; - if let Ok(Ok(recipients)) = recip { - let delta = crate::encrypted_pin::encrypt_and_pin( - &ipfs_api, - &repo_path_clone, - &db_clone, - &repo_id, - &node_seed, - &recipients, - ) - .await; - - // Option B3: anchor a per-push manifest of the blobs sealed - // this push to Arweave, so the oid->cid index survives total - // node loss. Best-effort; never fails the push. - if !delta.is_empty() && !irys_url.is_empty() { - let owner_short = crate::db::normalize_owner_key(&owner_did); - let repo_slug = format!("{owner_short}/{repo_name}"); - let ts = chrono::Utc::now().to_rfc3339(); - let manifest = crate::arweave::EncryptedManifest { - repo: &repo_slug, - owner_did: &owner_did, - node_did: &node_did_str, - timestamp: &ts, - blobs: &delta, - }; - match crate::arweave::anchor_encrypted_manifest( - &http_client, - &irys_url, - &manifest, - ) - .await - { - Ok(tx) if !tx.is_empty() => tracing::info!( - repo = %repo_slug, - tx_id = %tx, - "anchored encrypted manifest to Arweave" - ), - Ok(_) => {} - Err(e) => tracing::warn!( - repo = %repo_slug, - err = %e, - "encrypted manifest anchor failed" - ), - } - } - } - } - }); + crate::state::BeginOutcome::Admitted(inflight_guard) => { + let ctx = EncryptTaskCtx { + ipfs_api: state.config.ipfs_api.clone(), + repo_path: disk_path.clone(), + db: state.db.clone(), + repo_id: record.id.clone(), + owner_did: record.owner_did.clone(), + repo_name: record.name.clone(), + irys_url: state.config.irys_url.clone(), + http_client: std::sync::Arc::clone(&state.http_client), + node_did: state.node_did.to_string(), + node_keypair: std::sync::Arc::clone(&state.node_keypair), + git_bin: state.git_bin.clone(), + git_timeout: std::time::Duration::from_secs( + state.config.git_service_timeout_secs, + ), + encrypt_sem: state.git_encrypt_semaphore.clone(), + }; + tokio::spawn(run_encrypt_pin_task( + ctx, + inflight_guard, + object_list.clone(), + rules_opt.clone(), + record.is_public, + )); } } } @@ -5197,9 +5365,9 @@ mod tests { let mut admitted = Vec::new(); let mut coalesced = 0usize; for _ in 0..K { - match inflight.try_begin(repo) { - Some(g) => admitted.push(g), - None => coalesced += 1, + match inflight.try_begin(repo, vec![]) { + crate::state::BeginOutcome::Admitted(g) => admitted.push(g), + crate::state::BeginOutcome::Coalesced => coalesced += 1, } } @@ -5238,12 +5406,15 @@ mod tests { /// one repo in flight cannot starve a second repo's recovery copy. #[test] fn u4_distinct_repos_each_admit_one_encrypt_task() { + use crate::state::BeginOutcome; let inflight = crate::state::EncryptInflight::new(); - let a = inflight.try_begin("owner/repo-a"); - let b = inflight.try_begin("owner/repo-b"); - let c = inflight.try_begin("owner/repo-c"); + let a = inflight.try_begin("owner/repo-a", vec![]); + let b = inflight.try_begin("owner/repo-b", vec![]); + let c = inflight.try_begin("owner/repo-c", vec![]); assert!( - a.is_some() && b.is_some() && c.is_some(), + matches!(&a, BeginOutcome::Admitted(_)) + && matches!(&b, BeginOutcome::Admitted(_)) + && matches!(&c, BeginOutcome::Admitted(_)), "three distinct repos each admit their own encryption task" ); assert_eq!(inflight.len(), 3, "one in-flight entry per distinct repo"); @@ -5258,19 +5429,24 @@ mod tests { /// survives normal completion AND a panic, so no permanent skip / no leaked key. #[test] fn u4_coalesced_repo_is_reprocessed_after_task_ends_not_permanently_skipped() { + use crate::state::{BeginOutcome, FinishOutcome}; let inflight = crate::state::EncryptInflight::new(); let repo = "did:key:z6MkDurableRepoBBBBBBBBBBBBBBBBBBBBBBBBB/repo"; // Push #1 admits and "spawns". A concurrent push #2 (task #1 still in flight) - // coalesces — no duplicate spawn. - let guard1 = inflight.try_begin(repo).expect("first push admits"); + // coalesces — no duplicate spawn; its (empty) tip set is recorded, not lost. + let guard1 = admit(&inflight, repo); assert!( - inflight.try_begin(repo).is_none(), + matches!(inflight.try_begin(repo, vec![]), BeginOutcome::Coalesced), "while task #1 is in flight, push #2 to the same repo coalesces" ); - // Task #1 finishes (encrypt_and_pin ran or errored): guard drops, key released. - drop(guard1); + // Task #1 finishes normally: nothing pending (push #2 carried no tips), so + // the empty-pending check removes the key in its critical section. + assert!( + matches!(guard1.finish_or_take_pending(), FinishOutcome::Finished(_)), + "no pending tips — the task exits and releases the key" + ); assert_eq!( inflight.len(), 0, @@ -5280,18 +5456,21 @@ mod tests { // A LATER push for the SAME repo is admitted again (processed, not skipped // forever). This is what coalesce->shed breaks: shed drops the job and no sweep // re-derives the missing copy, so the recovery copy is permanently lost. - let guard2 = inflight.try_begin(repo).expect( - "a later push for a coalesced repo MUST be re-admitted — durability: the \ - deferred recovery copy is produced eventually, never dropped", - ); + let guard2 = admit(&inflight, repo); + // An errored task (guard dropped without finishing) still releases the key. drop(guard2); assert_eq!(inflight.len(), 0); - // Durability across PANIC: a task that panics mid-walk must still release its key - // (Drop runs on unwind), so one crashed walk never permanently locks a repo out - // of future recovery copies. + // Durability across PANIC: a task that panics mid-walk must still release its + // key (the still-armed guard's Drop runs on unwind), so one crashed walk never + // permanently locks a repo out of future recovery copies. Coalesce real tips + // first: the panic loses them (logged), and the loss must not corrupt the set. let panicked = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - let _g = inflight.try_begin(repo).expect("admit before the panic"); + let _g = admit(&inflight, repo); + assert!(matches!( + inflight.try_begin(repo, vec![("old1".to_string(), "new1".to_string())]), + BeginOutcome::Coalesced + )); assert_eq!(inflight.len(), 1); panic!("simulate the detached encryption task panicking mid-walk"); })); @@ -5302,9 +5481,12 @@ mod tests { "a panicked encryption task still releases its repo key (Drop on unwind) — no \ permanent leak that would block every future recovery copy for the repo" ); + // The next push is re-admitted, and the panicked task's pending tips did NOT + // survive into it (they are lost-and-logged, recovered only by a later push). + let guard3 = admit(&inflight, repo); assert!( - inflight.try_begin(repo).is_some(), - "after a panicked task the repo can still be admitted — durability preserved" + matches!(guard3.finish_or_take_pending(), FinishOutcome::Finished(_)), + "the pre-panic pending tips must not leak into the re-admitted task" ); } @@ -5315,11 +5497,25 @@ mod tests { let inflight = crate::state::EncryptInflight::new(); assert!(inflight.is_empty(), "cold set is empty"); assert!( - inflight.try_begin("owner/first").is_some(), + matches!( + inflight.try_begin("owner/first", vec![]), + crate::state::BeginOutcome::Admitted(_) + ), "the first push on a cold in-flight set must admit (never falsely coalesce)" ); } + /// Unwrap an Admitted outcome (panic on Coalesced) — the u4/u5 suites' shorthand. + fn admit( + inflight: &crate::state::EncryptInflight, + repo: &str, + ) -> crate::state::EncryptInflightGuard { + match inflight.try_begin(repo, vec![]) { + crate::state::BeginOutcome::Admitted(g) => g, + crate::state::BeginOutcome::Coalesced => panic!("expected {repo} to admit"), + } + } + /// Model of the pre-fix / mutated code: no coalescing check, so every push spawns. /// Returns the count of tasks spawned (== the size of the unbounded outstanding set /// the fix prevents), used as the RED comparison in the bound test above. @@ -5327,6 +5523,567 @@ mod tests { (0..pushes).count() } + // ---- #174 U5 (F5): a push that loses try_begin is REQUEUED, never dropped ---- + // + // F5: the in-flight task pins only its own pre-spawn object-list snapshot, so a + // push B arriving while task A is in flight used to be SKIPPED outright (the old + // None arm) — B's pins and recovery copies were silently absent until an + // unrelated later push re-walked the repo. U5 records B's (old, new) tip pairs + // into the in-flight key's pending slot in the SAME critical section as the + // presence check, and A's task loop-drains them before releasing the key. + + /// The F5 lost-update repro. Task A is in flight past its snapshot; push B + /// coalesces carrying its tip pair; when A finishes its snapshot iteration the + /// tracker must hand A exactly B's recorded work with the key retained — and only + /// an empty pending check may remove the key. On pre-U5 code this is RED: the + /// coalesce arm records B's work nowhere and there is no drain surface at all. + #[test] + fn u5_coalesced_push_work_is_drained_by_the_inflight_task() { + use crate::state::{BeginOutcome, FinishOutcome, PendingWork}; + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkF5LostUpdateCCCCCCCCCCCCCCCCCCCCCCCC/repo"; + + // Push A admits; its spawned task is "in flight past its snapshot". + let guard_a = match inflight.try_begin(repo, vec![]) { + BeginOutcome::Admitted(g) => g, + BeginOutcome::Coalesced => panic!("first push must admit"), + }; + + // Push B lands while A is in flight: coalesced, tip pair recorded. + let b_pair = ( + "b0ldb0ldb0ldb0ldb0ldb0ldb0ldb0ldb0ldb0ld".to_string(), + "bnewbnewbnewbnewbnewbnewbnewbnewbnewbnew".to_string(), + ); + match inflight.try_begin(repo, vec![b_pair.clone()]) { + BeginOutcome::Coalesced => {} + BeginOutcome::Admitted(_) => panic!("push B must coalesce while A is in flight"), + } + + // A finishes its snapshot iteration: it must be handed B's work (drained), + // not exit — an exit here is exactly the F5 silent loss. + match guard_a.finish_or_take_pending() { + FinishOutcome::Pending(guard_a, work) => { + assert_eq!( + work, + PendingWork::Tips(vec![b_pair]), + "A drains exactly B's recorded tip pair" + ); + assert_eq!(inflight.len(), 1, "the key is retained while A iterates"); + // Nothing further pending: A now exits and releases the key. + match guard_a.finish_or_take_pending() { + FinishOutcome::Finished(_) => {} + FinishOutcome::Pending(..) => panic!("no second batch was recorded"), + } + assert_eq!( + inflight.len(), + 0, + "an empty pending check at task end releases the key" + ); + } + FinishOutcome::Finished(_) => panic!( + "F5: B's coalesced work vanished — the in-flight task exited without draining it" + ), + } + } + + /// Drain-vs-admit race, both orderings driven deterministically through the lock + /// API (the check+merge and check+remove are each ONE critical section, so a push + /// can only land on one side of A's final pending check — never inside it): + /// before it, the push is merged and A drains it; after it, the key is gone and + /// the push is admitted as a fresh task. Neither ordering loses the work. A + /// check-then-record split (merge moved outside try_begin's critical section) + /// turns ordering 1 RED: the work recorded after A's check is never drained. + #[test] + fn u5_drain_vs_admit_race_loses_no_work_in_either_ordering() { + use crate::state::{BeginOutcome, FinishOutcome, PendingWork}; + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkF5RaceOrderDDDDDDDDDDDDDDDDDDDDDDDDD/repo"; + let pair = ("cold".to_string(), "cnew".to_string()); + + // Ordering 1: push C lands BEFORE A's final pending check → merged in + // try_begin's critical section → A must drain it (key retained). + let guard_a = admit(&inflight, repo); + assert!(matches!( + inflight.try_begin(repo, vec![pair.clone()]), + BeginOutcome::Coalesced + )); + match guard_a.finish_or_take_pending() { + FinishOutcome::Pending(g, work) => { + assert_eq!(work, PendingWork::Tips(vec![pair.clone()])); + assert!(matches!( + g.finish_or_take_pending(), + FinishOutcome::Finished(_) + )); + } + FinishOutcome::Finished(_) => { + panic!("a push merged before the final check must be drained, not lost") + } + } + assert!(inflight.is_empty()); + + // Ordering 2: push C lands AFTER A's final pending check removed the key → + // it must be ADMITTED as a fresh task (its own snapshot covers its work). + let guard_a = admit(&inflight, repo); + assert!(matches!( + guard_a.finish_or_take_pending(), + FinishOutcome::Finished(_) + )); + match inflight.try_begin(repo, vec![pair]) { + BeginOutcome::Admitted(g) => drop(g), + BeginOutcome::Coalesced => panic!( + "a push landing after the key was removed must admit a new task — a \ + coalesce here records work no task will ever drain" + ), + } + } + + /// Exit-vs-successor (the double-remove hazard). A's normal exit removes the key + /// and disarms the guard in ONE critical section, and the disarmed guard is + /// handed back — so its eventual Drop lands in the real remove→drop window. A + /// successor task B admitted inside that window must keep ITS key when A's guard + /// finally drops. With the disarm reverted (Drop removing unconditionally) this + /// is RED: dropping A's guard deletes B's key and the third push falsely admits + /// a second task for the repo. + #[test] + fn u5_disarmed_guard_drop_never_removes_a_successor_key() { + use crate::state::{BeginOutcome, FinishOutcome}; + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkF5DisarmEEEEEEEEEEEEEEEEEEEEEEEEEEEE/repo"; + + // A admits and exits normally; HOLD the disarmed guard to keep the window open. + let guard_a = admit(&inflight, repo); + let disarmed = match guard_a.finish_or_take_pending() { + FinishOutcome::Finished(g) => g, + FinishOutcome::Pending(..) => panic!("nothing was pending"), + }; + assert!(inflight.is_empty(), "A's exit released the key"); + + // Successor B is admitted inside the remove→drop window. + let guard_b = admit(&inflight, repo); + assert_eq!(inflight.len(), 1); + + // A's disarmed guard now drops. B's key must SURVIVE: a third push still + // coalesces against B's in-flight task. + drop(disarmed); + assert_eq!( + inflight.len(), + 1, + "dropping A's disarmed guard must not remove successor B's key" + ); + assert!( + matches!(inflight.try_begin(repo, vec![]), BeginOutcome::Coalesced), + "B's task is still the (only) in-flight task — at-most-one-per-repo holds" + ); + drop(guard_b); + assert!(inflight.is_empty()); + } + + /// Pending overflow: past the 1024-pair bound the slot degrades to the FullScan + /// marker (bounded memory under a hostile push burst); at exactly the bound it + /// stays a Tips batch. The marker is an explicit variant, never an empty tip + /// list — an empty-tips encoding would drain to an empty delta and pin nothing. + #[test] + fn u5_pending_overflow_degrades_to_full_scan_marker() { + use crate::state::{BeginOutcome, FinishOutcome, PendingWork}; + let inflight = crate::state::EncryptInflight::new(); + let pair = |i: usize| (format!("old{i}"), format!("new{i}")); + + // At the bound: exactly 1024 pairs stay a Tips batch. + let repo_at = "owner/at-bound"; + let g = admit(&inflight, repo_at); + assert!(matches!( + inflight.try_begin(repo_at, (0..1024).map(pair).collect()), + BeginOutcome::Coalesced + )); + match g.finish_or_take_pending() { + FinishOutcome::Pending(g, PendingWork::Tips(v)) => { + assert_eq!(v.len(), 1024, "at the bound the pairs are kept verbatim"); + assert!(matches!( + g.finish_or_take_pending(), + FinishOutcome::Finished(_) + )); + } + other => panic!( + "expected a Tips batch at the bound, got {:?}", + match other { + FinishOutcome::Pending(_, w) => Some(w), + FinishOutcome::Finished(_) => None, + } + ), + } + + // Past the bound: the accumulated slot degrades to FullScan and later + // merges are absorbed (still one bounded marker, not a growing list). + let repo_over = "owner/over-bound"; + let g = admit(&inflight, repo_over); + assert!(matches!( + inflight.try_begin(repo_over, (0..1024).map(pair).collect()), + BeginOutcome::Coalesced + )); + assert!(matches!( + inflight.try_begin(repo_over, vec![pair(9999)]), + BeginOutcome::Coalesced + )); + assert!(matches!( + inflight.try_begin(repo_over, vec![pair(10000)]), + BeginOutcome::Coalesced + )); + match g.finish_or_take_pending() { + FinishOutcome::Pending(g, work) => { + assert_eq!( + work, + PendingWork::FullScan, + "overflow degrades to the explicit FullScan marker" + ); + assert!(matches!( + g.finish_or_take_pending(), + FinishOutcome::Finished(_) + )); + } + FinishOutcome::Finished(_) => panic!("the overflowed pending work vanished"), + } + } + + // ---- u5 drain-pipeline fixtures: a real git repo + a DB repo row ---- + + fn u5_git(dir: &std::path::Path, args: &[&str]) -> String { + let out = std::process::Command::new("git") + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + + fn u5_init_repo(dir: &std::path::Path) { + u5_git(dir, &["init", "-q", "-b", "main"]); + u5_git(dir, &["config", "user.email", "t@t"]); + u5_git(dir, &["config", "user.name", "t"]); + } + + /// Commit `name` (parent dirs created) with `body`; returns the commit sha. + fn u5_commit_file(dir: &std::path::Path, name: &str, body: &str) -> String { + let path = dir.join(name); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(&path, body).unwrap(); + u5_git(dir, &["add", name]); + u5_git(dir, &["commit", "-qm", &format!("add {name}")]); + u5_git(dir, &["rev-parse", "HEAD"]) + } + + /// A drain-task context over the test state and an on-disk repo. Empty + /// `ipfs_api`/`irys_url` keep the pin/anchor stages inert (no network). + fn u5_ctx( + state: &AppState, + rec: &crate::db::RepoRecord, + repo_path: std::path::PathBuf, + git_bin: &str, + sem: std::sync::Arc, + ) -> EncryptTaskCtx { + EncryptTaskCtx { + ipfs_api: String::new(), + repo_path, + db: state.db.clone(), + repo_id: rec.id.clone(), + owner_did: rec.owner_did.clone(), + repo_name: rec.name.clone(), + irys_url: String::new(), + http_client: std::sync::Arc::clone(&state.http_client), + node_did: state.node_did.to_string(), + node_keypair: std::sync::Arc::clone(&state.node_keypair), + git_bin: git_bin.to_string(), + git_timeout: std::time::Duration::from_secs(600), + encrypt_sem: sem, + } + } + + /// The drain resolves a coalesced push's tip pair to exactly that push's + /// introduced objects (delta semantics — the F5 observable: push B's pins are + /// recorded by the drain, and pre-existing objects are not re-listed). + #[sqlx::test] + async fn u5_drain_resolves_coalesced_tips_to_their_objects(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + u5_init_repo(tmp.path()); + let c1 = u5_commit_file(tmp.path(), "a.txt", "one\n"); + let c2 = u5_commit_file(tmp.path(), "b.txt", "two\n"); + state + .db + .upsert_mirror_repo("z6u5delta", "d", "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6u5delta", "d").await.unwrap().unwrap(); + let ctx = u5_ctx( + &state, + &rec, + tmp.path().to_path_buf(), + "git", + std::sync::Arc::new(tokio::sync::Semaphore::new(4)), + ); + + // Push B advanced main c1 -> c2 and lost try_begin; its pair was coalesced. + let (list, _rules, is_public) = resolve_drain_object_list( + &ctx, + crate::state::PendingWork::Tips(vec![(c1.clone(), c2.clone())]), + ) + .await + .expect("a public repo drains to a pin list"); + assert!(is_public, "mirror rows are public"); + let got: std::collections::HashSet = list.into_iter().collect(); + let new_blob = u5_git(tmp.path(), &["rev-parse", "HEAD:b.txt"]); + let old_blob = u5_git(tmp.path(), &["rev-parse", &format!("{c1}:a.txt")]); + assert!( + got.contains(&c2) && got.contains(&new_blob), + "B's commit and blob are in the drained pin list (the F5 fix)" + ); + assert!( + !got.contains(&c1) && !got.contains(&old_blob), + "pre-existing objects are not re-listed (delta, not full scan)" + ); + } + + /// The FullScan marker drains through the FLAGGED full-scan path to a NON-EMPTY + /// candidate set. RED arm of the encoding: were the marker a plain empty-tips + /// call, the deletion-only fast path would return an empty delta and the drain + /// would pin nothing (the F5 silent loss resurfacing). + #[sqlx::test] + async fn u5_drain_full_scan_marker_yields_nonempty_candidates(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + u5_init_repo(tmp.path()); + let c1 = u5_commit_file(tmp.path(), "a.txt", "one\n"); + state + .db + .upsert_mirror_repo("z6u5full", "f", "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6u5full", "f").await.unwrap().unwrap(); + let ctx = u5_ctx( + &state, + &rec, + tmp.path().to_path_buf(), + "git", + std::sync::Arc::new(tokio::sync::Semaphore::new(4)), + ); + + let (list, _rules, _pub) = + resolve_drain_object_list(&ctx, crate::state::PendingWork::FullScan) + .await + .expect("a public repo drains to a pin list"); + let got: std::collections::HashSet = list.into_iter().collect(); + assert!( + !got.is_empty(), + "the FullScan drain must enumerate the repo — an empty list means the \ + marker collapsed into the empty-tips fast path" + ); + let blob = u5_git(tmp.path(), &["rev-parse", "HEAD:a.txt"]); + assert!( + got.contains(&c1) && got.contains(&blob), + "the full-scan drain covers the repo's commit and blob" + ); + } + + /// Rules tightened between the coalesced push and its drain are honored, fail + /// closed: the drain re-fetches rules/is_public fresh, so (1) a newly-withheld + /// blob is NOT pinned, and (2) a repo whose root became unreadable to the + /// anonymous public drains to nothing at all. + #[sqlx::test] + async fn u5_drain_honors_rules_tightened_after_the_push(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + u5_init_repo(tmp.path()); + u5_commit_file(tmp.path(), "pub.txt", "public\n"); + let c2 = u5_commit_file(tmp.path(), "secret/hidden.txt", "sealed\n"); + state + .db + .upsert_mirror_repo("z6u5tight", "t", "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6u5tight", "t").await.unwrap().unwrap(); + let ctx = u5_ctx( + &state, + &rec, + tmp.path().to_path_buf(), + "git", + std::sync::Arc::new(tokio::sync::Semaphore::new(4)), + ); + let pending = || crate::state::PendingWork::Tips(vec![(ZERO_SHA.to_string(), c2.clone())]); + + // At push time the repo had no rules. TIGHTEN before the drain: /secret/** + // becomes reader-gated. The drain must re-fetch and withhold the new blob. + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &[READER_DID.to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + let (list, _rules, _pub) = resolve_drain_object_list(&ctx, pending()) + .await + .expect("still announceable at root"); + let got: std::collections::HashSet = list.into_iter().collect(); + let pub_blob = u5_git(tmp.path(), &["rev-parse", "HEAD:pub.txt"]); + let secret_blob = u5_git(tmp.path(), &["rev-parse", "HEAD:secret/hidden.txt"]); + assert!( + got.contains(&pub_blob), + "the still-public blob is pinned by the drain" + ); + assert!( + !got.contains(&secret_blob), + "a blob withheld by a rule added AFTER the push must NOT be pinned by \ + the drain (fresh rules, fail closed)" + ); + + // Tighten further: root becomes reader-gated → not announceable to the + // anonymous public → the drain pins nothing at all. + state + .db + .set_visibility_rule( + &rec.id, + "/", + crate::db::VisibilityMode::A, + &[READER_DID.to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + assert!( + resolve_drain_object_list(&ctx, pending()).await.is_none(), + "a repo no longer announceable under current rules drains to nothing \ + (fail closed)" + ); + } + + /// Hot-repo drain at encrypt-pool size 1: the task loop holds NO task-level + /// permit, so per-iteration helper acquires (withheld walk, candidate scan, + /// recipients walk) each get the pool's single permit in turn and BOTH the + /// snapshot iteration and the coalesced-drain iteration complete. RED if the + /// loop takes a task-level permit: the first helper acquire nests over the + /// same exhausted semaphore and the task parks forever. + #[sqlx::test] + async fn u5_hot_repo_drain_completes_at_pool_size_one(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + u5_init_repo(tmp.path()); + u5_commit_file(tmp.path(), "pub.txt", "public\n"); + let c2 = u5_commit_file(tmp.path(), "secret/hidden.txt", "sealed\n"); + state + .db + .upsert_mirror_repo("z6u5hot", "h", "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6u5hot", "h").await.unwrap().unwrap(); + // A path-scoped rule so every gated walk actually runs (withheld walk on + // the drain, recipients walk on both iterations). READER_DID carries no + // resolvable key, so the encrypt stage plans no seal and stays offline. + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &[READER_DID.to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + let rules = state.db.list_visibility_rules(&rec.id).await.unwrap(); + + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + let ctx = u5_ctx(&state, &rec, tmp.path().to_path_buf(), "git", sem); + + let inflight = crate::state::EncryptInflight::new(); + let guard = admit(&inflight, &rec.id); + assert!(matches!( + inflight.try_begin(&rec.id, vec![(ZERO_SHA.to_string(), c2)]), + crate::state::BeginOutcome::Coalesced + )); + + tokio::time::timeout( + std::time::Duration::from_secs(60), + run_encrypt_pin_task(ctx, guard, Vec::new(), Some(rules), true), + ) + .await + .expect( + "the drain must complete at pool size 1 — a task-level permit would \ + deadlock the helper-internal acquires", + ); + assert!( + inflight.is_empty(), + "the drained task released its repo key on exit" + ); + } + + /// The task LOOP is load-bearing: work coalesced during the snapshot iteration + /// is drained (its candidate scan runs git) before the task exits. RED under + /// the drain-loop revert (task drops its guard after the snapshot without + /// checking pending): the fake git never runs and the marker is absent. + #[cfg(unix)] + #[sqlx::test] + async fn u5_task_drains_coalesced_work_before_exiting(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + let marker = tmp.path().join("git.ran"); + // The drain's candidate scan probes the tip type then walks: report a + // commit tip and an empty rev-list, recording every invocation. + let body = format!( + "#!/bin/sh\necho ran >> \"{}\"\ncase \"$1\" in\n cat-file) echo commit ;;\n *) : ;;\nesac\nexit 0\n", + marker.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + state + .db + .upsert_mirror_repo("z6u5loop", "l", "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6u5loop", "l").await.unwrap().unwrap(); + let ctx = u5_ctx( + &state, + &rec, + tmp.path().to_path_buf(), + &git_bin, + std::sync::Arc::new(tokio::sync::Semaphore::new(4)), + ); + + let inflight = crate::state::EncryptInflight::new(); + let guard = admit(&inflight, &rec.id); + // Push B coalesces mid-flight with a real (created-ref) tip pair. + assert!(matches!( + inflight.try_begin( + &rec.id, + vec![( + ZERO_SHA.to_string(), + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef".to_string(), + )] + ), + crate::state::BeginOutcome::Coalesced + )); + + // Snapshot: empty object list, no rules — the snapshot iteration itself + // spawns no git, so any git invocation below belongs to the DRAIN. + run_encrypt_pin_task(ctx, guard, Vec::new(), None, true).await; + assert!( + marker.exists(), + "the task must drain B's coalesced tips (candidate scan runs git) \ + before exiting — an absent marker is the F5 skip" + ); + assert!( + inflight.is_empty(), + "the empty pending check at task end released the key" + ); + } + /// #174 SC2 (per-source key, U1): the per-caller read sub-cap keys on the /// resolved source IP, NOT the signed DID, so a disposable-DID farm cannot /// multiply its budget. Fill the source IP's single read slot, then drive two diff --git a/crates/gitlawb-node/src/git/push_delta.rs b/crates/gitlawb-node/src/git/push_delta.rs index 18f2c4da..27064357 100644 --- a/crates/gitlawb-node/src/git/push_delta.rs +++ b/crates/gitlawb-node/src/git/push_delta.rs @@ -288,6 +288,13 @@ pub struct PinCandidateSet { /// rev-list, and the full-scan fallback — run under ONE permit held for the /// whole blocking scan. The deletion-only fast path (no new tips) spawns no git /// and never parks. +/// +/// `force_full_scan` forces the full-scan fallback regardless of the tips — the +/// coalesced-drain `PendingWork::FullScan` marker (#174 F5). It composes with +/// the env kill-switch (either forces), and it disqualifies the empty-tips fast +/// path: a forced-scan call with no tips must still enumerate the repo, never +/// silently resolve to an empty delta (which would pin nothing and lose the +/// drained pushes' work — the F5 bug shape). pub async fn resolve_candidates_for_push( scan_sem: std::sync::Arc, repo_path: PathBuf, @@ -295,11 +302,13 @@ pub async fn resolve_candidates_for_push( old_tips: Vec, git_bin: String, timeout: Duration, + force_full_scan: bool, ) -> PinCandidateSet { + let force = force_full_scan || self::force_full_scan(); // No-git fast path: a deletion-only push resolves to an empty delta without - // spawning any git child, so it must not park on the scan pool. The - // kill-switch disqualifies it — forcing the full scan spawns git. - if new_tips.is_empty() && !force_full_scan() { + // spawning any git child, so it must not park on the scan pool. A forced + // full scan (caller flag or kill-switch) disqualifies it — the scan spawns git. + if new_tips.is_empty() && !force { tracing::info!(delta = 0usize, repo = %repo_path.display(), "pin candidate set from push delta"); return PinCandidateSet { candidates: Vec::new(), @@ -327,7 +336,16 @@ pub async fn resolve_candidates_for_push( let deadline = Instant::now() + timeout; let new_refs: Vec<&str> = new_tips.iter().map(String::as_str).collect(); let old_refs: Vec<&str> = old_tips.iter().map(String::as_str).collect(); - match resolve_push_delta(&repo_path, &new_refs, &old_refs, &git_bin, deadline) { + // `force` already folds in the env kill-switch, so the forced arm skips the + // delta machinery outright; the unforced arm goes through the normal + // resolver (whose own env read is false here by construction). + let resolved = if force { + tracing::debug!("full scan forced (coalesced-drain marker or kill-switch)"); + PinCandidates::FullScanRequired + } else { + resolve_push_delta(&repo_path, &new_refs, &old_refs, &git_bin, deadline) + }; + match resolved { PinCandidates::Delta(objs) => { tracing::info!(delta = objs.len(), repo = %repo_path.display(), "pin candidate set from push delta"); PinCandidateSet { candidates: objs, full_scan: false } @@ -586,6 +604,7 @@ mod tests { vec![c1.clone()], "git".to_string(), std::time::Duration::from_secs(600), + false, ) .await; assert!(!set.full_scan, "happy-path delta is not a full scan"); @@ -616,6 +635,7 @@ mod tests { vec![], "git".to_string(), std::time::Duration::from_secs(600), + false, ) .await; assert!(set.full_scan, "non-commit tip is signalled as a full scan"); @@ -665,6 +685,7 @@ mod tests { vec![], git_bin.clone(), Duration::from_secs(5), + false, ), ) .await; @@ -686,6 +707,7 @@ mod tests { vec![], git_bin, Duration::from_secs(5), + false, ) .await; assert!( @@ -730,6 +752,7 @@ mod tests { vec!["deadbeefdeadbeefdeadbeefdeadbeefdeadbeef".to_string()], fake.to_str().unwrap().to_string(), Duration::from_secs(5), + false, ), ) .await @@ -744,6 +767,56 @@ mod tests { assert!(!marker.exists(), "the fast path must spawn no git at all"); } + /// #174 F5: the coalesced-drain FullScan marker is signalled via the explicit + /// `force_full_scan` flag, and a flagged call with NO tips must still enumerate + /// the whole repo (full_scan=true, non-empty candidates). The RED arm of the + /// encoding question: if the marker were encoded as a plain empty-tips call + /// (flag off — the fast path), the drain would resolve to an empty delta and + /// pin nothing, silently losing the coalesced pushes' work. + #[tokio::test] + async fn forced_full_scan_with_no_tips_enumerates_the_repo() { + let repo = Repo::new(); + repo.commit_file("a.txt", "one\n"); + let all: HashSet = list_all_objects(&repo.path, "git", td()) + .unwrap() + .into_iter() + .collect(); + assert!(!all.is_empty(), "fixture repo has objects"); + + let set = resolve_candidates_for_push( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), + repo.path.clone(), + vec![], + vec![], + "git".to_string(), + std::time::Duration::from_secs(600), + true, + ) + .await; + assert!(set.full_scan, "the forced call is signalled as a full scan"); + let got: HashSet = set.candidates.into_iter().collect(); + assert_eq!( + got, all, + "a forced full scan with no tips enumerates the repo — never an empty delta" + ); + + // The discriminator: the SAME empty-tips call without the flag is the + // deletion-only fast path (empty delta, pin nothing). The two must differ, + // or the marker encoding has collapsed into the silent-loss shape. + let unforced = resolve_candidates_for_push( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), + repo.path.clone(), + vec![], + vec![], + "git".to_string(), + std::time::Duration::from_secs(600), + false, + ) + .await; + assert!(!unforced.full_scan); + assert!(unforced.candidates.is_empty()); + } + #[test] fn missing_oid_tip_forces_full_scan() { let repo = Repo::new(); diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 0badf802..4f8ffb0e 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -132,8 +132,9 @@ pub struct AppState { /// detached tasks *spawn and park* on that semaphore's `acquire_owned().await`. /// Before spawning a per-push encryption task, the receive-pack handler consults /// this set: if the repo already has a task in flight it coalesces (skips the - /// duplicate spawn) rather than parking a new unbounded waiter. Coalescing only - /// delays a duplicate walk — it never drops the withheld-blob recovery copy, which + /// duplicate spawn) rather than parking a new unbounded waiter, and its tip pairs + /// are recorded for that task's drain loop (#174 F5). Coalescing only delays the + /// coalesced push's walk — it never drops the withheld-blob recovery copy, which /// `2a54c15` deliberately kept fail-closed (there is no reconciliation sweep to /// re-derive a dropped copy). See [`EncryptInflight`]. pub encrypt_inflight: EncryptInflight, @@ -230,23 +231,80 @@ impl AppState { /// unbounded outstanding set. /// /// This tracks the repo keys with an in-flight encryption task. Before spawning, the -/// handler calls [`try_begin`](Self::try_begin): if a task for the repo is already -/// in-flight it returns `None` and the handler SKIPS spawning a duplicate (coalesce — -/// the newer push's objects are covered by the pending/next walk over the same repo's -/// history). This bounds the outstanding set to <=1 pending task per repo WITHOUT -/// dropping work: coalescing only delays a duplicate walk, it never sheds the recovery -/// copy (there is no reconciliation sweep, so a *dropped* job would be lost forever). +/// handler calls [`try_begin`](Self::try_begin) with the push's (old, new) tip pairs: +/// if no task is in-flight the push is [`Admitted`](BeginOutcome::Admitted) and spawns +/// one; if a task IS in-flight the push [`Coalesces`](BeginOutcome::Coalesced) — no +/// duplicate spawn — and its tip pairs are merged into the in-flight key's pending +/// slot in the SAME critical section as the presence check. The in-flight task pins +/// only its own pre-spawn object-list snapshot, so the merge is what keeps coalescing +/// lossless (#174 F5): the task loop-drains the pending slot via +/// [`EncryptInflightGuard::finish_or_take_pending`] before releasing the key, so a +/// coalesced push's pins and recovery copies are delayed, never dropped (there is no +/// reconciliation sweep, so a *dropped* job would be lost forever). Check-then-record +/// as two lock acquisitions would race the task's final pending check — hence one +/// critical section for both. /// -/// The returned [`EncryptInflightGuard`] is moved into the detached task and removes -/// the repo key on drop — on normal completion, error, OR panic (Drop runs on unwind) -/// — so one crashed walk can never permanently lock a repo out of future recovery -/// copies. +/// The returned [`EncryptInflightGuard`] is moved into the detached task. On normal +/// exit the key is removed (and the guard disarmed) inside `finish_or_take_pending`'s +/// empty-pending critical section; the guard's Drop is the PANIC backstop (Drop runs +/// on unwind), so one crashed walk can never permanently lock a repo out of future +/// recovery copies. #[derive(Clone, Default)] pub struct EncryptInflight { - // std::sync::Mutex: only ever held for O(1) HashSet insert/remove in a sync - // context (right before `tokio::spawn`, and in the guard's Drop) — never across - // an await, so a std Mutex is correct and cheaper than a tokio one. - repos: Arc>>, + // std::sync::Mutex: only ever held for O(1)-ish map ops (insert/remove/merge — + // the merge is an O(pairs) Vec extend bounded by MAX_PENDING_TIP_PAIRS) in a + // sync context, never across an await, so a std Mutex is correct and cheaper + // than a tokio one. Key present == task in flight; the value is the work + // recorded by pushes that coalesced against it. + repos: Arc>>, +} + +/// Cap on the accumulated coalesced tip pairs per repo. Past it the pending slot +/// degrades to [`PendingWork::FullScan`], so a hostile pusher cannot grow the slot +/// without bound while a walk is in flight; the drain then costs one full-repo +/// enumeration instead (the same already-tested fallback the push path uses). +const MAX_PENDING_TIP_PAIRS: usize = 1024; + +/// Work recorded by pushes that coalesced against an in-flight encryption task, +/// drained by that task one batch per loop iteration. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PendingWork { + /// The coalesced pushes' raw (old_sha, new_sha) ref-update pairs, zeros + /// included — the drain strips the create/delete sentinels exactly like the + /// handler tail does. An EMPTY vec is "nothing pending", never a work item. + Tips(Vec<(String, String)>), + /// The pair bound overflowed: drain with a FORCED full-repo scan. This must be + /// signalled explicitly (the `force_full_scan` flag on + /// `resolve_candidates_for_push`), never encoded as an empty tip list — empty + /// tips resolve to an empty delta and would pin nothing (the F5 loss again). + FullScan, +} + +/// Outcome of [`EncryptInflight::try_begin`]. +pub enum BeginOutcome { + /// No task was in flight: the caller spawns one, moving the guard into it. The + /// push's own tip pairs are NOT recorded — the caller's pre-spawn snapshot + /// covers them; the pending slot starts empty. + Admitted(EncryptInflightGuard), + /// A task is in flight; this push's tip pairs were merged into its pending + /// slot (same critical section as the presence check). The in-flight task's + /// drain loop will process them. + Coalesced, +} + +/// Outcome of [`EncryptInflightGuard::finish_or_take_pending`]. +pub enum FinishOutcome { + /// Coalesced work was pending: it is handed back with the still-armed guard + /// (the repo key is retained) and the task must run another drain iteration. + Pending(EncryptInflightGuard, PendingWork), + /// Nothing was pending: the repo key was removed AND the guard disarmed in one + /// critical section, so dropping the returned guard is inert. The task exits. + /// Remove-then-drop as two steps would double-remove: a successor task admitted + /// between them would have ITS key deleted by the late Drop. The disarmed guard + /// is handed back rather than dropped internally so that remove→drop window is + /// real and the disarm is testable; production just lets it fall out of scope + /// (hence the allow). + Finished(#[allow(dead_code)] EncryptInflightGuard), } impl EncryptInflight { @@ -254,19 +312,25 @@ impl EncryptInflight { Self::default() } - /// Try to begin an encryption task for `repo_id`. Returns `Some(guard)` if no task - /// for the repo was in-flight (the caller should spawn), or `None` if one already - /// is (the caller should COALESCE — skip spawning a duplicate). The guard releases - /// the repo key on drop. - pub fn try_begin(&self, repo_id: &str) -> Option { - let mut set = self.repos.lock().expect("encrypt_inflight mutex poisoned"); - if set.insert(repo_id.to_string()) { - Some(EncryptInflightGuard { - repos: Arc::clone(&self.repos), - repo_id: repo_id.to_string(), - }) - } else { - None + /// Begin-or-coalesce an encryption task for `repo_id`, in one critical section. + /// `tip_pairs` is this push's raw (old_sha, new_sha) ref-update list; it is + /// merged into the pending slot only on the [`Coalesced`](BeginOutcome::Coalesced) + /// arm (an admitted caller's own snapshot already covers its pairs). + pub fn try_begin(&self, repo_id: &str, tip_pairs: Vec<(String, String)>) -> BeginOutcome { + let mut map = self.repos.lock().expect("encrypt_inflight mutex poisoned"); + match map.entry(repo_id.to_string()) { + std::collections::hash_map::Entry::Vacant(slot) => { + slot.insert(PendingWork::Tips(Vec::new())); + BeginOutcome::Admitted(EncryptInflightGuard { + repos: Arc::clone(&self.repos), + repo_id: repo_id.to_string(), + armed: true, + }) + } + std::collections::hash_map::Entry::Occupied(mut slot) => { + merge_pending(slot.get_mut(), tip_pairs); + BeginOutcome::Coalesced + } } } @@ -287,21 +351,95 @@ impl EncryptInflight { } } -/// RAII guard removing a repo key from [`EncryptInflight`] when the detached -/// encryption task finishes (drop on completion, error, or panic-unwind). Move-only — -/// there is no reason to clone a guard, and cloning would double-remove. +/// Merge a coalesced push's tip pairs into a repo's pending slot. FullScan absorbs +/// everything; a Tips slot that would exceed [`MAX_PENDING_TIP_PAIRS`] degrades to +/// FullScan rather than growing without bound. +fn merge_pending(slot: &mut PendingWork, pairs: Vec<(String, String)>) { + match slot { + PendingWork::FullScan => {} + PendingWork::Tips(acc) => { + if acc.len().saturating_add(pairs.len()) > MAX_PENDING_TIP_PAIRS { + *slot = PendingWork::FullScan; + } else { + acc.extend(pairs); + } + } + } +} + +/// Guard owned by the detached encryption task for its repo key. Move-only — there +/// is no reason to clone a guard, and cloning would double-remove. Normal exit goes +/// through [`finish_or_take_pending`](Self::finish_or_take_pending); Drop is the +/// panic-path backstop only. pub struct EncryptInflightGuard { - repos: Arc>>, + repos: Arc>>, repo_id: String, + /// True until the normal-exit path removes the key. A disarmed guard's Drop is + /// a no-op: the key slot may already belong to a successor task admitted after + /// our removal, and removing THAT key would break at-most-one-task-per-repo. + armed: bool, +} + +impl EncryptInflightGuard { + /// The task's end-of-iteration step, one critical section: if coalesced work is + /// pending, take it and hand the still-armed guard back (key retained — iterate); + /// if nothing is pending, remove the key and disarm the guard (task exits; the + /// returned guard's Drop is inert). The atomicity is load-bearing both ways: a + /// push landing before this call is merged and therefore drained here; a push + /// landing after it finds the key gone and is admitted as a fresh task. No + /// interleaving can lose the work or admit two tasks for one repo. + pub fn finish_or_take_pending(mut self) -> FinishOutcome { + let mut map = self.repos.lock().expect("encrypt_inflight mutex poisoned"); + match map.get_mut(&self.repo_id) { + Some(PendingWork::Tips(acc)) if acc.is_empty() => { + map.remove(&self.repo_id); + self.armed = false; + drop(map); + FinishOutcome::Finished(self) + } + Some(slot) => { + let work = std::mem::replace(slot, PendingWork::Tips(Vec::new())); + drop(map); + FinishOutcome::Pending(self, work) + } + None => { + // Unreachable while armed (only this method removes a live key), + // but never panic in the release path: treat as finished. + self.armed = false; + drop(map); + FinishOutcome::Finished(self) + } + } + } } impl Drop for EncryptInflightGuard { fn drop(&mut self) { - // A poisoned lock (a prior panic while holding it) still lets us take the inner - // set via into_inner-on-guard; but poisoning here is not expected because the - // only critical sections are the O(1) ops above. Remove best-effort. - if let Ok(mut set) = self.repos.lock() { - set.remove(&self.repo_id); + // Normal exit disarmed us inside finish_or_take_pending's critical section; + // an armed drop means the task ended abnormally (panic-unwind, or a future + // code path that returns without finishing). Release the key so the repo is + // not permanently locked out, and log any pending work this loses — there + // is no sweep, so it stays lost until a later push re-walks the repo. + if !self.armed { + return; + } + // A poisoned lock is not expected (the critical sections above are small + // and panic-free); remove best-effort. + if let Ok(mut map) = self.repos.lock() { + match map.remove(&self.repo_id) { + Some(PendingWork::Tips(acc)) if !acc.is_empty() => tracing::warn!( + repo = %self.repo_id, + lost_tip_pairs = acc.len(), + "encryption task ended abnormally with coalesced pushes pending; \ + their pins/recovery copies are lost until a later push" + ), + Some(PendingWork::FullScan) => tracing::warn!( + repo = %self.repo_id, + "encryption task ended abnormally with a pending full-scan drain; \ + it is lost until a later push" + ), + _ => {} + } } } } diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index bcd0758e..75285f21 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -88,6 +88,23 @@ fn inv22_concurrency_gates_present_and_not_bypassed() { encrypt_inflight.try_begin to coalesce per repo (bound the outstanding-task set)" ); + // F5: coalescing must REQUEUE, not shed. The in-flight task pins only its own + // pre-spawn snapshot, so a coalesced push's tip pairs are recorded on its key and + // the task must loop-drain them (`finish_or_take_pending`) before releasing it. + // Removing the drain reverts to the silent loss: a coalesced push's pins and + // recovery copies are absent until an unrelated later push. Scan only the + // production half of the file — the u5 tests in its `mod tests` also name the + // drain call, and matching them would make this check vacuous. + let repos_production = repos + .split("#[cfg(test)]") + .next() + .expect("split always yields a first chunk"); + assert!( + repos_production.contains("finish_or_take_pending"), + "F5 gate missing: the post-push encryption task must loop-drain coalesced \ + pushes via finish_or_take_pending before releasing its repo key" + ); + // F4: every post-receive scan helper acquires a `git_encrypt_semaphore` permit // BEFORE its spawn_blocking git walk, so a push burst cannot accumulate unbounded // concurrent scans once the write permit is released. Structural check: within each From 48ed0a741f16904e15124033c0618480be3a9932 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 17 Jul 2026 20:16:04 -0500 Subject: [PATCH 049/124] fix(review): close the round's review findings on the #174 fix series object_type now distinguishes repo-level git failures (not-a-repository, error:-prefixed corruption) from genuine object absence, so a corrupt repo taints the /ipfs scan into the retryable 503 instead of vouching for a definitive 404; the drain reads visibility rules by the freshly fetched record id, closing a latent fail-open on id divergence. The four budget gates collapse into one stage-labelled helper, the three scan sites share state::acquire_scan_permit (inv22 tripwire repointed and sever-verified), and the Tigris stall tests use a local accept-and-park listener instead of a non-routable address. --- crates/gitlawb-node/src/api/ipfs.rs | 229 ++++++++++++++++------ crates/gitlawb-node/src/api/repos.rs | 50 ++--- crates/gitlawb-node/src/git/push_delta.rs | 21 +- crates/gitlawb-node/src/git/store.rs | 16 +- crates/gitlawb-node/src/state.rs | 31 +++ crates/gitlawb-node/tests/inv22_gates.rs | 33 +++- 6 files changed, 259 insertions(+), 121 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 28b9cc67..014411e7 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -182,14 +182,35 @@ pub async fn get_by_cid( } } - // Remaining request budget (F3), or None once exhausted. A stage is never - // started with zero remaining; the probe and read subprocesses carry no - // internal duration clamp, so the check before them is their entire bound - // (documented overshoot: at most one unclamped cat-file plus the in-flight - // clamped stage's kill/reap slack). - fn budget_remaining(deadline: std::time::Instant) -> Option { + // Budget gate shared by the four per-stage checks (F3): the remaining + // request budget, or — once exhausted — None, after logging the stage and + // knob and tainting "budget"; the call site only breaks (the scan STOPS, + // leaving this and every later candidate unproven, never a false 404). A + // stage is never started with zero remaining; the probe and read + // subprocesses carry no internal duration clamp, so this pre-start check is + // their entire bound (documented overshoot: at most one unclamped cat-file + // plus the in-flight clamped stage's kill/reap slack). The acquire and walk + // stages clamp their deadlines to the returned remainder. + fn budget_gate( + truncated_by: &mut Vec<&'static str>, + deadline: std::time::Instant, + budget_secs: u64, + repo_name: &str, + stage: &'static str, + ) -> Option { let left = deadline.saturating_duration_since(std::time::Instant::now()); - (!left.is_zero()).then_some(left) + if left.is_zero() { + tracing::warn!( + repo = %repo_name, + stage, + budget_secs, + "/ipfs request budget exhausted before the stage \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); stopping the scan without a verdict" + ); + taint(truncated_by, "budget"); + return None; + } + Some(left) } // Cap on EXPENSIVE walks only (F2): counts the repos that actually require the @@ -216,18 +237,15 @@ pub async fn get_by_cid( continue; } - // Budget gate for the acquire stage (F3): once the request budget is - // exhausted the scan STOPS, leaving this and every later candidate - // unproven (taint, never a false 404). Checked ahead of the visit + // Budget gate for the acquire stage (F3), checked ahead of the visit // bookkeeping so an unstarted acquire is not counted as a visit. - let Some(budget_left) = budget_remaining(request_deadline) else { - tracing::warn!( - repo = %repo.name, - budget_secs = state.config.ipfs_request_budget_secs, - "/ipfs request budget exhausted before the repo acquire \ - (GITLAWB_IPFS_REQUEST_BUDGET_SECS); stopping the scan without a verdict" - ); - taint(&mut truncated_by, "budget"); + let Some(budget_left) = budget_gate( + &mut truncated_by, + request_deadline, + state.config.ipfs_request_budget_secs, + &repo.name, + "repo acquire", + ) else { break; }; @@ -276,13 +294,15 @@ pub async fn get_by_cid( // Budget gate for the probe stage (F3). The probe subprocess has no // internal duration clamp, so this pre-start check is its entire bound. - if budget_remaining(request_deadline).is_none() { - tracing::warn!( - repo = %repo.name, - "/ipfs request budget exhausted before the object-type probe; \ - stopping the scan without a verdict" - ); - taint(&mut truncated_by, "budget"); + if budget_gate( + &mut truncated_by, + request_deadline, + state.config.ipfs_request_budget_secs, + &repo.name, + "object-type probe", + ) + .is_none() + { break; } @@ -311,14 +331,13 @@ pub async fn get_by_cid( // started walk runs its git children under a deadline clamped // to the remainder (the min below), so a walk can never // complete past the budget. - let Some(budget_left) = budget_remaining(request_deadline) else { - tracing::warn!( - repo = %repo.name, - budget_secs = state.config.ipfs_request_budget_secs, - "/ipfs request budget exhausted before the visibility walk \ - (GITLAWB_IPFS_REQUEST_BUDGET_SECS); stopping the scan without a verdict" - ); - taint(&mut truncated_by, "budget"); + let Some(budget_left) = budget_gate( + &mut truncated_by, + request_deadline, + state.config.ipfs_request_budget_secs, + &repo.name, + "visibility walk", + ) else { break; }; // Walk cap (F2), checked at the one site that actually spends a walk: @@ -394,13 +413,15 @@ pub async fn get_by_cid( // serving keeps the terminal arm honest and the stop unconditional; the // retryable 503 tells the caller to come back rather than letting an // over-budget request keep spending. - if budget_remaining(request_deadline).is_none() { - tracing::warn!( - repo = %repo.name, - "/ipfs request budget exhausted before the content read; \ - stopping the scan without a verdict" - ); - taint(&mut truncated_by, "budget"); + if budget_gate( + &mut truncated_by, + request_deadline, + state.config.ipfs_request_budget_secs, + &repo.name, + "content read", + ) + .is_none() + { break; } @@ -600,6 +621,24 @@ mod tests { .to_string() } + /// A local endpoint whose TCP accept succeeds instantly but that never writes + /// an HTTP response, so a Tigris HEAD against it stalls deterministically + /// until the caller's timeout. (A non-routable address hangs only if the + /// network blackholes the SYN — a fast RST would end the stall early.) The + /// accepted sockets are parked in the spawned task, which dies with the + /// test's runtime, so the peer never sees a close mid-test. + async fn silent_tigris_endpoint() -> String { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { + let mut held = Vec::new(); + while let Ok((sock, _)) = listener.accept().await { + held.push(sock); + } + }); + endpoint + } + /// Fake git for the WALK only (`state.git_bin`): empty refs, `rev-parse` /// resolves, and each `rev-list` appends one line to `log` and prints nothing — /// every walked repo yields an EMPTY allowed-set (path-gate deny verdict) and @@ -854,9 +893,9 @@ mod tests { } /// F2 acquire taint: a repo row with NO local copy over a Tigris backend that - /// stalls (non-routable endpoint — the connect just hangs) hits the 1s acquire - /// timeout at the read-acquire site. The skip carries no verdict, so the scan is - /// truncated: retryable 503 + Retry-After, never the old silent-skip 404. + /// stalls (a silent local endpoint — accepted, never answered) hits the 1s + /// acquire timeout at the read-acquire site. The skip carries no verdict, so the + /// scan is truncated: retryable 503 + Retry-After, never the old silent-skip 404. /// MUTATION (RED): drop the taint on the acquire-timeout arm and this decays to /// a 404. #[sqlx::test] @@ -866,12 +905,12 @@ mod tests { std::fs::create_dir_all(&repos_dir).unwrap(); let mut state = crate::test_support::test_state(pool.clone()).await; // Endpoint-pinned test client (no AWS_* env reads — env is racy under a - // parallel test run); 10.255.255.1 is non-routable so the HEAD hangs. - let tigris = crate::git::tigris::TigrisClient::for_testing_with_endpoint( - "test-bucket", - "http://10.255.255.1:9000", - ) - .await; + // parallel test run); the silent local endpoint stalls the HEAD + // deterministically. + let endpoint = silent_tigris_endpoint().await; + let tigris = + crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) + .await; state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; let mut cfg = (*state.config).clone(); @@ -911,9 +950,10 @@ mod tests { /// `object_type` is Err. That is not an absence verdict, so the scan is /// truncated: 503, never 404. A second, real repo probes clean (absent verdict) /// — the one bad row is what taints. NOTE: the probe shells to the real `git` - /// (not `state.git_bin`) and maps a NONZERO cat-file exit to `Ok(None)` (an - /// absent verdict), so a fake git exiting nonzero cannot reach this arm; only a - /// spawn failure is Err, hence the missing-dir recipe. MUTATION (RED): drop the + /// (not `state.git_bin`), and a clean missing/invalid-object nonzero exit is + /// still `Ok(None)` (an absent verdict) — this arm needs a probe that could + /// not RUN, hence the missing-dir spawn failure here; the corrupt-repo test + /// below drives the stderr-discriminated Err. MUTATION (RED): drop the /// taint on the probe-error arm and this decays to a 404. #[sqlx::test] async fn get_by_cid_probe_error_taints_scan_to_503(pool: sqlx::PgPool) { @@ -951,6 +991,76 @@ mod tests { ); } + /// F2 probe taint, corrupt-repo arm: a repo whose git dir EXISTS but is broken + /// (objects/ removed, HEAD garbage) makes the real `cat-file -t` die with the + /// repo-level `fatal: not a git repository` — a probe that could not examine + /// the object store, not an absence verdict, so `object_type` must map it to + /// Err and the scan must shed the probe-tainted 503, never the silent-absence + /// 404. A second, real repo probes clean (absent verdict) — the corrupt row is + /// what taints. MUTATION (RED): map every nonzero cat-file exit back to + /// `Ok(None)` in `object_type` (drop the stderr discrimination) and this + /// decays to a 404. + #[sqlx::test] + async fn get_by_cid_corrupt_repo_dir_probe_error_taints_scan_to_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // Older row: a real repo that probes clean. Newer row: a bare repo whose + // git dir exists on disk but is corrupt at the repo level. + seed_repo_with_blob(&state, tmp.path(), "z6f2corrupt", "real", b"probe clean\n").await; + state + .db + .upsert_mirror_repo("z6f2corrupt", "broken", "/unused-broken", None, false) + .await + .unwrap(); + let rec = state + .db + .get_repo("z6f2corrupt", "broken") + .await + .unwrap() + .unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + std::fs::create_dir_all(&bare).unwrap(); + run_git(&["init", "-q", "--bare", "--object-format=sha256"], &bare); + std::fs::remove_dir_all(bare.join("objects")).unwrap(); + std::fs::write(bare.join("HEAD"), b"junk\n").unwrap(); + + let peer: SocketAddr = "203.0.113.68:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a repo-level cat-file fatal leaves the repo unproven — the scan must \ + shed 503, not report the object absent" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("probe"), + "the shed must name the probe taint; got: {body}" + ); + } + /// F2 read taint: the gate passes (the probe reads the truncated loose object's /// intact "blob 64" header) but the content read fails (`cat-file blob` dies on /// the deflate stream cut mid-content) — the probe just said the object EXISTS @@ -1074,7 +1184,7 @@ mod tests { /// (`ipfs_request_budget_secs`) bounds the whole admitted scan; per-repo /// stages may not each draw a fresh timeout past it. Budget 1s, per-iteration /// acquire timeout 2s; the NEWER row is a Tigris-backed ghost (no local copy, - /// non-routable endpoint) whose acquire stalls, the OLDER row is a plain + /// silent local endpoint) whose acquire stalls, the OLDER row is a plain /// public repo carrying the blob. The ghost's acquire runs clamped to the ~1s /// remainder and times out; at the next repo the budget gate sees zero /// remaining, taints "budget", and STOPS the scan, so the blob repo is never @@ -1105,13 +1215,12 @@ mod tests { .await; // Swap in a Tigris-backed store over the SAME repos_dir (the seeded bare // repo stays a fast local hit) and add a NEWER ghost row with no local - // copy: its acquire consults the non-routable endpoint and stalls - // (endpoint-pinned test client, no AWS_* env reads; 10.255.255.1 hangs). - let tigris = crate::git::tigris::TigrisClient::for_testing_with_endpoint( - "test-bucket", - "http://10.255.255.1:9000", - ) - .await; + // copy: its acquire consults the silent local endpoint and stalls past + // the budget (endpoint-pinned test client, no AWS_* env reads). + let endpoint = silent_tigris_endpoint().await; + let tigris = + crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) + .await; state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); state .db diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 2b9cf1d4..2770ffb9 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -74,30 +74,14 @@ async fn replication_withheld_set( // that off the async worker thread. Some(rules) => { let owner_did = owner_did.to_string(); - // Scan admission (#174 F4): DEFER (await), never shed — dropping the - // walk would skip the vetting and fail the push's replication closed - // for no reason. Accepted residuals, stated honestly: (1) the park - // wait is queue-depth multiplied — post-receive tails are no longer - // admission-bounded once the write permit is released, so N landed - // pushes can queue N walks and the last waits N walk-durations; - // (2) a client-timeout disconnect while parked HERE drops the - // handler future and silently loses this push's replication work — - // the encrypt_inflight coalescing requeue does NOT cover it, because - // this park precedes the `try_begin` spawn gate. - let parked = std::time::Instant::now(); - let permit = encrypt_sem - .acquire_owned() - .await - .expect("git_encrypt_semaphore is never closed"); - tracing::debug!( - repo = %disk_path.display(), - queue_wait_ms = parked.elapsed().as_millis() as u64, - "post-push withheld walk admitted to the scan pool" - ); + // Scan admission (#174 F4): DEFER, never shed — dropping the walk + // would skip the vetting and fail the push's replication closed for + // no reason. Residuals at `acquire_scan_permit`. + let permit = + crate::state::acquire_scan_permit(encrypt_sem, &disk_path, "withheld walk").await; tokio::task::spawn_blocking(move || { // The permit lives inside the blocking closure: a started walk - // always completes holding it (a disconnect cannot cancel - // spawn_blocking or leak the permit mid-walk). + // always completes holding it. let _permit = permit; crate::git::visibility_pack::withheld_blob_oids_bounded( &disk_path, &git_bin, timeout, &rules, is_public, &owner_did, None, @@ -137,9 +121,7 @@ async fn replication_withheld_set( /// /// Always walks (there is no no-git arm), so the whole blocking scan runs under /// one `git_encrypt_semaphore` admission permit (#174 F4) — see -/// `replication_withheld_set`'s acquire for the defer rationale and the honest -/// residuals (queue-depth-multiplied park wait; a disconnect while parked loses -/// this push's replication work, uncovered by the coalescing requeue). +/// `acquire_scan_permit` for the defer rationale and the honest residuals. #[allow(clippy::too_many_arguments)] async fn fail_closed_full_scan_objects( encrypt_sem: std::sync::Arc, @@ -153,16 +135,8 @@ async fn fail_closed_full_scan_objects( ) -> Vec { // Scan admission (#174 F4): DEFER, never shed; the permit moves into the // closure so a started scan always completes holding it. - let parked = std::time::Instant::now(); - let permit = encrypt_sem - .acquire_owned() - .await - .expect("git_encrypt_semaphore is never closed"); - tracing::debug!( - repo = %disk_path.display(), - queue_wait_ms = parked.elapsed().as_millis() as u64, - "post-push fail-closed full scan admitted to the scan pool" - ); + let permit = + crate::state::acquire_scan_permit(encrypt_sem, &disk_path, "fail-closed full scan").await; tokio::task::spawn_blocking(move || -> anyhow::Result> { let _permit = permit; let allowed = crate::git::visibility_pack::replicable_blob_set_bounded( @@ -925,7 +899,11 @@ async fn resolve_drain_object_list( return None; } }; - let rules_opt = ctx.db.list_visibility_rules(&ctx.repo_id).await.ok(); + // record.id, never the spawn-time ctx.repo_id: the record above is re-fetched + // fresh by owner/name, and a delete+re-create between spawn and drain gives + // the row a NEW id — rules read against the stale id come back empty and + // would fail open for the new row. + let rules_opt = ctx.db.list_visibility_rules(&record.id).await.ok(); let (_announce, withheld) = replication_withheld_set( ctx.encrypt_sem.clone(), rules_opt.clone(), diff --git a/crates/gitlawb-node/src/git/push_delta.rs b/crates/gitlawb-node/src/git/push_delta.rs index 27064357..0b569693 100644 --- a/crates/gitlawb-node/src/git/push_delta.rs +++ b/crates/gitlawb-node/src/git/push_delta.rs @@ -315,21 +315,12 @@ pub async fn resolve_candidates_for_push( full_scan: false, }; } - // Scan admission (#174 F4): DEFER (await), never shed — a dropped scan would - // silently under-pin this push. The permit moves into the blocking closure so - // a started scan always completes holding it. Residuals as documented at - // `replication_withheld_set`'s acquire: park wait is queue-depth multiplied, - // and a client disconnect while parked loses this push's replication work. - let parked = Instant::now(); - let permit = scan_sem - .acquire_owned() - .await - .expect("git_encrypt_semaphore is never closed"); - tracing::debug!( - repo = %repo_path.display(), - queue_wait_ms = parked.elapsed().as_millis() as u64, - "pin-candidate scan admitted to the scan pool" - ); + // Scan admission (#174 F4): DEFER, never shed — a dropped scan would + // silently under-pin this push. The permit moves into the blocking closure + // so a started scan always completes holding it. Residuals at + // `acquire_scan_permit`. + let permit = + crate::state::acquire_scan_permit(scan_sem, &repo_path, "pin-candidate scan").await; tokio::task::spawn_blocking(move || { let _permit = permit; // ONE shared deadline for the whole scan, per jatmn ("the same deadline"). diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index 229ee695..5697c229 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -271,7 +271,8 @@ pub struct TreeEntry { /// `/ipfs/` is computed from these same content bytes via /// `gitlawb_core::cid::Cid::from_git_object_bytes`. /// -/// Get just the object type. Returns `None` if the object doesn't exist. +/// Get just the object type. Returns `None` if the object doesn't exist; a +/// probe that could not examine the object store is `Err`, never `None`. pub fn object_type(repo_path: &Path, sha256_hex: &str) -> Result> { let type_output = Command::new("git") .args(["cat-file", "-t", sha256_hex]) @@ -280,6 +281,19 @@ pub fn object_type(repo_path: &Path, sha256_hex: &str) -> Result> .context("failed to run git cat-file -t")?; if !type_output.status.success() { + // A nonzero exit is an ABSENCE verdict only when git could examine the + // object store: missing-object and invalid-oid probes die with a single + // clean `fatal:` line. A broken repo dir (`fatal: not a git repository`) + // or a corrupt object (`error: inflate` / `error: unable to unpack` + // lines before the fatal) proves nothing about absence, so it must + // surface as Err — the /ipfs scan taints on Err rather than treating + // the repo as probed-clean. + let stderr = String::from_utf8_lossy(&type_output.stderr); + if stderr.contains("not a git repository") + || stderr.lines().any(|l| l.starts_with("error:")) + { + bail!("git cat-file -t failed: {}", stderr.trim()); + } return Ok(None); } diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 4f8ffb0e..c4d2b079 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -443,3 +443,34 @@ impl Drop for EncryptInflightGuard { } } } + +/// Admit a post-receive git scan to the shared `git_encrypt_semaphore` pool +/// (#174 F4): DEFER (await), never shed — a dropped scan would lose the push's +/// recovery copy or silently under-pin it. The returned permit must move into +/// the blocking closure so a started scan always completes holding it (a +/// disconnect cannot cancel `spawn_blocking` or leak the permit mid-walk). +/// Accepted residuals, stated once for every caller: (1) the park wait is +/// queue-depth multiplied — post-receive tails are no longer admission-bounded +/// once the write permit is released, so N landed pushes can queue N scans and +/// the last waits N scan-durations; (2) a client-timeout disconnect while +/// parked HERE drops the handler future and silently loses this push's +/// replication work — the `encrypt_inflight` coalescing requeue does NOT cover +/// it, because this park precedes the `try_begin` spawn gate. +pub async fn acquire_scan_permit( + scan_sem: Arc, + repo: &std::path::Path, + stage: &'static str, +) -> tokio::sync::OwnedSemaphorePermit { + let parked = std::time::Instant::now(); + let permit = scan_sem + .acquire_owned() + .await + .expect("git_encrypt_semaphore is never closed"); + tracing::debug!( + repo = %repo.display(), + stage, + queue_wait_ms = parked.elapsed().as_millis() as u64, + "post-receive scan admitted to the scan pool" + ); + permit +} diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index 75285f21..cbfa0d4c 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -105,12 +105,25 @@ fn inv22_concurrency_gates_present_and_not_bypassed() { pushes via finish_or_take_pending before releasing its repo key" ); - // F4: every post-receive scan helper acquires a `git_encrypt_semaphore` permit - // BEFORE its spawn_blocking git walk, so a push burst cannot accumulate unbounded - // concurrent scans once the write permit is released. Structural check: within each - // helper the first `acquire_owned` precedes the first `spawn_blocking`. Removing a - // gate pushes the next `acquire_owned` occurrence past the helper's own - // `spawn_blocking` (or off the end of the file), turning the assertion red. + // F4: every post-receive scan helper admits itself to the shared scan pool via + // `crate::state::acquire_scan_permit` BEFORE its spawn_blocking git walk, so a + // push burst cannot accumulate unbounded concurrent scans once the write permit + // is released. Two halves, both load-bearing: the helper body must actually + // acquire the pool (state.rs sits the helper at the end of the file, so the + // definition tail contains no other `acquire_owned` to match vacuously), and + // within each scan helper the first qualified call precedes the first + // `spawn_blocking`. Severing a call site pushes the next occurrence past the + // helper's own `spawn_blocking` (or off the end of the file), turning the + // assertion red; comments name the helper without the `crate::state::` prefix, + // so this targets the real call sites. + let state_rs = src("state.rs"); + let helper_def = state_rs + .find("fn acquire_scan_permit") + .expect("F4 gate missing: state.rs no longer defines acquire_scan_permit"); + assert!( + state_rs[helper_def..].contains("acquire_owned"), + "F4 gate gutted: acquire_scan_permit must acquire the scan pool via acquire_owned" + ); let push_delta = src("git/push_delta.rs"); for (file_src, file, helper) in [ (&repos, "api/repos.rs", "fn replication_withheld_set"), @@ -125,9 +138,11 @@ fn inv22_concurrency_gates_present_and_not_bypassed() { .find(helper) .unwrap_or_else(|| panic!("{file}: `{helper}` not found")); let tail = &file_src[start..]; - let acquire = tail.find("acquire_owned").unwrap_or_else(|| { - panic!("F4 gate missing: {file} `{helper}` no longer acquires a scan permit") - }); + let acquire = tail + .find("crate::state::acquire_scan_permit(") + .unwrap_or_else(|| { + panic!("F4 gate missing: {file} `{helper}` no longer acquires a scan permit") + }); let spawn = tail.find("spawn_blocking").unwrap_or_else(|| { panic!("{file}: `{helper}` lost its spawn_blocking walk — update this guard") }); From b6d835d61228c01e7055ba6b9c6188e711d3a6e6 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 17 Jul 2026 20:42:48 -0500 Subject: [PATCH 050/124] fix(review): state the probe/read stages' unbounded-hang residual honestly; net the acquire-taint found path The budget-knob docs claimed the overshoot was bounded by one unclamped probe; an unclamped subprocess is not a bound, so the README, .env.example, config help, and module comments now say plainly that a hung git probe holds the walk slot for the hang's duration. New regression: an acquire-tainting ghost row ahead of a healthy public copy must not stop the scan; the later found short-circuit still serves 200. --- .env.example | 12 ++-- README.md | 2 +- crates/gitlawb-node/src/api/ipfs.rs | 102 ++++++++++++++++++++++++++-- crates/gitlawb-node/src/config.rs | 12 ++-- 4 files changed, 111 insertions(+), 17 deletions(-) diff --git a/.env.example b/.env.example index 94685069..8f9bfa2e 100644 --- a/.env.example +++ b/.env.example @@ -180,11 +180,13 @@ GITLAWB_IPFS_MAX_REPOS_WALKED=64 # Default 1024. GITLAWB_IPFS_MAX_REPO_VISITS=1024 # Absolute wall-clock budget for one admitted /ipfs request's acquire+walk -# lifetime (all stages of the whole scan). Bounds how LONG one request may hold -# its walk slot: per-stage timeouts are clamped to the remaining budget and no -# stage starts once it is exhausted (the scan then sheds a retryable 503). -# Residual overshoot: the in-flight clamped stage's kill/reap slack plus one -# unclamped cat-file probe subprocess. Default 600. +# lifetime (all stages of the whole scan). Per-stage clamps bound the acquire +# and walk stages to the remaining budget, and no stage starts once it is +# exhausted (the scan then sheds a retryable 503). The object-type probe and +# content-read cat-file subprocesses are budget-checked before starting but +# have NO duration bound of their own: a hung git probe (corrupt pack, stuck +# filesystem) holds the request's walk slot for the full duration of the hang. +# Default 600. GITLAWB_IPFS_REQUEST_BUDGET_SECS=600 # Max /ipfs/{cid} requests per client IP per hour (route flood brake, distinct # from the concurrency caps above). 0 disables. Default 600. diff --git a/README.md b/README.md index bc338339..cc32bcf7 100644 --- a/README.md +++ b/README.md @@ -349,7 +349,7 @@ Important node settings: | `GITLAWB_IPFS_WALK_PER_SOURCE` | Max concurrent `/ipfs` walks a single source IP may hold. Default 4. | | `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max expensive path-scope visibility walks per `/ipfs/{cid}` request; over-cap repos are skipped and the scan continues, shedding a retryable 503 (not a false 404) if the object is then found nowhere. Default 64. | | `GITLAWB_IPFS_MAX_REPO_VISITS` | Ceiling on repos one `/ipfs/{cid}` request may visit (acquire + probe) past the visibility gate — also the worst-case per-request Tigris fetch count. On exhaustion the scan stops with a retryable 503. Default 1024. | -| `GITLAWB_IPFS_REQUEST_BUDGET_SECS` | Absolute wall-clock budget for one admitted `/ipfs/{cid}` request's acquire+walk lifetime. Per-stage timeouts are clamped to the remaining budget and no stage starts once it is exhausted; the scan then stops with a retryable 503. Overshoot is bounded by the in-flight clamped stage's kill/reap slack plus one unclamped `cat-file` probe. Default 600. | +| `GITLAWB_IPFS_REQUEST_BUDGET_SECS` | Absolute wall-clock budget for one admitted `/ipfs/{cid}` request's acquire+walk lifetime. Per-stage clamps bound the acquire and walk stages to the remaining budget, and no stage starts once it is exhausted; the scan then stops with a retryable 503. The object-type probe and content-read `cat-file` subprocesses are budget-checked before starting but have no duration bound of their own, so a hung git probe (corrupt pack, stuck filesystem) holds the request's walk slot for the full duration of the hang. Default 600. | | `GITLAWB_IPFS_RATE_LIMIT` | Max `/ipfs/{cid}` requests per client IP per hour (route flood brake). 0 disables. Default 600. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 014411e7..41080833 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -64,9 +64,9 @@ use crate::visibility::{visibility_check, Decision}; /// Request budget (F3): one absolute clock (`ipfs_request_budget_secs`) spans /// the whole admitted request. No stage (acquire, probe, walk, content read) /// starts once it is exhausted, and the acquire wait and walk deadline are -/// clamped to the remainder, so an admitted request cannot hold its scarce walk -/// slot past the budget plus the in-flight stage's kill/reap slack and one -/// unclamped probe subprocess. +/// clamped to the remainder. The probe and content-read subprocesses have no +/// duration bound of their own past their pre-start budget check, so a hung +/// git probe holds the request's walk slot for the full duration of the hang. /// /// Scope: this closes the direct unauthenticated scan, including the dangling /// case. A stale-public mirror row still serves withheld content (tracked @@ -188,9 +188,9 @@ pub async fn get_by_cid( // leaving this and every later candidate unproven, never a false 404). A // stage is never started with zero remaining; the probe and read // subprocesses carry no internal duration clamp, so this pre-start check is - // their entire bound (documented overshoot: at most one unclamped cat-file - // plus the in-flight clamped stage's kill/reap slack). The acquire and walk - // stages clamp their deadlines to the returned remainder. + // their entire bound (a hung one holds the request's walk slot for the + // duration of the hang). The acquire and walk stages clamp their deadlines + // to the returned remainder. fn budget_gate( truncated_by: &mut Vec<&'static str>, deadline: std::time::Instant, @@ -944,6 +944,96 @@ mod tests { ); } + /// F2 found-beats-taint on the acquire arm: an acquire timeout taints the + /// scan but must NOT stop it — the loop `continue`s, and a later repo that + /// genuinely carries the object still serves. The NEWER row (visited first + /// under `list_all_repos`' updated_at DESC) is a Tigris-backed ghost whose + /// acquire stalls against the silent endpoint and times out at 1s; the + /// OLDER row is a plain public repo carrying the blob, reached next and + /// served from a cheap probe — found beats taint: 200 with the blob bytes, + /// never the truncation 503. MUTATION (RED): turn the acquire-timeout arm's + /// `continue` into a `break` and the public copy never serves (503). + #[sqlx::test] + async fn get_by_cid_acquire_taint_does_not_block_later_public_copy(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Seed the blob repo through a LOCAL-ONLY store first, so seeding never + // consults the (deliberately unreachable) Tigris endpoint. + state.repo_store = + crate::git::repo_store::RepoStore::for_testing(repos_dir.clone(), pool.clone()); + let content = b"acquire taint continue proof\n"; + let (_, oid) = + seed_repo_with_blob(&state, tmp.path(), "z6f2acqcont", "pubcopy", content).await; + // Swap in a Tigris-backed store over the SAME repos_dir (the seeded bare + // repo stays a fast local hit) and add a NEWER ghost row with no local + // copy: its acquire consults the silent local endpoint and stalls to the + // 1s timeout (endpoint-pinned test client, no AWS_* env reads). + let endpoint = silent_tigris_endpoint().await; + let tigris = + crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) + .await; + state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); + state + .db + .upsert_mirror_repo("z6f2acqcont", "ghost", "/unused-ghost", None, false) + .await + .unwrap(); + let mut cfg = (*state.config).clone(); + cfg.git_acquire_timeout_secs = 1; + state.config = Arc::new(cfg); + + // Ordering precondition: the ghost must be iterated FIRST (updated_at + // DESC — it was upserted after the blob repo), otherwise the pubcopy + // would serve before the taint ever fires and the continue-vs-break + // distinction would go untested. + let order: Vec = state + .db + .list_all_repos() + .await + .unwrap() + .into_iter() + .map(|r| r.name) + .collect(); + let ghost_pos = order.iter().position(|n| n == "ghost").unwrap(); + let pub_pos = order.iter().position(|n| n == "pubcopy").unwrap(); + assert!( + ghost_pos < pub_pos, + "precondition: the stalling ghost must be iterated before the blob repo; got {order:?}" + ); + + let peer: SocketAddr = "203.0.113.73:5000".parse().unwrap(); + let started = std::time::Instant::now(); + let resp = ipfs_router(state) + .oneshot(get_cid(&cid_for_oid(&oid), Some(peer))) + .await + .unwrap(); + // The taint arm demonstrably FIRED on this run: the response can only + // arrive after the ghost's stalled acquire burned its full 1s timeout + // (a cheap skip or a deny verdict would answer near-instantly). + assert!( + started.elapsed() >= std::time::Duration::from_millis(900), + "the ghost's acquire must stall to its timeout before the scan continues; \ + got {:?}", + started.elapsed() + ); + assert_eq!( + resp.status(), + StatusCode::OK, + "an acquire taint must not stop the scan — the later public copy serves" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + assert_eq!( + &body[..], + content.as_slice(), + "the served body must be the blob content from the later public copy" + ); + } + /// F2 probe taint: a repo row whose local dir does not exist (no Tigris) — /// `RepoStore::acquire` returns the path anyway (local passthrough), and the /// `cat-file -t` probe cannot even spawn (missing working dir), so diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index d564a554..87c5d65c 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -420,11 +420,13 @@ pub struct Config { /// acquire wait and walk deadline are clamped to `min(their own timeout, /// remaining budget)`; a stage is never started with zero remaining. On /// exhaustion the scan stops without a verdict and the request sheds a - /// retryable 503 + Retry-After rather than a false 404. Residual overshoot - /// past the budget is bounded by the kill/reap slack of the one in-flight - /// clamped stage (the walk watchdog's SIGTERM grace + SIGKILL settle) plus - /// the `object_type` / `read_object_content` probe subprocesses, which are - /// budget-checked before they start but carry no internal duration clamp. + /// retryable 503 + Retry-After rather than a false 404. The clamps bound + /// only the acquire and walk stages (overshoot there is the walk watchdog's + /// SIGTERM grace + SIGKILL settle); the `object_type` / + /// `read_object_content` probe subprocesses are budget-checked before they + /// start but have NO duration bound of their own, so a hung git probe + /// (corrupt pack, stuck filesystem) holds the request's walk slot for the + /// full duration of the hang. /// Must be positive. Default: 600s (10 min), matching /// `git_service_timeout_secs` so a single full-length walk still fits. #[arg( From 8d79692555758496126370814579b5b9ec0ff8be Mon Sep 17 00:00:00 2001 From: t Date: Sat, 18 Jul 2026 09:51:10 -0500 Subject: [PATCH 051/124] fix(node): make write-acquire cancellation-safe and connection-affine (#174 F1) acquire_write took the pg advisory lock, then awaited Tigris, then built the guard. An outer tokio::time::timeout firing during the Tigris await dropped the future with the lock held and no guard, so no unlock ran and later pushes to the repo spun 60s. release() also unlocked on an arbitrary pooled connection, not the one that locked (session advisory locks are connection-affine), so even the normal release was unreliable. Pin a PoolConnection in RepoWriteGuard, build the guard before the lock query, run the lock loop through it, unlock on that same connection in release(), and add a Drop backstop that spawns a detached affine unlock (Handle::try_current so an off-runtime drop logs instead of panicking). Two distinct-session tests (drop-without-release, affine release) go RED without the fix, GREEN with it. Pinning a connection per active write is a DB-pool DoS unless the pool clears the concurrent-write cap: the shipped default (20) was below max_concurrent_git_pushes (32). Raise the db_max_connections default to 48 and add Config::validate() to reject db_max_connections < max_concurrent_git_pushes + 8 headroom at boot; wire it into startup. --- crates/gitlawb-node/src/config.rs | 76 ++++++++- crates/gitlawb-node/src/git/repo_store.rs | 199 +++++++++++++++++++--- crates/gitlawb-node/src/main.rs | 7 + 3 files changed, 259 insertions(+), 23 deletions(-) diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 87c5d65c..5fb66b55 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -206,11 +206,15 @@ pub struct Config { /// Maximum connections in the PostgreSQL pool. This is a cap, not a floor /// (connections open lazily). Size against the database server's - /// max_connections, remembering admin tooling opens its own pool. + /// max_connections, remembering admin tooling opens its own pool. Each + /// concurrent write pins one pooled connection for its whole duration (the + /// advisory lock in `repo_store::acquire_write` is connection-affine), so this + /// must exceed `max_concurrent_git_pushes` by `DB_POOL_APP_HEADROOM` or slow + /// pushes starve every other DB path — enforced by `Config::validate`. #[arg( long, env = "GITLAWB_DB_MAX_CONNECTIONS", - default_value_t = 20, + default_value_t = 48, value_parser = clap::value_parser!(u32).range(1..) )] pub db_max_connections: u32, @@ -461,6 +465,36 @@ impl Config { } PathBuf::from(&self.key_path) } + + /// DB connections reserved for everything other than held write-locks: auth + /// lookups, visibility-rule reads, the post-receive tail's own DB writes, and + /// admin tooling. A write pins one pooled connection for its whole duration, so + /// the pool must clear the concurrent-write cap by at least this margin. + pub const DB_POOL_APP_HEADROOM: u32 = 8; + + /// Cross-field boot validation. Single-field ranges are enforced by clap; this + /// catches combinations that ship a denial-of-service under otherwise-valid + /// values. Call once at startup and fail fast on `Err`. + pub fn validate(&self) -> Result<(), String> { + // A write pins one pooled connection for its whole duration (the + // connection-affine advisory lock in repo_store::acquire_write), and + // concurrent writes are capped at max_concurrent_git_pushes. If the pool + // does not exceed that cap by DB_POOL_APP_HEADROOM, a burst of slow pushes + // drains every connection and every other DB path 503s. (#174 F1) + let floor = (self.max_concurrent_git_pushes as u64) + (Self::DB_POOL_APP_HEADROOM as u64); + if (self.db_max_connections as u64) < floor { + return Err(format!( + "GITLAWB_DB_MAX_CONNECTIONS ({}) must be at least max_concurrent_git_pushes ({}) \ + + {} headroom = {}: each concurrent write pins one pooled connection for its whole \ + duration, so a smaller pool lets a burst of slow pushes starve every other DB path.", + self.db_max_connections, + self.max_concurrent_git_pushes, + Self::DB_POOL_APP_HEADROOM, + floor + )); + } + Ok(()) + } } #[cfg(test)] @@ -666,4 +700,42 @@ mod tests { 1_048_576 ); } + + /// #174 F1: a connection-affine write lock pins a pooled connection per + /// concurrent write, so the pool must clear `max_concurrent_git_pushes` by + /// `DB_POOL_APP_HEADROOM` or a push burst starves every other DB path. + /// `validate()` must reject an under-sized pool at boot. + #[test] + fn db_pool_must_clear_the_git_push_cap() { + // Shipped defaults validate (48 >= 32 + 8). + Config::parse_from(["gitlawb-node"]) + .validate() + .expect("default config must validate"); + + // An under-sized pool relative to the push cap is rejected (20 < 32 + 8). + let under = Config::parse_from([ + "gitlawb-node", + "--db-max-connections", + "20", + "--max-concurrent-git-pushes", + "32", + ]); + assert!( + under.validate().is_err(), + "db_max_connections 20 below max_concurrent_git_pushes 32 + headroom must be rejected" + ); + + // Exactly at the floor validates (40 == 32 + 8). + let at_floor = Config::parse_from([ + "gitlawb-node", + "--db-max-connections", + "40", + "--max-concurrent-git-pushes", + "32", + ]); + assert!( + at_floor.validate().is_ok(), + "db_max_connections at the floor (pushes + headroom) must validate" + ); + } } diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index a5c367e9..86a1ee8b 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -13,7 +13,8 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use anyhow::{Context, Result}; -use sqlx::PgPool; +use sqlx::pool::PoolConnection; +use sqlx::{PgPool, Postgres}; use tokio::sync::Mutex; use tracing::{debug, info, warn}; @@ -157,13 +158,42 @@ impl RepoStore { let (owner_slug, local_path) = self.local_path(owner_did, repo_name)?; let lock_key = advisory_lock_key(&owner_slug, repo_name); - // Acquire Postgres advisory lock with retry using pg_try_advisory_lock - // to avoid blocking indefinitely on stale locks from crashed connections. + // Pin a dedicated pooled connection and build the guard holding it BEFORE + // issuing the lock query. Session-level pg advisory locks are + // connection-affine (they can only be released on the session that took + // them), so the guard must own the locking connection; and building the + // guard first means any cancellation after the lock is taken — a + // `tokio::time::timeout` firing during the Tigris download below — drops a + // guard that CAN release, closing the leak the outer timeout otherwise + // opened (#174 F1). + let conn = self + .pool + .acquire() + .await + .context("acquiring db connection for the write advisory lock")?; + let mut guard = RepoWriteGuard { + owner_slug: owner_slug.clone(), + repo_name: repo_name.to_string(), + local_path: local_path.clone(), + lock_key, + conn: Some(conn), + locked: false, + released: false, + tigris: self.tigris.clone(), + }; + + // Acquire the advisory lock with retry, through the guard's OWN connection, + // so the matching unlock (in release, or the Drop backstop) runs on the same + // session — pg_advisory_unlock on a different pooled connection is a no-op. let mut acquired = false; for attempt in 0..60 { + let c = guard + .conn + .as_deref_mut() + .expect("write guard holds its connection during acquisition"); let row: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") .bind(lock_key) - .fetch_one(&self.pool) + .fetch_one(&mut *c) .await .context("trying advisory lock")?; if row.0 { @@ -177,9 +207,11 @@ impl RepoStore { if !acquired { anyhow::bail!("could not acquire advisory lock after 60s — possible stale lock for {owner_slug}/{repo_name}"); } + guard.locked = true; - // Always download the latest from Tigris before writing. - // Local disk may be stale if another machine pushed since our last access. + // Always download the latest from Tigris before writing. Local disk may be + // stale if another machine pushed since our last access. The guard already + // owns the lock + its connection, so a cancellation here drops through Drop. if let Some(ref tigris) = self.tigris { if tigris.exists(&owner_slug, repo_name).await.unwrap_or(false) { debug!(repo = %repo_name, "write acquire: downloading latest from tigris"); @@ -197,14 +229,7 @@ impl RepoStore { } } - Ok(RepoWriteGuard { - owner_slug, - repo_name: repo_name.to_string(), - local_path, - lock_key, - pool: self.pool.clone(), - tigris: self.tigris.clone(), - }) + Ok(guard) } /// Initialize a new bare repo on local disk and upload to Tigris. @@ -354,7 +379,16 @@ pub struct RepoWriteGuard { repo_name: String, pub local_path: PathBuf, lock_key: i64, - pool: PgPool, + /// The pooled connection that took the advisory lock. Session-level pg + /// advisory locks are connection-affine, so the guard pins that connection + /// for its whole lifetime and unlocks on it (in `release`, or the `Drop` + /// backstop). `None` only after the connection has been taken to unlock. + conn: Option>, + /// Set once the advisory lock has actually been taken. A guard dropped + /// before the lock is held (or after `release`) performs no unlock. + locked: bool, + /// Set once `release` has run its unlock, making the `Drop` backstop inert. + released: bool, tigris: Option, } @@ -369,7 +403,7 @@ impl RepoWriteGuard { /// half-applied or otherwise inconsistent repo would propagate corruption to /// Tigris (and to every node that later downloads it). The lock is always /// released regardless, to avoid stale locks blocking future writes. - pub async fn release(self, success: bool) { + pub async fn release(mut self, success: bool) { // Upload to Tigris only on success. if success { if let Some(ref tigris) = self.tigris { @@ -384,11 +418,57 @@ impl RepoWriteGuard { warn!(repo = %self.repo_name, "write failed — skipping tigris upload to avoid propagating an inconsistent repo"); } - // Release advisory lock - let _ = sqlx::query("SELECT pg_advisory_unlock($1)") - .bind(self.lock_key) - .execute(&self.pool) - .await; + // Release the advisory lock on the SAME connection that took it (session + // advisory locks are connection-affine). Taking the connection here makes + // the Drop backstop inert. + if self.locked { + if let Some(mut conn) = self.conn.take() { + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(self.lock_key) + .execute(&mut *conn) + .await; + } + } + self.released = true; + } +} + +impl Drop for RepoWriteGuard { + /// Cancellation-safe backstop: if the guard is dropped while still holding the + /// advisory lock (a `tokio::time::timeout` cancelled `acquire_write`, or a + /// handler future was dropped before `release`), unlock on the pinned + /// connection. `Drop` cannot await, so spawn a detached unlock — it runs on the + /// same session (connection-affine). An off-runtime drop falls back to a log; + /// the ~60s stale-lock retry loop in `acquire_write` reclaims it. + fn drop(&mut self) { + if self.released || !self.locked { + return; + } + let Some(mut conn) = self.conn.take() else { + return; + }; + let lock_key = self.lock_key; + let repo_name = self.repo_name.clone(); + match tokio::runtime::Handle::try_current() { + Ok(handle) => { + handle.spawn(async move { + if let Err(e) = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(lock_key) + .execute(&mut *conn) + .await + { + warn!(repo = %repo_name, err = %e, "detached advisory-unlock on write-guard drop failed"); + } + }); + } + Err(_) => { + warn!( + repo = %repo_name, + "RepoWriteGuard dropped off a Tokio runtime; advisory lock not released \ + synchronously — the stale-lock retry loop will reclaim it" + ); + } + } } } @@ -562,4 +642,81 @@ mod tests { ); } } + + // ── advisory-lock cancellation-safety (#174 F1, RED-before/GREEN-after) ── + + /// F1 (P1): dropping a `RepoWriteGuard` WITHOUT calling `release()` — the + /// state a `tokio::time::timeout` cancellation leaves `acquire_write` in when + /// it fires during the Tigris await — must still release the session advisory + /// lock. A checker connection is held OUT of the pool first, so `acquire_write` + /// is forced onto a distinct session; the checker (a different session) then + /// probes the lock, so advisory-lock re-entrancy cannot mask a leak. + /// + /// Load-bearing: RED today (no `Drop` releases the lock → held by + /// `acquire_write`'s session → checker's `pg_try_advisory_lock` returns + /// false). GREEN after the connection-affine `Drop` backstop. + #[sqlx::test] + async fn write_guard_drop_without_release_frees_the_lock(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone()); + let owner = "did:key:z6MkDropBackstopProofAAAAAAAAAAAAAAAAAAAAAA"; + let name = "leaktest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + // Distinct session for the probe: hold it out of the pool BEFORE acquiring, + // so acquire_write cannot use it and a re-entrant probe cannot falsely read free. + let mut checker = pool.acquire().await.expect("checker connection"); + + let guard = store.acquire_write(owner, name).await.expect("acquire"); + // The cancellation shape: drop without release(). + drop(guard); + // Let the detached unlock task run. + tokio::time::sleep(std::time::Duration::from_millis(400)).await; + + let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *checker) + .await + .unwrap(); + assert!( + free, + "advisory lock must be released when the guard is dropped without release()" + ); + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } + + /// F1 (latent non-affine release): `release()` must unlock on the SAME + /// session that locked, so the lock is freed regardless of which pooled + /// connection would service a fresh query. Observed from a distinct session. + #[sqlx::test] + async fn write_guard_release_frees_the_lock_from_a_distinct_session(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone()); + let owner = "did:key:z6MkAffineReleaseProofBBBBBBBBBBBBBBBBBBBB"; + let name = "affinetest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + let mut checker = pool.acquire().await.expect("checker connection"); + let guard = store.acquire_write(owner, name).await.expect("acquire"); + guard.release(false).await; + + let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *checker) + .await + .unwrap(); + assert!( + free, + "release() must free the advisory lock via connection-affine unlock" + ); + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } } diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index ffac65a2..b7bc758e 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -74,6 +74,13 @@ async fn main() -> Result<()> { // bootstrap peers. Operators can opt out via GITLAWB_BOOTSTRAP_DISABLE_SEEDS. bootstrap::merge_seeds(&mut config); + // Fail fast on config combinations that are individually in-range but jointly + // unsafe — notably a DB pool too small for the concurrent-write cap, which + // would let a push burst starve every other DB path (#174 F1). + config + .validate() + .map_err(|e| anyhow::anyhow!("invalid configuration: {e}"))?; + if !config.public_read { warn!( "GITLAWB_PUBLIC_READ=false is reserved; per-repository private-read enforcement is not wired in alpha" From 65d7af2a741a6c8104a1463da02c6fd1d3cdddef Mon Sep 17 00:00:00 2001 From: t Date: Sat, 18 Jul 2026 10:14:27 -0500 Subject: [PATCH 052/124] fix(node): own the post-receive replication tail in a detached task (#174 F2) The post-receive replication tail (withheld/candidate/full-scan resolution, then the encrypt+recovery spawn and the announce spawn) ran inline in the request future and parked on git_encrypt_semaphore before the durable try_begin gate. A client/proxy disconnect while parked dropped the future and silently lost this push's pins, recovery copy, and announcements, with no reconciliation sweep to recover them (state.rs documented this exact residual). Move the whole tail into an independently owned tokio task and return the git response immediately, so a disconnect can no longer drop the parked work. Each push owns its own tail, including its own always-spawned announce, so the announce is never brought under the per-repo encrypt coalescing (which would drop a coalesced push's per-ref announcements). Update state.rs to retire the disconnect-loses-work residual. Test: receive_pack_landed_push_returns_without_parking_on_scan_pool asserts the handler returns 200 while the scan pool is held (RED with the tail inline: the request future parks and the timeout fires; GREEN detached). The burst serialization test is retimed to poll for scan completion since the withheld walk is now detached too. --- crates/gitlawb-node/src/api/repos.rs | 645 ++++++++++++++------------- crates/gitlawb-node/src/state.rs | 14 +- 2 files changed, 352 insertions(+), 307 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 2770ffb9..98e395f6 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1645,314 +1645,336 @@ pub async fn git_receive_pack( } } - // Replication enforcement (Phase 2): decide once per push whether the public - // may read this repo at all and, if so, which blob OIDs must not leave the - // node. `withheld == None` means replicate nothing (private / mode A / - // undetermined): skip every pin so even commit and tree objects (which - // withheld_blob_oids never lists) stay local. `announce` gates the - // network-facing announcements. Fail closed: a private or undetermined repo - // never leaks. - let rules_opt = state.db.list_visibility_rules(&record.id).await.ok(); - let (announce, withheld) = replication_withheld_set( - state.git_encrypt_semaphore.clone(), - rules_opt.clone(), - &record.owner_did, - record.is_public, - disk_path.clone(), - state.git_bin.clone(), - std::time::Duration::from_secs(state.config.git_service_timeout_secs), - ) - .await; - - // Resolve the per-push pin candidate set once, off the async worker, then - // filter to what may actually replicate. Delta path: the reachable-only - // `withheld` set suffices (delta objects are reachable). Full-scan path: the - // candidate set can include dangling blobs the withheld set never classified, - // so fail closed — replicate a blob only if it is reachable AND - // visibility-allowed (#99). Only computed when something will actually - // replicate; every degraded path logs rather than failing silently. - let object_list: Vec = if let Some(withheld_set) = withheld.clone() { - let new_tips: Vec = ref_updates - .iter() - .map(|u| u.new_sha.clone()) - .filter(|s| s != ZERO_SHA) - .collect(); - let old_tips: Vec = ref_updates - .iter() - .map(|u| u.old_sha.clone()) - .filter(|s| s != ZERO_SHA) - .collect(); - let pin_set = crate::git::push_delta::resolve_candidates_for_push( + // #174 F2: move the whole post-receive replication tail into an independently owned + // task. It parks on `git_encrypt_semaphore` (withheld / candidate / full-scan + // resolution), so leaving it in the request future means a client/proxy disconnect + // while parked silently drops this push's pins, recovery copy, and announcements — + // the residual `state.rs` documented, because the durable `try_begin` gate sat after + // the park. The request future now returns the git response immediately and the tail + // runs detached; a disconnect can no longer drop it. Each push owns its own tail, + // including its own always-spawned announce, so per-push announcements are never + // coalesced away (the announce spawn stays out of the per-repo encrypt coalescing). + let did = did.to_string(); + tokio::spawn(async move { + // Replication enforcement (Phase 2): decide once per push whether the public + // may read this repo at all and, if so, which blob OIDs must not leave the + // node. `withheld == None` means replicate nothing (private / mode A / + // undetermined): skip every pin so even commit and tree objects (which + // withheld_blob_oids never lists) stay local. `announce` gates the + // network-facing announcements. Fail closed: a private or undetermined repo + // never leaks. + let rules_opt = state.db.list_visibility_rules(&record.id).await.ok(); + let (announce, withheld) = replication_withheld_set( state.git_encrypt_semaphore.clone(), + rules_opt.clone(), + &record.owner_did, + record.is_public, disk_path.clone(), - new_tips, - old_tips, state.git_bin.clone(), std::time::Duration::from_secs(state.config.git_service_timeout_secs), - false, ) .await; - if pin_set.full_scan { - fail_closed_full_scan_objects( + + // Resolve the per-push pin candidate set once, off the async worker, then + // filter to what may actually replicate. Delta path: the reachable-only + // `withheld` set suffices (delta objects are reachable). Full-scan path: the + // candidate set can include dangling blobs the withheld set never classified, + // so fail closed — replicate a blob only if it is reachable AND + // visibility-allowed (#99). Only computed when something will actually + // replicate; every degraded path logs rather than failing silently. + let object_list: Vec = if let Some(withheld_set) = withheld.clone() { + let new_tips: Vec = ref_updates + .iter() + .map(|u| u.new_sha.clone()) + .filter(|s| s != ZERO_SHA) + .collect(); + let old_tips: Vec = ref_updates + .iter() + .map(|u| u.old_sha.clone()) + .filter(|s| s != ZERO_SHA) + .collect(); + let pin_set = crate::git::push_delta::resolve_candidates_for_push( state.git_encrypt_semaphore.clone(), disk_path.clone(), - rules_opt.clone().unwrap_or_default(), - record.is_public, - record.owner_did.clone(), - pin_set.candidates, + new_tips, + old_tips, state.git_bin.clone(), std::time::Duration::from_secs(state.config.git_service_timeout_secs), + false, ) - .await - } else { - crate::git::visibility_pack::replicable_objects(pin_set.candidates, &withheld_set) - } - } else { - Vec::new() - }; - - // Pin new git objects to the local IPFS node (no-op if ipfs_api is empty). - // Skipped entirely when the public cannot read the repo (withheld == None). - // - // Coalesce-and-requeue per repo (#174 P2-2 + F5): the spawned task's walks park - // on `git_encrypt_semaphore` (which DEFERS when the pool is full rather than - // dropping the recovery copy). To bound the OUTSTANDING task set, at most one - // task per repo is in flight; a push arriving while one is in flight does NOT - // spawn a duplicate — and is NOT dropped either. The in-flight task pins only - // its own pre-spawn object-list snapshot, so this push's (old, new) tip pairs - // are merged into the in-flight key's pending slot in the same critical section - // as the presence check, and the task loop-drains them (fresh rules, fail - // closed) before releasing the key. Without the requeue a coalesced push's pins - // and recovery copies would be silently absent until an unrelated later push - // (the F5 loss). The guard still releases the key on panic (Drop on unwind), so - // a crashed walk never permanently locks the repo out. - if withheld.is_some() { - let tip_pairs: Vec<(String, String)> = ref_updates - .iter() - .map(|u| (u.old_sha.clone(), u.new_sha.clone())) - .collect(); - match state.encrypt_inflight.try_begin(&record.id, tip_pairs) { - crate::state::BeginOutcome::Coalesced => { - tracing::debug!( - repo = %record.id, - "post-push encryption task already in flight for this repo; coalesced \ - — this push's tip pairs are queued for that task's drain" - ); - } - crate::state::BeginOutcome::Admitted(inflight_guard) => { - let ctx = EncryptTaskCtx { - ipfs_api: state.config.ipfs_api.clone(), - repo_path: disk_path.clone(), - db: state.db.clone(), - repo_id: record.id.clone(), - owner_did: record.owner_did.clone(), - repo_name: record.name.clone(), - irys_url: state.config.irys_url.clone(), - http_client: std::sync::Arc::clone(&state.http_client), - node_did: state.node_did.to_string(), - node_keypair: std::sync::Arc::clone(&state.node_keypair), - git_bin: state.git_bin.clone(), - git_timeout: std::time::Duration::from_secs( - state.config.git_service_timeout_secs, - ), - encrypt_sem: state.git_encrypt_semaphore.clone(), - }; - tokio::spawn(run_encrypt_pin_task( - ctx, - inflight_guard, - object_list.clone(), - rules_opt.clone(), + .await; + if pin_set.full_scan { + fail_closed_full_scan_objects( + state.git_encrypt_semaphore.clone(), + disk_path.clone(), + rules_opt.clone().unwrap_or_default(), record.is_public, - )); - } - } - } - - // Pin new git objects to Pinata, then record branch→CID and gossip. - // - // #174 P2-2 scope note: this SECOND detached spawn is deliberately NOT brought - // under the per-repo encryption coalescing above. Two reasons: (1) it does not - // park on `git_encrypt_semaphore` (or any semaphore) — the Pinata `pin_new_objects` - // is a bounded reqwest round-trip, so it does not form the unbounded PARKED-waiter - // set that is the P2-2 residual; it runs to completion under the HTTP client's - // network timeouts. (2) Unlike the idempotent recovery-copy walk, this task does - // PER-PUSH, PER-REF work — branch→CID upserts, gossip publish, GraphQL subscription - // broadcast, Arweave anchoring, and peer notify, each keyed to THIS push's - // ref_updates. Coalescing it against an in-flight task for the same repo would DROP - // a later push's ref-update announcements (a correctness regression), not merely - // delay a duplicate. So it is scoped out with rationale, not brought under the bound. - { - let pinata_jwt = state.config.pinata_jwt.clone(); - let pinata_upload_url = state.config.pinata_upload_url.clone(); - let repo_path_clone = disk_path.clone(); - let db_clone = state.db.clone(); - let http_client = Arc::clone(&state.http_client); - let node_did_str = state.node_did.to_string(); - let repo_slug = format!( - "{}/{}", - crate::db::normalize_owner_key(&record.owner_did), - record.name - ); - let ref_updates_clone = ref_updates - .iter() - .map(|u| (u.ref_name.clone(), u.old_sha.clone(), u.new_sha.clone())) - .collect::>(); - let p2p_handle = state.p2p.clone(); - let pusher_did_clone = did.to_string(); - let db_for_peers = state.db.clone(); - let ref_update_tx = state.ref_update_tx.clone(); - let irys_url = state.config.irys_url.clone(); - let owner_did_for_arweave = record.owner_did.clone(); - let self_public_url = state.config.public_url.clone(); - let node_keypair = Arc::clone(&state.node_keypair); - let object_list_pinata = object_list; - let do_pinata_replication = withheld.is_some(); - tokio::spawn(async move { - let pinned = if do_pinata_replication { - crate::pinata::pin_new_objects( - &http_client, - &pinata_upload_url, - &pinata_jwt, - &repo_path_clone, - object_list_pinata, - &db_clone, + record.owner_did.clone(), + pin_set.candidates, + state.git_bin.clone(), + std::time::Duration::from_secs(state.config.git_service_timeout_secs), ) .await } else { - Vec::new() - }; + crate::git::visibility_pack::replicable_objects(pin_set.candidates, &withheld_set) + } + } else { + Vec::new() + }; - if !pinned.is_empty() { - tracing::info!(count = pinned.len(), "pinned git objects to Pinata"); + // Pin new git objects to the local IPFS node (no-op if ipfs_api is empty). + // Skipped entirely when the public cannot read the repo (withheld == None). + // + // Coalesce-and-requeue per repo (#174 P2-2 + F5): the spawned task's walks park + // on `git_encrypt_semaphore` (which DEFERS when the pool is full rather than + // dropping the recovery copy). To bound the OUTSTANDING task set, at most one + // task per repo is in flight; a push arriving while one is in flight does NOT + // spawn a duplicate — and is NOT dropped either. The in-flight task pins only + // its own pre-spawn object-list snapshot, so this push's (old, new) tip pairs + // are merged into the in-flight key's pending slot in the same critical section + // as the presence check, and the task loop-drains them (fresh rules, fail + // closed) before releasing the key. Without the requeue a coalesced push's pins + // and recovery copies would be silently absent until an unrelated later push + // (the F5 loss). The guard still releases the key on panic (Drop on unwind), so + // a crashed walk never permanently locks the repo out. + if withheld.is_some() { + let tip_pairs: Vec<(String, String)> = ref_updates + .iter() + .map(|u| (u.old_sha.clone(), u.new_sha.clone())) + .collect(); + match state.encrypt_inflight.try_begin(&record.id, tip_pairs) { + crate::state::BeginOutcome::Coalesced => { + tracing::debug!( + repo = %record.id, + "post-push encryption task already in flight for this repo; coalesced \ + — this push's tip pairs are queued for that task's drain" + ); + } + crate::state::BeginOutcome::Admitted(inflight_guard) => { + let ctx = EncryptTaskCtx { + ipfs_api: state.config.ipfs_api.clone(), + repo_path: disk_path.clone(), + db: state.db.clone(), + repo_id: record.id.clone(), + owner_did: record.owner_did.clone(), + repo_name: record.name.clone(), + irys_url: state.config.irys_url.clone(), + http_client: std::sync::Arc::clone(&state.http_client), + node_did: state.node_did.to_string(), + node_keypair: std::sync::Arc::clone(&state.node_keypair), + git_bin: state.git_bin.clone(), + git_timeout: std::time::Duration::from_secs( + state.config.git_service_timeout_secs, + ), + encrypt_sem: state.git_encrypt_semaphore.clone(), + }; + tokio::spawn(run_encrypt_pin_task( + ctx, + inflight_guard, + object_list.clone(), + rules_opt.clone(), + record.is_public, + )); + } } + } + + // Pin new git objects to Pinata, then record branch→CID and gossip. + // + // #174 P2-2 scope note: this SECOND detached spawn is deliberately NOT brought + // under the per-repo encryption coalescing above. Two reasons: (1) it does not + // park on `git_encrypt_semaphore` (or any semaphore) — the Pinata `pin_new_objects` + // is a bounded reqwest round-trip, so it does not form the unbounded PARKED-waiter + // set that is the P2-2 residual; it runs to completion under the HTTP client's + // network timeouts. (2) Unlike the idempotent recovery-copy walk, this task does + // PER-PUSH, PER-REF work — branch→CID upserts, gossip publish, GraphQL subscription + // broadcast, Arweave anchoring, and peer notify, each keyed to THIS push's + // ref_updates. Coalescing it against an in-flight task for the same repo would DROP + // a later push's ref-update announcements (a correctness regression), not merely + // delay a duplicate. So it is scoped out with rationale, not brought under the bound. + { + let pinata_jwt = state.config.pinata_jwt.clone(); + let pinata_upload_url = state.config.pinata_upload_url.clone(); + let repo_path_clone = disk_path.clone(); + let db_clone = state.db.clone(); + let http_client = Arc::clone(&state.http_client); + let node_did_str = state.node_did.to_string(); + let repo_slug = format!( + "{}/{}", + crate::db::normalize_owner_key(&record.owner_did), + record.name + ); + let ref_updates_clone = ref_updates + .iter() + .map(|u| (u.ref_name.clone(), u.old_sha.clone(), u.new_sha.clone())) + .collect::>(); + let p2p_handle = state.p2p.clone(); + let pusher_did_clone = did.to_string(); + let db_for_peers = state.db.clone(); + let ref_update_tx = state.ref_update_tx.clone(); + let irys_url = state.config.irys_url.clone(); + let owner_did_for_arweave = record.owner_did.clone(); + let self_public_url = state.config.public_url.clone(); + let node_keypair = Arc::clone(&state.node_keypair); + let object_list_pinata = object_list; + let do_pinata_replication = withheld.is_some(); + tokio::spawn(async move { + let pinned = if do_pinata_replication { + crate::pinata::pin_new_objects( + &http_client, + &pinata_upload_url, + &pinata_jwt, + &repo_path_clone, + object_list_pinata, + &db_clone, + ) + .await + } else { + Vec::new() + }; - // Build sha→cid map from pinned objects - let cid_map: std::collections::HashMap = pinned.into_iter().collect(); + if !pinned.is_empty() { + tracing::info!(count = pinned.len(), "pinned git objects to Pinata"); + } - // Record branch→CID for each ref update and publish gossip - for (ref_name, old_sha, new_sha) in &ref_updates_clone { - let cid = cid_map.get(new_sha).map(|s| s.as_str()); + // Build sha→cid map from pinned objects + let cid_map: std::collections::HashMap = + pinned.into_iter().collect(); - if let Some(cid_str) = cid { - let _ = db_clone - .upsert_branch_cid(&repo_slug, ref_name, new_sha, cid_str, &node_did_str) - .await; + // Record branch→CID for each ref update and publish gossip + for (ref_name, old_sha, new_sha) in &ref_updates_clone { + let cid = cid_map.get(new_sha).map(|s| s.as_str()); + + if let Some(cid_str) = cid { + let _ = db_clone + .upsert_branch_cid( + &repo_slug, + ref_name, + new_sha, + cid_str, + &node_did_str, + ) + .await; + } + + if announce { + if let Some(p2p) = &p2p_handle { + p2p.publish_ref_update(crate::p2p::RefUpdateEvent { + node_did: node_did_str.clone(), + pusher_did: pusher_did_clone.clone(), + repo: repo_slug.clone(), + ref_name: ref_name.clone(), + old_sha: old_sha.clone(), + new_sha: new_sha.clone(), + timestamp: chrono::Utc::now().to_rfc3339(), + cert_id: None, + cid: cid.map(|s| s.to_string()), + }) + .await; + } + } } + // Broadcast ref update to GraphQL subscription listeners — one per ref. + // Gated on `announce`: /graphql/ws is unauthenticated (mounted after + // the optional_signature layer), and the subscription resolver has no + // caller to gate against, so only publicly-readable ref updates may + // reach anonymous subscribers. Mirrors the gossip (above) and Arweave + // (below) sends, which are already `announce`-gated. Without this a + // private-repo push would leak live ref metadata over the socket — + // the subscription analog of #112/#114. + let now_ts = chrono::Utc::now().to_rfc3339(); if announce { - if let Some(p2p) = &p2p_handle { - p2p.publish_ref_update(crate::p2p::RefUpdateEvent { - node_did: node_did_str.clone(), - pusher_did: pusher_did_clone.clone(), + for (ref_name, old_sha, new_sha) in &ref_updates_clone { + let _ = ref_update_tx.send(crate::state::RefUpdateBroadcast { repo: repo_slug.clone(), ref_name: ref_name.clone(), old_sha: old_sha.clone(), new_sha: new_sha.clone(), - timestamp: chrono::Utc::now().to_rfc3339(), - cert_id: None, - cid: cid.map(|s| s.to_string()), - }) - .await; + pusher_did: pusher_did_clone.clone(), + node_did: node_did_str.clone(), + timestamp: now_ts.clone(), + }); } } - } - - // Broadcast ref update to GraphQL subscription listeners — one per ref. - // Gated on `announce`: /graphql/ws is unauthenticated (mounted after - // the optional_signature layer), and the subscription resolver has no - // caller to gate against, so only publicly-readable ref updates may - // reach anonymous subscribers. Mirrors the gossip (above) and Arweave - // (below) sends, which are already `announce`-gated. Without this a - // private-repo push would leak live ref metadata over the socket — - // the subscription analog of #112/#114. - let now_ts = chrono::Utc::now().to_rfc3339(); - if announce { - for (ref_name, old_sha, new_sha) in &ref_updates_clone { - let _ = ref_update_tx.send(crate::state::RefUpdateBroadcast { - repo: repo_slug.clone(), - ref_name: ref_name.clone(), - old_sha: old_sha.clone(), - new_sha: new_sha.clone(), - pusher_did: pusher_did_clone.clone(), - node_did: node_did_str.clone(), - timestamp: now_ts.clone(), - }); - } - } - // Arweave permanent anchoring — fire for each ref update. - // Suppressed for repos the public cannot read (public permanent ledger). - if announce && !irys_url.is_empty() { - for (ref_name, old_sha, new_sha) in &ref_updates_clone { - let cid = cid_map.get(new_sha).cloned(); - let anchor = crate::arweave::RefAnchor { - repo: repo_slug.clone(), - owner_did: owner_did_for_arweave.clone(), - ref_name: ref_name.clone(), - old_sha: old_sha.clone(), - new_sha: new_sha.clone(), - cid: cid.clone(), - timestamp: now_ts.clone(), - node_did: node_did_str.clone(), - }; - match crate::arweave::anchor_ref_update(&http_client, &irys_url, &anchor).await - { - Ok(tx_id) if !tx_id.is_empty() => { - let arweave_url = crate::arweave::arweave_url(&tx_id); - let _ = db_clone - .record_arweave_anchor(&crate::db::RecordAnchorInput { - repo: &repo_slug, - owner_did: &owner_did_for_arweave, - ref_name, - old_sha, - new_sha, - cid: cid.as_deref(), - irys_tx_id: &tx_id, - arweave_url: &arweave_url, - node_did: &node_did_str, - }) - .await; + // Arweave permanent anchoring — fire for each ref update. + // Suppressed for repos the public cannot read (public permanent ledger). + if announce && !irys_url.is_empty() { + for (ref_name, old_sha, new_sha) in &ref_updates_clone { + let cid = cid_map.get(new_sha).cloned(); + let anchor = crate::arweave::RefAnchor { + repo: repo_slug.clone(), + owner_did: owner_did_for_arweave.clone(), + ref_name: ref_name.clone(), + old_sha: old_sha.clone(), + new_sha: new_sha.clone(), + cid: cid.clone(), + timestamp: now_ts.clone(), + node_did: node_did_str.clone(), + }; + match crate::arweave::anchor_ref_update(&http_client, &irys_url, &anchor) + .await + { + Ok(tx_id) if !tx_id.is_empty() => { + let arweave_url = crate::arweave::arweave_url(&tx_id); + let _ = db_clone + .record_arweave_anchor(&crate::db::RecordAnchorInput { + repo: &repo_slug, + owner_did: &owner_did_for_arweave, + ref_name, + old_sha, + new_sha, + cid: cid.as_deref(), + irys_tx_id: &tx_id, + arweave_url: &arweave_url, + node_did: &node_did_str, + }) + .await; + } + Ok(_) => {} + Err(e) => { + tracing::warn!(repo=%repo_slug, err=%e, "Arweave anchor failed") + } } - Ok(_) => {} - Err(e) => tracing::warn!(repo=%repo_slug, err=%e, "Arweave anchor failed"), } } - } - // HTTP peer notification — notify all known peers to pull from us. - // This is the reliable fallback when Gossipsub p2p is not yet connected. - // Suppressed for repos the public cannot read. Runs last so a slow or - // unreachable peer cannot delay the local GraphQL broadcast or Arweave - // anchoring above; this is the lowest-priority best-effort step. - if announce { - if let Ok(peers) = db_for_peers.list_peers().await { - for peer in peers { - if peer.http_url.is_empty() { - continue; - } - let peer_url = peer.http_url.trim_end_matches('/'); - if let Some(self_url) = self_public_url.as_deref() { - if peer_url == self_url.trim_end_matches('/') { + // HTTP peer notification — notify all known peers to pull from us. + // This is the reliable fallback when Gossipsub p2p is not yet connected. + // Suppressed for repos the public cannot read. Runs last so a slow or + // unreachable peer cannot delay the local GraphQL broadcast or Arweave + // anchoring above; this is the lowest-priority best-effort step. + if announce { + if let Ok(peers) = db_for_peers.list_peers().await { + for peer in peers { + if peer.http_url.is_empty() { continue; } + let peer_url = peer.http_url.trim_end_matches('/'); + if let Some(self_url) = self_public_url.as_deref() { + if peer_url == self_url.trim_end_matches('/') { + continue; + } + } + let notify_url = format!("{peer_url}{SYNC_NOTIFY_PATH}"); + notify_peer_of_refs( + &http_client, + node_keypair.as_ref(), + &peer.did, + ¬ify_url, + &repo_slug, + &ref_updates_clone, + &node_did_str, + &pusher_did_clone, + ) + .await; } - let notify_url = format!("{peer_url}{SYNC_NOTIFY_PATH}"); - notify_peer_of_refs( - &http_client, - node_keypair.as_ref(), - &peer.did, - ¬ify_url, - &repo_slug, - &ref_updates_clone, - &node_did_str, - &pusher_did_clone, - ) - .await; } } - } - }); - } + }); + } + }); Ok(result) } @@ -5179,9 +5201,23 @@ mod tests { assert_eq!(a.status(), 200, "push A lands 200 despite scan contention"); assert_eq!(b.status(), 200, "push B lands 200 despite scan contention"); - // Let the detached recipients walks drain through the same pool before - // reading the detector files. - tokio::time::sleep(std::time::Duration::from_millis(800)).await; + // Wait for both pushes' detached scan tails to drain through the pool of 1 + // before reading the detector files. The WHOLE tail (withheld walk included) now + // runs detached (#174 F2), so poll until every expected scan has run rather than a + // fixed sleep, which is load-sensitive under a parallel test run. + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); + loop { + let ran = std::fs::read_to_string(&ranfile) + .unwrap_or_default() + .lines() + .count(); + if ran >= 6 || std::time::Instant::now() >= deadline { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + // Small settle so the last scan's rmdir has landed before the overlap check. + tokio::time::sleep(std::time::Duration::from_millis(100)).await; assert!( !overlap.exists(), "with a scan pool of 1, no two scans' git may ever be alive at once \ @@ -5252,9 +5288,20 @@ mod tests { /// F4 scenario 4 — landed-push-never-fails: a push whose post-receive walk must /// park (pool held elsewhere) DEFERS and then returns the receive-pack success /// once admission frees; contention never converts the landed push into a 5xx. + /// #174 F2 (RED-before/GREEN-after): the post-receive replication tail parks on + /// `git_encrypt_semaphore` (withheld/candidate/full-scan resolution). Leaving it in the + /// request future means a client/proxy disconnect while parked silently loses this + /// push's pins, recovery copy, and announcements (state.rs documented this residual). + /// The fix moves the whole tail into an independently owned task, so the handler + /// returns its receive-pack 200 WITHOUT waiting on the scan pool and a disconnect can + /// no longer drop the work. + /// + /// Load-bearing: with the tail inline (pre-fix) the handler parks while the pool is + /// held and does NOT return within the bound (RED — the timeout fires). With the + /// detached tail it returns 200 promptly (GREEN). #[cfg(unix)] #[sqlx::test] - async fn receive_pack_landed_push_defers_on_scan_contention_never_errors(pool: sqlx::PgPool) { + async fn receive_pack_landed_push_returns_without_parking_on_scan_pool(pool: sqlx::PgPool) { use axum::extract::{Path, State}; use axum::Extension; use std::net::SocketAddr; @@ -5268,40 +5315,38 @@ mod tests { f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f4park", "p1", true).await; let sem = Arc::new(Semaphore::new(1)); state.git_encrypt_semaphore = sem.clone(); - // The test holds the pool's only permit: the push's withheld walk must park. + // Hold the pool's only permit: the post-receive scan would park if it ran in the + // request future. let held = sem.clone().acquire_owned().await.unwrap(); let peer: SocketAddr = "203.0.113.74:5000".parse().unwrap(); - let mut handle = tokio::spawn(git_receive_pack( - State(state), - Path(("z6f4park".to_string(), "p1".to_string())), - Extension(crate::auth::AuthenticatedDid( - "did:key:z6MkF4ParkPusherAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), - )), - crate::rate_limit::PeerAddr(Some(peer)), - axum::http::HeaderMap::new(), - ref_update_body("2222222222222222222222222222222222222222"), - )); - - // Parked, not errored: the handler must NOT complete while the pool is held. - let parked = tokio::time::timeout(std::time::Duration::from_millis(700), &mut handle).await; - assert!( - parked.is_err(), - "the post-receive walk must defer while the scan pool is held; got {parked:?}" - ); - - // Release admission: the SAME push now finishes with the receive-pack 200. - drop(held); - let resp = tokio::time::timeout(std::time::Duration::from_secs(30), handle) - .await - .expect("the deferred push must complete once admission frees") - .expect("the push task must not panic") - .expect("contention must never convert a landed push into an error"); + // The handler must return its receive-pack 200 WITHOUT waiting on the held scan + // pool — the tail is owned by a detached task. Pre-fix this times out. + let resp = tokio::time::timeout( + std::time::Duration::from_secs(5), + git_receive_pack( + State(state), + Path(("z6f4park".to_string(), "p1".to_string())), + Extension(crate::auth::AuthenticatedDid( + "did:key:z6MkF4ParkPusherAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + )), + crate::rate_limit::PeerAddr(Some(peer)), + axum::http::HeaderMap::new(), + ref_update_body("2222222222222222222222222222222222222222"), + ), + ) + .await + .expect("the handler must return without parking on the held scan pool") + .expect("contention must never convert a landed push into an error"); assert_eq!( resp.status(), 200, - "the response is the receive-pack success, not a contention 5xx" + "the response is the receive-pack success, returned before the detached tail runs" ); + + // The detached tail is still owned: release admission and let it drain cleanly. + drop(held); + tokio::time::sleep(std::time::Duration::from_millis(300)).await; } // ---- #174 U4 (P2-2): post-push encryption task set bounded by per-repo coalescing ---- diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index c4d2b079..8396a369 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -449,13 +449,13 @@ impl Drop for EncryptInflightGuard { /// recovery copy or silently under-pin it. The returned permit must move into /// the blocking closure so a started scan always completes holding it (a /// disconnect cannot cancel `spawn_blocking` or leak the permit mid-walk). -/// Accepted residuals, stated once for every caller: (1) the park wait is -/// queue-depth multiplied — post-receive tails are no longer admission-bounded -/// once the write permit is released, so N landed pushes can queue N scans and -/// the last waits N scan-durations; (2) a client-timeout disconnect while -/// parked HERE drops the handler future and silently loses this push's -/// replication work — the `encrypt_inflight` coalescing requeue does NOT cover -/// it, because this park precedes the `try_begin` spawn gate. +/// Accepted residual, stated once for every caller: the park wait is queue-depth +/// multiplied — post-receive tails are no longer admission-bounded once the write +/// permit is released, so N landed pushes can queue N scans and the last waits N +/// scan-durations. A client-timeout disconnect no longer loses the work (#174 F2): +/// the whole post-receive replication tail runs in an independently owned task, so +/// dropping the request future cannot drop this parked scan — the park no longer +/// precedes any durable-record gate in a cancellable future. pub async fn acquire_scan_permit( scan_sem: Arc, repo: &std::path::Path, From 759c9b6aa387f15cc0f2f88ed3dfb1f0d13a8cc8 Mon Sep 17 00:00:00 2001 From: t Date: Sat, 18 Jul 2026 10:36:16 -0500 Subject: [PATCH 053/124] fix(node): bound the /ipfs cat-file probe and content read (#174 F3) The GET /ipfs/{cid} object-type probe and content read ran bare synchronous Command::output() on the async worker, budget-checked only before they started. A hung or corrupt object store therefore pinned a Tokio runtime worker and both held IPFS admission permits indefinitely, so enough requests exhausted the route despite GITLAWB_IPFS_REQUEST_BUDGET_SECS. The walk beside them already used spawn_blocking + the reaped bounded runner; these two stages were left bare. Extract run_bounded_git_raw (returns ExitStatus/stdout/stderr so callers can classify exit codes; run_bounded_git stays a thin bail-on-nonzero wrapper for the walk callers) and add store::object_type_bounded / read_object_content_bounded on it, preserving object_type's Ok(None)-on-absence vs Err classification so the serve path's 404-vs-503 semantics are unchanged. Run both /ipfs stages in spawn_blocking under a deadline clamped to min(git timeout, remaining budget), with the IPFS permits held across the awaited spawn_blocking. Test: get_by_cid_hung_probe_is_reaped_and_sheds_503 hangs git on a FIFO alternates with no feeder; the reaped probe sheds a 503 in bounded time (GREEN), while a neutralized deadline blocks the handler forever (RED). The existing probe-error / corrupt-repo tests still assert the absence-vs-error classification at the handler. --- crates/gitlawb-node/src/api/ipfs.rs | 158 +++++++++++------- crates/gitlawb-node/src/git/store.rs | 56 +++++++ .../gitlawb-node/src/git/visibility_pack.rs | 39 ++++- 3 files changed, 185 insertions(+), 68 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 41080833..43e418ae 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -292,32 +292,52 @@ pub async fn get_by_cid( } }; - // Budget gate for the probe stage (F3). The probe subprocess has no - // internal duration clamp, so this pre-start check is its entire bound. - if budget_gate( + // Budget gate for the probe stage (F3): a probe is never STARTED with zero + // remaining, and a started probe now runs its git child under a deadline + // clamped to the remainder (below), so it can never complete past the budget. + let Some(probe_budget) = budget_gate( &mut truncated_by, request_deadline, state.config.ipfs_request_budget_secs, &repo.name, "object-type probe", - ) - .is_none() - { + ) else { break; - } + }; // Check whether the object exists in this repo before any expensive // reachability walk. This prevents random-CID spray from triggering // full-history git walks on repos that don't carry the object. Absent - // (`Ok(None)`) is a VERDICT; a probe that could not run is not. - let obj_type = match store::object_type(&repo_path, &sha256_hex) { - Ok(Some(t)) => t, - Ok(None) => continue, - Err(e) => { + // (`Ok(None)`) is a VERDICT; a probe that could not run is not. The + // `git cat-file -t` shells out, so run it OFF the async worker under the + // reaped bounded runner (#174 F3) — a hung/corrupt object store cannot pin a + // runtime worker or the held IPFS permits past the deadline. + let probe_deadline = std::time::Instant::now() + + std::cmp::min( + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + probe_budget, + ); + // The probe shells to the real `git` (as `object_type` historically did), + // independent of `state.git_bin` (which tests point at a fake walk git). + let probe_path = repo_path.clone(); + let probe_sha = sha256_hex.clone(); + let obj_type = match tokio::task::spawn_blocking(move || { + store::object_type_bounded("git", &probe_path, &probe_sha, probe_deadline) + }) + .await + { + Ok(Ok(Some(t))) => t, + Ok(Ok(None)) => continue, + Ok(Err(e)) => { tracing::warn!(repo = %repo.name, err = %e, "object-type probe failed during /ipfs scan; skipping repo without a verdict"); taint(&mut truncated_by, "probe"); continue; } + Err(join_err) => { + tracing::warn!(repo = %repo.name, err = %join_err, "object-type probe task panicked during /ipfs scan; skipping repo without a verdict"); + taint(&mut truncated_by, "probe"); + continue; + } }; // Per-blob gating only applies when a path-scoped rule exists (KTD4). @@ -413,28 +433,52 @@ pub async fn get_by_cid( // serving keeps the terminal arm honest and the stop unconditional; the // retryable 503 tells the caller to come back rather than letting an // over-budget request keep spending. - if budget_gate( + let Some(read_budget) = budget_gate( &mut truncated_by, request_deadline, state.config.ipfs_request_budget_secs, &repo.name, "content read", - ) - .is_none() - { + ) else { break; - } + }; // Now that we've passed the gate, read the content. A failed read after a // passed gate is not an absence verdict — the probe just said the object - // exists here — so the skip taints the scan. - let content = match store::read_object_content(&repo_path, &sha256_hex, &obj_type) { - Ok(c) => c, - Err(e) => { + // exists here — so the skip taints the scan. Like the probe, the read shells + // out to `git cat-file `, so run it OFF the async worker under the reaped + // bounded runner clamped to the remaining budget (#174 F3). + let read_deadline = std::time::Instant::now() + + std::cmp::min( + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + read_budget, + ); + // Real `git`, as the read historically used, independent of `state.git_bin`. + let read_path = repo_path.clone(); + let read_sha = sha256_hex.clone(); + let read_type = obj_type.clone(); + let content = match tokio::task::spawn_blocking(move || { + store::read_object_content_bounded( + "git", + &read_path, + &read_sha, + &read_type, + read_deadline, + ) + }) + .await + { + Ok(Ok(c)) => c, + Ok(Err(e)) => { tracing::warn!(repo = %repo.name, err = %e, "object content read failed during /ipfs scan; skipping repo without a verdict"); taint(&mut truncated_by, "read"); continue; } + Err(join_err) => { + tracing::warn!(repo = %repo.name, err = %join_err, "object content read task panicked during /ipfs scan; skipping repo without a verdict"); + taint(&mut truncated_by, "read"); + continue; + } }; // 3. Return the content with IPFS-style headers @@ -1539,22 +1583,22 @@ mod tests { ); } - /// F3 expiry between probe and walk: an object PROBED PRESENT in a - /// path-scoped repo is still not served once the budget is gone; the - /// walk-stage gate taints and stops before any walk starts. The probe is - /// stalled past the 1s budget while still SUCCEEDING via a FIFO at - /// `objects/info/alternates`: real `git cat-file -t` blocks opening it at - /// odb setup until a plain OS thread feeds it an empty alternates list at - /// ~2s, after which the probe finds the genuine loose object and reports - /// "blob". The walk-needing repo thus reaches the walk gate with zero - /// remaining: 503 naming the budget, never a 404 (the walk-gate deny of the - /// fake git's empty allowed-set would have been a verdict) and never a - /// serve, and the fake-git walk log stays EMPTY (no walk ever started). - /// MUTATION (RED): drop the walk-stage budget gate and this decays to the - /// walked 404 with one logged walk. + /// #174 F3 hung-probe reap (RED-before/GREEN-after): the `git cat-file -t` + /// probe runs OFF the async worker under the reaped bounded runner, so a hung or + /// corrupt object store cannot pin a runtime worker or the held IPFS permits. + /// `objects/info/alternates` is a FIFO with no writer, so real `git cat-file -t` + /// blocks at odb setup forever. With the probe bounded to + /// `min(git_service_timeout, remaining budget)` (~1s here), the watchdog tears the + /// git process group down at the deadline and the probe returns Err — a taint, not + /// a verdict — so the scan sheds a retryable 503 naming the probe, no walk ever + /// starts, and the whole request returns in bounded time. + /// + /// Load-bearing: with the probe on the bare async worker (pre-fix) this FIFO blocks + /// the handler forever (no feeder frees it) and the request hangs — the wrapping + /// timeout fires (RED). With the reaped bounded probe it returns 503 promptly. #[cfg(unix)] #[sqlx::test] - async fn get_by_cid_budget_expiry_between_probe_and_walk_sheds_503(pool: sqlx::PgPool) { + async fn get_by_cid_hung_probe_is_reaped_and_sheds_503(pool: sqlx::PgPool) { let tmp = tempfile::TempDir::new().unwrap(); let mut state = crate::test_support::test_state(pool.clone()).await; let repos_dir = tmp.path().join("repos"); @@ -1587,12 +1631,10 @@ mod tests { .await .unwrap(); - // Stall the REAL-git probe past the budget while still letting it - // succeed: `objects/info/alternates` as a FIFO blocks cat-file's odb - // setup until a writer appears. The feeder thread supplies an EMPTY - // alternates list (open + close, no bytes) after 2s, so the probe then - // resolves the genuine loose object. Non-blocking open with retries so - // a handler that never probes cannot wedge the feeder forever. + // Hang the REAL-git probe indefinitely: `objects/info/alternates` as a FIFO + // with no writer blocks `git cat-file -t` at odb setup forever. There is no + // feeder — the reaped bounded runner must tear the git process group down at + // the deadline; a bare unbounded probe would block the handler here. let rec = state .db .get_repo("z6f3probe", "gated") @@ -1611,28 +1653,16 @@ mod tests { 0, "mkfifo(objects/info/alternates) must succeed" ); - let feeder = std::thread::spawn(move || { - use std::os::unix::fs::OpenOptionsExt; - std::thread::sleep(std::time::Duration::from_secs(2)); - for _ in 0..300 { - if std::fs::OpenOptions::new() - .write(true) - .custom_flags(libc::O_NONBLOCK) - .open(&fifo) - .is_ok() - { - return; - } - std::thread::sleep(std::time::Duration::from_millis(50)); - } - }); - let peer: SocketAddr = "203.0.113.72:5000".parse().unwrap(); - let resp = ipfs_router(state) - .oneshot(get_cid(&cid_for_oid(&oid), Some(peer))) - .await - .unwrap(); - feeder.join().unwrap(); + // The request must return in bounded time: the reaped probe sheds a 503; a + // bare unbounded probe would block on the FIFO forever (no feeder frees it). + let resp = tokio::time::timeout( + std::time::Duration::from_secs(15), + ipfs_router(state).oneshot(get_cid(&cid_for_oid(&oid), Some(peer))), + ) + .await + .expect("the hung probe must be reaped, not block the handler") + .unwrap(); assert_eq!( resp.status(), StatusCode::SERVICE_UNAVAILABLE, @@ -1651,8 +1681,8 @@ mod tests { .unwrap(); let body = String::from_utf8_lossy(&body); assert!( - body.contains("budget"), - "the shed must name the budget taint; got: {body}" + body.contains("probe"), + "the shed must name the reaped-probe taint; got: {body}" ); let walks = std::fs::read_to_string(&walk_log) .map(|s| s.lines().count()) diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index 5697c229..d5a0d1a1 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -320,6 +320,62 @@ pub fn read_object_content(repo_path: &Path, sha256_hex: &str, obj_type: &str) - Ok(content_output.stdout) } +/// Bounded, reaped variant of [`object_type`] for the async `/ipfs` serve path +/// (#174 F3): runs `git cat-file -t` off the caller's runtime through the +/// process-group + watchdog reaper, so a hung or corrupt object store cannot pin a +/// runtime worker or an IPFS admission permit past `deadline`. Exit classification +/// is identical to [`object_type`] — a missing object is `Ok(None)`, a store-access +/// failure is `Err` — so the serve path's 404-vs-503 semantics are unchanged. +pub fn object_type_bounded( + git_bin: &str, + repo_path: &Path, + sha256_hex: &str, + deadline: std::time::Instant, +) -> Result> { + let (status, stdout, stderr) = crate::git::visibility_pack::run_bounded_git_raw( + git_bin, + &["cat-file", "-t", sha256_hex], + repo_path, + &[], + deadline, + )?; + if !status.success() { + // Same classification as `object_type` (a nonzero exit is an ABSENCE verdict + // only when git could examine the object store); U5 refines the absence arm + // with an out-of-band readability check. + let stderr = String::from_utf8_lossy(&stderr); + if stderr.contains("not a git repository") + || stderr.lines().any(|l| l.starts_with("error:")) + { + bail!("git cat-file -t failed: {}", stderr.trim()); + } + return Ok(None); + } + Ok(Some(String::from_utf8_lossy(&stdout).trim().to_string())) +} + +/// Bounded, reaped variant of [`read_object_content`] for the async `/ipfs` serve +/// path (#174 F3). Same teardown guarantees as [`object_type_bounded`]. +pub fn read_object_content_bounded( + git_bin: &str, + repo_path: &Path, + sha256_hex: &str, + obj_type: &str, + deadline: std::time::Instant, +) -> Result> { + let (status, stdout, stderr) = crate::git::visibility_pack::run_bounded_git_raw( + git_bin, + &["cat-file", obj_type, sha256_hex], + repo_path, + &[], + deadline, + )?; + if !status.success() { + bail!("git cat-file failed: {}", String::from_utf8_lossy(&stderr)); + } + Ok(stdout) +} + /// Read a git object by its SHA-256 hex object ID. /// /// Returns `(object_type, content_bytes)` where `content_bytes` is the raw diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index 9f07020d..e69c4ceb 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -67,13 +67,13 @@ fn child_terminated_without_reaping(pid: i32) -> bool { } #[cfg(unix)] -pub(crate) fn run_bounded_git( +pub(crate) fn run_bounded_git_raw( git_bin: &str, args: &[&str], repo_path: &Path, stdin_bytes: &[u8], deadline: Instant, -) -> Result> { +) -> Result<(std::process::ExitStatus, Vec, Vec)> { use std::io::{Read, Write}; use std::os::unix::process::CommandExt; use std::sync::mpsc::RecvTimeoutError; @@ -191,6 +191,23 @@ pub(crate) fn run_bounded_git( if killed && !status.success() { return Err(crate::git::smart_http::GitServiceTimeout.into()); } + Ok((status, out, err)) +} + +/// Bounded git returning only stdout, `bail!`ing on any nonzero exit. The thin +/// wrapper the walk callers use. Probes that must distinguish exit classes — +/// `git cat-file` absence vs an object-store access failure — call +/// [`run_bounded_git_raw`] and classify the status/stderr themselves. +#[cfg(unix)] +pub(crate) fn run_bounded_git( + git_bin: &str, + args: &[&str], + repo_path: &Path, + stdin_bytes: &[u8], + deadline: Instant, +) -> Result> { + let label = args.first().copied().unwrap_or("git"); + let (status, out, err) = run_bounded_git_raw(git_bin, args, repo_path, stdin_bytes, deadline)?; if !status.success() { anyhow::bail!("git {label} failed: {}", String::from_utf8_lossy(&err)); } @@ -210,13 +227,13 @@ pub(crate) fn run_bounded_git( /// the Unix version's signature and result semantics so every caller compiles on all /// targets (#174). #[cfg(not(unix))] -pub(crate) fn run_bounded_git( +pub(crate) fn run_bounded_git_raw( git_bin: &str, args: &[&str], repo_path: &Path, stdin_bytes: &[u8], deadline: Instant, -) -> Result> { +) -> Result<(std::process::ExitStatus, Vec, Vec)> { use std::io::{Read, Write}; use std::sync::mpsc::RecvTimeoutError; @@ -283,6 +300,20 @@ pub(crate) fn run_bounded_git( if killed && !status.success() { return Err(crate::git::smart_http::GitServiceTimeout.into()); } + Ok((status, out, err)) +} + +/// Non-Unix thin wrapper matching the Unix [`run_bounded_git`] semantics. +#[cfg(not(unix))] +pub(crate) fn run_bounded_git( + git_bin: &str, + args: &[&str], + repo_path: &Path, + stdin_bytes: &[u8], + deadline: Instant, +) -> Result> { + let label = args.first().copied().unwrap_or("git"); + let (status, out, err) = run_bounded_git_raw(git_bin, args, repo_path, stdin_bytes, deadline)?; if !status.success() { anyhow::bail!("git {label} failed: {}", String::from_utf8_lossy(&err)); } From 6dc2b2e912587b14d1b80c8db87426231d29be53 Mon Sep 17 00:00:00 2001 From: t Date: Sat, 18 Jul 2026 10:49:18 -0500 Subject: [PATCH 054/124] fix(node): share one whole-scan deadline across both full-scan phases (#174 F4) fail_closed_full_scan_objects ran phase 1 (replicable_blob_set_bounded) then phase 2 (all_blob_oids) with a FRESH Instant::now() + timeout for phase 2, under one held scan permit. A large-but-successful phase 1 plus a full phase 2 held the permit ~2x the configured budget instead of the one whole-scan budget. Compute one deadline before phase 1 and share it: phase 2 takes the shared Instant directly, phase 1 takes the remaining-until-deadline. Total occupancy is now ~1x. The completeness cost is honest: a repo whose phase 1 nears the budget under-pins this push (fail closed) rather than over-holding, so size the budget so both phases normally fit. Test: full_scan_shares_one_deadline_across_both_phases drives a slow-but- successful phase 1 that eats the budget; the shared deadline reaps phase 2 and the scan fails closed (empty). RED before: phase 2's fresh budget completed and kept the non-blob candidate. --- crates/gitlawb-node/src/api/repos.rs | 64 +++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 2 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 98e395f6..ede6c5e1 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -139,10 +139,25 @@ async fn fail_closed_full_scan_objects( crate::state::acquire_scan_permit(encrypt_sem, &disk_path, "fail-closed full scan").await; tokio::task::spawn_blocking(move || -> anyhow::Result> { let _permit = permit; + // One whole-scan deadline shared across both phases (#174 F4). A fresh + // `Instant::now() + timeout` for phase 2 let a large-but-successful phase 1 plus + // a full phase 2 hold the scan permit ~2x the configured budget. Sharing the + // deadline caps total occupancy at ~1x: phase 1 runs against the remaining + // budget, and if it consumes the budget phase 2 gets what is left and fails + // closed (pins nothing) rather than over-holding — the safe direction. The cost + // is honest: a genuinely large repo whose phase 1 nears the budget under-pins + // this push rather than the previous silent ~2x hold; size the budget so both + // phases normally fit. + let deadline = std::time::Instant::now() + timeout; let allowed = crate::git::visibility_pack::replicable_blob_set_bounded( - &disk_path, &git_bin, timeout, &rules, is_public, &owner_did, + &disk_path, + &git_bin, + deadline.saturating_duration_since(std::time::Instant::now()), + &rules, + is_public, + &owner_did, )?; - let all_blobs = crate::git::push_delta::all_blob_oids(&disk_path, &git_bin, std::time::Instant::now() + timeout)?; + let all_blobs = crate::git::push_delta::all_blob_oids(&disk_path, &git_bin, deadline)?; Ok(crate::git::visibility_pack::replicable_objects_fail_closed( candidates, &allowed, &all_blobs, )) @@ -5036,6 +5051,51 @@ mod tests { ); } + /// #174 F4 (RED-before/GREEN-after): the two full-scan phases share ONE whole-scan + /// deadline. Phase 1 (`replicable_blob_set_bounded`) succeeds but consumes almost + /// the whole budget; phase 2 (`all_blob_oids`) then gets only the remainder. With a + /// shared deadline phase 2 is reaped and the scan fails closed (pins nothing) — the + /// safe direction. With a FRESH `Instant::now() + timeout` for phase 2 (pre-fix) it + /// gets a full second budget, completes with an empty blob set, and the non-blob + /// candidate is kept — so the result is NON-empty (RED) and the permit is held ~2x. + #[cfg(unix)] + #[tokio::test] + async fn full_scan_shares_one_deadline_across_both_phases() { + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + // Phase 1 (ls-tree) sleeps 1.5s and succeeds (empty tree); phase 2 + // (cat-file --batch-all-objects) sleeps 1.5s. With a 2s whole-scan budget the + // shared deadline leaves phase 2 only ~0.5s, so it is reaped; a fresh 2s budget + // would let it finish. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-parse) echo deadbeef ;;\n rev-list) echo aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ;;\n ls-tree) sleep 1.5 ;;\n cat-file) case \"$*\" in *--batch-all-objects*) sleep 1.5 ;; *) : ;; esac ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + // A candidate that is NOT a blob (never appears in all_blob_oids): kept by + // replicable_objects_fail_closed only if phase 2 actually ran to completion. + let candidates = vec!["cccccccccccccccccccccccccccccccccccccccc".to_string()]; + + let sem: Arc = Arc::new(Semaphore::new(1)); + let objs = fail_closed_full_scan_objects( + sem, + tmp.path().to_path_buf(), + vec![vis_rule("/secret/**", &[])], + true, + OWNER_DID.to_string(), + candidates, + git_bin, + Duration::from_secs(2), + ) + .await; + assert!( + objs.is_empty(), + "a large-but-successful phase 1 must leave phase 2 only the SHARED whole-scan \ + remainder, so it reaps and the scan fails closed; got {objs:?} (a fresh phase-2 \ + budget completed the scan and kept the candidate — the ~2x-budget bug)" + ); + } + /// Shared fixture for the F4 handler-layer tests: a state whose repo_store and /// git_bin point at the given tempdir/fake-git, plus a seeded on-disk repo, /// optionally with a path-scoped rule (so the post-receive walks actually run). From d39fc8de9cf5a3f15c42d23675fc5741f0412302 Mon Sep 17 00:00:00 2001 From: t Date: Sat, 18 Jul 2026 10:57:16 -0500 Subject: [PATCH 055/124] fix(node): disambiguate absence from an unreadable object store in the /ipfs probe (#174 F5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit object_type_bounded treated any bare `fatal:` cat-file exit as absence. A packed object whose pack/idx is transiently unreadable (permissions, or a mid-repack race) emits `could not get object info` — byte-identical to a genuine miss — so a present object could return Ok(None) and the /ipfs scan a definitive 404 instead of the intended retryable 503. The stderr cannot separate the two (the CID is pre-validated, so that string is also the only legitimate absence path), so the disambiguation must be out of band. On the collided fatal, probe object_store_readable (objects/ listable, every pack/idx openable). Unreadable -> Err (taint -> 503). Readable -> re-probe once; still absent -> Ok(None), now present -> the type. This narrows but does not close the concurrent-repack TOCTOU (the check samples a different instant), stated honestly rather than claimed deterministic. Test: object_type_bounded_unreadable_pack_is_error_not_absence packs a real repo, chmod 000 the pack, and asserts Err (root-guarded). RED with the readability check disabled: the present blob returned Ok(None). --- crates/gitlawb-node/src/git/store.rs | 199 +++++++++++++++++++++++++-- 1 file changed, 188 insertions(+), 11 deletions(-) diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index d5a0d1a1..f9d65e9d 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -339,19 +339,86 @@ pub fn object_type_bounded( &[], deadline, )?; - if !status.success() { - // Same classification as `object_type` (a nonzero exit is an ABSENCE verdict - // only when git could examine the object store); U5 refines the absence arm - // with an out-of-band readability check. - let stderr = String::from_utf8_lossy(&stderr); - if stderr.contains("not a git repository") - || stderr.lines().any(|l| l.starts_with("error:")) - { - bail!("git cat-file -t failed: {}", stderr.trim()); + if status.success() { + return Ok(Some(String::from_utf8_lossy(&stdout).trim().to_string())); + } + // A nonzero exit whose stderr proves git could not EXAMINE the object store — + // "not a git repository", or any `error:` line (corrupt loose object, bad idx) — + // is not an absence verdict, so surface it as Err (#173/#174: never a false 404). + let stderr = String::from_utf8_lossy(&stderr); + if stderr.contains("not a git repository") || stderr.lines().any(|l| l.starts_with("error:")) { + bail!("git cat-file -t failed: {}", stderr.trim()); + } + // Any other bare `fatal:` ("could not get object info") is the absence-vs- + // unreadable-pack COLLISION (#174 F5): a genuinely missing object and a packed + // object whose pack/idx is unreadable (permissions, or a mid-repack race) emit the + // identical string. Disambiguate OUT OF BAND: if the object store is not readable, + // this is not an absence verdict -> Err (taint -> retryable 503). + if !object_store_readable(repo_path) { + bail!( + "git cat-file -t inconclusive: object store not readable at {} (not an absence verdict)", + repo_path.display() + ); + } + // Store readable: re-probe once. An object still absent on a confirmed-readable + // store is very likely truly absent (Ok(None)); if a mid-repack race resolved and + // the re-probe now finds it, return the type. This narrows, but cannot fully close, + // the concurrent-repack window (the readability check samples a different instant). + let (status2, stdout2, stderr2) = crate::git::visibility_pack::run_bounded_git_raw( + git_bin, + &["cat-file", "-t", sha256_hex], + repo_path, + &[], + deadline, + )?; + if status2.success() { + return Ok(Some(String::from_utf8_lossy(&stdout2).trim().to_string())); + } + let stderr2 = String::from_utf8_lossy(&stderr2); + if stderr2.contains("not a git repository") || stderr2.lines().any(|l| l.starts_with("error:")) + { + bail!("git cat-file -t failed: {}", stderr2.trim()); + } + Ok(None) +} + +/// Best-effort check that a repo's object store is readable, used to disambiguate a +/// genuine missing-object `git cat-file` fatal from an unreadable or racing pack +/// (both emit "could not get object info"). Returns false on any unreadable +/// `objects/` dir or any pack/idx that cannot be opened (EACCES / EIO), so the +/// caller surfaces an error rather than a false absence. Cheap — a couple of readdir +/// plus open probes. It narrows, but does not close, the concurrent-repack TOCTOU: it +/// samples a different instant than the failing cat-file. +fn object_store_readable(repo_path: &Path) -> bool { + let objects = repo_path.join("objects"); + // The objects dir itself must be listable; drain the iterator so a mid-listing + // EACCES/EIO surfaces, not just the initial open. + let Ok(entries) = std::fs::read_dir(&objects) else { + return false; + }; + for entry in entries { + if entry.is_err() { + return false; } - return Ok(None); } - Ok(Some(String::from_utf8_lossy(&stdout).trim().to_string())) + // Every pack file and its index must be openable for read. A loose-only store + // (no pack dir) is fine — the objects readdir above already proved reachability. + if let Ok(pack_entries) = std::fs::read_dir(objects.join("pack")) { + for entry in pack_entries { + let Ok(entry) = entry else { + return false; + }; + let path = entry.path(); + if matches!( + path.extension().and_then(|s| s.to_str()), + Some("pack") | Some("idx") + ) && std::fs::File::open(&path).is_err() + { + return false; + } + } + } + true } /// Bounded, reaped variant of [`read_object_content`] for the async `/ipfs` serve @@ -571,4 +638,114 @@ mod tests { "unchanged file must not appear: {names:?}" ); } + + /// #174 F5 (RED-before/GREEN-after): a packed object whose pack/idx is unreadable + /// makes `git cat-file -t` emit "could not get object info" — byte-identical to a + /// genuine miss. `object_type_bounded` must report absence ONLY when the object + /// store is confirmed readable; an unreadable store is Err (-> retryable 503), + /// never Ok(None) (-> a wrong 404 for a present object). + #[cfg(unix)] + #[test] + fn object_type_bounded_unreadable_pack_is_error_not_absence() { + use std::os::unix::fs::PermissionsExt; + let td = tempfile::TempDir::new().unwrap(); + let work = td.path().join("work"); + let bare = td.path().join("bare.git"); + std::fs::create_dir_all(&work).unwrap(); + let g = |args: &[&str], dir: &Path| { + assert!( + Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + g(&["init", "-q", "--object-format=sha256", "."], &work); + g(&["config", "user.email", "t@t"], &work); + g(&["config", "user.name", "t"], &work); + std::fs::write(work.join("file.txt"), b"packed f5 content\n").unwrap(); + g(&["add", "file.txt"], &work); + g(&["commit", "-qm", "c1"], &work); + let blob = String::from_utf8( + Command::new("git") + .args(["rev-parse", "HEAD:file.txt"]) + .current_dir(&work) + .output() + .unwrap() + .stdout, + ) + .unwrap() + .trim() + .to_string(); + g( + &[ + "clone", + "-q", + "--bare", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + td.path(), + ); + g(&["gc", "-q"], &bare); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + + // Readable store: the packed blob probes present; a genuine miss is Ok(None). + assert_eq!( + super::object_type_bounded("git", &bare, &blob, deadline) + .unwrap() + .as_deref(), + Some("blob"), + "a packed blob on a readable store must probe present" + ); + assert!( + super::object_type_bounded("git", &bare, &"0".repeat(64), deadline) + .unwrap() + .is_none(), + "a genuinely-absent object on a readable store must be Ok(None)" + ); + + // Make the pack unreadable: cat-file -t now emits the collided fatal for the + // PRESENT blob. It must surface as Err, not a false Ok(None). + let pack_dir = bare.join("objects").join("pack"); + let set_pack_mode = |mode: u32| { + for e in std::fs::read_dir(&pack_dir).unwrap() { + let p = e.unwrap().path(); + if matches!( + p.extension().and_then(|s| s.to_str()), + Some("pack") | Some("idx") + ) { + let mut perms = std::fs::metadata(&p).unwrap().permissions(); + perms.set_mode(mode); + std::fs::set_permissions(&p, perms).unwrap(); + } + } + }; + set_pack_mode(0o000); + // Root bypasses file permissions, so the chmod won't block reads there; only + // assert the error path when the pack is genuinely unreadable to this process. + let a_pack = std::fs::read_dir(&pack_dir) + .unwrap() + .filter_map(|e| e.ok()) + .map(|e| e.path()) + .find(|p| p.extension().and_then(|s| s.to_str()) == Some("pack")); + let genuinely_unreadable = a_pack + .as_ref() + .map(|p| std::fs::File::open(p).is_err()) + .unwrap_or(false); + let res = super::object_type_bounded("git", &bare, &blob, deadline); + set_pack_mode(0o644); // restore so TempDir cleanup succeeds + + if genuinely_unreadable { + assert!( + res.is_err(), + "an unreadable pack must surface as Err (-> retryable 503), not Ok(None) \ + (-> a wrong 404 for a present object); got {res:?}" + ); + } + } } From 448d38458defead064141573626410a3d68ff9bd Mon Sep 17 00:00:00 2001 From: t Date: Sat, 18 Jul 2026 11:13:32 -0500 Subject: [PATCH 056/124] fix(node): bound concurrent pin-loop memory; document the per-repo task cap (#174 F6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EncryptInflight bounds the outstanding post-push encryption-task set to one per repo, but its state.rs doc implied a global cap it does not deliver, and each pin loop (ipfs_pin + pinata pin_new_objects) holds a full per-push object-id list while walking it — so N distinct authenticated first-pushes could hold N MB-scale lists at once, unbounded. Add a global pin_semaphore (GITLAWB_MAX_CONCURRENT_PIN_TASKS, default 8) and gate BOTH pin loops through it: they DEFER when the pool is full, never drop a pin. Tighten the state.rs comment to state the bound is PER-REPO, with the cross-repo residual (throttled by auth + rate limits, memory bounded by the pin permit) called out honestly. Also correct .env.example GITLAWB_DB_MAX_CONNECTIONS (20 -> 48): with the #174 F1 boot validation, 20 is below max_concurrent_git_pushes(32)+headroom(8) and would fail startup — closing the INV-24 doc gap from the F1 change. Tests: pin_new_objects_gated_defers_when_pin_pool_exhausted (RED without the permit acquire); max_concurrent_pin_tasks range/default. --- .env.example | 14 ++++- crates/gitlawb-node/src/api/repos.rs | 83 ++++++++++++++++++++++++- crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/config.rs | 38 +++++++++++ crates/gitlawb-node/src/main.rs | 3 + crates/gitlawb-node/src/state.rs | 29 ++++++--- crates/gitlawb-node/src/test_support.rs | 1 + 7 files changed, 155 insertions(+), 14 deletions(-) diff --git a/.env.example b/.env.example index 8f9bfa2e..9913b41b 100644 --- a/.env.example +++ b/.env.example @@ -24,8 +24,11 @@ DATABASE_URL=postgresql://gitlawb:changeme@localhost:5432/gitlawb # ── Database pool & startup resilience ──────────────────────────────────── # Maximum connections in the PostgreSQL pool. A cap, not a floor — # connections open lazily. Size against the DB server's max_connections, -# remembering admin tooling opens its own pool. -GITLAWB_DB_MAX_CONNECTIONS=20 +# remembering admin tooling opens its own pool. Each concurrent write pins one +# connection for its whole duration (the connection-affine advisory lock), so the +# node REJECTS at boot any value below GITLAWB_MAX_CONCURRENT_GIT_PUSHES + 8 +# headroom — keep this comfortably above that (default 48 for pushes 32). +GITLAWB_DB_MAX_CONNECTIONS=48 # Seconds a request waits for a pool connection before failing with 503. GITLAWB_DB_ACQUIRE_TIMEOUT_SECS=5 # Upper bound on each startup connect+migrate attempt, in seconds. Keep it @@ -141,6 +144,13 @@ GITLAWB_MAX_CONCURRENT_GIT_OPS=128 # Retry-After. Default 32. GITLAWB_MAX_CONCURRENT_GIT_PUSHES=32 +# Max concurrent post-push pin loops (IPFS + Pinata pin_new_objects) across all +# repos. Each loop holds a full per-push object-id list while pinning, so this +# bounds that MB-scale memory even though the per-repo encrypt-task set already +# caps the task COUNT. A loop DEFERS (waits) when the pool is full, never drops a +# pin. Default 8. +GITLAWB_MAX_CONCURRENT_PIN_TASKS=8 + # Max concurrent read ops (upload-pack + the upload-pack info/refs advertisement) # a single caller may hold, so one caller cannot monopolize the read pool. Keyed # on the resolved SOURCE IP, never the DID: a signature does not move a caller off diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index ede6c5e1..6e6d5f3a 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -849,6 +849,7 @@ struct EncryptTaskCtx { git_bin: String, git_timeout: std::time::Duration, encrypt_sem: Arc, + pin_sem: Arc, } /// The detached post-push pin/encrypt task (#174 P2-2 + F5): run this push's own @@ -990,14 +991,40 @@ async fn resolve_drain_object_list( /// The pin/encrypt pipeline shared by the snapshot iteration and the /// coalesced-drain iterations: local IPFS pin, then (path-scoped rules only) the /// admission-gated recipients walk → encrypt-then-pin → Arweave manifest anchor. +/// Pin `object_list` to the local IPFS node under the global pin-admission permit +/// (#174 F6). `EncryptInflight` bounds the pin-task COUNT to one per repo, but each +/// pin loop holds a full per-push object-id list while walking it; this permit bounds +/// how many such MB-scale lists are held at once across all repos. DEFERS (waits) when +/// the pool is full — never drops, since a dropped pin loses the replication copy. +async fn pin_new_objects_gated( + pin_sem: &Arc, + ipfs_api: &str, + repo_path: &std::path::Path, + object_list: Vec, + db: &Arc, +) -> Vec<(String, String)> { + let _permit = pin_sem + .clone() + .acquire_owned() + .await + .expect("pin_semaphore is never closed"); + crate::ipfs_pin::pin_new_objects(ipfs_api, repo_path, object_list, db).await +} + async fn pin_and_encrypt_objects( ctx: &EncryptTaskCtx, object_list: Vec, rules: Option>, is_public: bool, ) { - let pinned = - crate::ipfs_pin::pin_new_objects(&ctx.ipfs_api, &ctx.repo_path, object_list, &ctx.db).await; + let pinned = pin_new_objects_gated( + &ctx.pin_sem, + &ctx.ipfs_api, + &ctx.repo_path, + object_list, + &ctx.db, + ) + .await; if !pinned.is_empty() { tracing::info!(count = pinned.len(), "pinned git objects to IPFS"); for (sha, cid) in &pinned { @@ -1782,6 +1809,7 @@ pub async fn git_receive_pack( state.config.git_service_timeout_secs, ), encrypt_sem: state.git_encrypt_semaphore.clone(), + pin_sem: state.pin_semaphore.clone(), }; tokio::spawn(run_encrypt_pin_task( ctx, @@ -1833,8 +1861,16 @@ pub async fn git_receive_pack( let node_keypair = Arc::clone(&state.node_keypair); let object_list_pinata = object_list; let do_pinata_replication = withheld.is_some(); + // Same global pin-admission bound as the IPFS loop (#174 F6): the Pinata pin + // loop also holds this push's full object-id list while pinning it, so it must + // share the cap. It DEFERS on a full pool rather than dropping the pin. + let pin_sem_pinata = state.pin_semaphore.clone(); tokio::spawn(async move { let pinned = if do_pinata_replication { + let _pin_permit = pin_sem_pinata + .acquire_owned() + .await + .expect("pin_semaphore is never closed"); crate::pinata::pin_new_objects( &http_client, &pinata_upload_url, @@ -5096,6 +5132,48 @@ mod tests { ); } + /// #174 F6 (RED-before/GREEN-after): a post-push pin loop holds this push's full + /// object-id list while walking it, so concurrent pin loops across many repos must + /// be bounded by a global permit, not just the per-repo task count. `pin_new_objects_gated` + /// DEFERS (waits) when the pin pool is exhausted rather than running unbounded. + /// + /// Load-bearing: without the permit acquire the pin loop runs immediately even with + /// the pool held (RED — the deferral assertion fails). With it, it parks. + #[sqlx::test] + async fn pin_new_objects_gated_defers_when_pin_pool_exhausted(pool: sqlx::PgPool) { + use std::sync::Arc; + use tokio::sync::Semaphore; + + let state = crate::test_support::test_state(pool).await; + let db = state.db.clone(); + let tmp = tempfile::TempDir::new().unwrap(); + let pin_sem = Arc::new(Semaphore::new(1)); + // Hold the only pin permit. + let held = pin_sem.clone().acquire_owned().await.unwrap(); + + // Empty ipfs_api makes the pin itself a no-op, but the loop must still DEFER on + // the exhausted pin pool rather than run. + let blocked = tokio::time::timeout( + std::time::Duration::from_millis(500), + pin_new_objects_gated(&pin_sem, "", tmp.path(), vec![], &db), + ) + .await; + assert!( + blocked.is_err(), + "a pin loop must defer while the pin pool is exhausted (#174 F6)" + ); + + // Release admission: the SAME call now completes. + drop(held); + let out = tokio::time::timeout( + std::time::Duration::from_secs(5), + pin_new_objects_gated(&pin_sem, "", tmp.path(), vec![], &db), + ) + .await + .expect("the pin loop completes once admission frees"); + assert!(out.is_empty(), "an empty ipfs_api pins nothing"); + } + /// Shared fixture for the F4 handler-layer tests: a state whose repo_store and /// git_bin point at the given tempdir/fake-git, plus a seeded on-disk repo, /// optionally with a path-scoped rule (so the post-receive walks actually run). @@ -5884,6 +5962,7 @@ mod tests { git_bin: git_bin.to_string(), git_timeout: std::time::Duration::from_secs(600), encrypt_sem: sem, + pin_sem: std::sync::Arc::new(tokio::sync::Semaphore::new(64)), } } diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 912108a0..c1fa6624 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -524,6 +524,7 @@ mod tests { git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + pin_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), encrypt_inflight: crate::state::EncryptInflight::new(), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 5fb66b55..d70be5ea 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -319,6 +319,24 @@ pub struct Config { )] pub max_concurrent_git_pushes: usize, + /// Max concurrent post-push pin loops (`ipfs_pin` and `pinata` + /// `pin_new_objects`) across all repos. `EncryptInflight` bounds the outstanding + /// pin-task COUNT to one per repo, but each pin loop holds a full per-push + /// object-id list (up to `git_max_pack_bytes` worth of OIDs) while it walks it, + /// so N distinct repos could hold N such lists at once. This caps how many run + /// concurrently, bounding that MB-scale memory regardless of how many repos an + /// authenticated actor pushes to (#174 F6). Beyond it a pin loop DEFERS (waits), + /// never drops — a dropped pin would lose the object's replication copy. + /// + /// Default: 8. Must be between 1 and 1_048_576. + #[arg( + long, + env = "GITLAWB_MAX_CONCURRENT_PIN_TASKS", + default_value_t = 8, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub max_concurrent_pin_tasks: usize, + /// Maximum concurrent read operations (`upload-pack` and the upload-pack /// `info/refs` advertisement) a single caller may hold at once, so one caller /// cannot monopolize the `max_concurrent_git_ops` read pool (#174). Callers are @@ -518,6 +536,26 @@ mod tests { ); } + #[test] + fn max_concurrent_pin_tasks_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).max_concurrent_pin_tasks, + 8 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-pin-tasks", "2"]) + .max_concurrent_pin_tasks, + 2 + ); + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-pin-tasks", "0"]).is_err() + ); + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-pin-tasks", "1048577"]) + .is_err() + ); + } + #[test] fn max_concurrent_git_ops_defaults_and_rejects_out_of_range() { assert_eq!( diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index b7bc758e..4c4c1da8 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -402,6 +402,9 @@ async fn main() -> Result<()> { git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new( config.max_concurrent_git_pushes, )), + // Bounds concurrent post-push pin loops (their MB-scale object-id lists) across + // all repos (#174 F6), independent of the per-repo encrypt-task coalescing below. + pin_semaphore: Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_pin_tasks)), // Coalesces the DETACHED post-push encryption tasks per repo so a rapid pusher // cannot grow the outstanding parked-waiter set past one task per repo (#174 // P2-2). No knob: it is a natural cap (one entry per distinct repo), not a diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 8396a369..77799bd0 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -127,16 +127,25 @@ pub struct AppState { /// walk must not hold a foreground write slot, and a handler already holding a /// write permit that needed a second would self-deadlock at pool size 1. pub git_encrypt_semaphore: Arc, - /// Bounds the OUTSTANDING post-push encryption-task set by per-repo coalescing - /// (#174 P2-2). `git_encrypt_semaphore` caps *active* walks; this caps how many - /// detached tasks *spawn and park* on that semaphore's `acquire_owned().await`. - /// Before spawning a per-push encryption task, the receive-pack handler consults - /// this set: if the repo already has a task in flight it coalesces (skips the - /// duplicate spawn) rather than parking a new unbounded waiter, and its tip pairs - /// are recorded for that task's drain loop (#174 F5). Coalescing only delays the - /// coalesced push's walk — it never drops the withheld-blob recovery copy, which - /// `2a54c15` deliberately kept fail-closed (there is no reconciliation sweep to - /// re-derive a dropped copy). See [`EncryptInflight`]. + /// Bounds concurrent post-push pin loops (`ipfs_pin` / `pinata` `pin_new_objects`) + /// across all repos (#174 F6). `encrypt_inflight` caps the pin-task COUNT to one + /// per repo, but each pin loop holds a full per-push object-id list while walking + /// it, so N distinct repos could hold N such MB-scale lists at once. This caps how + /// many run concurrently; a loop DEFERS (waits) when the pool is full, never drops. + pub pin_semaphore: Arc, + /// Bounds the outstanding post-push encryption-task set to at most one PER REPO by + /// coalescing (#174 P2-2). This is NOT a global cap: N distinct repos still admit N + /// tasks; the cross-repo residual (an authenticated actor pushing to many repos + /// leaves many parked tasks) is throttled by auth plus the per-IP/per-DID rate + /// limits, and its real cost — the MB-scale per-push object-id list each pin loop + /// holds — is bounded by `pin_semaphore`, not this. `git_encrypt_semaphore` caps + /// *active* walks; this caps duplicate SPAWNS per repo. Before spawning a per-push + /// encryption task, the receive-pack handler consults this set: if the repo already + /// has a task in flight it coalesces (skips the duplicate spawn) rather than parking + /// a new waiter, and its tip pairs are recorded for that task's drain loop (#174 F5). + /// Coalescing only delays the coalesced push's walk — it never drops the withheld-blob + /// recovery copy, which `2a54c15` deliberately kept fail-closed (there is no + /// reconciliation sweep to re-derive a dropped copy). See [`EncryptInflight`]. pub encrypt_inflight: EncryptInflight, /// Per-caller concurrency sub-cap on the read pool: each caller (keyed on the /// resolved source IP, #174 U1) may hold at most `max_concurrent_reads_per_caller` diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index df095b44..cd58f19e 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -87,6 +87,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + pin_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), encrypt_inflight: crate::state::EncryptInflight::new(), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( From 93cdf308baad7b31043e4f8c96e96ab2e24f4d44 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 09:14:01 -0500 Subject: [PATCH 057/124] fix(node): make RepoWriteGuard::release cancellation-safe (#174 F4) Await pg_advisory_unlock while the pooled connection is still owned by self; take it and set released only after the await resolves. A cancel mid-unlock now leaves the Drop backstop armed (conn still Some), so the session-level advisory lock is no longer leaked onto a pooled connection. Adds a cancel-mid-unlock regression test (via a test-only pre-unlock gate) and an append-only inv22 gate row asserting the take/released orderings. --- crates/gitlawb-node/src/git/repo_store.rs | 151 +++++++++++++++++++++- crates/gitlawb-node/tests/inv22_gates.rs | 47 +++++++ 2 files changed, 195 insertions(+), 3 deletions(-) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 86a1ee8b..d88594e7 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -180,6 +180,8 @@ impl RepoStore { locked: false, released: false, tigris: self.tigris.clone(), + #[cfg(test)] + test_pre_unlock_gate: None, }; // Acquire the advisory lock with retry, through the guard's OWN connection, @@ -390,6 +392,13 @@ pub struct RepoWriteGuard { /// Set once `release` has run its unlock, making the `Drop` backstop inert. released: bool, tigris: Option, + /// Test-only seam: when set, `release` parks on this gate at the exact point + /// it is about to await `pg_advisory_unlock` (connection still owned, not yet + /// released). Dropping the `release` future while it is parked reproduces a + /// mid-unlock cancellation, so a test can assert the `Drop` backstop still + /// frees the session lock. Never set outside tests. + #[cfg(test)] + test_pre_unlock_gate: Option>, } impl RepoWriteGuard { @@ -419,10 +428,22 @@ impl RepoWriteGuard { } // Release the advisory lock on the SAME connection that took it (session - // advisory locks are connection-affine). Taking the connection here makes - // the Drop backstop inert. + // advisory locks are connection-affine). Unlock through the connection + // while it is STILL owned by `self` — do not `take()` it first. If this + // future is cancelled during the unlock await, `self` is dropped with + // `conn == Some(..)` and `released == false`, so the `Drop` backstop still + // runs the detached unlock. `released` is set only AFTER the await + // resolves, so a cancellation cannot make the backstop inert (#174 F4). if self.locked { - if let Some(mut conn) = self.conn.take() { + #[cfg(test)] + let pre_unlock_gate = self.test_pre_unlock_gate.clone(); + if let Some(conn) = self.conn.as_deref_mut() { + // Test-only: park right before the unlock await so a test can drop + // this future mid-unlock (connection owned, not yet released). + #[cfg(test)] + if let Some(gate) = pre_unlock_gate { + gate.notified().await; + } let _ = sqlx::query("SELECT pg_advisory_unlock($1)") .bind(self.lock_key) .execute(&mut *conn) @@ -719,4 +740,128 @@ mod tests { .execute(&mut *checker) .await; } + + // ── cancellation-safe unlock (#174 F4, RED-before/GREEN-after) ────────── + + /// F4 (P1): a cancellation DURING the unlock await must still free the session + /// advisory lock. `release` unlocks through the connection while `self` still + /// owns it, so if the future is dropped mid-unlock, `Drop` sees `conn == Some` + /// + `locked && !released` and runs its detached-unlock backstop. A test-only + /// gate parks `release` at the exact pre-unlock point; dropping the future + /// there reproduces the cancellation. + /// + /// Load-bearing: RED on the original ordering (`self.conn.take()` before the + /// await → at cancellation `self.conn == None` → `Drop` skips → the local + /// connection returns to the pool with the session lock still held → the + /// checker's `pg_try_advisory_lock` returns false). GREEN after the reorder. + #[sqlx::test] + async fn write_guard_release_cancelled_mid_unlock_frees_the_lock(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone()); + let owner = "did:key:z6MkCancelMidUnlockProofCCCCCCCCCCCCCCCCCC"; + let name = "canceltest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + // Distinct session for the probe, held out of the pool before acquiring. + let mut checker = pool.acquire().await.expect("checker connection"); + + let mut guard = store.acquire_write(owner, name).await.expect("acquire"); + // Arm the pre-unlock gate; it is never notified, so `release` parks on it + // with the connection still owned and `released` still false. + let gate = Arc::new(tokio::sync::Notify::new()); + guard.test_pre_unlock_gate = Some(gate); + + // Box the future so we can drop it ourselves (tokio::pin! keeps it alive to + // end of scope, which would defer the guard's Drop past the assertions). + let mut fut = Box::pin(guard.release(false)); + let parked = + tokio::time::timeout(std::time::Duration::from_millis(300), fut.as_mut()).await; + assert!( + parked.is_err(), + "release should park on the pre-unlock gate, not complete" + ); + // Cancel mid-unlock: dropping the boxed future runs RepoWriteGuard::drop. + drop(fut); + + // Let the Drop backstop's detached unlock task run. + tokio::time::sleep(std::time::Duration::from_millis(400)).await; + + let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *checker) + .await + .unwrap(); + assert!( + free, + "advisory lock must be freed when release is cancelled mid-unlock — the \ + connection must stay owned by the guard so Drop's backstop can run" + ); + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } + + /// F4: the ordinary success path still frees the lock, and a second write + /// acquire on the same repo returns promptly (no stale-lock retry loop). + #[sqlx::test] + async fn write_guard_release_true_frees_lock_and_second_acquire_succeeds(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone()); + let owner = "did:key:z6MkReleaseTrueProofDDDDDDDDDDDDDDDDDDDDDD"; + let name = "reltruetest"; + + let guard = store + .acquire_write(owner, name) + .await + .expect("first acquire"); + guard.release(true).await; + + let again = tokio::time::timeout( + std::time::Duration::from_secs(2), + store.acquire_write(owner, name), + ) + .await + .expect("second acquire_write must not hit the ~60s stale-lock retry loop") + .expect("second acquire"); + again.release(true).await; + } + + /// F4: releasing a guard that never took the lock (`locked == false`, the state + /// acquire_write leaves after a failed acquisition) must not unlock or panic. + #[sqlx::test] + async fn write_guard_release_when_not_locked_does_not_unlock_or_panic(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let owner = "did:key:z6MkNotLockedProofEEEEEEEEEEEEEEEEEEEEEE"; + let name = "notlockedtest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + let guard = RepoWriteGuard { + owner_slug: slug, + repo_name: name.to_string(), + local_path: dir.path().to_path_buf(), + lock_key: key, + conn: Some(pool.acquire().await.expect("conn")), + locked: false, + released: false, + tigris: None, + test_pre_unlock_gate: None, + }; + // Must complete without panic and issue no unlock. + guard.release(false).await; + + let mut checker = pool.acquire().await.expect("checker"); + let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *checker) + .await + .unwrap(); + assert!(free, "release on an unlocked guard must not touch the key"); + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } } diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index cbfa0d4c..d21ec194 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -153,3 +153,50 @@ fn inv22_concurrency_gates_present_and_not_bypassed() { ); } } + +/// F4 (repo_store advisory-unlock cancellation safety): `RepoWriteGuard::release` +/// must await `pg_advisory_unlock` while `self` still owns the pooled connection, +/// and must not mark itself `released` until that await resolves. Either shape, +/// reintroduced, re-opens the mid-unlock cancellation leak: taking the connection +/// early leaves `Drop` with `conn == None`, and setting `released = true` early +/// leaves the `Drop` backstop inert — both strand the session lock on cancellation. +/// +/// Scoped to the `release` fn body: the `Drop` impl legitimately takes the +/// connection and unlocks, so a whole-file scan would match it and read as a false +/// pass. Reverting either ordering turns this red (proven load-bearing). +#[test] +fn f4_release_keeps_conn_owned_until_unlock_resolves() { + let repo_store = src("git/repo_store.rs"); + + let rel_start = repo_store + .find("pub async fn release(mut self") + .expect("F4 gate: repo_store.rs no longer defines RepoWriteGuard::release"); + let rel_end = repo_store[rel_start..] + .find("impl Drop for RepoWriteGuard") + .map(|off| rel_start + off) + .expect("F4 gate: release fn / Drop impl markers moved — update this guard"); + let release_body = &repo_store[rel_start..rel_end]; + + let unlock = release_body + .find("pg_advisory_unlock") + .expect("F4 gate: release must still issue pg_advisory_unlock"); + let before_unlock = &release_body[..unlock]; + + // (a) the connection must still be owned by `self` at the unlock await. + assert!( + !before_unlock.contains("self.conn.take()"), + "F4 regression: RepoWriteGuard::release takes self.conn BEFORE awaiting \ + pg_advisory_unlock. A cancellation during the unlock await then strands the \ + session advisory lock (Drop sees conn == None and skips its backstop). \ + Unlock through the still-owned connection instead." + ); + // (b) `released` must not be set before the unlock await — the other + // reintroduction shape a single-reorder check on (a) alone is blind to. + assert!( + !before_unlock.contains("released = true"), + "F4 regression: RepoWriteGuard::release sets `released = true` BEFORE awaiting \ + pg_advisory_unlock. A cancellation during the await then leaves the Drop \ + backstop inert (it early-returns on released). Set released only AFTER the \ + unlock await resolves." + ); +} From e0e207e6455dd3b51aa85085611e6b06c2c3f770 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 09:36:09 -0500 Subject: [PATCH 058/124] fix(node): classify /ipfs object probe with a version-stable signal (#174 F5) Probe existence via `git cat-file --batch-check` (structured ` missing` on exit 0) instead of matching English `cat-file -t` prose, so a git wording change cannot flip a miss into an error or vice-versa. A bad-config/corrupt repo on a readable store is now a terminal, non-retryable 500 (not a false 404, and not a retryable 503 that would fan out cat-file retries); a transient unreadable store stays 503. Error bodies are opaque (no raw git stderr). Layers onto the existing object_store_readable + re-probe disambiguation. --- crates/gitlawb-node/src/api/ipfs.rs | 126 ++++++++- crates/gitlawb-node/src/git/store.rs | 369 +++++++++++++++++++++++---- 2 files changed, 445 insertions(+), 50 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 43e418ae..1910a14b 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -61,6 +61,13 @@ use crate::visibility::{visibility_check, Decision}; /// the truncation sources — existing content is never misreported absent /// because of unrelated repos or transient faults. /// +/// Deterministic fault (F5/U4): a candidate repo that is persistently broken (a +/// corrupt repo, a bad `.git/config`) also yields no absence verdict, but a retry +/// cannot fix it, so a scan that found nothing sheds a TERMINAL, non-retryable 500 +/// (opaque body) rather than the retryable 503 — checked first so a deterministic +/// fault is never downgraded, and gated on nothing-served so a healthy repo that +/// carries the object still serves. +/// /// Request budget (F3): one absolute clock (`ipfs_request_budget_secs`) spans /// the whole admitted request. No stage (acquire, probe, walk, content read) /// starts once it is exhausted, and the acquire wait and walk deadline are @@ -181,6 +188,13 @@ pub async fn get_by_cid( truncated_by.push(source); } } + // A DETERMINISTIC probe fault (a corrupt repo / bad `.git/config`; #174 F5/U4) is + // separate from a transient taint: a retry cannot fix it, so a scan that found + // nothing must NOT shed the retryable 503 (which would invite a conformant client + // to retry-storm a fresh `git cat-file` per attempt against the broken repo). It + // sheds a terminal, non-retryable 500 instead — but only if nothing served, so one + // corrupt repo never masks a healthy repo that carries the object. + let mut deterministic_fault = false; // Budget gate shared by the four per-stage checks (F3): the remaining // request budget, or — once exhausted — None, after logging the stage and @@ -328,11 +342,22 @@ pub async fn get_by_cid( { Ok(Ok(Some(t))) => t, Ok(Ok(None)) => continue, - Ok(Err(e)) => { - tracing::warn!(repo = %repo.name, err = %e, "object-type probe failed during /ipfs scan; skipping repo without a verdict"); + // Transient probe fault (unreadable/mid-repack store): unproven, retryable. + Ok(Err(store::ProbeError::Transient(e))) => { + tracing::warn!(repo = %repo.name, err = %e, "object-type probe hit a transient store fault during /ipfs scan; skipping repo without a verdict"); taint(&mut truncated_by, "probe"); continue; } + // Deterministic probe fault (corrupt repo / bad config): a retry cannot fix + // it, so it does NOT taint (which would shed a retryable 503). It records a + // terminal condition that the terminal arm renders as a non-retryable 500 + // only if nothing served. The raw git detail stays in the log; the client + // body is opaque. + Ok(Err(store::ProbeError::Deterministic(e))) => { + tracing::warn!(repo = %repo.name, err = %e, "object-type probe hit a deterministic fault (corrupt repo/config) during /ipfs scan; skipping repo without a verdict"); + deterministic_fault = true; + continue; + } Err(join_err) => { tracing::warn!(repo = %repo.name, err = %join_err, "object-type probe task panicked during /ipfs scan; skipping repo without a verdict"); taint(&mut truncated_by, "probe"); @@ -500,6 +525,21 @@ pub async fn get_by_cid( return Ok((StatusCode::OK, headers, content).into_response()); } + // Deterministic fault (F5/U4): a candidate repo is persistently broken (corrupt + // repo / bad `.git/config`), so the object is not proven absent AND a retry cannot + // change that. Shed a TERMINAL, non-retryable 500 rather than the retryable 503 + // below — a 503 would let a conformant client retry-storm a fresh `git cat-file` + // per attempt against the broken repo. This is checked before the transient 503 so + // a deterministic fault is never downgraded to a retryable status. The body is + // opaque (a generic message via `AppError::Git` -> 500, no Retry-After): the raw + // git stderr — which leaks filesystem paths / config — was logged at the probe, and + // never reaches the client. + if deterministic_fault { + return Err(AppError::Git( + "ipfs object probe could not complete: a candidate repository is corrupt".into(), + )); + } + // Truncated scan (F2): at least one candidate repo yielded no verdict, so the // object is not proven absent. A 404 here would misreport existing content, so // shed retryable instead — Overloaded is the single 503 + Retry-After site in @@ -1195,6 +1235,88 @@ mod tests { ); } + /// #174 F5/U4 (RED-before/GREEN-after): a candidate repo with a corrupt + /// `.git/config` makes `git cat-file` die with `fatal: bad config line N` while + /// `objects/` stays readable. That is a DETERMINISTIC fault, not an absence, and a + /// retry cannot fix it — so the scan must shed a TERMINAL, non-retryable 500, never + /// the old false 404 (`Ok(None)` fell through) and never the retryable 503 (which + /// would invite a conformant client to retry-storm a fresh `cat-file` per attempt). + /// A second, healthy repo probes clean (absent verdict); the corrupt row is what + /// forces the 500. The body must be OPAQUE — no raw git stderr, no filesystem path. + /// MUTATION (RED): route the deterministic fault back to `Ok(None)` in + /// `object_type_bounded` and this decays to a 404; classify it Transient and it + /// decays to a retryable 503. + #[sqlx::test] + async fn get_by_cid_bad_config_repo_is_terminal_500_not_404_or_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // A healthy repo that probes clean (would give a definitive 404 on its own) plus + // a repo whose bare git dir has a corrupt config (objects/ intact). + seed_repo_with_blob(&state, tmp.path(), "z6f5clean", "real", b"probe clean\n").await; + state + .db + .upsert_mirror_repo("z6f5badcfg", "broken", "/unused-badcfg", None, false) + .await + .unwrap(); + let rec = state + .db + .get_repo("z6f5badcfg", "broken") + .await + .unwrap() + .unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + std::fs::create_dir_all(&bare).unwrap(); + run_git(&["init", "-q", "--bare", "--object-format=sha256"], &bare); + // Corrupt the config; leave objects/ readable (the readable-store + git-fails + // combination is exactly what makes this deterministic, not transient). + { + use std::io::Write; + let mut cfg = std::fs::OpenOptions::new() + .append(true) + .open(bare.join("config")) + .unwrap(); + cfg.write_all(b"\n[broken section\nnot a valid = = = line\n") + .unwrap(); + } + + let peer: SocketAddr = "203.0.113.69:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::INTERNAL_SERVER_ERROR, + "a bad-config (deterministic) repo fault must shed a terminal 500, never a \ + 404 (false absence) or a retryable 503" + ); + assert!( + resp.headers().get("retry-after").is_none(), + "a terminal 500 must NOT advertise a retry (that is the whole point vs 503)" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + !body.contains("bad config") + && !body.contains(tmp.path().to_str().unwrap()) + && !body.contains(".git") + && !body.contains("fatal"), + "the 500 body must be opaque — no raw git stderr / config text / filesystem \ + path; got: {body}" + ); + } + /// F2 read taint: the gate passes (the probe reads the truncated loose object's /// intact "blob 64" header) but the content read fails (`cat-file blob` dies on /// the deflate stream cut mid-content) — the probe just said the object EXISTS diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index f9d65e9d..a18ab472 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -320,66 +320,164 @@ pub fn read_object_content(repo_path: &Path, sha256_hex: &str, obj_type: &str) - Ok(content_output.stdout) } -/// Bounded, reaped variant of [`object_type`] for the async `/ipfs` serve path -/// (#174 F3): runs `git cat-file -t` off the caller's runtime through the -/// process-group + watchdog reaper, so a hung or corrupt object store cannot pin a -/// runtime worker or an IPFS admission permit past `deadline`. Exit classification -/// is identical to [`object_type`] — a missing object is `Ok(None)`, a store-access -/// failure is `Err` — so the serve path's 404-vs-503 semantics are unchanged. -pub fn object_type_bounded( +/// Why an `/ipfs` existence probe could not return an absence verdict (#174 F5/U4). +/// The caller (`api::ipfs`) maps the variant to an HTTP status, and the split exists +/// only for that mapping: a `Transient` fault is retryable (503), a `Deterministic` +/// fault is terminal (500). +/// +/// The discriminator is object-store readability, NOT any English `git` wording, so a +/// future `git` message change cannot silently reclassify a fault (KTD-4): if the +/// store cannot be read (an unreadable or mid-repack pack, a removed `objects/` dir, +/// a permissions fault) the fault may clear on its own -> `Transient`; if the store IS +/// readable yet `git` still fails (a corrupt repo, a bad `.git/config`) a retry cannot +/// fix it -> `Deterministic`, so a conformant client is told not to retry-storm a +/// fresh `git cat-file` per attempt against a persistently broken repo. +#[derive(Debug)] +pub enum ProbeError { + /// Retryable (-> 503): the object store could not be read right now. + Transient(anyhow::Error), + /// Terminal (-> 500): a persistent, deterministic fault a retry cannot fix. + Deterministic(anyhow::Error), +} + +impl std::fmt::Display for ProbeError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ProbeError::Transient(e) => write!(f, "transient probe fault: {e}"), + ProbeError::Deterministic(e) => write!(f, "deterministic probe fault: {e}"), + } + } +} + +impl std::error::Error for ProbeError {} + +/// Structured outcome of one `git cat-file --batch-check` existence probe. +enum BatchProbe { + /// Present: git printed ` ` on exit 0. + Present(String), + /// A structured, CLEAN absence: git printed ` missing` on exit 0 with no + /// `error:` diagnostics on stderr. This is the ONLY signal that can become an + /// `Ok(None)` (404) absence verdict — and even then only after the store-readable + /// disambiguation below, since an unreadable pack ALSO prints a clean `missing`. + Missing, + /// git could not honestly examine the object store: a hard exit (bad config, not a + /// git repo) OR an exit-0 `missing` accompanied by `error:` diagnostics (a corrupt + /// loose object still prints `missing` on stdout but complains on stderr). Never an + /// absence verdict. Carries the raw git detail for the server log only. + Fault(anyhow::Error), +} + +/// One bounded, reaped `git cat-file --batch-check` probe (the oid is fed on stdin, so +/// no prose is matched to decide presence — the `missing` token and the exit code are +/// the structured signal). See [`object_type_bounded`] for the surrounding teardown +/// guarantees. +fn batch_check_probe( git_bin: &str, repo_path: &Path, sha256_hex: &str, deadline: std::time::Instant, -) -> Result> { +) -> std::result::Result { + let stdin = format!("{sha256_hex}\n"); let (status, stdout, stderr) = crate::git::visibility_pack::run_bounded_git_raw( git_bin, - &["cat-file", "-t", sha256_hex], + &["cat-file", "--batch-check"], repo_path, - &[], + stdin.as_bytes(), deadline, - )?; - if status.success() { - return Ok(Some(String::from_utf8_lossy(&stdout).trim().to_string())); - } - // A nonzero exit whose stderr proves git could not EXAMINE the object store — - // "not a git repository", or any `error:` line (corrupt loose object, bad idx) — - // is not an absence verdict, so surface it as Err (#173/#174: never a false 404). + ) + // A spawn/timeout failure of the reaped child is not deterministic — retry it. + .map_err(ProbeError::Transient)?; + let stderr = String::from_utf8_lossy(&stderr); - if stderr.contains("not a git repository") || stderr.lines().any(|l| l.starts_with("error:")) { - bail!("git cat-file -t failed: {}", stderr.trim()); - } - // Any other bare `fatal:` ("could not get object info") is the absence-vs- - // unreadable-pack COLLISION (#174 F5): a genuinely missing object and a packed - // object whose pack/idx is unreadable (permissions, or a mid-repack race) emit the - // identical string. Disambiguate OUT OF BAND: if the object store is not readable, - // this is not an absence verdict -> Err (taint -> retryable 503). - if !object_store_readable(repo_path) { - bail!( - "git cat-file -t inconclusive: object store not readable at {} (not an absence verdict)", - repo_path.display() - ); + // A corrupt object makes `--batch-check` print `missing` on stdout (exit 0) yet + // emit `error:` lines on stderr; those diagnostics disqualify a clean-absence read + // regardless of the exit code, so they are checked before anything else. + let has_error_diag = stderr.lines().any(|l| l.starts_with("error:")); + + if status.success() && !has_error_diag { + let line = String::from_utf8_lossy(&stdout); + let line = line.trim(); + // ` missing` is the structured absence token; ` ` is a + // hit. Anything else on a "success" exit is unexpected and not an absence. + if line + .rsplit(' ') + .next() + .is_some_and(|last| last == "missing") + { + return Ok(BatchProbe::Missing); + } + let mut parts = line.split_whitespace(); + if let (Some(_oid), Some(ty), Some(_size)) = (parts.next(), parts.next(), parts.next()) { + return Ok(BatchProbe::Present(ty.to_string())); + } + return Ok(BatchProbe::Fault(anyhow::anyhow!( + "unexpected git cat-file --batch-check output: {line:?}" + ))); } - // Store readable: re-probe once. An object still absent on a confirmed-readable - // store is very likely truly absent (Ok(None)); if a mid-repack race resolved and - // the re-probe now finds it, return the type. This narrows, but cannot fully close, - // the concurrent-repack window (the readability check samples a different instant). - let (status2, stdout2, stderr2) = crate::git::visibility_pack::run_bounded_git_raw( - git_bin, - &["cat-file", "-t", sha256_hex], - repo_path, - &[], - deadline, - )?; - if status2.success() { - return Ok(Some(String::from_utf8_lossy(&stdout2).trim().to_string())); + + Ok(BatchProbe::Fault(anyhow::anyhow!( + "git cat-file --batch-check failed (exit {:?}): {}", + status.code(), + stderr.trim() + ))) +} + +/// Bounded, reaped variant of [`object_type`] for the async `/ipfs` serve path +/// (#174 F3/F5): runs `git cat-file --batch-check` off the caller's runtime through the +/// process-group + watchdog reaper, so a hung or corrupt object store cannot pin a +/// runtime worker or an IPFS admission permit past `deadline`. +/// +/// Absence is keyed on `--batch-check`'s STRUCTURED ` missing` token on exit 0, +/// never on any English `fatal:` wording (KTD-4): a genuinely-absent object is the only +/// `Ok(None)` (404) path. A probe that could not honestly examine the store is a +/// [`ProbeError`], split by object-store readability into `Transient` (retryable 503) +/// and `Deterministic` (terminal 500) so the serve path can shed the right status. +pub fn object_type_bounded( + git_bin: &str, + repo_path: &Path, + sha256_hex: &str, + deadline: std::time::Instant, +) -> std::result::Result, ProbeError> { + match batch_check_probe(git_bin, repo_path, sha256_hex, deadline)? { + BatchProbe::Present(ty) => Ok(Some(ty)), + BatchProbe::Fault(detail) => Err(classify_store_fault(repo_path, detail)), + BatchProbe::Missing => { + // A clean `missing` is the absence-vs-unreadable-pack COLLISION (#174 F5): + // a genuinely missing object AND a packed object whose pack/idx is + // unreadable (permissions, or a mid-repack race) both print an identical + // clean `missing`. Disambiguate OUT OF BAND on store readability — an + // unreadable store is not an absence verdict (taint -> retryable 503). + if !object_store_readable(repo_path) { + return Err(ProbeError::Transient(anyhow::anyhow!( + "git cat-file inconclusive: object store not readable at {} (not an absence verdict)", + repo_path.display() + ))); + } + // Store readable: re-probe once. Still `missing` on a confirmed-readable + // store is very likely truly absent (Ok(None)); a mid-repack race that + // resolved returns the type. This narrows, but cannot fully close, the + // concurrent-repack window (the readability check samples a different + // instant than the failing probe). + match batch_check_probe(git_bin, repo_path, sha256_hex, deadline)? { + BatchProbe::Present(ty) => Ok(Some(ty)), + BatchProbe::Fault(detail) => Err(classify_store_fault(repo_path, detail)), + BatchProbe::Missing => Ok(None), + } + } } - let stderr2 = String::from_utf8_lossy(&stderr2); - if stderr2.contains("not a git repository") || stderr2.lines().any(|l| l.starts_with("error:")) - { - bail!("git cat-file -t failed: {}", stderr2.trim()); +} + +/// Classify a probe fault by object-store readability (#174 F5/U4). An unreadable store +/// may be a transient permissions/mid-repack condition (retryable 503); a readable +/// store on which git still fails is a persistent, deterministic fault — a corrupt repo +/// or a bad `.git/config` — that a retry cannot fix (terminal 500). The `detail` is +/// carried for the server log; the client-facing body is opaque (set by the caller). +fn classify_store_fault(repo_path: &Path, detail: anyhow::Error) -> ProbeError { + if object_store_readable(repo_path) { + ProbeError::Deterministic(detail) + } else { + ProbeError::Transient(detail) } - Ok(None) } /// Best-effort check that a repo's object store is readable, used to disambiguate a @@ -748,4 +846,179 @@ mod tests { ); } } + + /// Shared setup: a bare sha256 repo carrying one committed blob. Returns the repo + /// path and the blob's oid. + #[cfg(unix)] + fn bare_repo_with_blob(td: &std::path::Path) -> (std::path::PathBuf, String) { + let work = td.join("work"); + let bare = td.join("bare.git"); + std::fs::create_dir_all(&work).unwrap(); + let g = |args: &[&str], dir: &Path| { + assert!( + Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + g(&["init", "-q", "--object-format=sha256", "."], &work); + g(&["config", "user.email", "t@t"], &work); + g(&["config", "user.name", "t"], &work); + std::fs::write(work.join("file.txt"), b"f5 u4 content\n").unwrap(); + g(&["add", "file.txt"], &work); + g(&["commit", "-qm", "c1"], &work); + let blob = String::from_utf8( + Command::new("git") + .args(["rev-parse", "HEAD:file.txt"]) + .current_dir(&work) + .output() + .unwrap() + .stdout, + ) + .unwrap() + .trim() + .to_string(); + g( + &[ + "clone", + "-q", + "--bare", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + td, + ); + (bare, blob) + } + + /// #174 F5/U4 (RED-before/GREEN-after, the CORE regression guard): a repo with a + /// corrupt `.git/config` makes `git cat-file` die with `fatal: bad config line N` + /// (exit 128, NO `error:` line) while `objects/` stays fully readable. The old + /// `-t` path let that fall through to `Ok(None)` — a false 404 for content that + /// may well exist. `object_type_bounded` must instead classify it as a + /// DETERMINISTIC fault (a retry cannot fix it) so the serve path renders a + /// terminal 500, never a 404 and never a retryable 503. + /// + /// LOAD-BEARING: revert the classification (route `BatchProbe::Fault` on a readable + /// store back to `Ok(None)`, or drop the `has_error_diag`/exit checks so a hard + /// `fatal:` is read as `missing`) and this goes RED — the probe reports the corrupt + /// repo as an absent object. + #[cfg(unix)] + #[test] + fn object_type_bounded_bad_config_is_deterministic_not_absence() { + let td = tempfile::TempDir::new().unwrap(); + let (bare, blob) = bare_repo_with_blob(td.path()); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + + // Baseline on the healthy repo: present blob probes present, genuine miss is + // the clean Ok(None) absence verdict. + assert_eq!( + super::object_type_bounded("git", &bare, &blob, deadline) + .unwrap() + .as_deref(), + Some("blob"), + "a present blob on a healthy readable store must probe present" + ); + assert!( + super::object_type_bounded("git", &bare, &"0".repeat(64), deadline) + .unwrap() + .is_none(), + "a genuinely-absent object on a healthy readable store must be Ok(None) (404)" + ); + + // Corrupt the config; objects/ is untouched (and stays readable). + { + use std::io::Write; + let mut cfg = std::fs::OpenOptions::new() + .append(true) + .open(bare.join("config")) + .unwrap(); + cfg.write_all(b"\n[broken section\nnot a valid = = = line\n") + .unwrap(); + } + assert!( + super::object_store_readable(&bare), + "config corruption must leave objects/ readable (that is the whole point: \ + a readable store + a git failure == deterministic, not transient)" + ); + + // Probing the PRESENT blob under the bad config must be a DETERMINISTIC fault, + // never Ok(None) (the old false 404) and never a Transient (retryable 503). + let res = super::object_type_bounded("git", &bare, &blob, deadline); + assert!( + matches!(res, Err(super::ProbeError::Deterministic(_))), + "a bad-config fatal on a readable store must be a terminal Deterministic \ + fault (-> 500), never Ok(None) (-> false 404) or Transient (-> 503); got {res:?}" + ); + // And a genuinely-absent oid under the bad config is ALSO not an absence verdict. + let res_absent = super::object_type_bounded("git", &bare, &"0".repeat(64), deadline); + assert!( + matches!(res_absent, Err(super::ProbeError::Deterministic(_))), + "even a would-be-absent oid must not read as Ok(None) once the config is \ + corrupt; got {res_absent:?}" + ); + } + + /// #174 F5/U4: a corrupt LOOSE object makes `git cat-file --batch-check` print + /// ` missing` on stdout (exit 0) yet emit `error:` diagnostics on stderr. The + /// clean-`missing` absence path must NOT fire here — the `error:` line disqualifies + /// a clean-absence read — so the probe surfaces a fault, not a false Ok(None) 404. + /// The object store is readable (a corrupt object file still opens), so this is a + /// Deterministic fault. LOAD-BEARING: drop the `has_error_diag` guard and a corrupt + /// object reads as `missing` -> Ok(None) -> false 404 (RED). + #[cfg(unix)] + #[test] + fn object_type_bounded_corrupt_loose_object_is_fault_not_absence() { + use std::os::unix::fs::PermissionsExt; + let td = tempfile::TempDir::new().unwrap(); + let work = td.path().join("loose"); + std::fs::create_dir_all(&work).unwrap(); + let g = |args: &[&str]| { + assert!( + Command::new("git") + .args(args) + .current_dir(&work) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + g(&["init", "-q", "--object-format=sha256", "."]); + g(&["config", "user.email", "t@t"]); + g(&["config", "user.name", "t"]); + std::fs::write(work.join("f.txt"), b"loose object content\n").unwrap(); + g(&["add", "f.txt"]); + g(&["commit", "-qm", "c1"]); + let blob = String::from_utf8( + Command::new("git") + .args(["rev-parse", "HEAD:f.txt"]) + .current_dir(&work) + .output() + .unwrap() + .stdout, + ) + .unwrap() + .trim() + .to_string(); + + // Overwrite the loose object file with non-zlib garbage (it is 0o444 by default). + let obj = work.join(".git/objects").join(&blob[0..2]).join(&blob[2..]); + let mut perms = std::fs::metadata(&obj).unwrap().permissions(); + perms.set_mode(0o644); + std::fs::set_permissions(&obj, perms).unwrap(); + std::fs::write(&obj, b"garbage not a zlib stream").unwrap(); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + let res = super::object_type_bounded("git", &work, &blob, deadline); + assert!( + res.is_err(), + "a corrupt loose object (error: on stderr, `missing` on stdout) must be Err, \ + never a false Ok(None) 404; got {res:?}" + ); + } } From 9180467973c5b25b3e2a20f401c7bd532d91abf1 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 10:00:40 -0500 Subject: [PATCH 059/124] fix(node): clamp the initial /ipfs metadata queries to the request budget (#174 F6) list_all_repos and list_visibility_rules_for_repos ran as bare awaits while holding the scarce IPFS walk permits, so a query blocked in Postgres pinned the slots past GITLAWB_IPFS_REQUEST_BUDGET_SECS and shed later requests as 503s. Wrap both in the remaining request-deadline budget; on timeout the RAII permits drop and the handler sheds a retryable budget 503. Fails closed: a timeout on the visibility-rules query denies the request rather than serving an unfiltered listing. Appends an inv22 gate row asserting both queries stay deadline-wrapped. --- crates/gitlawb-node/src/api/ipfs.rs | 263 ++++++++++++++++++++++- crates/gitlawb-node/tests/inv22_gates.rs | 41 ++++ 2 files changed, 293 insertions(+), 11 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 1910a14b..7f977a81 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -148,22 +148,65 @@ pub async fn get_by_cid( None => None, }; - // 2. Search all repos for an object with this SHA-256 - let repos = state - .db - .list_all_repos() - .await - .map_err(AppError::Internal)?; + // 2. Search all repos for an object with this SHA-256. + // + // F6/KTD-5: both initial metadata queries run while the scarce walk permits + // acquired above are ALREADY held (RAII, for the whole request) and BEFORE the + // per-repo loop's first budget gate. With no pool statement_timeout, a query + // blocked in Postgres would otherwise pin those walk slots for the entire stall + // — past the request budget — capacity-503'ing later requests. Clamp BOTH to the + // remaining request budget. On timeout return the same retryable budget 503 the + // later stages shed (the "budget" source); returning here drops the RAII permits, + // freeing the slot. list_visibility_rules_for_repos is the access-control query, + // so its timeout returns BEFORE the loop — the scan can NEVER run with an empty + // rule map and serve an unfiltered listing that exposes private repos (FAIL CLOSED). + let repos = match tokio::time::timeout( + request_deadline.saturating_duration_since(std::time::Instant::now()), + state.db.list_all_repos(), + ) + .await + { + Ok(Ok(repos)) => repos, + Ok(Err(e)) => return Err(AppError::Internal(e)), + Err(_elapsed) => { + tracing::warn!( + budget_secs = state.config.ipfs_request_budget_secs, + "/ipfs list_all_repos exceeded the request budget \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); shedding a retryable 503 and freeing the walk permit" + ); + return Err(AppError::Overloaded(format!( + "ipfs scan incomplete (budget) for CID {cid_str}; retry shortly" + ))); + } + }; // Fetch every repo's visibility rules in one query rather than one per row // (the gate runs each row against its OWN rules — KTD2a). A row absent from // the map has no rules. let repo_ids: Vec = repos.iter().map(|r| r.id.clone()).collect(); - let rules_by_repo = state - .db - .list_visibility_rules_for_repos(&repo_ids) - .await - .map_err(AppError::Internal)?; + let rules_by_repo = match tokio::time::timeout( + request_deadline.saturating_duration_since(std::time::Instant::now()), + state.db.list_visibility_rules_for_repos(&repo_ids), + ) + .await + { + Ok(Ok(rules)) => rules, + Ok(Err(e)) => return Err(AppError::Internal(e)), + // FAIL CLOSED (security-critical): a timeout on the access-control query must + // DENY. Returning here — before the loop — means the scan can never fall + // through and apply an empty rule map, which would serve an unfiltered listing + // exposing private repos. It sheds the same retryable budget 503 as above. + Err(_elapsed) => { + tracing::warn!( + budget_secs = state.config.ipfs_request_budget_secs, + "/ipfs list_visibility_rules_for_repos exceeded the request budget \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); denying (fail closed) and freeing the walk permit" + ); + return Err(AppError::Overloaded(format!( + "ipfs scan incomplete (budget) for CID {cid_str}; retry shortly" + ))); + } + }; // Request-scoped memo of the per-repo allowed-blob set (KTD1, #126). The // caller is constant for one request, so `repo.id` alone is a safe, @@ -2416,4 +2459,202 @@ mod tests { "a different IP must not be braked by another IP's exhausted bucket" ); } + + /// F6/KTD-5: the two initial metadata queries (`list_all_repos`, + /// `list_visibility_rules_for_repos`) run AFTER the scarce walk permits are + /// acquired (held RAII for the whole request) but BEFORE the per-repo loop's + /// first budget gate. Pre-fix they were bare awaits with no deadline, so a query + /// blocked in Postgres pinned the walk slot for the whole stall, past the request + /// budget. Here we hold an ACCESS EXCLUSIVE lock on `repos` so `list_all_repos` + /// blocks; with the budget clamp the request sheds a retryable budget 503 within + /// ~budget and FREES the walk permit, and a follow-up (lock released) is served. + /// + /// Load-bearing: pre-fix the bare await blocks on the lock until the 10s wrapping + /// timeout fires (RED — "never returned within budget"). After the fix it returns + /// the 503 at ~1s and the permit is free again. MUTATION (RED): drop the + /// `tokio::time::timeout` around `list_all_repos` and this hangs past the wrap. + #[sqlx::test] + async fn get_by_cid_stalled_metadata_query_frees_walk_permit(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Global walk pool of 1 so the held/freed permit is directly observable; + // per-source cap permissive so only the global pool matters. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + let mut cfg = (*state.config).clone(); + cfg.ipfs_request_budget_secs = 1; + state.config = Arc::new(cfg); + + let sem = state.git_ipfs_walk_semaphore.clone(); + let router = ipfs_router(state); + + // Hold an ACCESS EXCLUSIVE lock on `repos` on a dedicated pooled connection: + // `list_all_repos`' SELECT needs ACCESS SHARE, which conflicts, so it blocks + // at lock acquisition regardless of row count. + let mut lock_conn = pool.acquire().await.unwrap(); + sqlx::raw_sql("BEGIN; LOCK TABLE repos IN ACCESS EXCLUSIVE MODE;") + .execute(&mut *lock_conn) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.80:5000".parse().unwrap(); + let started = std::time::Instant::now(); + let resp = tokio::time::timeout( + std::time::Duration::from_secs(10), + router.clone().oneshot(get_cid(&valid_cid(), Some(peer))), + ) + .await + .expect("the budget clamp must return within budget; a bare await hangs on the lock") + .unwrap(); + let elapsed = started.elapsed(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a metadata query blocked past the request budget must shed a retryable 503" + ); + assert!( + elapsed < std::time::Duration::from_secs(3), + "the clamp must end the request at ~budget (1s); got {elapsed:?} \ + (pre-fix the bare await blocks on the lock for the whole stall)" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("budget"), + "the shed must name the budget taint so it maps to \ + GITLAWB_IPFS_REQUEST_BUDGET_SECS; got: {body}" + ); + // The scarce walk permit was RAII-dropped on the early return, not pinned for + // the stall: the slot is free again the instant the request returns. + assert_eq!( + sem.available_permits(), + 1, + "the walk permit must be freed on the budget-shed path, not held for the stall" + ); + + // Release the lock; a follow-up request is now SERVED (404 — empty DB), never + // capacity-503'd, proving the slot was not left pinned. + sqlx::raw_sql("ROLLBACK") + .execute(&mut *lock_conn) + .await + .unwrap(); + drop(lock_conn); + let resp2 = router + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp2.status(), + StatusCode::NOT_FOUND, + "with the permit freed and the lock released, a follow-up is served (404), \ + not capacity-503'd" + ); + } + + /// F6/KTD-5 FAIL CLOSED (security-critical): `list_visibility_rules_for_repos` is + /// the access-control query. If its timeout let the handler fall through with an + /// empty rule map, the loop would apply no visibility rules and serve an unfiltered + /// listing — exposing a public repo's path-restricted blob. Here a PUBLIC repo + /// carries the blob under a path-scoped rule that denies anon; `visibility_rules` + /// is locked ACCESS EXCLUSIVE so the rule query blocks. The fix returns the budget + /// 503 BEFORE the loop, so the handler NEVER serves (never 200). + /// + /// Load-bearing: pre-fix the bare await blocks on the lock until the 10s wrap fires + /// (RED). After the fix it sheds the 503 at ~1s. The `assert_ne!(200)` is the + /// fail-closed guard: a naive fix that `unwrap_or_default()`s the rules on timeout + /// and falls through would serve the blob 200 and trip it. + #[sqlx::test] + async fn get_by_cid_visibility_rule_timeout_fails_closed(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool.clone()); + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + + // A PUBLIC repo (upsert_mirror_repo sets is_public=true) carrying the blob, + // with a path-scoped rule restricting src/** to a reader that is NOT the anon + // caller. Rules applied => the blob is denied; rules skipped (fall-through) => + // the public repo serves it (the exposure this guard forbids). + let (repo_id, oid) = + seed_repo_with_blob(&state, tmp.path(), "z6f3failclosed", "gated", b"private\n").await; + state + .db + .set_visibility_rule( + &repo_id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderDDDDDDDDDDDDDDDDDDDDDDDD".to_string()], + "z6f3failclosed", + ) + .await + .unwrap(); + + let mut cfg = (*state.config).clone(); + cfg.ipfs_request_budget_secs = 1; + state.config = Arc::new(cfg); + let sem = state.git_ipfs_walk_semaphore.clone(); + let router = ipfs_router(state); + + // Lock `visibility_rules` ACCESS EXCLUSIVE: list_all_repos (on `repos`) still + // succeeds, but list_visibility_rules_for_repos blocks on the rule query. + let mut lock_conn = pool.acquire().await.unwrap(); + sqlx::raw_sql("BEGIN; LOCK TABLE visibility_rules IN ACCESS EXCLUSIVE MODE;") + .execute(&mut *lock_conn) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.81:5000".parse().unwrap(); + let started = std::time::Instant::now(); + let resp = tokio::time::timeout( + std::time::Duration::from_secs(10), + router.oneshot(get_cid(&cid_for_oid(&oid), Some(peer))), + ) + .await + .expect("the budget clamp must return within budget; a bare await hangs on the lock") + .unwrap(); + let elapsed = started.elapsed(); + + let status = resp.status(); + // Fail closed: the handler must NEVER emit the listing with no rules applied. + assert_ne!( + status, + StatusCode::OK, + "a visibility-rule query timeout must DENY, never serve the path-restricted \ + blob from the public repo (that would expose private content)" + ); + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "a visibility-rule query blocked past the budget must shed the retryable budget 503" + ); + assert!( + elapsed < std::time::Duration::from_secs(3), + "the clamp must end the request at ~budget (1s); got {elapsed:?}" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("budget"), + "the fail-closed shed must name the budget taint; got: {body}" + ); + assert_eq!( + sem.available_permits(), + 1, + "the walk permit must be freed on the fail-closed budget-shed path" + ); + + sqlx::raw_sql("ROLLBACK") + .execute(&mut *lock_conn) + .await + .unwrap(); + drop(lock_conn); + } } diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index d21ec194..756806c0 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -200,3 +200,44 @@ fn f4_release_keeps_conn_owned_until_unlock_resolves() { unlock await resolves." ); } + +/// F6/KTD-5 (initial IPFS metadata queries deadline-wrapped): `get_by_cid` acquires +/// the scarce walk permits (RAII, held for the whole request) BEFORE its two initial +/// metadata queries, and the per-repo loop's first budget gate runs only later. So +/// both `list_all_repos` and `list_visibility_rules_for_repos` must be clamped to the +/// remaining request budget — otherwise a query blocked in Postgres pins the walk slot +/// for the whole stall, past the budget. This scans the PRODUCTION half of `api/ipfs.rs` +/// (the `mod tests` half names the same calls in its own harness and would make the +/// check vacuous) and asserts each query call is immediately preceded by a +/// `tokio::time::timeout(` wrapper. Removing either wrapper turns this red (proven +/// load-bearing). +#[test] +fn f6_ipfs_metadata_queries_are_deadline_wrapped() { + let ipfs = src("api/ipfs.rs"); + let production = ipfs + .split("#[cfg(test)]") + .next() + .expect("split always yields a first chunk"); + + for call in [".list_all_repos()", ".list_visibility_rules_for_repos("] { + assert_eq!( + production.matches(call).count(), + 1, + "F6 guard stale: `{call}` must appear exactly once in the production half \ + of api/ipfs.rs (the deadline-wrapped handler call) — update this guard" + ); + let idx = production + .find(call) + .unwrap_or_else(|| panic!("F6 gate: api/ipfs.rs no longer calls `{call}`")); + // The wrapper opens a few lines above the call (match tokio::time::timeout( ... + // remaining budget ..., )); a 240-char lookback covers it without + // reaching the previous statement. + let window = &production[idx.saturating_sub(240)..idx]; + assert!( + window.contains("tokio::time::timeout("), + "F6 gate missing: `{call}` must be wrapped in tokio::time::timeout(...) \ + clamped to the remaining request budget. An unwrapped await pins the held \ + walk permit for the whole DB stall, past GITLAWB_IPFS_REQUEST_BUDGET_SECS." + ); + } +} From a2be8bab136396c382d8810d77355290a49c16bb Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 10:19:50 -0500 Subject: [PATCH 060/124] fix(node): bound post-receive Pinata replication memory (#174 F2) The detached Pinata task moved each push's full object-id list into the closure and parked it on pin_semaphore.acquire_owned(); a tokio semaphore bounds active holders, not the waiter queue, so a slow Pinata backend let N queued pushes each retain an MB-scale OID list (unbounded memory/tasks). Capture only the small ref-update tuples and re-derive the object set via the existing reaped/deadline-bounded git helpers AFTER the pin permit is acquired, so retained memory is O(ref tuples), not O(object lists). Per-ref effects are unchanged (still one spawn per push, no coalescing or shedding, so no dropped announcements). Adds an inv22 gate row forbidding a retained object list across the acquire, plus re-derivation-equivalence and reaping tests. --- crates/gitlawb-node/src/api/repos.rs | 276 +++++++++++++++++++++-- crates/gitlawb-node/tests/inv22_gates.rs | 65 ++++++ 2 files changed, 327 insertions(+), 14 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 6e6d5f3a..0f12e720 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -988,6 +988,96 @@ async fn resolve_drain_object_list( Some((object_list, rules_opt, record.is_public)) } +/// Re-derive the Pinata replication object set for a push from its ref-update +/// tuples (#174 F2 / KTD-3). +/// +/// The detached Pinata task used to MOVE the push's full pre-resolved object list +/// into its closure and hold it across the `pin_semaphore` await. Under a slow +/// Pinata backend every later push then parked a fresh task each still retaining +/// an MB-scale OID list, so outstanding memory grew O(pushes x object-list) — +/// unbounded. The task now captures only the small `(ref, old, new)` tuples and +/// calls this once a pin slot frees, re-deriving the SAME OID set via +/// `git rev-list` (the delta scan) filtered by the current withheld set. Retained +/// memory is O(ref tuples); the object list is materialized only inside the +/// pin-bounded section, so at most `pin_semaphore` permits' worth exist at once. +/// +/// Coalescing / shedding were rejected because the task's per-ref work is +/// non-idempotent (branch->CID upsert, gossip, GraphQL broadcast, Arweave anchor, +/// peer-notify); dropping a later push's task drops its announcements. Only the +/// retained object list is dropped here, not the task, so every push's effects +/// still fire exactly once. +/// +/// Fresh by design, exactly like `resolve_drain_object_list`: the withheld set +/// and candidate set are recomputed from the current rules, so a rule tightened +/// since the push is honored and fails closed (a newly-withheld blob is not +/// pinned; a no-longer-announceable repo pins nothing). Every git child runs +/// through the same INV-22 bounded, process-group-reaped helpers the sibling +/// post-receive scans use (`replication_withheld_set`, +/// `resolve_candidates_for_push`, `fail_closed_full_scan_objects`). +#[allow(clippy::too_many_arguments)] +async fn pinata_object_list_for_refs( + encrypt_sem: Arc, + disk_path: std::path::PathBuf, + ref_updates: &[(String, String, String)], + rules_opt: Option>, + is_public: bool, + owner_did: String, + git_bin: String, + timeout: std::time::Duration, +) -> Vec { + let (_announce, withheld) = replication_withheld_set( + encrypt_sem.clone(), + rules_opt.clone(), + &owner_did, + is_public, + disk_path.clone(), + git_bin.clone(), + timeout, + ) + .await; + // Not announceable, or the withheld walk failed: replicate nothing (fail + // closed) — mirrors the receive-pack tail's `withheld == None` handling. + let withheld_set = match withheld { + Some(w) => w, + None => return Vec::new(), + }; + let new_tips: Vec = ref_updates + .iter() + .map(|(_, _, new)| new.clone()) + .filter(|s| s != ZERO_SHA) + .collect(); + let old_tips: Vec = ref_updates + .iter() + .map(|(_, old, _)| old.clone()) + .filter(|s| s != ZERO_SHA) + .collect(); + let pin_set = crate::git::push_delta::resolve_candidates_for_push( + encrypt_sem.clone(), + disk_path.clone(), + new_tips, + old_tips, + git_bin.clone(), + timeout, + false, + ) + .await; + if pin_set.full_scan { + fail_closed_full_scan_objects( + encrypt_sem, + disk_path, + rules_opt.unwrap_or_default(), + is_public, + owner_did, + pin_set.candidates, + git_bin, + timeout, + ) + .await + } else { + crate::git::visibility_pack::replicable_objects(pin_set.candidates, &withheld_set) + } +} + /// The pin/encrypt pipeline shared by the snapshot iteration and the /// coalesced-drain iterations: local IPFS pin, then (path-scoped rules only) the /// admission-gated recipients walk → encrypt-then-pin → Arweave manifest anchor. @@ -1825,16 +1915,20 @@ pub async fn git_receive_pack( // Pin new git objects to Pinata, then record branch→CID and gossip. // // #174 P2-2 scope note: this SECOND detached spawn is deliberately NOT brought - // under the per-repo encryption coalescing above. Two reasons: (1) it does not - // park on `git_encrypt_semaphore` (or any semaphore) — the Pinata `pin_new_objects` - // is a bounded reqwest round-trip, so it does not form the unbounded PARKED-waiter - // set that is the P2-2 residual; it runs to completion under the HTTP client's - // network timeouts. (2) Unlike the idempotent recovery-copy walk, this task does - // PER-PUSH, PER-REF work — branch→CID upserts, gossip publish, GraphQL subscription - // broadcast, Arweave anchoring, and peer notify, each keyed to THIS push's - // ref_updates. Coalescing it against an in-flight task for the same repo would DROP - // a later push's ref-update announcements (a correctness regression), not merely - // delay a duplicate. So it is scoped out with rationale, not brought under the bound. + // under the per-repo encryption coalescing above, because unlike the idempotent + // recovery-copy walk it does PER-PUSH, PER-REF work — branch→CID upserts, gossip + // publish, GraphQL subscription broadcast, Arweave anchoring, and peer notify, each + // keyed to THIS push's ref_updates. Coalescing (or shedding) it against an in-flight + // task for the same repo would DROP a later push's ref-update announcements (a + // correctness regression), not merely delay a duplicate. So the task stays one per + // push and every push's effects fire exactly once. + // + // #174 F2 / KTD-3: {bounded memory, no dropped effects, no handler latency} are + // jointly unsatisfiable by coalesce/shed/block, so instead of retaining the full + // object list we bound the thing that actually accumulates. The task captures only + // the small ref tuples and RE-DERIVES the object set inside the worker once a pin + // slot frees (see `pinata_object_list_for_refs`); the MB-scale OID list is never + // held by a parked task. { let pinata_jwt = state.config.pinata_jwt.clone(); let pinata_upload_url = state.config.pinata_upload_url.clone(); @@ -1859,11 +1953,22 @@ pub async fn git_receive_pack( let owner_did_for_arweave = record.owner_did.clone(); let self_public_url = state.config.public_url.clone(); let node_keypair = Arc::clone(&state.node_keypair); - let object_list_pinata = object_list; let do_pinata_replication = withheld.is_some(); + // #174 F2 / KTD-3: capture only the small inputs the re-derivation needs; the + // MB-scale object list is NOT moved in. `pinata_object_list_for_refs` recomputes + // it from these once a pin slot frees. rules/owner/is_public drive the fresh + // fail-closed withheld filter; encrypt_sem + git_bin + timeout keep the re-derive + // git children under the same INV-22 bounded, group-reaped scan admission. + let pinata_rules_opt = rules_opt.clone(); + let pinata_owner_did = record.owner_did.clone(); + let pinata_is_public = record.is_public; + let pinata_git_bin = state.git_bin.clone(); + let pinata_git_timeout = + std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let pinata_encrypt_sem = state.git_encrypt_semaphore.clone(); // Same global pin-admission bound as the IPFS loop (#174 F6): the Pinata pin - // loop also holds this push's full object-id list while pinning it, so it must - // share the cap. It DEFERS on a full pool rather than dropping the pin. + // loop holds a re-derived object-id list while pinning it, so it shares the cap. + // It DEFERS on a full pool rather than dropping the pin. let pin_sem_pinata = state.pin_semaphore.clone(); tokio::spawn(async move { let pinned = if do_pinata_replication { @@ -1871,12 +1976,28 @@ pub async fn git_receive_pack( .acquire_owned() .await .expect("pin_semaphore is never closed"); + // Re-derive the object set now that a pin slot is free (#174 F2 / + // KTD-3). A parked task retained only `ref_updates_clone` (O(ref + // tuples)), never this list, so a slow Pinata backend cannot grow + // outstanding memory O(pushes x object-list). Fresh + fail-closed; + // each git child is INV-22 bounded and process-group reaped. + let object_list = pinata_object_list_for_refs( + pinata_encrypt_sem, + repo_path_clone.clone(), + &ref_updates_clone, + pinata_rules_opt, + pinata_is_public, + pinata_owner_did, + pinata_git_bin, + pinata_git_timeout, + ) + .await; crate::pinata::pin_new_objects( &http_client, &pinata_upload_url, &pinata_jwt, &repo_path_clone, - object_list_pinata, + object_list, &db_clone, ) .await @@ -6127,6 +6248,133 @@ mod tests { ); } + /// #174 F2 / KTD-3 re-derivation equivalence: the object set the Pinata worker + /// re-derives from ONLY the ref tuples (`pinata_object_list_for_refs`, run once a + /// pin slot frees) must equal exactly what the old retained `object_list` would + /// have pinned — the inline-resolved delta, filtered by the withheld set. If the + /// two differ, the memory fix changed what gets pinned; they must not. + #[tokio::test] + async fn f2_pinata_rederivation_equals_retained_object_list() { + let tmp = tempfile::TempDir::new().unwrap(); + u5_init_repo(tmp.path()); + let c1 = u5_commit_file(tmp.path(), "a.txt", "one\n"); + let c2 = u5_commit_file(tmp.path(), "b.txt", "two\n"); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(4)); + let timeout = std::time::Duration::from_secs(600); + + // What the OLD retained-list task WOULD have pinned: the inline pipeline the + // receive-pack tail ran before moving `object_list` into the closure — the + // delta for main c1 -> c2, filtered by the (empty) withheld set. + let candidates = crate::git::push_delta::resolve_candidates_for_push( + sem.clone(), + tmp.path().to_path_buf(), + vec![c2.clone()], + vec![c1.clone()], + "git".to_string(), + timeout, + false, + ) + .await; + assert!( + !candidates.full_scan, + "the c1 -> c2 push is a delta, not a full scan" + ); + let retained: std::collections::HashSet = + crate::git::visibility_pack::replicable_objects( + candidates.candidates, + &std::collections::HashSet::new(), + ) + .into_iter() + .collect(); + + // What the worker re-derives from only the (ref, old, new) tuples. Empty rules + // + is_public => announceable, withheld = {} (the common Pinata case). + let ref_updates = vec![("refs/heads/main".to_string(), c1.clone(), c2.clone())]; + let rederived: std::collections::HashSet = pinata_object_list_for_refs( + sem.clone(), + tmp.path().to_path_buf(), + &ref_updates, + Some(Vec::new()), + true, + "z6MkPinataOwnerAAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + "git".to_string(), + timeout, + ) + .await + .into_iter() + .collect(); + + let new_blob = u5_git(tmp.path(), &["rev-parse", "HEAD:b.txt"]); + assert!( + retained.contains(&c2) && retained.contains(&new_blob), + "the push introduced the new commit and blob" + ); + assert_eq!( + rederived, retained, + "the worker's git rev-list re-derivation must yield exactly the object set \ + the retained list would have pinned — the memory fix must not change what pins" + ); + } + + /// #174 F2 / KTD-3 reaped + deadline-bounded: the worker's re-derivation git children + /// run through the same INV-22 bounded, process-group-reaped helpers the sibling scans + /// use. On a git that hangs on both `rev-list` and `--batch-all-objects`, + /// `pinata_object_list_for_refs` must RETURN within the watchdog budget (the group is + /// SIGKILLed + reaped at the deadline), not block. A bare `Command::output()` here + /// would hang past the ceiling (RED). + #[cfg(unix)] + #[tokio::test] + async fn f2_pinata_rederivation_is_deadline_bounded_and_reaped() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::TempDir::new().unwrap(); + // Empty rules => replication_withheld_set short-circuits (no git). The tip + // peel (`cat-file -t`) reports a commit so the delta stage proceeds; rev-list + // and the full-scan `cat-file --batch-all-objects` both hang (bounded 30s so a + // broken test cannot leak a permanent orphan). + let fake = dir.path().join("fakegit"); + std::fs::write( + &fake, + "#!/bin/sh\ncase \"$1\" in\n \ + cat-file) case \"$*\" in *--batch-all-objects*) i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;; *) echo commit ;; esac ;;\n \ + rev-list) i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;;\n \ + *) : ;;\nesac\nexit 0\n", + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + let git_bin = fake.to_str().unwrap().to_string(); + + let ref_updates = vec![( + "refs/heads/main".to_string(), + ZERO_SHA.to_string(), + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef".to_string(), + )]; + let got = tokio::time::timeout( + std::time::Duration::from_secs(10), + pinata_object_list_for_refs( + std::sync::Arc::new(tokio::sync::Semaphore::new(4)), + dir.path().to_path_buf(), + &ref_updates, + Some(Vec::new()), + true, + "z6MkPinataOwnerAAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + git_bin, + std::time::Duration::from_millis(400), + ), + ) + .await + .expect( + "pinata_object_list_for_refs must return within the watchdog budget — a hang \ + means the re-derivation git is not deadline-bounded / group-reaped (RED)", + ); + assert!( + got.is_empty(), + "a hung git yields nothing this push (the reconciliation sweep backstops)" + ); + } + /// Hot-repo drain at encrypt-pool size 1: the task loop holds NO task-level /// permit, so per-iteration helper acquires (withheld walk, candidate scan, /// recipients walk) each get the pool's single permit in turn and BOTH the diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index 756806c0..2962dc4b 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -241,3 +241,68 @@ fn f6_ipfs_metadata_queries_are_deadline_wrapped() { ); } } + +/// #174 F2 / KTD-3: the detached post-receive Pinata replication task must enqueue +/// only the push's `(ref, old, new)` tuples and RE-DERIVE its object set inside the +/// worker once a pin slot frees — it must NOT move the full per-push object list into +/// the closure and hold it across the `pin_semaphore` acquire. Retaining the list makes +/// every parked task (under a slow Pinata backend) hold an MB-scale OID list, so +/// outstanding memory grows O(pushes x object-list). Coalescing/shedding the task is +/// forbidden (its per-ref effects are non-idempotent), so the fix bounds the retained +/// data, not the task count. +/// +/// Two load-bearing checks, both against the PRODUCTION half of `api/repos.rs` (the +/// `mod tests` half names the same identifiers in its own harness and would make the +/// scan vacuous): (a) the closure-local `object_list_pinata` binding — the retain form — +/// must be gone; reintroducing `let object_list_pinata = object_list;` turns this red. +/// (b) the re-derivation (`pinata_object_list_for_refs`) must appear AFTER the Pinata +/// pin permit is acquired, so the object list is materialized only inside the pin-bounded +/// section and a parked task holds O(ref tuples). +#[test] +fn f2_pinata_enqueues_refs_not_retained_object_lists() { + let repos = src("api/repos.rs"); + let production = repos + .split("#[cfg(test)]") + .next() + .expect("split always yields a first chunk"); + + // (a) the retained-list form must be gone from production. + assert!( + !production.contains("object_list_pinata"), + "F2/KTD-3 regression: the Pinata task retains a full per-push object list \ + (`object_list_pinata`) across the pin-permit acquire. Enqueue only the ref \ + tuples and re-derive the object set inside the worker (pinata_object_list_for_refs)." + ); + + // (b) the re-derivation runs AFTER the pin permit is acquired. Anchor on the + // Pinata pin-admission clone so the window is the Pinata task, not the sibling + // IPFS/encrypt spawn (which shares `pin_semaphore` but never re-derives). + let anchor = production + .find("let pin_sem_pinata") + .expect("F2 gate stale: the Pinata pin-admission clone (pin_sem_pinata) moved"); + let tail = &production[anchor..]; + let acquire = tail + .find(".acquire_owned()") + .expect("F2 gate: the Pinata task no longer acquires a pin permit (acquire_owned)"); + let rederive = tail.find("pinata_object_list_for_refs(").expect( + "F2 gate missing: the Pinata task must re-derive its object set via \ + pinata_object_list_for_refs; it can no longer move a pre-resolved list into the closure", + ); + assert!( + acquire < rederive, + "F2 gate bypassed: the Pinata object-set re-derivation must run AFTER the pin \ + permit is acquired, so a parked task never holds the MB-scale object list" + ); + + // The re-derivation is driven by the push's ref tuples (the enqueued unit), not a + // retained object list — tie "enqueue ref_updates" to the call explicitly. + assert!( + tail[rederive..].starts_with("pinata_object_list_for_refs(") + && tail[rederive..] + .get(..400) + .map(|w| w.contains("ref_updates_clone")) + .unwrap_or(false), + "F2 gate: pinata_object_list_for_refs must re-derive from the push's ref tuples \ + (ref_updates_clone), the small unit the parked task retains" + ); +} From a333ddc1fc7b68830ad6d2d7a55967f64256b0b0 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 10:58:37 -0500 Subject: [PATCH 061/124] fix(node): serialize a disconnected push's repo until its group is reaped (#174 F3) On client disconnect, RepoWriteGuard::Drop released the per-repo advisory lock immediately (it holds no child/pgid), while KillGroupOnDrop's detached reaper was still tearing down the git process group for ~4s. A second same-node push could then acquire the repo and race the first's still-writing objects/ dir and Tigris upload. Add an in-process per-repo write lease (block-and-wait, keyed like the advisory lock) that SUPPLEMENTS the retained cluster-wide PG lock. The lease rides into the disconnect reaper via a clone on the write-path AdmissionGuard (None on all read paths) and a second clone held across guard.release() for the clean-path Tigris upload, so it frees only after the group is reaped. A bounded-wait steal reclaims a leaked lease (block-and-wait has no coalesce degradation). The lease is always taken before the PG lock and released after, so the two serializers cannot invert. Deterministic fake-git race test + inv22 gate row. --- crates/gitlawb-node/src/api/repos.rs | 262 ++++++++++++++++++++- crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/git/smart_http.rs | 20 ++ crates/gitlawb-node/src/main.rs | 5 + crates/gitlawb-node/src/state.rs | 274 ++++++++++++++++++++++ crates/gitlawb-node/src/test_support.rs | 1 + crates/gitlawb-node/tests/inv22_gates.rs | 50 ++++ 7 files changed, 612 insertions(+), 1 deletion(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 0f12e720..9774148e 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1626,6 +1626,25 @@ pub async fn git_receive_pack( } } + // Per-repo in-process write lease (#174 U2/F3): SUPPLEMENTS the cluster-wide pg + // advisory lock. Acquire it BEFORE acquire_write (one consistent order everywhere, + // so the two serializers can never invert into a self-hang) so a second SAME-NODE + // push to this repo blocks here rather than racing a disconnected first push's git + // group while its detached reaper is still tearing it down over the shared local + // objects/ dir. Taking it before the pg lock also means a blocked second writer pins + // no pooled pg connection while it waits. The lease rides the write-path + // AdmissionGuard into the reaper (clone (a)) and spans the clean-path Tigris upload + // in guard.release (clone (b)); it frees only when the LAST clone drops. steal_after + // is set well above any legitimate hold (a full receive-pack under + // git_service_timeout + the ~4s reaper cap + the Tigris upload), so the bounded-wait + // reclaim only ever fires for a genuinely leaked lease, never a merely-slow push. + let lease_steal_after = + std::time::Duration::from_secs(state.config.git_service_timeout_secs * 2 + 60); + let lease = state + .repo_write_leases + .acquire(&record.id, lease_steal_after) + .await; + tracing::debug!(repo = %name, "acquiring write lock"); // Bound the write acquire under `git_acquire_timeout_secs`. acquire_write's // advisory-lock loop already caps at ~60s, but its per-iteration @@ -1661,7 +1680,13 @@ pub async fn git_receive_pack( // instant a disconnect drops this future while the detached reaper runs (#174 P1-a). // The handler keeps no copy. This is independent of the write-lock `guard.release` // below: admission tracks the git process lifetime, the write lock tracks the repo. - let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + // Clone (a) of the write lease rides this AdmissionGuard: on a client disconnect the + // guard moves into KillGroupOnDrop's detached reaper, so the lease frees only after + // the receive-pack group is reaped — NOT at the disconnect instant (which is exactly + // when RepoWriteGuard::Drop frees the pg lock). Tying the lease to RepoWriteGuard + // instead would drop it at disconnect and reopen the F3 race. + let admission = + smart_http::AdmissionGuard::new(_permit, _caller_permit).with_lease(lease.clone()); let receive_result = smart_http::receive_pack( &state.git_bin, &disk_path, @@ -1675,6 +1700,12 @@ pub async fn git_receive_pack( // from blocking subsequent pushes. Only upload to Tigris when the push // succeeded; uploading a half-applied repo would propagate corruption. guard.release(receive_result.is_ok()).await; + // Clean path: clone (a) already dropped inside run_git_service when the receive-pack + // group was reaped; clone (b) held here spanned the success-only Tigris upload that + // ran inside release() above. Drop it now so a second same-repo push proceeds the + // moment this write is durable, rather than at end of the (longer) handler tail. On + // the disconnect path this line is never reached — clone (a) rides the reaper (F3). + drop(lease); let result = receive_result.map_err(|e| { let app = git_service_app_error(&e); @@ -6665,4 +6696,233 @@ mod tests { "repo creation must be IP-throttled before signature verification" ); } + + // ── #174 U2 / F3: second same-repo push serialized until a disconnected first ── + // push's git process GROUP is reaped (RepoWriteLease riding the disconnect reaper). + + /// `kill(pid, 0)` liveness probe (same-uid here, so EPERM never applies). + #[cfg(unix)] + fn f3_alive(pid: i32) -> bool { + unsafe { libc::kill(pid, 0) == 0 } + } + + /// F3 (P1, RED-before/GREEN-after): on a client disconnect DURING receive-pack the + /// disconnected push's git group is torn down by KillGroupOnDrop's detached reaper + /// (~4s TERM/grace/KILL/reap), while RepoWriteGuard::Drop releases the pg advisory + /// lock at the disconnect INSTANT. Without the in-process write lease, a second + /// same-node push then acquires the freed pg lock and mutates the shared local repo + /// WHILE the first group is still writing — a torn snapshot. The lease is held by the + /// write-path AdmissionGuard, which rides that reaper, so the second push must not run + /// its receive-pack (mutate the repo) until the first group is reaped. + /// + /// The fake git labels the pushes by receive-pack arrival order (atomic mkdir): the + /// first (push A) forks a SIGTERM-IGNORING descendant, records its pid, then hangs + /// (so A can be dropped mid-transfer and its group survives the SIGTERM grace); the + /// second (push B, a DIFFERENT source) records that its receive-pack ran — i.e. that + /// B mutated the repo. The load-bearing invariant is strictly ordered, not + /// time-windowed: B's marker must NEVER appear while A's descendant is still alive. + /// + /// Load-bearing: pre-fix (no lease) A's disconnect frees the pg lock, B's + /// acquire_write succeeds within its ~1s retry, and B's receive-pack runs (marker + /// appears) WHILE A's descendant is still alive — RED. With the lease the reaper + /// holds it until the group is ESRCH-gone, so B's marker appears only AFTER — GREEN. + #[cfg(unix)] + #[sqlx::test] + async fn f3_second_push_serialized_until_disconnected_group_reaped(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let tmp = tempfile::TempDir::new().unwrap(); + let seq_a = tmp.path().join("seq_a"); // first receive-pack wins this mkdir = push A + let descfile = tmp.path().join("desc.pid"); // A's SIGTERM-ignoring descendant pid + let b_ran = tmp.path().join("b.ran"); // set when B's receive-pack runs (B mutates) + // receive-pack: first invocation (A) forks a TERM-ignoring descendant (bounded + // loop so a RED run leaks no permanent orphan), records its pid, and hangs in + // `wait`; second (B) records that it ran. rev-parse feeds any tail probe. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + receive-pack)\n\ + cat >/dev/null 2>/dev/null\n\ + if mkdir \"{seq}\" 2>/dev/null; then\n\ + sh -c 'trap \"\" TERM; echo $$ > \"{desc}\"; i=0; while [ $i -lt 60 ]; do sleep 0.1; i=$((i+1)); done' &\n\ + wait\n\ + else\n\ + echo 1 > \"{bran}\"\n\ + fi ;;\n\ + rev-parse) echo deadbeef ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + seq = seq_a.display(), + desc = descfile.display(), + bran = b_ran.display(), + ); + let git_bin = write_fake_git(tmp.path(), &body); + // One repo; A and B push to it (same record.id -> same lease key). Non-path-scoped + // + flush-only body -> no post-receive scans to muddy the observation. + let state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f3repo", "r1", false).await; + let did = "did:key:z6MkF3PusherAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + + // Push A: drive its handler future in slices until it reaches receive-pack and the + // fake records its descendant pid (A now holds the lease and is hung). + let mut fut_a = Box::pin(git_receive_pack( + State(state.clone()), + Path(("z6f3repo".to_string(), "r1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some("203.0.113.81:5000".parse::().unwrap())), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + )); + let mut desc: Option = None; + for _ in 0..1000 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut_a).await; + if let Some(p) = std::fs::read_to_string(&descfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + desc = Some(p); + break; + } + } + let desc = desc.expect("push A must reach receive-pack and record its descendant pid"); + assert!( + f3_alive(desc), + "A's descendant must be alive before the disconnect" + ); + + // Push B: a DIFFERENT source, same repo. It blocks on the lease A holds. + let state_b = state.clone(); + let handle_b = tokio::spawn(async move { + git_receive_pack( + State(state_b), + Path(("z6f3repo".to_string(), "r1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some( + "203.0.113.82:5000".parse::().unwrap(), + )), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await + }); + // Give B time to reach and block on the lease acquire. + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + assert!( + !b_ran.exists(), + "B must not have mutated the repo while A legitimately holds the lease" + ); + + // Client disconnect on A: drop its future. RepoWriteGuard::Drop frees the pg lock + // immediately; the write-path AdmissionGuard (carrying the lease's clone (a)) + // rides KillGroupOnDrop's detached reaper, which now tears down A's group. + drop(fut_a); + + // Load-bearing ordering invariant: while A's descendant is still alive (group not + // yet reaped), B must NOT have run its receive-pack. Poll until the descendant is + // gone; every step it is alive, B's marker must be absent. Pre-fix, B's marker + // appears here (RED); with the lease it can only appear after the reap (GREEN). + let mut reaped = false; + for _ in 0..800 { + if !f3_alive(desc) { + reaped = true; + break; + } + assert!( + !b_ran.exists(), + "F3 RED: push B mutated the repo while push A's disconnected git group \ + was still alive (descendant pid {desc}) — the second writer must be \ + serialized until the first group is reaped" + ); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + // Safety net so a RED run never leaks the orphan. + unsafe { + libc::kill(desc, libc::SIGKILL); + } + assert!( + reaped, + "A's disconnected group must be reaped within the teardown cap" + ); + + // GREEN tail: once the group is reaped the lease frees and B proceeds — its + // receive-pack runs (marker appears) and it returns 200. + let mut b_mutated = false; + for _ in 0..1000 { + if b_ran.exists() { + b_mutated = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert!( + b_mutated, + "push B must proceed and mutate the repo once A's group is reaped" + ); + let resp = tokio::time::timeout(std::time::Duration::from_secs(30), handle_b) + .await + .expect("push B must complete once the lease frees") + .expect("push B task must not panic") + .expect("push B must succeed"); + assert_eq!(resp.status(), 200, "push B lands 200 after serialization"); + } + + /// F3 clean-path no-regression: a clean push (no disconnect) releases the lease after + /// the receive-pack group is reaped and the (success-only) Tigris upload in + /// guard.release runs, so the per-repo lease entry is GC'd and a second same-repo + /// push proceeds immediately. A lease that failed to free on the clean path would + /// wedge every subsequent push to the repo. + #[cfg(unix)] + #[sqlx::test] + async fn f3_clean_push_frees_lease_and_second_push_proceeds(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let tmp = tempfile::TempDir::new().unwrap(); + // Clean receive-pack: drain stdin, exit 0. No hang, no descendant. + let body = "#!/bin/sh\ncase \"$1\" in\n receive-pack) cat >/dev/null 2>/dev/null ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + let state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f3clean", "c1", false).await; + let did = "did:key:z6MkF3CleanPusherAAAAAAAAAAAAAAAAAAAAAAAA"; + + let push = |st: AppState, peer: &'static str| async move { + tokio::time::timeout( + std::time::Duration::from_secs(30), + git_receive_pack( + State(st), + Path(("z6f3clean".to_string(), "c1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(peer.parse::().unwrap())), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ), + ) + .await + .expect("a clean push must not wedge on the lease") + .expect("the push must succeed") + }; + + let a = push(state.clone(), "203.0.113.91:5000").await; + assert_eq!(a.status(), 200, "clean push A lands 200"); + // The clean push freed its lease (both clones dropped) -> the entry GC'd. + assert!( + state.repo_write_leases.is_empty(), + "a clean push must free the per-repo lease (Drop-frees-key) so it never wedges" + ); + + let b = push(state.clone(), "203.0.113.92:5000").await; + assert_eq!( + b.status(), + 200, + "a second same-repo push proceeds after a clean first" + ); + assert!( + state.repo_write_leases.is_empty(), + "the lease entry must be freed again after the second clean push" + ); + } } diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index c1fa6624..431ac983 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -526,6 +526,7 @@ mod tests { git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), pin_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), encrypt_inflight: crate::state::EncryptInflight::new(), + repo_write_leases: crate::state::RepoWriteLeases::new(), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index f8b46ce8..9ac742ec 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -33,17 +33,37 @@ pub struct AdmissionGuard { // 'static` so the guard can move into the detached reaper task. _global: Option>, _caller: Option>, + // Per-repo write lease (#174 U2/F3), `Some` ONLY on the receive-pack write path + // (via [`with_lease`](Self::with_lease)); `None` on every read path and every + // non-receive-pack write path. It rides this guard into `KillGroupOnDrop`'s detached + // reaper, so on a client disconnect the lease frees only after the disconnected + // push's git group is reaped — serializing a second same-node push against the + // still-writing group. A `RepoWriteLease` clone is `Send + 'static` and holds no pg + // connection, so it travels with the reaper cleanly. A stray `Some` on a READ path + // would wrongly serialize upload-pack against pushes, hence the None-everywhere-else + // discipline. + _lease: Option, } impl AdmissionGuard { /// Take ownership of the global permit and an optional per-caller permit. Both are /// erased to `Box` — the guard's only job is to hold them until it drops. + /// No lease is attached here; the receive-pack write path adds one via + /// [`with_lease`](Self::with_lease), so every other call site is lease-free. pub fn new(global: impl Send + 'static, caller: Option) -> Self { Self { _global: Some(Box::new(global)), _caller: caller.map(|c| Box::new(c) as Box), + _lease: None, } } + + /// Attach the per-repo write lease (#174 U2/F3). Called ONLY on the receive-pack + /// write path, so the lease rides the disconnect reaper; read paths never call this. + pub fn with_lease(mut self, lease: crate::state::RepoWriteLease) -> Self { + self._lease = Some(lease); + self + } } /// Handle `GET /:owner/:repo/info/refs?service=git-upload-pack` diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 4c4c1da8..d6c0df46 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -410,6 +410,11 @@ async fn main() -> Result<()> { // P2-2). No knob: it is a natural cap (one entry per distinct repo), not a // sized pool. encrypt_inflight: crate::state::EncryptInflight::new(), + // Per-repo in-process write-lease serializer (#174 U2/F3): supplements the pg + // advisory lock so a disconnected push's still-reaping git group can't be raced + // by a second same-node push. Natural cap (one entry per contended repo, freed + // when unreferenced), no sized knob. + repo_write_leases: crate::state::RepoWriteLeases::new(), git_read_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( config.max_concurrent_reads_per_caller, ), diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 77799bd0..213b8b35 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -147,6 +147,20 @@ pub struct AppState { /// recovery copy, which `2a54c15` deliberately kept fail-closed (there is no /// reconciliation sweep to re-derive a dropped copy). See [`EncryptInflight`]. pub encrypt_inflight: EncryptInflight, + /// Per-repo in-process write serializer that SUPPLEMENTS the cluster-wide pg + /// advisory lock on the receive-pack path (#174 U2/F3). On a client disconnect + /// mid-`receive-pack`, `RepoWriteGuard::Drop` releases the pg advisory lock at the + /// disconnect instant, but the disconnected push's git process GROUP is still + /// being torn down by `KillGroupOnDrop`'s detached reaper (~4s TERM/grace/KILL/reap) + /// over the shared LOCAL objects/ dir — so a second SAME-NODE push could acquire + /// the repo and race the still-writing group into a torn snapshot. This lease is + /// held by the write-path `AdmissionGuard`, which rides that reaper, so a second + /// same-repo push blocks until the first group is reaped. It is per-NODE (the + /// corruption is same-node: shared local objects/ + in-process reaper, and the + /// disconnect path uploads nothing to Tigris), so it needs no cross-node counterpart + /// and does NOT replace the pg lock (which stays the genuine cluster-wide serializer). + /// See [`RepoWriteLeases`]. + pub repo_write_leases: RepoWriteLeases, /// Per-caller concurrency sub-cap on the read pool: each caller (keyed on the /// resolved source IP, #174 U1) may hold at most `max_concurrent_reads_per_caller` /// in-flight read ops, so one caller cannot monopolize `git_read_semaphore` @@ -453,6 +467,182 @@ impl Drop for EncryptInflightGuard { } } +/// Per-repo in-process write-lease serializer (#174 U2/F3). Keyed by the repo's DB +/// id (1:1 with the pg advisory lock's owner/name key), each entry is a one-permit +/// semaphore: the receive-pack handler takes it BEFORE `acquire_write` (see the acquire +/// order note on [`acquire`](Self::acquire)) and a second same-repo push BLOCKS on it — +/// block-and-wait, NOT coalesce. It mirrors [`EncryptInflight`]'s keyed-map + guard + +/// Drop-frees-key STRUCTURE; the semantics differ (block-and-wait, so there is no +/// lossy-coalesce degradation to fall back on). +#[derive(Clone, Default)] +pub struct RepoWriteLeases { + // std::sync::Mutex: held only for O(1) map ops (get-or-create + refcount) in a sync + // context, never across an await — the semaphore wait happens OUTSIDE this lock. + repos: Arc>>, +} + +/// A per-repo lease entry: the one-permit semaphore plus a refcount of the handlers +/// currently referencing it (holding or waiting). While `refs > 0` every acquirer +/// shares the SAME semaphore, so mutual exclusion holds; the entry is removed only when +/// `refs` hits 0 (no one references it), so a fresh entry can never split serialization. +struct LeaseSlot { + sem: Arc, + refs: usize, +} + +impl RepoWriteLeases { + pub fn new() -> Self { + Self::default() + } + + /// Acquire the per-repo write lease, blocking until it is free (a second same-repo + /// writer waits). `steal_after` bounds that wait: past it the acquirer STEALS + /// (proceeds permit-less) rather than block forever. + /// + /// Why a bounded steal: block-and-wait has no degradation of its own (unlike the + /// coalescing [`EncryptInflight`], whose lost key merely delays a best-effort copy), + /// and unlike the pg advisory lock (60s stale reclaim) an in-process waiter has no + /// reclaim — so a leaked/never-run Drop (runtime teardown without unwind, task abort, + /// `mem::forget`) would otherwise wedge the repo permanently. A stealer takes NO + /// permit and touches NO count, so a merely-slow holder that later drops can never + /// leave the semaphore over-counted; the caller must therefore set `steal_after` + /// safely ABOVE any legitimate hold (a full receive-pack under + /// `git_service_timeout_secs` + the ~4s reaper cap + the Tigris upload). + /// + /// Acquire order (one consistent order everywhere, so no inversion self-hang): the + /// lease is taken BEFORE the pg advisory lock (`acquire_write`) and released AFTER + /// it. Nothing anywhere takes the pg lock before this lease, so the two serializers + /// can never deadlock; taking the lease first also means a blocked second writer + /// pins no pooled pg connection while it waits. + pub async fn acquire(&self, repo_id: &str, steal_after: std::time::Duration) -> RepoWriteLease { + // Take the entry refcount BEFORE the await, so the entry cannot be GC'd out from + // under a waiter (a fresh entry for a new acquirer would split serialization). + let sem = { + let mut map = self.repos.lock().expect("repo_write_leases mutex poisoned"); + let slot = map.entry(repo_id.to_string()).or_insert_with(|| LeaseSlot { + sem: Arc::new(tokio::sync::Semaphore::new(1)), + refs: 0, + }); + slot.refs += 1; + Arc::clone(&slot.sem) + }; + // Cancellation-safe refcount: hold a reservation across the (cancellable) wait so + // that if this acquire future is DROPPED mid-wait — a client disconnect while a + // second same-repo push is blocked here — the reservation's Drop still decrements + // the ref it just took, rather than stranding it (which would defeat the + // Drop-frees-key GC). On success the reservation is `forget`-transferred into the + // returned guard, which then owns the single decrement. + let reservation = RefReservation { + repos: Arc::clone(&self.repos), + repo_id: repo_id.to_string(), + }; + let permit = match tokio::time::timeout(steal_after, Arc::clone(&sem).acquire_owned()).await + { + Ok(Ok(p)) => Some(p), + // The semaphore is never closed; treat the (unreachable) closed case as a + // steal so acquire always makes forward progress. + Ok(Err(_closed)) => None, + Err(_elapsed) => { + tracing::warn!( + repo = %repo_id, + steal_after_secs = steal_after.as_secs(), + "repo write-lease wait exceeded the steal bound; presuming a leaked \ + lease and proceeding permit-less (in-process serializer reclaim)" + ); + None + } + }; + // Transfer the ref from the reservation to the guard: forget the reservation (so + // it does NOT decrement) and let the guard own the single decrement on its Drop. + std::mem::forget(reservation); + RepoWriteLease(Arc::new(LeaseGuardInner { + repos: Arc::clone(&self.repos), + repo_id: repo_id.to_string(), + _permit: permit, + })) + } + + /// Number of repos with a live lease entry. Test/metrics observability. + #[allow(dead_code)] + pub fn len(&self) -> usize { + self.repos + .lock() + .expect("repo_write_leases mutex poisoned") + .len() + } + + #[allow(dead_code)] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +/// Shared-ownership handle to a held repo write lease (#174 U2/F3). `Clone` hands a +/// second holder a handle to the SAME inner guard; the lease (permit + map refcount) +/// frees only when the LAST clone drops. The receive-pack handler makes two: +/// * clone (a) rides the write-path [`AdmissionGuard`] into `KillGroupOnDrop`'s +/// detached reaper, so on a client disconnect it drops only AFTER the git group is +/// reaped (this is the F3 fix — a lease tied to `RepoWriteGuard` would instead drop +/// at the disconnect instant, reopening the race); +/// * clone (b) is held by the handler across `guard.release()`, so on the clean path +/// it spans the success-only Tigris upload that runs inside `release`, AFTER +/// `receive_pack` has already dropped clone (a) inside `run_git_service`. +/// +/// `Send + 'static` with NO pg connection (just an `Arc`), so it can ride the reaper. +#[derive(Clone)] +pub struct RepoWriteLease(#[allow(dead_code)] Arc); + +struct LeaseGuardInner { + repos: Arc>>, + repo_id: String, + // `None` only on the steal path (the bounded wait elapsed). Dropping `None` releases + // no permit, so a stealer never corrupts the semaphore's permit count. + _permit: Option, +} + +impl Drop for LeaseGuardInner { + fn drop(&mut self) { + // Runs exactly ONCE per handler acquisition — when the last `RepoWriteLease` + // clone drops (the Arc strong count hits 0) — so the refcount decrements once, + // however many clones existed. `_permit` drops after this body, releasing the + // semaphore permit so a waiting acquirer proceeds. + release_lease_ref(&self.repos, &self.repo_id); + } +} + +/// Holds the entry refcount across the cancellable wait inside +/// [`RepoWriteLeases::acquire`]. If that acquire future is dropped mid-wait, this Drop +/// decrements the ref it took; on success `acquire` `forget`s it and hands the ref to +/// the returned [`LeaseGuardInner`], so the ref is decremented exactly once either way. +struct RefReservation { + repos: Arc>>, + repo_id: String, +} + +impl Drop for RefReservation { + fn drop(&mut self) { + release_lease_ref(&self.repos, &self.repo_id); + } +} + +/// Decrement a lease entry's refcount and remove it once no handler references it, so +/// the map cannot grow without bound (Drop-frees-key, like `EncryptInflight`). Safe +/// under block-and-wait: while `refs > 0` every acquirer shares the SAME semaphore, and +/// a fresh entry is created only after `refs` hits 0, when no one references the old one. +fn release_lease_ref( + repos: &Arc>>, + repo_id: &str, +) { + if let Ok(mut map) = repos.lock() { + if let Some(slot) = map.get_mut(repo_id) { + slot.refs = slot.refs.saturating_sub(1); + if slot.refs == 0 { + map.remove(repo_id); + } + } + } +} + /// Admit a post-receive git scan to the shared `git_encrypt_semaphore` pool /// (#174 F4): DEFER (await), never shed — a dropped scan would lose the push's /// recovery copy or silently under-pin it. The returned permit must move into @@ -483,3 +673,87 @@ pub async fn acquire_scan_permit( ); permit } + +#[cfg(test)] +mod repo_write_lease_tests { + use super::RepoWriteLeases; + use std::time::Duration; + + /// #174 U2/F3 lease mechanics: block-and-wait serialization on the same repo, + /// no serialization across distinct repos, Drop-frees-key GC, and the bounded-wait + /// steal reclaim so a leaked (never-run Drop) holder cannot wedge the repo forever. + #[tokio::test] + async fn serializes_same_repo_frees_key_and_steals_on_leak() { + let leases = RepoWriteLeases::new(); + let big = Duration::from_secs(3600); + + // Block-and-wait: a second same-repo acquire waits while the first is held. + let a = leases.acquire("repo1", big).await; + let blocked = + tokio::time::timeout(Duration::from_millis(200), leases.acquire("repo1", big)).await; + assert!( + blocked.is_err(), + "a second same-repo acquire must block while the first lease is held" + ); + // ... and proceeds once the first frees. + drop(a); + let b = tokio::time::timeout(Duration::from_millis(500), leases.acquire("repo1", big)) + .await + .expect("the second acquire must proceed once the first lease frees"); + drop(b); + + // Drop-frees-key: with no holders the entry is removed (bounded map growth). + assert!( + leases.is_empty(), + "the lease entry must be removed once no handler references it" + ); + + // Distinct repos never serialize against each other. + let x = leases.acquire("repoX", big).await; + let _y = tokio::time::timeout(Duration::from_millis(200), leases.acquire("repoY", big)) + .await + .expect("distinct repos must not serialize"); + drop(x); + drop(_y); + + // Steal-on-timeout reclaim: a leaked holder (never-run Drop, simulated by + // mem::forget) must NOT wedge the repo — the bounded wait proceeds permit-less. + let leaked = leases.acquire("repoZ", big).await; + std::mem::forget(leaked); + let stolen = tokio::time::timeout( + Duration::from_secs(2), + leases.acquire("repoZ", Duration::from_millis(150)), + ) + .await + .expect("a leaked lease must be reclaimed by the bounded-wait steal, not hang forever"); + drop(stolen); + } + + /// Cancellation safety: dropping an acquire future while it is BLOCKED waiting for + /// the lease (a client disconnect on a second same-repo push) must not strand the + /// entry refcount — after the holder frees and the waiter is cancelled, the key GCs. + #[tokio::test] + async fn cancelled_waiter_does_not_strand_the_refcount() { + let leases = RepoWriteLeases::new(); + let big = Duration::from_secs(3600); + + let a = leases.acquire("repoC", big).await; + // A waiter blocks, then is cancelled (its acquire future dropped) mid-wait. + let cancelled = + tokio::time::timeout(Duration::from_millis(150), leases.acquire("repoC", big)).await; + assert!( + cancelled.is_err(), + "the waiter must be blocked, then cancelled" + ); + + // Release the holder. If the cancelled waiter had stranded its ref, the entry + // would never GC; assert it does once the holder frees. + drop(a); + // Let any pending Drop bookkeeping settle. + tokio::task::yield_now().await; + assert!( + leases.is_empty(), + "a cancelled waiter must not strand the entry refcount (key must GC)" + ); + } +} diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index cd58f19e..b88ae463 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -89,6 +89,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), pin_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), encrypt_inflight: crate::state::EncryptInflight::new(), + repo_write_leases: crate::state::RepoWriteLeases::new(), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( 8, diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index 2962dc4b..0ce54088 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -306,3 +306,53 @@ fn f2_pinata_enqueues_refs_not_retained_object_lists() { (ref_updates_clone), the small unit the parked task retains" ); } + +/// #174 U2 / F3 (second same-repo writer serialized until a disconnected first push's +/// git group is reaped): `git_receive_pack` must take the per-repo in-process write +/// lease and CARRY it on the write-path `AdmissionGuard` (via `.with_lease`), so the +/// lease rides `KillGroupOnDrop`'s detached reaper and frees only after the group is +/// reaped — supplementing the pg advisory lock, which `RepoWriteGuard::Drop` releases at +/// the disconnect instant. Tying the lease to `RepoWriteGuard` instead (or dropping the +/// `.with_lease` carry) reopens the race; the behavioral RED/GREEN test +/// (`f3_second_push_serialized_until_disconnected_group_reaped`) is the real bar, this +/// is the completeness tripwire. Scanned against the PRODUCTION half of `api/repos.rs` +/// (the `mod tests` half names these identifiers in its own harness). +#[test] +fn f3_second_writer_leased_until_reap() { + let repos = src("api/repos.rs"); + let smart_http = src("git/smart_http.rs"); + let repos_production = repos + .split("#[cfg(test)]") + .next() + .expect("split always yields a first chunk"); + + // The lease is acquired, then carried by the AdmissionGuard (.with_lease), before + // receive_pack runs the write. Severing any of the three turns this red. + let lease_acquire = repos_production.find("repo_write_leases").expect( + "F3 gate missing: git_receive_pack no longer takes the per-repo write lease \ + (state.repo_write_leases)", + ); + let with_lease = repos_production.find(".with_lease(").expect( + "F3 gate missing: the write-path AdmissionGuard no longer carries the lease \ + (.with_lease). The lease must ride the disconnect reaper via the AdmissionGuard, \ + NOT RepoWriteGuard (which drops at the disconnect instant, reopening F3).", + ); + let receive = repos_production + .find("smart_http::receive_pack(") + .expect("F3 gate stale: git_receive_pack no longer calls smart_http::receive_pack"); + assert!( + lease_acquire < with_lease && with_lease < receive, + "F3 gate bypassed: the write lease must be acquired, then carried by the \ + AdmissionGuard (.with_lease), BEFORE receive_pack runs the write" + ); + + // The AdmissionGuard must actually hold the lease (Option) via a + // with_lease setter, so it travels into the detached reaper. Removing the field or + // method turns this red. + assert!( + smart_http.contains("_lease: Option") + && smart_http.contains("pub fn with_lease("), + "F3 gate missing: AdmissionGuard must hold an Option set via \ + with_lease, so the lease rides the guard into KillGroupOnDrop's reaper" + ); +} From f02c5e1ebd6c365b2d29bdb1890e1398e8011338 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 11:54:09 -0500 Subject: [PATCH 062/124] fix(node): take the write permits after the per-repo lease, not before (#174 F3 review) Code review found a DoS the F3 lease introduced: git_receive_pack acquired the global and per-source write permits before block-waiting on the per-repo lease, so a second same-repo push that parked on the lease pinned a scarce global write-pool slot (1 of 32) for up to steal_after while sending zero bytes. A few hostile sources stacking same-repo pushes could hold every slot on idle lease-waiters and shed 503 on every push to every other repo node-wide. Move both permit acquisitions to after the lease acquire so a lease-blocked waiter pins no write-pool slot, and keep a cheap non-holding availability peek before the DB lookup so a push flood on a saturated pool still sheds 503 without touching Postgres (the authoritative held permit is taken after the lease). Adds a blocked-waiter-holds-no-permit regression test. --- crates/gitlawb-node/src/api/repos.rs | 192 ++++++++++++++++++++++++--- 1 file changed, 171 insertions(+), 21 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 9774148e..836e5625 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1536,27 +1536,16 @@ pub async fn git_receive_pack( body: Bytes, ) -> Result { let name = smart_http_repo_name(&repo)?; - // Per-source write sub-cap (#174 P1-d): before the global write permit so one - // source IP cannot occupy the whole write pool via many slow authenticated pushes - // and 503 every other source. Owner enforcement defaults off, so any valid did:key - // is accepted (auth != authz), and the 600/hour push limiter bounds arrival RATE, - // not in-flight concurrency — so without this a single host minting disposable DIDs - // saturates the pool. Keyed on the resolved source IP, NEVER the signed DID (a DID - // farm defeats a DID key); no resolvable key -> global write pool only. - let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); - let _caller_permit = acquire_read_caller_permit( - &state.git_write_per_caller, - caller_key.as_deref(), - name, - "receive-pack", - )?; - // Shed with a 503 before spawning git when the concurrency cap is saturated. - // Pushes draw from the dedicated WRITE pool, separate from reads, so a flood of - // anonymous reads cannot shed an authenticated push (#174). Taken after the - // per-source cap above so one source cannot occupy global slots it would be - // sub-cap-denied for; still before the Tigris acquire_write, bounding concurrent - // fresh acquires (INV-10); held for the whole op. - let _permit = git_permit(&state.git_write_semaphore)?; + // Fast-path shed BEFORE the DB lookup when the write pool is saturated, so a push + // flood on a full pool does not hit Postgres per request. Best-effort (racy) and + // NON-holding: the authoritative, held permit is taken after the per-repo lease below + // (so a lease-blocked waiter pins no write slot — #174 F3 review). This peek only + // restores the cheap pre-DB shed the permit reorder would otherwise have lost. + if state.git_write_semaphore.available_permits() == 0 { + return Err(AppError::Overloaded( + "git service at capacity, retry shortly".into(), + )); + } tracing::info!(owner = %owner, repo = %name, "receive-pack request"); let record = state .db @@ -1645,6 +1634,31 @@ pub async fn git_receive_pack( .acquire(&record.id, lease_steal_after) .await; + // Admission permits are taken HERE, AFTER the per-repo lease and BEFORE acquire_write. + // Ordering is the fix (#174 P2 DoS): the lease is a block-and-wait serializer, so a + // second same-repo push can park on `acquire` above for up to steal_after. Taking the + // scarce write permits only once we own the lease means a lease-blocked waiter pins NO + // write-pool slot while it waits. Otherwise a few hostile sources could stack same-repo + // pushes, hold every global slot on zero-byte lease-waiters, and shed 503 on every push + // to every OTHER repo node-wide. Still before acquire_write, so the git op stays + // admission-gated (INV-10) and a saturated pool sheds 503 before spawning git. + // + // Per-source sub-cap first (#174 P1-d): one source IP cannot occupy the whole write + // pool via many slow pushes. Owner enforcement defaults off, so any valid did:key is + // accepted (auth != authz) and the push rate limiter bounds arrival RATE, not in-flight + // concurrency. Keyed on the resolved source IP, NEVER the signed DID (a DID farm defeats + // a DID key); no resolvable key -> global write pool only. Then the global write permit: + // pushes draw from the dedicated WRITE pool, separate from reads, and it is held for the + // whole op (moved into the AdmissionGuard below). + let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); + let _caller_permit = acquire_read_caller_permit( + &state.git_write_per_caller, + caller_key.as_deref(), + name, + "receive-pack", + )?; + let _permit = git_permit(&state.git_write_semaphore)?; + tracing::debug!(repo = %name, "acquiring write lock"); // Bound the write acquire under `git_acquire_timeout_secs`. acquire_write's // advisory-lock loop already caps at ~60s, but its per-iteration @@ -6925,4 +6939,140 @@ mod tests { "the lease entry must be freed again after the second clean push" ); } + + /// F3 DoS (P2, RED-before/GREEN-after): a second same-repo push that BLOCKS on the + /// per-repo write lease must hold NO global write permit while it waits. The lease + /// is a block-and-wait serializer, so a lease-blocked waiter can sit for up to + /// steal_after (~a full git_service_timeout window). If it grabs a scarce global + /// write-pool slot BEFORE blocking, a handful of hostile sources can stack same-repo + /// pushes, pin every write slot on lease-waiters sending zero bytes, and shed 503 on + /// every push to every OTHER repo node-wide. The fix acquires the lease BEFORE the + /// two write permits, so a blocked waiter pins no slot. + /// + /// Load-bearing invariant: with the write pool sized to 2, push A holds the lease and + /// is in-flight in receive-pack (1 permit held), and same-repo push B is blocked on + /// the lease, `git_write_semaphore.available_permits()` must stay 1 (only A holds). + /// Pre-fix B takes its permit BEFORE blocking on the lease, draining the pool to 0 + /// (RED). With the reorder B blocks before any permit, so the pool stays at 1 (GREEN). + #[cfg(unix)] + #[sqlx::test] + async fn f3_lease_blocked_waiter_holds_no_write_permit(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let seq_a = tmp.path().join("seq_a"); // first receive-pack wins this mkdir = push A + let a_inpack = tmp.path().join("a.inpack"); // set when A reaches receive-pack (holds lease+permit) + let b_ran = tmp.path().join("b.ran"); // set when B's receive-pack runs (B got past the lease) + // receive-pack: first invocation (A) marks that it reached the pack and hangs in a + // bounded loop (so a RED run leaks no permanent orphan); second (B) marks it ran. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + receive-pack)\n\ + cat >/dev/null 2>/dev/null\n\ + if mkdir \"{seq}\" 2>/dev/null; then\n\ + echo 1 > \"{ainp}\"\n\ + i=0; while [ $i -lt 100 ]; do sleep 0.1; i=$((i+1)); done\n\ + else\n\ + echo 1 > \"{bran}\"\n\ + fi ;;\n\ + rev-parse) echo deadbeef ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + seq = seq_a.display(), + ainp = a_inpack.display(), + bran = b_ran.display(), + ); + let git_bin = write_fake_git(tmp.path(), &body); + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f3dos", "d1", false).await; + // Size the write pool to 2 so one in-flight holder (A) leaves exactly one slot + // free, and a pool-holding waiter (B, pre-fix) would drain it to zero. Sizing to + // 1 would 503 B on the pool before it could block on the lease, hiding the bug. + state.git_write_semaphore = Arc::new(Semaphore::new(2)); + let did = "did:key:z6MkF3DosPusherAAAAAAAAAAAAAAAAAAAAAAAAAA"; + + // Push A: drive its handler future in slices until it reaches receive-pack (it now + // holds the lease and one write permit and is hung). available_permits() drops to 1. + let mut fut_a = Box::pin(git_receive_pack( + State(state.clone()), + Path(("z6f3dos".to_string(), "d1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some("203.0.113.71:5000".parse::().unwrap())), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + )); + for _ in 0..1000 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut_a).await; + if a_inpack.exists() { + break; + } + } + assert!( + a_inpack.exists(), + "push A must reach receive-pack and hold the lease" + ); + assert_eq!( + state.git_write_semaphore.available_permits(), + 1, + "with the pool sized to 2, the single in-flight holder (A) leaves one slot free" + ); + + // Push B: a DIFFERENT source, same repo. It blocks on the lease A holds. + let state_b = state.clone(); + let handle_b = tokio::spawn(async move { + git_receive_pack( + State(state_b), + Path(("z6f3dos".to_string(), "d1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some( + "203.0.113.72:5000".parse::().unwrap(), + )), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await + }); + + // Load-bearing check: while B is a lease-blocked waiter the pool must stay at 1 + // (only A holds a permit). Poll the invariant across a full window; pre-fix B + // grabs the last slot within ms and the pool falls to 0 (RED), post-fix it never + // does (GREEN). A stable state, not a one-shot race: B stays blocked on the lease + // (steal_after is far larger than this window) so once it settles the pool holds. + for _ in 0..100 { + assert_eq!( + state.git_write_semaphore.available_permits(), + 1, + "F3 DoS RED: a lease-blocked same-repo waiter took a global write permit \ + while sending zero bytes, draining the pool — a blocked waiter must pin \ + no write-pool slot" + ); + assert!( + !b_ran.exists(), + "B must not have run receive-pack while A holds the lease" + ); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + // Non-vacuous: B is genuinely parked on the lease, not returned early. + assert!( + !handle_b.is_finished(), + "push B must still be blocked on the lease at this point" + ); + + // Client disconnect on A: drop its future. The write-path AdmissionGuard rides the + // reaper, freeing the lease once A's group is reaped; B then proceeds. + drop(fut_a); + let resp = tokio::time::timeout(std::time::Duration::from_secs(30), handle_b) + .await + .expect("push B must complete once A's group is reaped and the lease frees") + .expect("push B task must not panic") + .expect("push B must succeed"); + assert_eq!(resp.status(), 200, "push B lands 200 after serialization"); + assert!(b_ran.exists(), "push B ran its receive-pack once unblocked"); + } } From e2be6f3019a3fe2b6d941f0d94722fb10843ef2d Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 11:54:09 -0500 Subject: [PATCH 063/124] fix(node): don't 500 an /ipfs miss when a transient taint co-occurs (#174 F5 review) The deterministic-fault terminal 500 was checked before the transient truncated_by 503, unconditionally. When one repo is corrupt (deterministic) while a DIFFERENT repo is transiently skipped in the same scan, the requested CID may live in the transient repo -- a retryable 503 is correct, but the handler returned a terminal 500 and a conformant client won't retry, hiding retrievable content until the unrelated corrupt repo is repaired. Gate the 500 on truncated_by.is_empty(): it fires only when a deterministic fault is the SOLE reason nothing served; a co-occurring transient taint falls through to the retryable 503. A pure deterministic fault still terminally 500s an absent lookup by design. Adds a co-occurrence regression test. --- crates/gitlawb-node/src/api/ipfs.rs | 126 ++++++++++++++++++++++++++-- 1 file changed, 120 insertions(+), 6 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 7f977a81..5e7ae05a 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -572,12 +572,19 @@ pub async fn get_by_cid( // repo / bad `.git/config`), so the object is not proven absent AND a retry cannot // change that. Shed a TERMINAL, non-retryable 500 rather than the retryable 503 // below — a 503 would let a conformant client retry-storm a fresh `git cat-file` - // per attempt against the broken repo. This is checked before the transient 503 so - // a deterministic fault is never downgraded to a retryable status. The body is - // opaque (a generic message via `AppError::Git` -> 500, no Retry-After): the raw - // git stderr — which leaks filesystem paths / config — was logged at the probe, and - // never reaches the client. - if deterministic_fault { + // per attempt against the broken repo. The body is opaque (a generic message via + // `AppError::Git` -> 500, no Retry-After): the raw git stderr — which leaks + // filesystem paths / config — was logged at the probe, and never reaches the client. + // + // Gate on `truncated_by.is_empty()`: the terminal 500 fires only when a + // deterministic fault is the SOLE reason nothing served. When a TRANSIENT taint + // co-occurs (a DIFFERENT repo was skipped by budget / acquire / walk-cap / probe / + // visit-ceiling), the requested object may live in that transiently-skipped repo, + // so fall through to the retryable 503 below — a retry can re-probe it and serve + // the content. Reporting a terminal 500 there would wrongly tell a conformant + // client not to retry, leaving reachable content unreachable until the unrelated + // broken repo is repaired. + if deterministic_fault && truncated_by.is_empty() { return Err(AppError::Git( "ipfs object probe could not complete: a candidate repository is corrupt".into(), )); @@ -1360,6 +1367,113 @@ mod tests { ); } + /// #174 F5 co-occurrence (RED-before/GREEN-after): a deterministic fault on ONE + /// repo and a TRANSIENT taint on a DIFFERENT repo occur in the same scan, and the + /// requested CID is served by neither. The transiently-skipped repo could hold the + /// object, so a retry can surface it — the correct shed is the RETRYABLE 503, not + /// the terminal 500. Two broken repos drive it, both local so the outcome is + /// deterministic: a bad-`config` repo whose `objects/` stays readable is a + /// DETERMINISTIC probe fault (`deterministic_fault = true`), while a repo whose + /// `objects/` dir is removed is a TRANSIENT probe fault (taints "probe"). A third + /// healthy repo probes clean (absent verdict) so nothing serves. Before the fix the + /// terminal `if deterministic_fault` arm fired first and shed 500 unconditionally, + /// hiding the transiently-skipped repo behind a non-retryable status. MUTATION + /// (RED): drop the `&& truncated_by.is_empty()` gate and this shes 500 again. + #[sqlx::test] + async fn get_by_cid_deterministic_fault_with_cooccurring_transient_taint_is_503_not_500( + pool: sqlx::PgPool, + ) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // Healthy repo that probes clean (definitive absence on its own). + seed_repo_with_blob(&state, tmp.path(), "z6f5coclean", "real", b"probe clean\n").await; + + // Bad-config repo: objects/ readable, config corrupt -> DETERMINISTIC fault. + state + .db + .upsert_mirror_repo("z6f5cobadcfg", "broken", "/unused-badcfg", None, false) + .await + .unwrap(); + let rec = state + .db + .get_repo("z6f5cobadcfg", "broken") + .await + .unwrap() + .unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + std::fs::create_dir_all(&bare).unwrap(); + run_git(&["init", "-q", "--bare", "--object-format=sha256"], &bare); + { + use std::io::Write; + let mut cfg = std::fs::OpenOptions::new() + .append(true) + .open(bare.join("config")) + .unwrap(); + cfg.write_all(b"\n[broken section\nnot a valid = = = line\n") + .unwrap(); + } + + // Corrupt-dir repo: objects/ removed -> TRANSIENT probe fault (taints "probe"), + // a DIFFERENT repo than the deterministic one above. + state + .db + .upsert_mirror_repo("z6f5cocorrupt", "broken", "/unused-corrupt", None, false) + .await + .unwrap(); + let rec2 = state + .db + .get_repo("z6f5cocorrupt", "broken") + .await + .unwrap() + .unwrap(); + let bare2 = state + .repo_store + .acquire(&rec2.owner_did, &rec2.name) + .await + .unwrap(); + std::fs::create_dir_all(&bare2).unwrap(); + run_git(&["init", "-q", "--bare", "--object-format=sha256"], &bare2); + std::fs::remove_dir_all(bare2.join("objects")).unwrap(); + std::fs::write(bare2.join("HEAD"), b"junk\n").unwrap(); + + let peer: SocketAddr = "203.0.113.71:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a deterministic fault co-occurring with a transient taint on a DIFFERENT \ + repo must shed the retryable 503 (a retry can surface the object in the \ + transiently-skipped repo), never the terminal 500" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the co-occurrence 503 must carry Retry-After" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("probe"), + "the shed must name the transient probe taint; got: {body}" + ); + } + /// F2 read taint: the gate passes (the probe reads the truncated loose object's /// intact "blob 64" header) but the content read fails (`cat-file blob` dies on /// the deflate stream cut mid-content) — the probe just said the object EXISTS From e6f1fec861bb840e6883c92db1315cccce3aadb1 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 12:02:46 -0500 Subject: [PATCH 064/124] docs(node): correct three lease/replication comments from code review (#174) - steal_after: the reclaim is NOT a guarantee that only a leaked lease is reclaimed. A waiter's timeout starts at acquire(), not at the FIFO head, so a same-repo backlog whose cumulative wait exceeds steal_after can steal while an earlier waiter still writes; correctness rests on the retained pg advisory lock (which serializes the stealer at acquire_write), not on the bound. - pinata_object_list_for_refs: recomputes from the tail-start rules snapshot, not a fresh read at pin-worker time, so a rule tightened AFTER tail-start is not reflected (matches the old retained-list behavior; reconciliation sweep is the backstop). Removed the overstated 'rule tightened since the push is honored'. - RepoWriteGuard::Drop: note that on runtime shutdown the detached unlock task may be dropped before it polls, bounded by pool teardown / connection close. --- crates/gitlawb-node/src/api/repos.rs | 22 +++++++++++++++------- crates/gitlawb-node/src/git/repo_store.rs | 5 ++++- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 836e5625..f258a4bc 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1007,10 +1007,13 @@ async fn resolve_drain_object_list( /// retained object list is dropped here, not the task, so every push's effects /// still fire exactly once. /// -/// Fresh by design, exactly like `resolve_drain_object_list`: the withheld set -/// and candidate set are recomputed from the current rules, so a rule tightened -/// since the push is honored and fails closed (a newly-withheld blob is not -/// pinned; a no-longer-announceable repo pins nothing). Every git child runs +/// Exactly like `resolve_drain_object_list`: the withheld and candidate sets are +/// recomputed from the rules snapshot captured at post-receive tail start (NOT a +/// fresh read at pin-worker time), so a rule tightened up to that point is honored +/// and the filter always fails closed (a newly-withheld blob is not pinned; a +/// no-longer-announceable repo pins nothing). A tightening AFTER tail-start — before +/// a slow re-derivation runs — is not reflected, matching the old retained-list +/// behavior; the reconciliation sweep is the backstop. Every git child runs /// through the same INV-22 bounded, process-group-reaped helpers the sibling /// post-receive scans use (`replication_withheld_set`, /// `resolve_candidates_for_push`, `fail_closed_full_scan_objects`). @@ -1624,9 +1627,14 @@ pub async fn git_receive_pack( // no pooled pg connection while it waits. The lease rides the write-path // AdmissionGuard into the reaper (clone (a)) and spans the clean-path Tigris upload // in guard.release (clone (b)); it frees only when the LAST clone drops. steal_after - // is set well above any legitimate hold (a full receive-pack under - // git_service_timeout + the ~4s reaper cap + the Tigris upload), so the bounded-wait - // reclaim only ever fires for a genuinely leaked lease, never a merely-slow push. + // is sized above ONE legitimate hold (a full receive-pack under git_service_timeout + + // the ~4s reaper cap + the Tigris upload). It is NOT a guarantee that only a leaked + // lease is reclaimed: a waiter's timeout starts at acquire(), not at the head of the + // FIFO queue, so a same-repo backlog whose CUMULATIVE wait exceeds steal_after can + // steal while an earlier waiter is still writing. Correctness does not rest on the + // bound — on the non-disconnect path the retained pg advisory lock still serializes + // the stealer at acquire_write (a spurious 503, not a race); the only corruption-capable + // overlap is the ~4s disconnect/reap window, which the reaper-carried clone (a) covers. let lease_steal_after = std::time::Duration::from_secs(state.config.git_service_timeout_secs * 2 + 60); let lease = state diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index d88594e7..bd27a186 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -460,7 +460,10 @@ impl Drop for RepoWriteGuard { /// handler future was dropped before `release`), unlock on the pinned /// connection. `Drop` cannot await, so spawn a detached unlock — it runs on the /// same session (connection-affine). An off-runtime drop falls back to a log; - /// the ~60s stale-lock retry loop in `acquire_write` reclaims it. + /// the ~60s stale-lock retry loop in `acquire_write` reclaims it. On runtime + /// SHUTDOWN the spawned unlock task may be dropped before it polls, so the unlock + /// may not run — but shutdown tears down the pool, and closing the connection + /// releases the session-level advisory lock server-side, so this too is bounded. fn drop(&mut self) { if self.released || !self.locked { return; From 86828046eaa475795689c01969e63b930b7a0745 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 17:54:49 -0500 Subject: [PATCH 065/124] fix(node): hold read admission through the filtered-pack reaper The path-scoped upload-pack branch kept the read and per-caller permits as handler locals and called upload_pack_excluding with no AdmissionGuard, so a cancelled filtered clone released admission while KillGroupOnDrop was still reaping the git group, bypassing the read and per-caller concurrency caps. drive_git_child now returns the disarmed guard on success so one guard rides both build_filtered_pack stages (rev-list then pack-objects); the handler builds the guard from the permits as the plain path does. Adds a disconnect regression and an INV-22 gate row, both proven load-bearing. --- crates/gitlawb-node/src/api/repos.rs | 14 +- crates/gitlawb-node/src/git/smart_http.rs | 238 ++++++++++++++++++---- crates/gitlawb-node/tests/inv22_gates.rs | 16 ++ 3 files changed, 229 insertions(+), 39 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index cb6ed541..fb902f01 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -940,11 +940,15 @@ pub async fn git_upload_pack( smart_http::upload_pack(&state.git_bin, &disk_path, body, git_timeout, Some(admission)).await } else { tracing::info!(repo = %name, caller = ?caller, withheld = withheld.len(), "serving filtered pack"); - // upload_pack_excluding runs its own rev-list/pack-objects (both pass `None` - // admission internally); the walk's permits stay handler-locals held across - // this serve, as be0cdd6 established, and drop when the handler returns. - let _hold = (_permit, _caller_permit); - smart_http::upload_pack_excluding(&disk_path, body, &withheld, git_timeout).await + // Move both admission permits into the guard so they release only after the + // filtered serve's git group (rev-list then pack-objects) is reaped, on + // complete/timeout/disconnect — not the instant a disconnect drops this + // future. Without this, disconnect-spam on a path-scoped repo could hold PIDs + // past the concurrency cap while the permits were already freed (#174 P1-a, + // R2). The guard rides both stages inside upload_pack_excluding. + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + smart_http::upload_pack_excluding(&disk_path, body, &withheld, git_timeout, Some(admission)) + .await } } .map_err(|e| { diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index 64d3cb7f..ec115f1e 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -70,8 +70,9 @@ pub async fn info_refs( .arg("--stateless-rpc") .arg("--advertise-refs") .arg(repo_path); - // No request body — advertise-refs does not read stdin. - let stdout = + // No request body — advertise-refs does not read stdin. Single stage: the returned + // guard is dropped here (advertise-refs holds admission for its own child only). + let (stdout, _admission) = drive_git_child(command, Bytes::new(), timeout, "advertise-refs", admission).await?; let content_type = format!("application/x-{service}-advertisement"); @@ -331,23 +332,34 @@ async fn run_git_service( .arg(service_to_command(service)) .arg("--stateless-rpc") .arg(repo_path); - drive_git_child(command, input, timeout, service, admission).await + // Single stage: drop the returned guard when this function returns (its permits are + // released once the child's group is reaped, per drive_git_child). + let (out, _admission) = drive_git_child(command, input, timeout, service, admission).await?; + Ok(out) } /// Drive a spawned git child under `timeout` with process-group teardown, returning -/// its stdout. Shared core for [`run_git_service`] and [`info_refs`]: the caller -/// passes a `Command` with its args set; this adds piped stdio and `process_group(0)`. -/// On the deadline the whole group is torn down and reaped before returning -/// [`GitServiceTimeout`]; on a dropped future (client disconnect) the -/// [`KillGroupOnDrop`] guard fires. `input` is written to the child's stdin (empty -/// for the advertise-refs path, which has no request body); `what` labels errors. +/// its stdout AND the disarmed admission guard on success. Shared core for +/// [`run_git_service`] and [`info_refs`]: the caller passes a `Command` with its args +/// set; this adds piped stdio and `process_group(0)`. On the deadline the whole group +/// is torn down and reaped before returning [`GitServiceTimeout`]; on a dropped future +/// (client disconnect) the [`KillGroupOnDrop`] guard fires. `input` is written to the +/// child's stdin (empty for the advertise-refs path, which has no request body); `what` +/// labels errors. +/// +/// On success the guard is RETURNED rather than dropped internally (#174 KTD3), so a +/// caller running two sequential git stages under one admission — `build_filtered_pack`'s +/// rev-list then pack-objects — can hand the same guard from the first stage to the +/// second and keep the permits held across both, releasing them only when the second +/// stage's process group is reaped. Callers that run a single stage just let the +/// returned guard drop. async fn drive_git_child( mut command: Command, input: Bytes, timeout: Duration, what: &str, admission: Option, -) -> Result> { +) -> Result<(Vec, Option)> { command .stdin(Stdio::piped()) .stdout(Stdio::piped()) @@ -383,9 +395,9 @@ async fn drive_git_child( admission, }; // On non-unix there is no process-group teardown, so hold the admission guard here - // for the child's whole interaction; it drops when this function returns. + // for the child's whole interaction; it is returned to the caller below. #[cfg(not(unix))] - let _admission = admission; + let admission_holder = admission; let mut out = Vec::new(); let mut err = Vec::new(); @@ -419,15 +431,24 @@ async fn drive_git_child( }; let timed = tokio::time::timeout(timeout, interact).await; + // The disarmed admission guard, handed back to the caller on success so it can ride + // a following sequential git stage (#174 KTD3) instead of releasing between stages. + let admission_out: Option; let (write_result, status) = match timed { Ok(result) => { // The join runs all arms to completion, so the child is reaped: disarm // before surfacing any interaction error (a read/wait error), else the // guard's drop would reap an already-reaped child / signal a reused pgid. - // Dropping the returned admission guard here releases the permits at the - // earliest provably-free point on the success path (the op finished). + // The guard is returned (not dropped) so a two-stage caller keeps the + // permits held across both stages; a single-stage caller drops it on return. #[cfg(unix)] - drop(group_guard.disarm()); + { + admission_out = group_guard.disarm(); + } + #[cfg(not(unix))] + { + admission_out = admission_holder; + } result? } Err(_elapsed) => { @@ -460,7 +481,7 @@ async fn drive_git_child( write_result.context("failed to write to git stdin")?; - Ok(out) + Ok((out, admission_out)) } fn service_to_command(service: &str) -> &str { @@ -498,14 +519,18 @@ async fn rev_list_keep( repo_path: &Path, withheld: &HashSet, timeout: Duration, -) -> Result> { + admission: Option, +) -> Result<(Vec, Option)> { let mut command = Command::new(git_bin); command .args(["rev-list", "--objects", "--all"]) .current_dir(repo_path); - // The visibility-walk callers intentionally hold no admission permit (their - // admission is governed elsewhere), so pass `None` (#174 KTD2). - let stdout = drive_git_child(command, Bytes::new(), timeout, "rev-list", None).await?; + // First of `build_filtered_pack`'s two sequential stages: it holds the caller's + // admission for the rev-list child and hands the disarmed guard back so the same + // permits ride the pack-objects stage (#174 KTD3, R2). The inter-stage window holds + // no live git group, so nothing is reaped-late by carrying the guard between them. + let (stdout, admission) = + drive_git_child(command, Bytes::new(), timeout, "rev-list", admission).await?; let mut keep = Vec::new(); for line in String::from_utf8_lossy(&stdout).lines() { let oid = line.split_whitespace().next().unwrap_or(""); @@ -514,7 +539,7 @@ async fn rev_list_keep( } keep.push(oid.to_string()); } - Ok(keep) + Ok((keep, admission)) } /// Build a packfile containing every object reachable from all refs EXCEPT the @@ -532,15 +557,21 @@ pub async fn build_filtered_pack( repo_path: &Path, withheld: &HashSet, timeout: Duration, + admission: Option, ) -> Result> { // One deadline spans both git stages so a slow rev-list eats into the pack // budget rather than granting each stage a fresh `timeout` (2x the permit hold). let deadline = Instant::now() + timeout; - let keep = rev_list_keep( + // The caller's admission rides both stages: rev-list holds it, hands it back, then + // pack-objects holds it until ITS process group is reaped on disconnect. That closes + // the path-scoped cap bypass — permits release on group reap, not the instant the + // request future drops (#174 R2, KTD3). The plain upload-pack path already does this. + let (keep, admission) = rev_list_keep( git_bin, repo_path, withheld, deadline.saturating_duration_since(Instant::now()), + admission, ) .await?; let mut data = keep.join("\n").into_bytes(); @@ -549,15 +580,15 @@ pub async fn build_filtered_pack( command .args(["pack-objects", "--stdout"]) .current_dir(repo_path); - drive_git_child( + let (out, _admission) = drive_git_child( command, Bytes::from(data), deadline.saturating_duration_since(Instant::now()), "pack-objects", - // Visibility-walk pack build: no admission permit here (#174 KTD2). - None, + admission, ) - .await + .await?; + Ok(out) } /// Serve a clone/fetch with the withheld blobs removed from the response pack. @@ -588,12 +619,15 @@ pub async fn upload_pack_excluding( request_body: Bytes, withheld: &HashSet, timeout: Duration, + admission: Option, ) -> Result { - // The rev-list enumeration runs blocking off the runtime; the streaming - // pack-objects stage is duration-bounded and its process group is reaped on + // Both git stages are duration-bounded and their process groups are reaped on // disconnect via drive_git_child (#174), so a hung build no longer pins its - // concurrency slot and a client disconnect no longer orphans the git child. - let pack = build_filtered_pack("git", repo_path, withheld, timeout).await?; + // concurrency slot and a client disconnect no longer orphans the git child. The + // caller's read + per-caller admission is threaded through both stages so the + // permits are held until the pack-objects group is reaped, matching the plain path + // (#174 R2). + let pack = build_filtered_pack("git", repo_path, withheld, timeout, admission).await?; // The client lists its capabilities on the first `want` line. Honor // side-band-64k when offered (every modern smart-HTTP client offers it); @@ -712,7 +746,7 @@ mod tests { let mut withheld = std::collections::HashSet::new(); withheld.insert(secret.clone()); - let pack = build_filtered_pack("git", &bare, &withheld, Duration::from_secs(30)) + let pack = build_filtered_pack("git", &bare, &withheld, Duration::from_secs(30), None) .await .unwrap(); let ids = pack_object_ids(&pack); @@ -776,7 +810,7 @@ mod tests { b"0098want 0000000000000000000000000000000000000000 \ side-band-64k ofs-delta agent=git/2\n00000009done\n", ); - let resp = upload_pack_excluding(&bare, req, &withheld, Duration::from_secs(30)) + let resp = upload_pack_excluding(&bare, req, &withheld, Duration::from_secs(30), None) .await .unwrap(); let body = to_bytes(resp.into_body(), usize::MAX).await.unwrap(); @@ -837,7 +871,7 @@ mod tests { axum::extract::State(st): axum::extract::State>, body: Bytes, ) -> Response { - upload_pack_excluding(&st.repo, body, &st.withheld, Duration::from_secs(30)) + upload_pack_excluding(&st.repo, body, &st.withheld, Duration::from_secs(30), None) .await .unwrap() } @@ -1359,7 +1393,7 @@ mod tests { for i in 0..FAKE_GIT_RETRY_ATTEMPTS { let result = tokio::time::timeout( Duration::from_secs(10), - build_filtered_pack(git_bin, repo_path, withheld, stage_timeout), + build_filtered_pack(git_bin, repo_path, withheld, stage_timeout, None), ) .await .expect( @@ -1655,6 +1689,142 @@ mod tests { ); } + // #174 U1 (R2, KTD3, RED-before/GREEN-after): the path-scoped filtered-pack serve + // must hold read + per-caller admission until its pack-objects process group is + // reaped on a client disconnect, exactly as the plain upload_pack path does. Before + // the fix build_filtered_pack took no AdmissionGuard and the handler's `_hold` + // permits dropped the instant the request future was dropped, so disconnect-spam on + // a path-scoped repo could hold PIDs past the concurrency cap while the permits were + // already free (#174 P1-a, on the filtered path the plain path had already closed). + // + // A real AdmissionGuard built from two owned semaphore permits rides + // rev-list -> pack-objects. We drive the future until pack-objects has forked its + // grandchild (the streaming pack-writer stand-in), assert the permits are still held + // mid-serve, then DROP the future (client disconnect) and assert the permits are + // released only AFTER the group is ESRCH-confirmed gone — never while it is alive. + // Goes RED if the guard is not threaded into the pack-objects stage (it would then + // drop after rev-list, freeing the permits mid-serve or on the bare future drop). + #[cfg(unix)] + #[tokio::test] + async fn filtered_pack_holds_admission_until_group_reaped_on_disconnect() { + use std::sync::Arc; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let pidfile = tmp.path().join("pids"); + // rev-list returns one oid fast; pack-objects forks a grandchild (the streaming + // writer stand-in), records leader+grandchild pids, then hangs so the future + // parks mid-serve with the guard owned by the pack-objects KillGroupOnDrop. The + // grandchild inherits (holds open) the stdout pipe, so drive_git_child's + // read_to_end blocks and the future stays pending until we drop it. + let body = format!( + "#!/bin/sh\ncase \"$1\" in\n rev-list) echo deadbeefdeadbeefdeadbeefdeadbeefdeadbeef ;;\n pack-objects) sleep 300 &\nprintf '%s\\n%s\\n' \"$$\" \"$!\" > \"{}\"\nwait ;;\n *) exit 1 ;;\nesac\n", + pidfile.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + let withheld = HashSet::new(); + + // Retry the fake-git spawn race like the sibling disconnect test; each attempt + // gets a FRESH semaphore so a dropped losing attempt can't skew the winning + // attempt's permit accounting. Keep the winning attempt's future PENDING so the + // drop below exercises the client-disconnect teardown. + let (fut, sem, leader, grandchild) = { + let mut attempt = 0u64; + loop { + attempt += 1; + let _ = std::fs::remove_file(&pidfile); + // Semaphore(4): two owned permits model the handler's global-read + + // per-caller admission, leaving 2 available while the op is in flight. + let sem = Arc::new(Semaphore::new(4)); + let g = sem.clone().try_acquire_owned().unwrap(); + let c = sem.clone().try_acquire_owned().unwrap(); + let admission = AdmissionGuard::new(g, Some(c)); + let mut fut = Box::pin(build_filtered_pack( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_secs(60), + Some(admission), + )); + // Advance the future a slice at a time until the fake records its pids + // (i.e. pack-objects is running). `Ok(_)` means the future returned + // before the pidfile appeared (spawn error / early exit); stop polling + // then, since re-polling a completed future panics. + let mut pids = None; + for _ in 0..500 { + let finished = + tokio::time::timeout(Duration::from_millis(10), &mut fut).await.is_ok(); + if let Some(p) = read_two_pids(&pidfile) { + pids = Some(p); + break; + } + if finished { + break; + } + } + match pids { + Some((l, gch)) => break (fut, sem, l, gch), + None => { + // Transient spawn miss: drop the still-armed future so its guard + // reaps anything that spawned (and returns the permits), then + // back off before retrying. + drop(fut); + assert!( + attempt < FAKE_GIT_RETRY_ATTEMPTS, + "fake git failed to reach the pack-objects stage after \ + {FAKE_GIT_RETRY_ATTEMPTS} attempts (persistent failure, \ + not a transient parallel-runner miss)" + ); + tokio::time::sleep(Duration::from_millis( + FAKE_GIT_BACKOFF_STEP_MS * attempt, + )) + .await; + } + } + } + }; + let _cleanup = ReapOnPanic(vec![leader, grandchild]); + assert!(alive(grandchild), "grandchild must be running mid-serve"); + + // Mid-serve: the pack-objects stage owns the guard, so the two permits are still + // held. A build that dropped the guard after rev-list (unthreaded pack-objects + // stage) would have freed them here. + assert_eq!( + sem.available_permits(), + 2, + "admission permits must be held while the filtered serve is in flight" + ); + + // Client disconnect: drop the request future. The pack-objects KillGroupOnDrop + // must tear the group down AND hold the permits until the group is + // ESRCH-confirmed gone, releasing them only then. + drop(fut); + + let mut released_while_alive = false; + let mut released_after_reap = false; + for _ in 0..500 { + let released = sem.available_permits() == 4; + let group_alive = alive(grandchild); + if released && group_alive { + released_while_alive = true; + } + if released && !group_alive { + released_after_reap = true; + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert!( + !released_while_alive, + "admission permits were released while the git group was still alive — the \ + path-scoped concurrency-cap bypass (#174 P1-a) is open" + ); + assert!( + released_after_reap, + "admission permits must be released once the group is reaped on disconnect" + ); + } + // A request that runs to completion must DISARM the guard after reaping, so // no stray group SIGTERM fires. The fake exits non-zero (surfacing as Err) // but leaves a grandchild alive; the grandchild must survive. Goes RED if the diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index 7d3fd1c8..e750a48f 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -68,6 +68,22 @@ fn inv22_concurrency_gates_present_and_not_bypassed() { withheld_recipients_gated, which acquires git_encrypt_semaphore" ); + // U1 / R2 (#173 round-10): the path-scoped filtered-pack serve must thread the + // caller's AdmissionGuard through BOTH git stages so read + per-caller admission is + // held until the pack-objects group is reaped on disconnect, closing the cap bypass + // the plain upload_pack path already fixed. Two load-bearing markers: rev-list hands + // the disarmed guard back (its tuple return type), and build_filtered_pack forwards + // that guard into the pack-objects stage (the `admission` arg after the + // "pack-objects" label). Reverting either — dropping the guard between stages, or + // passing `None` to pack-objects — trips this. + assert!( + smart_http.contains("Result<(Vec, Option)>") + && smart_http.contains("\"pack-objects\",\n admission,"), + "U1/R2 gate missing: build_filtered_pack must thread the AdmissionGuard through \ + rev-list -> pack-objects so the permits are held until the pack-objects group \ + is reaped on disconnect (the path-scoped half of #174 P1-a)" + ); + // P1-e non-bypass tripwire: the bounded recipients walk is spawn_blocking'd nowhere // but inside withheld_recipients_gated. A second call site (count > 1) is a new // detached git walk that skips the admission gate — exactly the class U5 closed. From 757edf560cda78f264765db410e4d09956cb9a60 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 18:34:09 -0500 Subject: [PATCH 066/124] fix(node): own /ipfs admission for the life of the walk get_by_cid held the global walk permit and the per-source permit as handler locals, so dropping the request future released both while the spawn_blocking probe, visibility walk, and object read kept running; a cancel-spam or timeout client could exceed MAX_CONCURRENT_IPFS_WALKS and the per-source cap, and the bare cat-file probe/read children had no teardown at all. Move the gated serve pipeline into a detached task that owns an AdmissionGuard, awaited by the handler, so admission releases only after the bounded work is gone; back the /ipfs probe and read with run_bounded_git twins for process-group teardown. Adds cancel-mid-walk, timeout-reap, and cancel-spam regressions plus two load-bearing INV-22 rows. Also reflows one U1 test line to satisfy cargo fmt. --- crates/gitlawb-node/src/api/ipfs.rs | 749 +++++++++++++--------- crates/gitlawb-node/src/git/smart_http.rs | 5 +- crates/gitlawb-node/src/git/store.rs | 184 +++++- crates/gitlawb-node/tests/inv22_gates.rs | 33 + 4 files changed, 661 insertions(+), 310 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index c8911bac..016e130d 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -165,7 +165,7 @@ pub async fn get_by_cid( // admits any `did:key` unthrottled, so a DID key would be free to mint around); a // `None` key (no trusted header, no peer) is bounded by the global pool only, // never the per-source sub-cap. - let _ipfs_walk_permit = state + let ipfs_walk_permit = state .git_ipfs_walk_semaphore .clone() .try_acquire_owned() @@ -174,7 +174,7 @@ pub async fn get_by_cid( AppError::Overloaded("ipfs service at capacity, retry shortly".into()) })?; let source_key = crate::rate_limit::client_key(&headers, peer, state.push_limiter_trust); - let _ipfs_caller_permit = match &source_key { + let ipfs_caller_permit = match &source_key { Some(ip) => Some(state.git_ipfs_walk_per_caller.try_acquire(ip).ok_or_else(|| { tracing::warn!(key = %ip, "/ipfs per-source walk cap reached; shedding request with 503"); AppError::Overloaded("ipfs service at capacity for this source, retry shortly".into()) @@ -182,219 +182,253 @@ pub async fn get_by_cid( None => None, }; - // Resolve the content-addressed CID to the object's git oid(s). A real pin - // CID digests the raw object content (`Cid::from_git_object_bytes`), NOT the - // git oid (git frames content with a `" \0"` header first), so we - // map it back through `pinned_cids` rather than treating the digest as an oid - // (#173). The cid index is non-unique, so one CID can map to several oids (a - // tree and a blob whose raw bytes collide, or content pinned under two oids); - // we try each candidate below rather than pick one arbitrarily and false-404 - // when the chosen one is withheld or absent while another is readable (#173). - // An empty result is an opaque 404, uniform with a genuine not-found and a - // visibility denial. - let oids = state - .db - .oids_for_cid(&canonical_cid) - .await - .map_err(AppError::Internal)?; - if oids.is_empty() { - return Err(AppError::RepoNotFound(format!( - "no git object found for CID {cid_str}" - ))); - } - let caller = auth.as_ref().map(|e| e.0 .0.as_str()); - let caller_owned = caller.map(|c| c.to_string()); - - // Per-request walk budget + memos + throttle flag, shared by the provenance path - // and the legacy scan so both honor the same fan-out ceiling, per-repo memo, and - // IP brake. The caller is constant for one request, so `repo.id` alone keys the memo. - let mut walk = WalkState { - walks: 0, - probes: 0, - truncated: false, - allowed_blob_memo: HashMap::new(), - allowed_tree_memo: HashMap::new(), - reachable_ct_memo: HashMap::new(), - }; - // Set when a walk-requiring candidate is skipped because the source IP's walk quota - // is spent (#173 review, F-C): the scan keeps going so a later walk-free copy still - // serves; only if nothing is servable is it turned into the 429. - let mut throttled = false; - let rctx = ResolveCtx { - caller, - caller_owned: &caller_owned, - headers: &headers, - peer, - cid_str: &cid_str, - canonical_cid: &canonical_cid, - }; + // Caller DID (owned) resolved before the spawn: the detached task below cannot + // borrow the handler's `auth` extension. + let caller_owned = auth.as_ref().map(|e| e.0 .0.as_str().to_string()); + + // #173 round-10 (R1, KTD1): move BOTH admission permits into an AdmissionGuard OWNED + // by a detached tokio task that runs the whole gated serve pipeline, and have the + // handler await its JoinHandle. Dropping the request future on a client disconnect + // drops the JoinHandle, which DETACHES the task (tokio does not abort a task when its + // handle drops) rather than cancelling it — so the pipeline runs to its bounded + // completion and releases admission only then, instead of the permits dropping the + // instant the handler future is torn down while a spawn_blocking git probe/walk/read + // is still alive (the disconnect-spam cap bypass this closes, the /ipfs half of #174 + // P1-a). Every git child on this pipeline is duration-bounded (the run_bounded_git + // probe/read twins + the bounded walk), so the detached task cannot hold admission + // past ~git_service_timeout_secs. The client key is already captured (`source_key`) + // before the spawn; the detached task has no request extractors. + let admission = + crate::git::smart_http::AdmissionGuard::new(ipfs_walk_permit, ipfs_caller_permit); + let serve: tokio::task::JoinHandle> = tokio::spawn(async move { + // The guard is held for the whole task and drops LAST — after the response is + // built — releasing admission exactly once when the pipeline is truly done. + let _admission = admission; - // Legacy scan context (repos + rules + quarantined ids), loaded LAZILY only when a - // legacy NULL-provenance pin is hit — the provenance path must never trigger the - // O(repos) load (that fan-out is exactly what provenance removes, #173 round 2). - let mut scan_ctx: Option = None; - - for sha256_hex in &oids { - // A pinned object records EVERY repo it was pinned from (#173 round 8, F1). - // Resolve a PROVENANCED pin by trying each source repo (bounded to - // MAX_PIN_SOURCES) through the SAME gate; the first that authorizes serves — no - // scan fan-out. A shared object first pinned from a private/quarantined repo - // still serves from a later PUBLIC source. Deterministic (ORDER BY on the - // union), so no ordering can turn an authorized copy into a 404. - let sources = state + // Resolve the content-addressed CID to the object's git oid(s). A real pin + // CID digests the raw object content (`Cid::from_git_object_bytes`), NOT the + // git oid (git frames content with a `" \0"` header first), so we + // map it back through `pinned_cids` rather than treating the digest as an oid + // (#173). The cid index is non-unique, so one CID can map to several oids (a + // tree and a blob whose raw bytes collide, or content pinned under two oids); + // we try each candidate below rather than pick one arbitrarily and false-404 + // when the chosen one is withheld or absent while another is readable (#173). + // An empty result is an opaque 404, uniform with a genuine not-found and a + // visibility denial. + let oids = state .db - .pin_sources_for_oid(sha256_hex) + .oids_for_cid(&canonical_cid) .await .map_err(AppError::Internal)?; - // Provenance fast-path: try each recorded source repo through the SAME gate - // (bounded to first-pinner + MAX_PIN_SOURCES). Empty for a legacy NULL-provenance - // pin. The first source that authorizes serves — no scan fan-out on the common - // path. - for repo_id in &sources { - let repo = match state - .db - .get_repo_by_id(repo_id) - .await - .map_err(AppError::Internal)? - { - Some(r) => r, - // A source repo is gone: skip it; a later source or the scan fallback - // below may still resolve. - None => continue, - }; - let quarantined = state - .db - .is_repo_quarantined(repo_id) - .await - .map_err(AppError::Internal)?; - let rules_map = state - .db - .list_visibility_rules_for_repos(std::slice::from_ref(repo_id)) - .await - .map_err(AppError::Internal)?; - let rules = rules_map.get(repo_id).map(Vec::as_slice).unwrap_or(&[]); - match gate_and_serve( - &state, - &repo, - rules, - quarantined, - sha256_hex, - &rctx, - &mut walk, - false, - ) - .await - { - GateOutcome::Served(resp) => return Ok(resp), - GateOutcome::Throttled => { - throttled = true; - continue; - } - GateOutcome::Skip => continue, - } + if oids.is_empty() { + return Err(AppError::RepoNotFound(format!( + "no git object found for CID {cid_str}" + ))); } + let caller = caller_owned.as_deref(); + + // Per-request walk budget + memos + throttle flag, shared by the provenance path + // and the legacy scan so both honor the same fan-out ceiling, per-repo memo, and + // IP brake. The caller is constant for one request, so `repo.id` alone keys the memo. + let mut walk = WalkState { + walks: 0, + probes: 0, + truncated: false, + allowed_blob_memo: HashMap::new(), + allowed_tree_memo: HashMap::new(), + reachable_ct_memo: HashMap::new(), + }; + // Set when a walk-requiring candidate is skipped because the source IP's walk quota + // is spent (#173 review, F-C): the scan keeps going so a later walk-free copy still + // serves; only if nothing is servable is it turned into the 429. + let mut throttled = false; + let rctx = ResolveCtx { + caller, + caller_owned: &caller_owned, + headers: &headers, + peer, + cid_str: &cid_str, + canonical_cid: &canonical_cid, + }; + + // Legacy scan context (repos + rules + quarantined ids), loaded LAZILY only when a + // legacy NULL-provenance pin is hit — the provenance path must never trigger the + // O(repos) load (that fan-out is exactly what provenance removes, #173 round 2). + let mut scan_ctx: Option = None; - // Bounded legacy-scan fallback. Run it when the provenance set could not have - // served the caller AND may be INCOMPLETE: - // - empty -> a legacy NULL-provenance pin (recorded before provenance existed), or - // - at_cap -> `record_pin_source` stops inserting at MAX_PIN_SOURCES and drops - // later sources SILENTLY, so a full table may hide a servable source - // (e.g. a later PUBLIC pinner buried by 16 attacker sources — the - // pin-source griefing hole). The scan gates every repo through the - // real per-caller gate, so it finds that copy. - // A non-empty, non-full set is COMPLETE (every recorded source was just tried), so - // skip the scan and let the tail 404 — ordinary denials never fan out to O(repos) - // (INV-10 / F3). The at_cap query runs only on a provenance MISS (we return above - // on Served), so it never costs the serve path. - let needs_scan = sources.is_empty() - || state + for sha256_hex in &oids { + // A pinned object records EVERY repo it was pinned from (#173 round 8, F1). + // Resolve a PROVENANCED pin by trying each source repo (bounded to + // MAX_PIN_SOURCES) through the SAME gate; the first that authorizes serves — no + // scan fan-out. A shared object first pinned from a private/quarantined repo + // still serves from a later PUBLIC source. Deterministic (ORDER BY on the + // union), so no ordering can turn an authorized copy into a 404. + let sources = state .db - .pin_sources_at_cap(sha256_hex) + .pin_sources_for_oid(sha256_hex) .await .map_err(AppError::Internal)?; - if needs_scan { - // F3 (#173, INV-10/INV-15): peek the per-IP limiter WITHOUT consuming a token - // so an already-throttled source is shed BEFORE the O(repos) preload; the - // consuming per-probe charge inside gate_and_serve is left UNCHANGED (it is - // load-bearing for the across-request bound), so this adds no double-charge. - if let Some(key) = - crate::rate_limit::client_key(rctx.headers, rctx.peer, state.push_limiter_trust) - { - if state.ipfs_rate_limiter.is_throttled(&key).await { - throttled = true; - continue; - } - } - // Load the scan context once, lazily (shared across oid candidates). - if scan_ctx.is_none() { - #[cfg(test)] - bump_preload_queries(); - let repos = state + // Provenance fast-path: try each recorded source repo through the SAME gate + // (bounded to first-pinner + MAX_PIN_SOURCES). Empty for a legacy NULL-provenance + // pin. The first source that authorizes serves — no scan fan-out on the common + // path. + for repo_id in &sources { + let repo = match state .db - .list_all_repos() + .get_repo_by_id(repo_id) .await - .map_err(AppError::Internal)?; - let repo_ids: Vec = repos.iter().map(|r| r.id.clone()).collect(); - let rules_by_repo = state + .map_err(AppError::Internal)? + { + Some(r) => r, + // A source repo is gone: skip it; a later source or the scan fallback + // below may still resolve. + None => continue, + }; + let quarantined = state .db - .list_visibility_rules_for_repos(&repo_ids) + .is_repo_quarantined(repo_id) .await .map_err(AppError::Internal)?; - let quarantined: HashSet = state + let rules_map = state .db - .list_quarantined_repos() + .list_visibility_rules_for_repos(std::slice::from_ref(repo_id)) .await - .map_err(AppError::Internal)? - .into_iter() - .map(|r| r.id) - .collect(); - scan_ctx = Some((repos, rules_by_repo, quarantined)); - } - let (repos, rules_by_repo, quarantined) = scan_ctx.as_ref().unwrap(); - for repo in repos { - let rules = rules_by_repo - .get(&repo.id) - .map(Vec::as_slice) - .unwrap_or(&[]); - let is_quar = quarantined.contains(&repo.id); + .map_err(AppError::Internal)?; + let rules = rules_map.get(repo_id).map(Vec::as_slice).unwrap_or(&[]); match gate_and_serve( - &state, repo, rules, is_quar, sha256_hex, &rctx, &mut walk, true, + &state, + &repo, + rules, + quarantined, + sha256_hex, + &rctx, + &mut walk, + false, ) .await { GateOutcome::Served(resp) => return Ok(resp), - // A throttled walk-requiring candidate is skipped, not fatal: - // keep scanning for a later walk-free copy (#173 review, F-C). - GateOutcome::Throttled => throttled = true, - GateOutcome::Skip => {} + GateOutcome::Throttled => { + throttled = true; + continue; + } + GateOutcome::Skip => continue, + } + } + + // Bounded legacy-scan fallback. Run it when the provenance set could not have + // served the caller AND may be INCOMPLETE: + // - empty -> a legacy NULL-provenance pin (recorded before provenance existed), or + // - at_cap -> `record_pin_source` stops inserting at MAX_PIN_SOURCES and drops + // later sources SILENTLY, so a full table may hide a servable source + // (e.g. a later PUBLIC pinner buried by 16 attacker sources — the + // pin-source griefing hole). The scan gates every repo through the + // real per-caller gate, so it finds that copy. + // A non-empty, non-full set is COMPLETE (every recorded source was just tried), so + // skip the scan and let the tail 404 — ordinary denials never fan out to O(repos) + // (INV-10 / F3). The at_cap query runs only on a provenance MISS (we return above + // on Served), so it never costs the serve path. + let needs_scan = sources.is_empty() + || state + .db + .pin_sources_at_cap(sha256_hex) + .await + .map_err(AppError::Internal)?; + if needs_scan { + // F3 (#173, INV-10/INV-15): peek the per-IP limiter WITHOUT consuming a token + // so an already-throttled source is shed BEFORE the O(repos) preload; the + // consuming per-probe charge inside gate_and_serve is left UNCHANGED (it is + // load-bearing for the across-request bound), so this adds no double-charge. + if let Some(key) = + crate::rate_limit::client_key(rctx.headers, rctx.peer, state.push_limiter_trust) + { + if state.ipfs_rate_limiter.is_throttled(&key).await { + throttled = true; + continue; + } + } + // Load the scan context once, lazily (shared across oid candidates). + if scan_ctx.is_none() { + #[cfg(test)] + bump_preload_queries(); + let repos = state + .db + .list_all_repos() + .await + .map_err(AppError::Internal)?; + let repo_ids: Vec = repos.iter().map(|r| r.id.clone()).collect(); + let rules_by_repo = state + .db + .list_visibility_rules_for_repos(&repo_ids) + .await + .map_err(AppError::Internal)?; + let quarantined: HashSet = state + .db + .list_quarantined_repos() + .await + .map_err(AppError::Internal)? + .into_iter() + .map(|r| r.id) + .collect(); + scan_ctx = Some((repos, rules_by_repo, quarantined)); + } + let (repos, rules_by_repo, quarantined) = scan_ctx.as_ref().unwrap(); + for repo in repos { + let rules = rules_by_repo + .get(&repo.id) + .map(Vec::as_slice) + .unwrap_or(&[]); + let is_quar = quarantined.contains(&repo.id); + match gate_and_serve( + &state, repo, rules, is_quar, sha256_hex, &rctx, &mut walk, true, + ) + .await + { + GateOutcome::Served(resp) => return Ok(resp), + // A throttled walk-requiring candidate is skipped, not fatal: + // keep scanning for a later walk-free copy (#173 review, F-C). + GateOutcome::Throttled => throttled = true, + GateOutcome::Skip => {} + } } } } - } - // Nothing served — three distinct tails, in precedence order: - // 1. The scan was cut short by a cap (legacy probe ceiling or walk ceiling), so - // the object was NOT proven absent/unreadable everywhere → 503, retryable, and - // explicitly NOT a definitive not-found (#173, F2). This outranks the throttle: - // an incomplete search must not masquerade as a clean rate-limit outcome, and - // it carries only the caller-supplied CID (no object/OID/metadata leak). - // 2. A walk-requiring candidate was skipped for a spent IP quota while the scan - // otherwise completed → 429 (the brake bit; a cheaper copy was sought first). - // 3. A full scan under the caps found nothing readable → opaque 404, uniform with - // a genuine not-found and a visibility denial. - if walk.truncated { - return Err(AppError::SearchIncomplete(format!( - "CID {cid_str} search incomplete — retry" - ))); - } - if throttled { - return Err(AppError::TooManyRequests( - "ipfs retrieval rate limit exceeded — try again later".into(), - )); + // Nothing served — three distinct tails, in precedence order: + // 1. The scan was cut short by a cap (legacy probe ceiling or walk ceiling), so + // the object was NOT proven absent/unreadable everywhere → 503, retryable, and + // explicitly NOT a definitive not-found (#173, F2). This outranks the throttle: + // an incomplete search must not masquerade as a clean rate-limit outcome, and + // it carries only the caller-supplied CID (no object/OID/metadata leak). + // 2. A walk-requiring candidate was skipped for a spent IP quota while the scan + // otherwise completed → 429 (the brake bit; a cheaper copy was sought first). + // 3. A full scan under the caps found nothing readable → opaque 404, uniform with + // a genuine not-found and a visibility denial. + if walk.truncated { + return Err(AppError::SearchIncomplete(format!( + "CID {cid_str} search incomplete — retry" + ))); + } + if throttled { + return Err(AppError::TooManyRequests( + "ipfs retrieval rate limit exceeded — try again later".into(), + )); + } + Err(AppError::RepoNotFound(format!( + "no git object found for CID {cid_str}" + ))) + }); + + // Await the detached serve task. Dropping THIS future (client disconnect) drops the + // JoinHandle and detaches the task, which keeps running and releases admission only + // when it completes (KTD1). A JoinError here is a panic in the pipeline (the task is + // never aborted), surfaced as a 500 rather than a silent hang. + match serve.await { + Ok(result) => result, + Err(join_err) => Err(AppError::Internal(anyhow::anyhow!( + "ipfs serve task failed: {join_err}" + ))), } - Err(AppError::RepoNotFound(format!( - "no git object found for CID {cid_str}" - ))) } /// Outcome of gating one repo for one candidate oid. @@ -553,32 +587,34 @@ async fn gate_and_serve( let obj_type = { let rp = repo_path.clone(); let sha = sha256_hex.to_string(); - // Bound the blocking `git cat-file -t` under `git_service_timeout_secs`: this probe - // runs while the /ipfs walk permit is held, so a wedged cat-file (corrupt pack, NFS - // stall) would otherwise pin the global walk slot for the request's life. On timeout - // free the slot (mark truncated -> retryable 503) and skip the repo. spawn_blocking - // cannot be cancelled, so the child may linger on a blocking-pool thread, but it no - // longer holds the walk permit. - let probe_deadline = std::time::Duration::from_secs(state.config.git_service_timeout_secs); - match tokio::time::timeout( - probe_deadline, - tokio::task::spawn_blocking(move || store::object_type(&rp, &sha)), - ) + let git_bin = state.git_bin.clone(); + // Bound the probe CHILD itself (process-group teardown at `git_service_timeout_secs` + // via `object_type_bounded` -> `run_bounded_git`), not just an outer tokio timeout + // racing an uncancellable `spawn_blocking`: this probe runs while the /ipfs walk + // permit is held by the owning task, so a wedged cat-file (corrupt pack, NFS stall) + // must be REAPED at the deadline rather than left to linger and delay the task's + // completion — and thus admission release (#173 round-10, KTD2). No outer timeout, + // mirroring the bounded walk below; a `GitServiceTimeout` marks the search truncated + // (retryable 503). + let probe_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + match tokio::task::spawn_blocking(move || { + store::object_type_bounded(&git_bin, &rp, &sha, probe_timeout) + }) .await { - Ok(Ok(Ok(Some(t)))) => t, - Ok(Ok(Ok(None))) => return GateOutcome::Skip, - Ok(Ok(Err(e))) => { - tracing::warn!(repo = %repo.name, err = %e, "error checking git object type"); - return GateOutcome::Skip; - } + Ok(Ok(Some(t))) => t, + Ok(Ok(None)) => return GateOutcome::Skip, Ok(Err(e)) => { - tracing::warn!(repo = %repo.name, err = %e, "object-type probe task panicked; skipping repo"); + if e.is::() { + tracing::warn!(repo = %repo.name, "object-type probe timed out under the /ipfs walk permit; skipping repo"); + walk.truncated = true; + } else { + tracing::warn!(repo = %repo.name, err = %e, "error checking git object type"); + } return GateOutcome::Skip; } - Err(_elapsed) => { - tracing::warn!(repo = %repo.name, "object-type probe timed out under the /ipfs walk permit; skipping repo"); - walk.truncated = true; + Err(e) => { + tracing::warn!(repo = %repo.name, err = %e, "object-type probe task panicked; skipping repo"); return GateOutcome::Skip; } } @@ -710,45 +746,49 @@ async fn gate_and_serve( let read_sha = sha256_hex.to_string(); let read_type = obj_type.clone(); let want_cid = ctx.canonical_cid.to_string(); - // Bound the blocking size+read+verify under `git_service_timeout_secs` (same rationale - // as the object-type probe): a hung cat-file must not pin the held /ipfs walk permit. - let read_deadline = std::time::Duration::from_secs(state.config.git_service_timeout_secs); - let read = tokio::time::timeout( - read_deadline, - tokio::task::spawn_blocking(move || -> ServedRead { - match store::object_size(&read_repo, &read_sha) { - Ok(Some(size)) if size > max_bytes => return ServedRead::TooLarge(size), - Ok(Some(_)) => {} - // git ran and reported no such object (or an unparseable size): genuine - // not-found for this candidate. - Ok(None) => return ServedRead::Gone, - // git itself failed to run: an infra failure, not a not-found. - Err(e) => return ServedRead::ReadErr(e.to_string()), - } - let content = match store::read_object_content(&read_repo, &read_sha, &read_type) { - Ok(c) => c, - Err(e) => return ServedRead::ReadErr(e.to_string()), - }; - let served = gitlawb_core::cid::Cid::from_git_object_bytes(&content).to_string(); - if served != want_cid { - return ServedRead::Mismatch(served); - } - ServedRead::Ok(content) - }), - ) + let git_bin = state.git_bin.clone(); + // Bound the size+read CHILDREN themselves (process-group teardown at + // `git_service_timeout_secs` via the `*_bounded` twins), not an outer tokio timeout + // over an uncancellable `spawn_blocking`: a hung cat-file must be REAPED at the + // deadline rather than left to pin the held /ipfs walk permit (#173 round-10, KTD2). + // No outer timeout, mirroring the bounded walk; a `GitServiceTimeout` from either + // twin surfaces as `ServedRead::ReadErr` -> truncated (retryable 503). + let read_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let read = tokio::task::spawn_blocking(move || -> ServedRead { + match store::object_size_bounded(&git_bin, &read_repo, &read_sha, read_timeout) { + Ok(Some(size)) if size > max_bytes => return ServedRead::TooLarge(size), + Ok(Some(_)) => {} + // git ran and reported no such object (or an unparseable size): genuine + // not-found for this candidate. + Ok(None) => return ServedRead::Gone, + // git failed to run OR the bounded read timed out (GitServiceTimeout): an + // infra/timeout failure, not a not-found. + Err(e) => return ServedRead::ReadErr(e.to_string()), + } + let content = match store::read_object_content_bounded( + &git_bin, + &read_repo, + &read_sha, + &read_type, + read_timeout, + ) { + Ok(c) => c, + Err(e) => return ServedRead::ReadErr(e.to_string()), + }; + let served = gitlawb_core::cid::Cid::from_git_object_bytes(&content).to_string(); + if served != want_cid { + return ServedRead::Mismatch(served); + } + ServedRead::Ok(content) + }) .await; let served_read = match read { - Ok(Ok(sr)) => sr, - Ok(Err(e)) => { + Ok(sr) => sr, + Err(e) => { tracing::warn!(repo = %repo.name, err = %e, "object read task panicked"); walk.truncated = true; return GateOutcome::Skip; } - Err(_elapsed) => { - tracing::warn!(repo = %repo.name, "object read timed out under the /ipfs walk permit; skipping repo"); - walk.truncated = true; - return GateOutcome::Skip; - } }; let content = match served_read { ServedRead::Ok(c) => c, @@ -1063,38 +1103,33 @@ mod tests { drop(held); } - /// Retain-through-blocking (#174 F5, the load-bearing async property, on the - /// NEWLY-BOUNDED TREE path): the walk admission is held until the `spawn_blocking` - /// walk actually RETURNS, not when a tokio timeout fires. The requested CID - /// resolves to a TREE object under a path-scoped rule, so the gate runs - /// `allowed_tree_set_for_caller_bounded` — the walk this integration converts to - /// `run_bounded_git` — rather than the blob walk #174 already proved. With the - /// global pool at size 1, drive a request until its walk (a fake git that hangs on - /// `rev-list`) is in flight; the slot must stay held (`available_permits() == 0`) - /// and a replacement from a DIFFERENT source must shed 503 for as long as the - /// blocking walk runs — even though the request future is only `.await`ing the - /// blocking join. When the blocking walk ends the permit frees and a replacement - /// is admitted. The permit lives INSIDE the handler across the blocking `.await`; - /// move it out (drop before the walk) and the replacement would be admitted while - /// the walk still burns a blocking thread (the bug this guards). + /// Build the shared `/ipfs` TREE-walk fixture. A fake `git` whose `rev-list` records + /// its pid then sleeps ~6s (so the tree walk blocks deterministically inside + /// `run_bounded_git`) and whose `cat-file -t` answers "tree" (so the bounded + /// object-type probe, `object_type_bounded` on `state.git_bin`, routes into the + /// tree-gate arm); a real SHA-256 bare repo with a committed `src/` tree pinned WITH + /// provenance; and a path-scoped rule so the gate takes the tree-walk branch. Returns + /// the tempdir (keep it alive for the whole test), the state (the caller sets the walk + /// semaphores), the requested CID, and the rev-list pidfile path. #[cfg(unix)] - #[sqlx::test] - async fn get_by_cid_walk_permit_held_through_bounded_tree_walk(pool: sqlx::PgPool) { + async fn seed_tree_walk_fixture( + pool: sqlx::PgPool, + ) -> ( + tempfile::TempDir, + crate::state::AppState, + String, + std::path::PathBuf, + ) { use std::process::Command; let tmp = tempfile::TempDir::new().unwrap(); let revlist_pid = tmp.path().join("revlist.pid"); - // Fake git for the /ipfs TREE walk only (object_type/read_object_content use - // the real `git`, so the tree must genuinely exist below). `rev-parse` - // resolves (so the lenient enumeration appends HEAD) and `rev-list` records - // its pid then sleeps ~6s so the walk BLOCKS deterministically inside - // `run_bounded_git`. The sleep bounds the walk so a broken fix cannot wedge - // the suite. let body = format!( "#!/bin/sh\n\ case \"$1\" in\n\ for-each-ref) : ;;\n\ rev-parse) echo deadbeef ;;\n\ + cat-file) if [ \"$2\" = \"-t\" ]; then echo tree; fi ;;\n\ rev-list) echo $$ > \"{}\"; sleep 6 ;;\n\ *) : ;;\n\ esac\n\ @@ -1114,10 +1149,6 @@ mod tests { let repos_dir = tmp.path().join("repos"); std::fs::create_dir_all(&repos_dir).unwrap(); state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); - // Isolate the global walk pool at size 1; per-source cap permissive so only the - // held global permit can shed the replacement. - state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); - state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); state.git_bin = git_path.to_str().unwrap().to_string(); state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; @@ -1129,10 +1160,6 @@ mod tests { .await .unwrap(); let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); - // The exact bare path the handler's `acquire` resolves. Build a REAL SHA-256 - // bare repo there with a committed `src/` directory, so real - // `git cat-file -t ` classifies the requested object as a TREE and the - // handler routes into the tree-walk arm of the gate. let bare = state .repo_store .acquire(&rec.owner_did, &rec.name) @@ -1177,7 +1204,6 @@ mod tests { ], tmp.path(), ); - // The `src` directory's TREE oid — the object the request asks for. let tree_oid = { let out = Command::new("git") .args(["rev-parse", "HEAD:src"]) @@ -1187,8 +1213,6 @@ mod tests { assert!(out.status.success(), "rev-parse failed"); String::from_utf8_lossy(&out.stdout).trim().to_string() }; - // Precondition: real git classifies the object as a TREE (so the handler - // reaches the tree-walk arm, not the blob arm or an early `continue`). assert_eq!( crate::git::store::object_type(&bare, &tree_oid) .unwrap() @@ -1196,9 +1220,6 @@ mod tests { Some("tree"), "the seeded sha256 tree must exist so the handler reaches the tree walk" ); - // Pin the tree's content CID WITH provenance so the resolver targets this one - // repo (no legacy scan). A real pin CID digests the raw object content, not - // the git oid, so build it exactly as the pin path does (#173). let (_ty, raw) = crate::git::store::read_object(&bare, &tree_oid) .unwrap() .expect("tree object readable"); @@ -1208,8 +1229,6 @@ mod tests { .record_pinned_cid(&tree_oid, &cid, Some(&rec.id)) .await .unwrap(); - // A path-scoped rule so has_path_scoped_rule() is true (the tree-gate branch) - // without denying the "/" gate on the public repo. state .db .set_visibility_rule( @@ -1222,6 +1241,34 @@ mod tests { .await .unwrap(); + (tmp, state, cid, revlist_pid) + } + + /// Retain-through-blocking (#174 F5, the load-bearing async property, on the + /// NEWLY-BOUNDED TREE path): the walk admission is held until the `spawn_blocking` + /// walk actually RETURNS, not when a tokio timeout fires. The requested CID + /// resolves to a TREE object under a path-scoped rule, so the gate runs + /// `allowed_tree_set_for_caller_bounded` — the walk this integration converts to + /// `run_bounded_git` — rather than the blob walk #174 already proved. With the + /// global pool at size 1, drive a request until its walk (a fake git that hangs on + /// `rev-list`) is in flight; the slot must stay held (`available_permits() == 0`) + /// and a replacement from a DIFFERENT source must shed 503 for as long as the + /// blocking walk runs — even though the request future is only `.await`ing the + /// blocking join. When the blocking walk ends the permit frees and a replacement + /// is admitted. The permit lives INSIDE the handler across the blocking `.await`; + /// move it out (drop before the walk) and the replacement would be admitted while + /// the walk still burns a blocking thread (the bug this guards). + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_walk_permit_held_through_bounded_tree_walk(pool: sqlx::PgPool) { + let (tmp, mut state, cid, revlist_pid) = seed_tree_walk_fixture(pool).await; + // Isolate the global walk pool at size 1; per-source cap permissive so only the + // held global permit can shed the replacement. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + // Keep the fixture tempdir alive for the whole test (its Drop removes the repos). + let _tmp = tmp; + let sem = state.git_ipfs_walk_semaphore.clone(); assert_eq!( sem.available_permits(), @@ -1291,13 +1338,34 @@ mod tests { "a replacement must shed 503 while the prior request's blocking tree walk still runs" ); - // Drop the in-flight request; the detached blocking walk keeps running (a - // spawn_blocking cannot be cancelled), but the permit is a handler local, so - // dropping the future releases it once the blocking join is abandoned. Either - // way, kill the sleeping child so the slot frees promptly and poll for - // recovery — the point already proven above is that the slot stayed held for - // the duration of the blocking work. + // #173 round-10 (R1, KTD1): the NEW invariant. Drop the in-flight request. The + // gated serve pipeline runs in a DETACHED tokio task that OWNS the admission + // guard, so dropping the request future must NOT release admission — the task + // runs to its bounded completion first. Under the OLD handler-local permit the + // slot would free the instant the future dropped even though the blocking walk + // is still burning a thread (the cap bypass this closes). drop(fut); + // While the detached task is still inside the ~6s blocking tree walk, admission + // stays held. Poll a short window: a handler-local permit would already read 1. + tokio::time::sleep(std::time::Duration::from_millis(400)).await; + assert_eq!( + sem.available_permits(), + 0, + "dropping the request must NOT release admission while the detached walk \ + still runs (the spawned task owns the guard until its bounded work ends)" + ); + // A replacement from a different source STILL sheds 503 after the drop — the + // detached task holds the global slot, exactly as it did while the future lived. + let peer3: SocketAddr = "203.0.113.83:5000".parse().unwrap(); + let resp = router.clone().oneshot(make_req(peer3)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a replacement must still shed 503 after the original request was dropped, \ + while its detached walk holds admission" + ); + // Tear the walk down; the detached task finishes and drops the guard exactly + // once, freeing the single slot back to 1 (never double-freed to 2). unsafe { libc::kill(pid, libc::SIGKILL); } @@ -1311,7 +1379,112 @@ mod tests { } assert!( freed, - "once the blocking walk ends the walk permit must free the global slot" + "once the detached walk tears down, the task drops the guard and frees the slot" + ); + assert_eq!( + sem.available_permits(), + 1, + "admission released exactly once — the single slot is back, not double-freed" + ); + } + + /// Amplification negative (#173 round-10, R1): sequential cancel-spam from ONE source + /// cannot hold more than the per-source cap of concurrent detached walk tasks. A + /// detached serve task holds its per-source permit until its bounded work finishes (up + /// to `git_service_timeout_secs`), so with a per-source cap of 1 a second request from + /// the SAME source sheds 503 even though the GLOBAL pool has room — the source cannot + /// amplify its concurrent walk children past the cap by dropping-and-retrying. (The + /// worst case: a detached task can occupy its global/per-source permit for one + /// bound-interval, so distributed cancel-spam can hold the global pool that long — the + /// accepted bounded-admission tradeoff, not a leak.) + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_cancel_spam_bounded_by_per_source_cap(pool: sqlx::PgPool) { + let (tmp, mut state, cid, revlist_pid) = seed_tree_walk_fixture(pool).await; + // Global pool has ample room (4); the per-source cap is 1. So any shed of a + // same-source replacement is the PER-SOURCE cap, never global exhaustion. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(4)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + let _tmp = tmp; + + let sem = state.git_ipfs_walk_semaphore.clone(); + let per_caller = state.git_ipfs_walk_per_caller.clone(); + let router = ipfs_router(state); + let make_req = |peer: SocketAddr| { + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req + }; + + // Source S fires request 1; drive until its tree walk is in flight (the task now + // holds source S's single per-source permit). + let source_s: SocketAddr = "203.0.113.71:5000".parse().unwrap(); + let mut fut = Box::pin(router.clone().oneshot(make_req(source_s))); + let mut walk_pid: Option = None; + for _ in 0..500 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&revlist_pid) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + walk_pid = Some(p); + break; + } + } + let pid = walk_pid.expect("the fake git rev-list must have spawned"); + struct ReapOnDrop(i32); + impl Drop for ReapOnDrop { + fn drop(&mut self) { + unsafe { + libc::kill(self.0, libc::SIGKILL); + } + } + } + let _cleanup = ReapOnDrop(pid); + + // Cancel-spam: drop request 1's future. Its detached task keeps running the walk + // and KEEPS holding source S's single per-source permit. + drop(fut); + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + // A SECOND request from source S sheds 503. The global pool still has room (only 1 + // of 4 taken), so this is the per-source cap, not global exhaustion. + let resp = router.clone().oneshot(make_req(source_s)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a same-source cancel-spam replacement must shed 503 on the per-source cap \ + while the detached task still holds the source's permit" + ); + assert!( + sem.available_permits() >= 3, + "the shed was the per-source cap, not global exhaustion (global pool still has room)" + ); + assert_eq!( + per_caller.tracked_keys(), + 1, + "exactly one per-source permit is outstanding for the one source — no amplification" + ); + + // Tear the detached walk down; the task completes and releases source S's permit + // (tracked_keys returns to 0), so the source is no longer over the cap. + unsafe { + libc::kill(pid, libc::SIGKILL); + } + let mut released = false; + for _ in 0..400 { + if per_caller.tracked_keys() == 0 { + released = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + assert!( + released, + "once the detached task tears down it releases source S's per-source permit" ); } diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index ec115f1e..50145652 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -1752,8 +1752,9 @@ mod tests { // then, since re-polling a completed future panics. let mut pids = None; for _ in 0..500 { - let finished = - tokio::time::timeout(Duration::from_millis(10), &mut fut).await.is_ok(); + let finished = tokio::time::timeout(Duration::from_millis(10), &mut fut) + .await + .is_ok(); if let Some(p) = read_two_pids(&pidfile) { pids = Some(p); break; diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index 25f11246..a79d596a 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -290,26 +290,6 @@ pub fn object_type(repo_path: &Path, sha256_hex: &str) -> Result> )) } -/// Object size in bytes (`git cat-file -s`) WITHOUT reading the content, so an -/// oversized object can be rejected before it is buffered into memory (#173, F6). -/// `None` if the object does not exist or the size is unparseable. -pub fn object_size(repo_path: &Path, sha256_hex: &str) -> Result> { - // allow-unbounded-git: cheap cat-file -s header read (no content), holds no served-git - // permit and cannot hang; exact twin of object_type above. Not an INV-22 lifecycle op. - let out = Command::new("git") - .args(["cat-file", "-s", sha256_hex]) - .current_dir(repo_path) - .output() - .context("failed to run git cat-file -s")?; - if !out.status.success() { - return Ok(None); - } - Ok(String::from_utf8_lossy(&out.stdout) - .trim() - .parse::() - .ok()) -} - /// Read an object's content if its type is already known. pub fn read_object_content(repo_path: &Path, sha256_hex: &str, obj_type: &str) -> Result> { let content_output = Command::new("git") @@ -326,6 +306,90 @@ pub fn read_object_content(repo_path: &Path, sha256_hex: &str, obj_type: &str) - Ok(content_output.stdout) } +/// Bounded twin of [`object_type`] for the `GET /ipfs/{cid}` serve path (#173 +/// round-10, R1/KTD2). Runs `git cat-file -t` under +/// [`run_bounded_git`](crate::git::visibility_pack::run_bounded_git) so the child runs +/// in its own process group and a watchdog reaps it (SIGTERM -> grace -> SIGKILL) at +/// `timeout`. The bare [`object_type`] is a `spawn_blocking` `Command::output` that an +/// async timeout cannot cancel, so a wedged `cat-file` there pins the caller's held +/// /ipfs walk admission for the whole hang; this twin cannot. Semantics mirror +/// [`object_type`]: `Ok(Some(t))` for an existing object, `Ok(None)` when git exits +/// non-zero without timing out (the object is absent), and +/// `Err(`[`GitServiceTimeout`](crate::git::smart_http::GitServiceTimeout)`)` on the +/// deadline so the handler can mark the search truncated (retryable 503) rather than a +/// false not-found. Callers off the /ipfs path keep the bare helper. +pub fn object_type_bounded( + git_bin: &str, + repo_path: &Path, + sha256_hex: &str, + timeout: std::time::Duration, +) -> Result> { + let deadline = std::time::Instant::now() + timeout; + match crate::git::visibility_pack::run_bounded_git( + git_bin, + &["cat-file", "-t", sha256_hex], + repo_path, + b"", + deadline, + ) { + Ok(out) => Ok(Some(String::from_utf8_lossy(&out).trim().to_string())), + Err(e) if e.is::() => Err(e), + // A non-timeout failure is git reporting no such object (a non-zero exit), + // matching `object_type`'s `Ok(None)`. + Err(_) => Ok(None), + } +} + +/// Bounded `git cat-file -s` size read for the `GET /ipfs/{cid}` serve path (#173 +/// round-10, R1/KTD2): reads the object size WITHOUT its content (so an oversized object +/// is rejected before it is buffered, #173 F6), under +/// [`run_bounded_git`](crate::git::visibility_pack::run_bounded_git) so a wedged size +/// read is reaped at `timeout` instead of pinning the held /ipfs walk admission. +/// `Ok(Some(n))` on success, `Ok(None)` when the object is absent (a non-timeout +/// non-zero exit), `Err(GitServiceTimeout)` on the deadline. +pub fn object_size_bounded( + git_bin: &str, + repo_path: &Path, + sha256_hex: &str, + timeout: std::time::Duration, +) -> Result> { + let deadline = std::time::Instant::now() + timeout; + match crate::git::visibility_pack::run_bounded_git( + git_bin, + &["cat-file", "-s", sha256_hex], + repo_path, + b"", + deadline, + ) { + Ok(out) => Ok(String::from_utf8_lossy(&out).trim().parse::().ok()), + Err(e) if e.is::() => Err(e), + Err(_) => Ok(None), + } +} + +/// Bounded twin of [`read_object_content`] for the `GET /ipfs/{cid}` serve path (#173 +/// round-10, R1/KTD2): `git cat-file ` under +/// [`run_bounded_git`](crate::git::visibility_pack::run_bounded_git) so a wedged content +/// read is reaped at `timeout` instead of pinning the held /ipfs walk admission. Returns +/// the raw object bytes on success and an error (including `GitServiceTimeout` on the +/// deadline) otherwise, mirroring [`read_object_content`]. +pub fn read_object_content_bounded( + git_bin: &str, + repo_path: &Path, + sha256_hex: &str, + obj_type: &str, + timeout: std::time::Duration, +) -> Result> { + let deadline = std::time::Instant::now() + timeout; + crate::git::visibility_pack::run_bounded_git( + git_bin, + &["cat-file", obj_type, sha256_hex], + repo_path, + b"", + deadline, + ) +} + /// Read a git object by its SHA-256 hex object ID. /// /// Returns `(object_type, content_bytes)` where `content_bytes` is the raw @@ -521,4 +585,84 @@ mod tests { "unchanged file must not appear: {names:?}" ); } + + /// #173 round-10 (KTD2): `object_type_bounded` reaps a wedged `cat-file` child at its + /// deadline instead of blocking on it to natural exit, so a hung probe cannot pin the + /// /ipfs walk admission the owning task holds. A fake `git` records its pid and sleeps + /// far past the 1s deadline; the `run_bounded_git` watchdog (SIGTERM -> grace -> + /// SIGKILL of the process group) must kill it well before that natural exit, and the + /// call must surface `GitServiceTimeout`. REVERT PROOF (RED): swap the twin's + /// `run_bounded_git` for the bare `Command::output()` and the wedged child stays alive + /// past the deadline — the mid-flight liveness poll below reads it still running. + #[cfg(unix)] + #[test] + fn object_type_bounded_reaps_wedged_child_at_deadline() { + use std::time::Duration; + let tmp = tempfile::TempDir::new().unwrap(); + let pidfile = tmp.path().join("catfile.pid"); + // `cat-file` records its own pid then sleeps 8s (>> the 1s deadline) so the probe + // is genuinely wedged; the watchdog is what must end it. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + cat-file) echo $$ > \"{}\"; sleep 8 ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + pidfile.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + let repo = tmp.path().to_path_buf(); + let git = git_path.to_str().unwrap().to_string(); + + let alive = |pid: i32| unsafe { libc::kill(pid, 0) == 0 }; + + // The bounded probe blocks until the watchdog tears the child down, so run it on + // a worker thread and poll for the reap from here. + let handle = std::thread::spawn(move || { + super::object_type_bounded(&git, &repo, "deadbeef", Duration::from_secs(1)) + }); + + let mut pid = None; + for _ in 0..500 { + if let Some(p) = std::fs::read_to_string(&pidfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + pid = Some(p); + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + let pid = pid.expect("the fake cat-file must have spawned and recorded its pid"); + + // Past the 1s deadline + SIGTERM grace but well before the 8s natural exit: the + // watchdog must already have reaped the wedged group. A bare, unbounded read would + // leave it running here — the load-bearing RED. + std::thread::sleep(Duration::from_secs(3)); + let reaped = !alive(pid); + // Defensive reap so a RED run leaks no orphan. + unsafe { + libc::kill(pid, libc::SIGKILL); + } + assert!( + reaped, + "object_type_bounded must reap the wedged cat-file child at the deadline, \ + not leave it running to its natural exit" + ); + + let res = handle.join().expect("probe thread joins"); + let err = res.expect_err("a deadline overrun must be an error, not a value"); + assert!( + err.is::(), + "a deadline overrun must surface GitServiceTimeout, got: {err:?}" + ); + } } diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index e750a48f..5ec653fc 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -29,6 +29,7 @@ fn inv22_concurrency_gates_present_and_not_bypassed() { let repos = src("api/repos.rs"); let smart_http = src("git/smart_http.rs"); let vis = src("git/visibility_pack.rs"); + let ipfs = src("api/ipfs.rs"); // U1 / P1-a: run_bounded_git stands the watchdog down only after confirming the // child actually terminated (WNOWAIT), not on the raw stdout-drain EOF — otherwise @@ -84,6 +85,38 @@ fn inv22_concurrency_gates_present_and_not_bypassed() { is reaped on disconnect (the path-scoped half of #174 P1-a)" ); + // U2 / R1 (#173 round-10): the `GET /ipfs/{cid}` serve pipeline must run in a + // DETACHED tokio task that OWNS the AdmissionGuard built from BOTH walk permits, so a + // cancelled or timed-out request releases admission only after the spawned (bounded) + // work completes — not the instant the handler future drops (the /ipfs half of #174 + // P1-a). Two load-bearing markers: the guard is constructed from the two named + // permits, and the whole pipeline is moved into a `tokio::spawn`. Reverting to + // handler-local permits (dropping the guard/spawn) trips this. + assert!( + ipfs.contains("AdmissionGuard::new(ipfs_walk_permit, ipfs_caller_permit)") + && ipfs.contains( + "let serve: tokio::task::JoinHandle> = tokio::spawn(async move {" + ), + "U2/R1 gate missing: get_by_cid must move both /ipfs admission permits into an \ + AdmissionGuard owned by a detached tokio::spawn task so admission is released \ + only after the spawned serve work completes (the /ipfs half of #174 P1-a)" + ); + + // U2 / KTD2 (#173 round-10): the probe/read children on the /ipfs path must be the + // duration-bounded twins (process-group teardown via run_bounded_git), not the bare + // `store::object_type` / `read_object_content` (or an unbounded `cat-file -s`) a tokio + // timeout cannot cancel — otherwise a wedged cat-file lingers and pins the held + // admission past the deadline. Reverting any twin call site back to a bare read trips + // this. + assert!( + ipfs.contains("object_type_bounded(") + && ipfs.contains("object_size_bounded(") + && ipfs.contains("read_object_content_bounded("), + "U2/KTD2 gate missing: the /ipfs probe+read must call the run_bounded_git-backed \ + *_bounded twins so a wedged cat-file is reaped at the deadline, not left to pin \ + the held /ipfs walk admission" + ); + // P1-e non-bypass tripwire: the bounded recipients walk is spawn_blocking'd nowhere // but inside withheld_recipients_gated. A second call site (count > 1) is a new // detached git walk that skips the admission gate — exactly the class U5 closed. From 3351e09c082085bf33a0adfb382336d572fa0b0f Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 19:04:55 -0500 Subject: [PATCH 067/124] fix(node): requeue coalesced pushes instead of dropping them Per-repo coalescing dropped every push that arrived while a repo's encryption task was in flight, and no reconciliation ever processed it, so a withheld blob added by the coalesced push had its recovery copy permanently absent. The drop also silently skipped the coalesced push's local-IPFS pin work. EncryptInflight becomes a dirty-flag map; the detached task loops, and the atomic check-and-clear sits at the task tail, unconditional on the has_path_scoped_rule gate and walk success (a public or rules-free repo would otherwise exit before it ran). Each requeue re-reads repo state fresh and re-enumerates the pin half through the fail-closed full scan, never bare list_all_objects, so a coalesced rule change is honored and no withheld or dangling object leaks. At-most-one-task-per-repo is preserved. --- crates/gitlawb-node/src/api/repos.rs | 531 +++++++++++++++++++----- crates/gitlawb-node/src/state.rs | 124 ++++-- crates/gitlawb-node/src/test_support.rs | 351 ++++++++++++++++ 3 files changed, 857 insertions(+), 149 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index fb902f01..c9a2b3dd 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -790,6 +790,269 @@ async fn withheld_recipients_gated( .await } +/// Everything the detached post-push replication task needs that does not change +/// between requeue passes. Cloned once from `AppState` at the spawn site so the task +/// is self-contained (the handler keeps no copy). +struct PostPushReplication { + db: std::sync::Arc, + disk_path: std::path::PathBuf, + git_bin: String, + timeout: std::time::Duration, + ipfs_api: String, + repo_id: String, + encrypt_sem: std::sync::Arc, + node_seed: [u8; 32], + node_did: String, + repo_name: String, + irys_url: String, + http_client: std::sync::Arc, +} + +/// The requeue enumeration for the pin half: a fail-closed FULL scan of the current +/// object DB under the REFRESHED rules. The coalesced push's ref tips are gone by the +/// time we requeue, so the delta path is unavailable; the whole-repo scan is the safe +/// superset. Never pins the bare `list_all_objects` output — that includes +/// dangling/withheld blobs — it feeds it as CANDIDATES to the same fail-closed filter +/// the push path's full-scan branch uses, which drops dangling and visibility-withheld +/// blobs before anything is pinned. +async fn requeue_full_scan_object_list( + disk_path: &std::path::Path, + git_bin: &str, + timeout: std::time::Duration, + rules: Vec, + is_public: bool, + owner_did: String, +) -> Vec { + let disk = disk_path.to_path_buf(); + let gb = git_bin.to_string(); + let candidates = tokio::task::spawn_blocking(move || { + crate::git::push_delta::list_all_objects(&disk, &gb, std::time::Instant::now() + timeout) + }) + .await + .ok() + .and_then(|r| { + r.map_err(|e| { + tracing::warn!(err = %e, "requeue full-scan enumeration failed; pinning nothing this pass") + }) + .ok() + }) + .unwrap_or_default(); + fail_closed_full_scan_objects( + disk_path.to_path_buf(), + rules, + is_public, + owner_did, + candidates, + git_bin.to_string(), + timeout, + ) + .await +} + +/// The detached post-push encryption + local-IPFS pin task, as a REQUEUE LOOP. +/// +/// Pass one uses the spawn-time captures (`first_*`) — the delta the push handler +/// already computed. At the TASK TAIL (unconditional on the encrypt gate below and on +/// walk success) it consults the coalescing dirty flag: if a push coalesced during the +/// window it re-reads repo state (rules, is_public, owner_did, withheld) FRESH from the +/// DB, re-enumerates the pin set fail-closed, and runs another pass — so the coalesced +/// push is covered before the task exits. Otherwise it releases the key and returns. +/// +/// The tail placement is load-bearing: the encrypt+anchor block only runs under a +/// path-scoped rule, so a check-and-clear placed inside that gate would never run for a +/// public/rules-free repo or a failed walk, dropping exactly the pin-half push this +/// requeue must cover. +#[allow(clippy::too_many_arguments)] +async fn run_post_push_replication( + mut guard: crate::state::EncryptInflightGuard, + ctx: PostPushReplication, + first_object_list: Vec, + first_rules: Option>, + first_is_public: bool, + first_owner_did: String, + first_withheld: std::collections::HashSet, +) { + let mut object_list = first_object_list; + let mut rules_opt = first_rules; + let mut is_public = first_is_public; + let mut owner_did = first_owner_did; + // The task only spawns when `withheld.is_some()`, so pass one always replicates. + let mut withheld: Option> = Some(first_withheld); + + loop { + if withheld.is_some() { + // Pin new git objects to the local IPFS node (no-op if ipfs_api is empty). + crate::ipfs_pin::pin_new_objects( + &ctx.ipfs_api, + &ctx.disk_path, + object_list.clone(), + &ctx.db, + &ctx.repo_id, + ) + .await; + + // Option B1: encrypt-then-pin the withheld blobs. No path-scoped rule can + // withhold a blob, so a rules-free repo has nothing to seal; skip. Mirrors + // the has_path_scoped_rule gate on the other two withheld-walk sites. + if let Some(rules) = rules_opt + .clone() + .filter(|r| visibility_pack::has_path_scoped_rule(r)) + { + let recip = withheld_recipients_gated( + ctx.encrypt_sem.clone(), + ctx.disk_path.clone(), + ctx.git_bin.clone(), + ctx.timeout, + rules, + is_public, + owner_did.clone(), + ) + .await; + if let Ok(Ok(recipients)) = recip { + let delta = crate::encrypted_pin::encrypt_and_pin( + &ctx.ipfs_api, + &ctx.disk_path, + &ctx.db, + &ctx.repo_id, + &ctx.node_seed, + &recipients, + ) + .await; + + // Option B3: anchor a per-push manifest of the sealed blobs to + // Arweave. Best-effort; never fails the push. + if !delta.is_empty() && !ctx.irys_url.is_empty() { + let owner_short = crate::db::normalize_owner_key(&owner_did); + let repo_slug = format!("{owner_short}/{}", ctx.repo_name); + let ts = chrono::Utc::now().to_rfc3339(); + let manifest = crate::arweave::EncryptedManifest { + repo: &repo_slug, + owner_did: &owner_did, + node_did: &ctx.node_did, + timestamp: &ts, + blobs: &delta, + }; + match crate::arweave::anchor_encrypted_manifest( + &ctx.http_client, + &ctx.irys_url, + &manifest, + ) + .await + { + Ok(tx) if !tx.is_empty() => tracing::info!( + repo = %repo_slug, + tx_id = %tx, + "anchored encrypted manifest to Arweave" + ), + Ok(_) => {} + Err(e) => tracing::warn!( + repo = %repo_slug, + err = %e, + "encrypted manifest anchor failed" + ), + } + } + } + } + } + + // TASK TAIL — unconditional check-and-clear, atomic with the release decision. + if !guard.requeue_or_release() { + break; + } + + // A push coalesced during this pass. Re-read repo state FRESH (never the stale + // spawn-time captures) so a coalesced push that changed `.gitlawb` withholding + // is walked under the new policy, then re-enumerate the pin set fail-closed. + let (r_rules, r_is_public, r_owner) = match ctx.db.get_repo_by_id(&ctx.repo_id).await { + Ok(Some(rec)) => ( + ctx.db.list_visibility_rules(&ctx.repo_id).await.ok(), + rec.is_public, + rec.owner_did, + ), + Ok(None) => { + tracing::debug!(repo = %ctx.repo_id, "repo gone before requeue pass; releasing"); + (None, false, String::new()) + } + Err(e) => { + tracing::warn!(repo = %ctx.repo_id, err = %e, "requeue repo re-read failed; skipping this pass's work"); + (None, false, String::new()) + } + }; + let (_announce, r_withheld) = replication_withheld_set( + r_rules.clone(), + &r_owner, + r_is_public, + ctx.disk_path.clone(), + ctx.git_bin.clone(), + ctx.timeout, + ) + .await; + object_list = match &r_withheld { + Some(_) => { + requeue_full_scan_object_list( + &ctx.disk_path, + &ctx.git_bin, + ctx.timeout, + r_rules.clone().unwrap_or_default(), + r_is_public, + r_owner.clone(), + ) + .await + } + None => Vec::new(), + }; + rules_opt = r_rules; + is_public = r_is_public; + owner_did = r_owner; + withheld = r_withheld; + } +} + +/// Test-only entry point: build the `PostPushReplication` context from a test +/// `AppState` (with an overridable `ipfs_api` for a mock Kubo server and an explicit +/// `disk_path` for the fixture repo) and run the requeue loop. Keeps +/// `PostPushReplication` and `run_post_push_replication` private to this module. +#[cfg(test)] +#[allow(clippy::too_many_arguments)] +pub(crate) async fn run_post_push_replication_for_test( + state: &AppState, + guard: crate::state::EncryptInflightGuard, + disk_path: std::path::PathBuf, + repo_id: String, + ipfs_api: String, + is_public: bool, + owner_did: String, + object_list: Vec, + rules: Option>, + withheld: std::collections::HashSet, +) { + let ctx = PostPushReplication { + db: state.db.clone(), + disk_path: disk_path.clone(), + git_bin: state.git_bin.clone(), + timeout: std::time::Duration::from_secs(state.config.git_service_timeout_secs), + ipfs_api, + repo_id, + encrypt_sem: state.git_encrypt_semaphore.clone(), + node_seed: *state.node_keypair.to_seed(), + node_did: state.node_did.to_string(), + repo_name: String::new(), + irys_url: String::new(), + http_client: std::sync::Arc::clone(&state.http_client), + }; + run_post_push_replication( + guard, + ctx, + object_list, + rules, + is_public, + owner_did, + withheld, + ) + .await; +} + /// Map an error from a `smart_http` git service call to the right `AppError`: /// [`smart_http::GitServiceTimeout`] to 504, a malformed client request to 400, /// anything else to a 500 git error. Pure (no logging) so it is unit-testable; @@ -1434,128 +1697,44 @@ pub async fn git_receive_pack( // Pin new git objects to the local IPFS node (no-op if ipfs_api is empty). // Skipped entirely when the public cannot read the repo (withheld == None). // - // Coalesce per repo (#174 P2-2): this task parks on `git_encrypt_semaphore` - // (which DEFERS when the pool is full rather than dropping the recovery copy). To - // bound the OUTSTANDING parked-task set, only spawn if no encryption task for this - // repo is already in flight; otherwise skip — the pending/next walk over this - // repo's history already covers the newer push's objects, so the recovery copy is - // delayed, never lost. The guard removes the repo key when the task ends (success, - // error, or panic), so a later push is re-admitted (no permanent skip). - if withheld.is_some() { + // Coalesce per repo (#174 P2-2): only spawn a task if none is in flight for this + // repo; otherwise `try_begin` marks the repo DIRTY and the in-flight task requeues + // one more pass (re-reading fresh repo state) before it exits. This bounds the + // outstanding parked-task set to one per repo while covering every coalesced push — + // there is no reconciliation sweep, so a dropped job would be lost forever (#173 F3). + if let Some(withheld_set) = withheld.clone() { match state.encrypt_inflight.try_begin(&record.id) { None => { tracing::debug!( repo = %record.id, "post-push encryption task already in flight for this repo; coalescing \ - (the pending recovery-copy walk covers this push's objects)" + (the in-flight task will requeue one more pass to cover this push)" ); } Some(inflight_guard) => { - let object_list_ipfs = object_list.clone(); - let ipfs_api = state.config.ipfs_api.clone(); - let repo_path_clone = disk_path.clone(); - let db_clone = state.db.clone(); - let rules_for_enc = rules_opt.clone(); - let repo_id = record.id.clone(); - let owner_did = record.owner_did.clone(); - let is_public = record.is_public; - let irys_url = state.config.irys_url.clone(); - let http_client = std::sync::Arc::clone(&state.http_client); - let node_did_str = state.node_did.to_string(); - let node_seed = state.node_keypair.to_seed(); - let repo_name = record.name.clone(); - let enc_git_bin = state.git_bin.clone(); - let enc_timeout = - std::time::Duration::from_secs(state.config.git_service_timeout_secs); - let encrypt_sem = state.git_encrypt_semaphore.clone(); - tokio::spawn(async move { - // Held for the whole task; drop (on completion/error/panic) releases the - // repo's coalescing key so the next push for this repo can spawn again. - let _inflight_guard = inflight_guard; - let pinned = crate::ipfs_pin::pin_new_objects( - &ipfs_api, - &repo_path_clone, - object_list_ipfs, - &db_clone, - &repo_id, - ) - .await; - if !pinned.is_empty() { - tracing::info!(count = pinned.len(), "pinned git objects to IPFS"); - for (sha, cid) in &pinned { - tracing::info!(sha = %sha, %cid, "pinned"); - } - } - - // Option B1: encrypt-then-pin the withheld blobs so authorized - // readers can recover them when the origin cannot serve them. - // No path-scoped rule can withhold a blob, so withheld_blob_recipients - // would return an empty map after a full per-ref walk; skip it. Mirrors - // the has_path_scoped_rule gate on the other two withheld-walk sites. - if let Some(rules) = - rules_for_enc.filter(|r| visibility_pack::has_path_scoped_rule(r)) - { - // Bound the number of concurrent post-push encryption walks (#174 P1-e): - // acquire an admission permit before the full-history walk, deferring - // when the pool is full rather than shedding the recovery pin. - let recip = withheld_recipients_gated( - encrypt_sem.clone(), - repo_path_clone.clone(), - enc_git_bin.clone(), - enc_timeout, - rules, - is_public, - owner_did.clone(), - ) - .await; - if let Ok(Ok(recipients)) = recip { - let delta = crate::encrypted_pin::encrypt_and_pin( - &ipfs_api, - &repo_path_clone, - &db_clone, - &repo_id, - &node_seed, - &recipients, - ) - .await; - - // Option B3: anchor a per-push manifest of the blobs sealed - // this push to Arweave, so the oid->cid index survives total - // node loss. Best-effort; never fails the push. - if !delta.is_empty() && !irys_url.is_empty() { - let owner_short = crate::db::normalize_owner_key(&owner_did); - let repo_slug = format!("{owner_short}/{repo_name}"); - let ts = chrono::Utc::now().to_rfc3339(); - let manifest = crate::arweave::EncryptedManifest { - repo: &repo_slug, - owner_did: &owner_did, - node_did: &node_did_str, - timestamp: &ts, - blobs: &delta, - }; - match crate::arweave::anchor_encrypted_manifest( - &http_client, - &irys_url, - &manifest, - ) - .await - { - Ok(tx) if !tx.is_empty() => tracing::info!( - repo = %repo_slug, - tx_id = %tx, - "anchored encrypted manifest to Arweave" - ), - Ok(_) => {} - Err(e) => tracing::warn!( - repo = %repo_slug, - err = %e, - "encrypted manifest anchor failed" - ), - } - } - } - } - }); + let ctx = PostPushReplication { + db: state.db.clone(), + disk_path: disk_path.clone(), + git_bin: state.git_bin.clone(), + timeout: std::time::Duration::from_secs(state.config.git_service_timeout_secs), + ipfs_api: state.config.ipfs_api.clone(), + repo_id: record.id.clone(), + encrypt_sem: state.git_encrypt_semaphore.clone(), + node_seed: *state.node_keypair.to_seed(), + node_did: state.node_did.to_string(), + repo_name: record.name.clone(), + irys_url: state.config.irys_url.clone(), + http_client: std::sync::Arc::clone(&state.http_client), + }; + tokio::spawn(run_post_push_replication( + inflight_guard, + ctx, + object_list.clone(), + rules_opt.clone(), + record.is_public, + record.owner_did.clone(), + withheld_set, + )); } } } @@ -4573,6 +4752,130 @@ mod tests { ); } + // ---- U3 (#173 F3): dirty-flag requeue seam (the mechanics behind the end-to-end + // requeue tests in test_support.rs) ---- + + /// A coalesced push MARKS the repo dirty (not just "skip"), and the in-flight task's + /// tail check-and-clear then runs ONE more pass before releasing: `requeue_or_release` + /// returns `true` (loop) while dirty, clearing the flag, then `false` (release) when + /// clean. This is the mechanism that makes a coalesced push requeued, not dropped. + #[test] + fn u3_try_begin_marks_dirty_then_requeue_loops_once_then_releases() { + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkU3DirtyAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA/proj"; + + let mut guard = inflight.try_begin(repo).expect("first push admits"); + assert_eq!(inflight.dirty(repo), Some(false), "fresh task starts clean"); + + // A push coalesces during the in-flight window: marked dirty, no new task. + assert!( + inflight.try_begin(repo).is_none(), + "coalesced push does not spawn" + ); + assert_eq!( + inflight.dirty(repo), + Some(true), + "the coalesced push marked the repo dirty" + ); + + // Task tail, pass 1: dirty -> requeue (clear the flag, keep the key, loop). + assert!( + guard.requeue_or_release(), + "dirty repo requeues one more pass" + ); + assert_eq!( + inflight.dirty(repo), + Some(false), + "the dirty flag is cleared for the next pass" + ); + assert_eq!( + inflight.len(), + 1, + "the key is still held while the task loops" + ); + + // Task tail, pass 2: clean -> release (remove the key, exit). + assert!( + !guard.requeue_or_release(), + "a clean repo releases and exits" + ); + assert_eq!(inflight.len(), 0, "the key is removed on the clean release"); + assert_eq!( + inflight.dirty(repo), + None, + "no in-flight entry after release" + ); + } + + /// The check-and-clear is ATOMIC with the release decision, so no push lands in a + /// "checked clean but still present" gap (scenario 6, race gap). A push that arrives + /// BEFORE the tail check sets dirty -> the same pass requeues it. A push that arrives + /// AFTER a clean release finds an empty set -> `try_begin` spawns a fresh task. Both + /// directions covered; neither drops the push. + #[test] + fn u3_requeue_or_release_leaves_no_uncovered_gap() { + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkU3GapBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB/proj"; + + // Case A: push arrives before the tail check -> requeue covers it. + let mut guard = inflight.try_begin(repo).expect("admit"); + assert!( + inflight.try_begin(repo).is_none(), + "push arrives during the window" + ); + assert!( + guard.requeue_or_release(), + "the pre-check push is covered by a requeue" + ); + // Now clean: the task releases and exits. + assert!(!guard.requeue_or_release(), "clean -> release"); + assert_eq!(inflight.len(), 0); + + // Case B: a push arriving after release starts a brand-new task (not dropped). + let guard2 = inflight.try_begin(repo); + assert!( + guard2.is_some(), + "a post-release push spawns a fresh task, never dropped" + ); + } + + /// Drop is a PANIC BACKSTOP only. A task that panics before releasing still frees the + /// key (so a crashed walk never permanently locks the repo out); a task that releases + /// via `requeue_or_release` and then drops does not double-free. + #[test] + fn u3_drop_is_panic_backstop_for_unreleased_guard() { + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkU3PanicCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC/proj"; + + // Normal release then drop: key already gone, drop is a no-op. + let mut g = inflight.try_begin(repo).expect("admit"); + assert!(!g.requeue_or_release(), "clean release"); + assert_eq!(inflight.len(), 0); + drop(g); + assert_eq!( + inflight.len(), + 0, + "dropping an already-released guard does not resurrect a key" + ); + + // Panic before releasing: Drop-on-unwind frees the key. + let panicked = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _g = inflight.try_begin(repo).expect("admit before panic"); + assert_eq!(inflight.len(), 1); + panic!("task panics before reaching its tail check-and-clear"); + })); + assert!(panicked.is_err(), "the simulated task panicked"); + assert_eq!( + inflight.len(), + 0, + "a panic before release still frees the key (backstop), so the repo is not locked out" + ); + assert!( + inflight.try_begin(repo).is_some(), + "the repo can be admitted again after a panic" + ); + } + /// Model of the pre-fix / mutated code: no coalescing check, so every push spawns. /// Returns the count of tasks spawned (== the size of the unbounded outstanding set /// the fix prevents), used as the RED comparison in the bound test above. diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 10dbbf32..61bbec6a 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -253,24 +253,29 @@ impl AppState { /// repo spawn N parked tasks, each holding cloned object lists/rules/paths/keys — an /// unbounded outstanding set. /// -/// This tracks the repo keys with an in-flight encryption task. Before spawning, the -/// handler calls [`try_begin`](Self::try_begin): if a task for the repo is already -/// in-flight it returns `None` and the handler SKIPS spawning a duplicate (coalesce — -/// the newer push's objects are covered by the pending/next walk over the same repo's -/// history). This bounds the outstanding set to <=1 pending task per repo WITHOUT -/// dropping work: coalescing only delays a duplicate walk, it never sheds the recovery -/// copy (there is no reconciliation sweep, so a *dropped* job would be lost forever). +/// This tracks the repo keys with an in-flight encryption task, each carrying a +/// DIRTY flag. Before spawning, the handler calls [`try_begin`](Self::try_begin): if +/// a task for the repo is already in-flight it MARKS THE REPO DIRTY and returns `None` +/// so the handler SKIPS spawning a duplicate (coalesce). The in-flight task consults +/// the flag at its tail via [`requeue_or_release`](EncryptInflightGuard::requeue_or_release): +/// a set flag makes it run ONE MORE pass (re-reading repo state) before it exits, so a +/// push coalesced during the in-flight window is REQUEUED, never dropped. This bounds +/// the outstanding set to <=1 task per repo without losing work: there is no +/// reconciliation sweep, so a *dropped* job would be lost forever. /// -/// The returned [`EncryptInflightGuard`] is moved into the detached task and removes -/// the repo key on drop — on normal completion, error, OR panic (Drop runs on unwind) -/// — so one crashed walk can never permanently lock a repo out of future recovery -/// copies. +/// The returned [`EncryptInflightGuard`] is moved into the detached task. The normal +/// exit path is `requeue_or_release`, which removes the repo key ATOMICALLY with the +/// "clean" decision — no push can land in the gap between "checked clean" and "task +/// exits". `Drop` is a panic backstop only: if the task panics (or returns without +/// calling `requeue_or_release`) it still removes the key so a crashed walk can never +/// permanently lock a repo out of future recovery copies. #[derive(Clone, Default)] pub struct EncryptInflight { - // std::sync::Mutex: only ever held for O(1) HashSet insert/remove in a sync - // context (right before `tokio::spawn`, and in the guard's Drop) — never across - // an await, so a std Mutex is correct and cheaper than a tokio one. - repos: Arc>>, + // std::sync::Mutex: only ever held for O(1) HashMap ops in a sync context (before + // `tokio::spawn`, at the task tail, and in Drop) — never across an await, so a std + // Mutex is correct and cheaper than a tokio one. The value is the DIRTY flag: a + // coalesced push flips it true so the in-flight task requeues one more pass. + repos: Arc>>, } impl EncryptInflight { @@ -279,18 +284,24 @@ impl EncryptInflight { } /// Try to begin an encryption task for `repo_id`. Returns `Some(guard)` if no task - /// for the repo was in-flight (the caller should spawn), or `None` if one already - /// is (the caller should COALESCE — skip spawning a duplicate). The guard releases - /// the repo key on drop. + /// for the repo was in-flight (the caller should spawn). If one already is, MARKS + /// the repo dirty and returns `None` (the caller COALESCES — skips the duplicate + /// spawn; the in-flight task requeues one more pass to cover this push). pub fn try_begin(&self, repo_id: &str) -> Option { - let mut set = self.repos.lock().expect("encrypt_inflight mutex poisoned"); - if set.insert(repo_id.to_string()) { - Some(EncryptInflightGuard { - repos: Arc::clone(&self.repos), - repo_id: repo_id.to_string(), - }) - } else { - None + let mut map = self.repos.lock().expect("encrypt_inflight mutex poisoned"); + match map.get_mut(repo_id) { + Some(dirty) => { + *dirty = true; + None + } + None => { + map.insert(repo_id.to_string(), false); + Some(EncryptInflightGuard { + repos: Arc::clone(&self.repos), + repo_id: repo_id.to_string(), + released: false, + }) + } } } @@ -309,23 +320,66 @@ impl EncryptInflight { pub fn is_empty(&self) -> bool { self.len() == 0 } + + /// Test-only: read the dirty flag for `repo_id` (`None` if no task is in-flight). + #[cfg(test)] + pub fn dirty(&self, repo_id: &str) -> Option { + self.repos + .lock() + .expect("encrypt_inflight mutex poisoned") + .get(repo_id) + .copied() + } } -/// RAII guard removing a repo key from [`EncryptInflight`] when the detached -/// encryption task finishes (drop on completion, error, or panic-unwind). Move-only — -/// there is no reason to clone a guard, and cloning would double-remove. +/// RAII guard for one in-flight encryption task's repo key. The task drives it at its +/// tail with [`requeue_or_release`](Self::requeue_or_release); `Drop` is a panic +/// backstop. Move-only — cloning would double-release. pub struct EncryptInflightGuard { - repos: Arc>>, + repos: Arc>>, repo_id: String, + released: bool, +} + +impl EncryptInflightGuard { + /// TASK-TAIL check-and-clear, atomic with the release decision. If a push + /// coalesced since the last pass (dirty), clear the flag and return `true` (the + /// task loops one more pass). Otherwise remove the key and return `false` (the task + /// exits). Atomic under the mutex: a concurrent push either sets the flag BEFORE + /// this reads it (-> requeue covers it) or arrives AFTER the key is removed (-> a + /// fresh `try_begin` spawns a new task) — there is no window where the key is + /// present-but-clean while the task exits. + pub fn requeue_or_release(&mut self) -> bool { + let mut map = self.repos.lock().expect("encrypt_inflight mutex poisoned"); + match map.get_mut(&self.repo_id) { + Some(dirty) if *dirty => { + *dirty = false; + true + } + _ => { + map.remove(&self.repo_id); + self.released = true; + false + } + } + } } impl Drop for EncryptInflightGuard { fn drop(&mut self) { - // A poisoned lock (a prior panic while holding it) still lets us take the inner - // set via into_inner-on-guard; but poisoning here is not expected because the - // only critical sections are the O(1) ops above. Remove best-effort. - if let Ok(mut set) = self.repos.lock() { - set.remove(&self.repo_id); + // Panic backstop ONLY. The normal exit is `requeue_or_release` (which removed + // the key atomically and set `released`), so this does nothing then. If the + // task panicked or returned without releasing, remove the key so one crashed + // walk never permanently locks the repo out. + // + // Accepted residual: a panic with the dirty flag still set drops the requeued + // work — Drop cannot loop — the same loss class as the on-panic behavior before + // this change. No reconciliation sweep re-derives it. + if self.released { + return; + } + if let Ok(mut map) = self.repos.lock() { + map.remove(&self.repo_id); } } } diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 45b2b807..20e325ac 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -7306,4 +7306,355 @@ mod tests { "result includes the deep cert matching the prefix" ); } + + // ---- U3 (#173 F3): coalesced post-push work is REQUEUED, not dropped ---- + // + // The seam mechanics (dirty flag, atomic check-and-clear, Drop backstop) are unit + // tested next to the #174 coalescing tests in `api/repos.rs`. These drive the whole + // detached task through a mock Kubo node and a real git repo, so the requeue's + // fresh re-read (encrypt half) and fail-closed full-scan enumeration (pin half) are + // proven end to end, with DB-observable effects. Each test models a push that + // coalesced during the in-flight window by (a) marking the repo dirty via a second + // `try_begin` and (b) making the repo/policy dynamic so the FIRST pass's spawn-time + // captures are stale — a static-state test would pass vacuously over the gap. + mod u3_requeue { + use super::*; + use crate::db::VisibilityMode; + use std::collections::HashSet; + use std::path::{Path, PathBuf}; + use std::process::Command; + + fn git(args: &[&str], dir: &Path) { + let ok = Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(); + assert!(ok, "git {args:?} failed"); + } + fn oid(rev: &str, dir: &Path) -> String { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(dir) + .output() + .unwrap(); + assert!(out.status.success(), "rev-parse {rev}: {out:?}"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + struct Repo { + _td: tempfile::TempDir, + path: PathBuf, + } + fn init_repo() -> Repo { + let td = tempfile::TempDir::new().unwrap(); + let path = td.path().to_path_buf(); + git(&["init", "-q"], &path); + git(&["config", "user.email", "t@t"], &path); + git(&["config", "user.name", "t"], &path); + Repo { _td: td, path } + } + /// Commit `content` at `rel`, return the blob oid. + fn commit(repo: &Path, rel: &str, content: &str) -> String { + let full = repo.join(rel); + std::fs::create_dir_all(full.parent().unwrap()).unwrap(); + std::fs::write(&full, content).unwrap(); + git(&["add", "."], repo); + git(&["commit", "-qm", rel], repo); + oid(&format!("HEAD:{rel}"), repo) + } + /// Write a loose, UNREACHABLE blob (dangling object). + fn write_dangling_blob(repo: &Path, content: &str) -> String { + let out = Command::new("git") + .args(["hash-object", "-w", "--stdin"]) + .current_dir(repo) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap(); + use std::io::Write; + out.stdin + .as_ref() + .unwrap() + .write_all(content.as_bytes()) + .unwrap(); + let o = out.wait_with_output().unwrap(); + assert!(o.status.success()); + String::from_utf8_lossy(&o.stdout).trim().to_string() + } + fn new_did() -> String { + Keypair::generate().did().to_string() + } + + /// SCENARIO 2 + 5 (pin half, TAIL-PLACEMENT guard). A coalesced push on a PUBLIC + /// repo with NO path-scoped rule must still requeue its pin half: the second + /// push's new object is pinned after the task. RED without the loop (stale spawn + /// object_list never lists obj2), and RED if the check-and-clear sits inside the + /// `has_path_scoped_rule` block (a rules-free repo would never reach it). + #[sqlx::test] + async fn u3_rules_free_public_repo_requeues_pin_half(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u3-pin"); + state.db.create_repo(&repo).await.expect("seed repo"); + let git_repo = init_repo(); + let obj1 = commit(&git_repo.path, "a.txt", "one\n"); + // The coalesced push B adds obj2 (present at requeue time, NOT in the stale + // push-A spawn object_list). + let obj2 = commit(&git_repo.path, "b.txt", "two\n"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // Push A admits (guard); push B coalesces (marks dirty). + let guard = state + .encrypt_inflight + .try_begin(&repo.id) + .expect("push A admits"); + assert!( + state.encrypt_inflight.try_begin(&repo.id).is_none(), + "push B coalesces while A is in flight" + ); + + // Spawn-time (push A) captures are STALE: object_list lists only obj1, no rule. + crate::api::repos::run_post_push_replication_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + server.url(), + true, + owner.clone(), + vec![obj1.clone()], + Some(vec![]), + HashSet::new(), + ) + .await; + + assert!( + state.db.is_pinned(&obj1).await.unwrap(), + "push A's object is pinned on the first pass" + ); + assert!( + state.db.is_pinned(&obj2).await.unwrap(), + "the coalesced push's new object is pinned by the REQUEUE full scan (RED \ + without the loop, or if the check-and-clear sits inside the encrypt gate)" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is released once the task exits clean" + ); + } + + /// SCENARIO 1 + 3 (encrypt half, FRESH re-read). A coalesced push adds a + /// path-scoped rule withholding a blob. The task must re-read rules FRESH on + /// requeue and seal the newly-withheld blob's recovery copy. RED without the loop + /// (pass one's stale empty rule set seals nothing). + #[sqlx::test] + async fn u3_requeue_seals_blob_withheld_by_coalesced_rule_change(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let reader = new_did(); + let repo = seed_repo(&owner, "u3-enc"); + state.db.create_repo(&repo).await.expect("seed repo"); + let git_repo = init_repo(); + let _pub_oid = commit(&git_repo.path, "public/a.txt", "public\n"); + let secret_oid = commit(&git_repo.path, "secret/b.txt", "TOP SECRET\n"); + + // Coalesced push B changes .gitlawb: withhold /secret/** from anon, grant reader. + state + .db + .set_visibility_rule(&repo.id, "/secret/**", VisibilityMode::B, &[reader], &owner) + .await + .expect("set rule"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + let guard = state + .encrypt_inflight + .try_begin(&repo.id) + .expect("push A admits"); + assert!( + state.encrypt_inflight.try_begin(&repo.id).is_none(), + "push B coalesces" + ); + + // Push A captures are STALE: no rule, empty withheld set (public repo). + crate::api::repos::run_post_push_replication_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + server.url(), + true, + owner.clone(), + vec![], + Some(vec![]), + HashSet::new(), + ) + .await; + + assert!( + state + .db + .encrypted_blob_recipients_tag(&repo.id, &secret_oid) + .await + .unwrap() + .is_some(), + "the coalesced push's newly-withheld blob is sealed after the REQUEUE re-read \ + (RED without the loop: pass one's stale empty rules seal nothing)" + ); + assert!(state.encrypt_inflight.is_empty(), "guard key released"); + } + + /// SCENARIO 4 (visibility-leak negative). The requeue full scan must feed + /// `list_all_objects` through the fail-closed filter, never pin it bare: a + /// withheld secret blob and a dangling blob must NOT land in the public pin set. + #[sqlx::test] + async fn u3_requeue_full_scan_does_not_publicly_pin_withheld_or_dangling(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let reader = new_did(); + let repo = seed_repo(&owner, "u3-leak"); + state.db.create_repo(&repo).await.expect("seed repo"); + let git_repo = init_repo(); + let pub_oid = commit(&git_repo.path, "public/a.txt", "public\n"); + let secret_oid = commit(&git_repo.path, "secret/b.txt", "TOP SECRET\n"); + state + .db + .set_visibility_rule(&repo.id, "/secret/**", VisibilityMode::B, &[reader], &owner) + .await + .expect("set rule"); + // Coalesced push adds a new public object and a dangling blob. + let new_pub_oid = commit(&git_repo.path, "public/c.txt", "more public\n"); + let dangling_oid = write_dangling_blob(&git_repo.path, "orphan bytes\n"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + let rules = state.db.list_visibility_rules(&repo.id).await.unwrap(); + let mut withheld = HashSet::new(); + withheld.insert(secret_oid.clone()); + + let guard = state + .encrypt_inflight + .try_begin(&repo.id) + .expect("push A admits"); + assert!( + state.encrypt_inflight.try_begin(&repo.id).is_none(), + "push B coalesces" + ); + + crate::api::repos::run_post_push_replication_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + server.url(), + true, + owner.clone(), + vec![pub_oid.clone()], + Some(rules), + withheld, + ) + .await; + + assert!( + state.db.is_pinned(&new_pub_oid).await.unwrap(), + "the coalesced push's new PUBLIC object is pinned by the requeue" + ); + assert!( + !state.db.is_pinned(&secret_oid).await.unwrap(), + "a WITHHELD blob is never publicly pinned by the requeue enumeration (leak guard)" + ); + assert!( + !state.db.is_pinned(&dangling_oid).await.unwrap(), + "a DANGLING blob is never publicly pinned by the requeue enumeration (leak guard)" + ); + // The withheld blob still gets its ENCRYPTED recovery copy (not a public pin). + assert!( + state + .db + .encrypted_blob_recipients_tag(&repo.id, &secret_oid) + .await + .unwrap() + .is_some(), + "withheld blob is sealed as an encrypted recovery copy, not pinned in the clear" + ); + } + + /// SCENARIO 8 (no-coalesce happy path). A single push with no coalesced follower + /// runs exactly one pass, pins its object, and releases the key. No requeue. + #[sqlx::test] + async fn u3_no_coalesce_single_pass_pins_and_releases(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u3-happy"); + state.db.create_repo(&repo).await.expect("seed repo"); + let git_repo = init_repo(); + let obj1 = commit(&git_repo.path, "a.txt", "one\n"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // No second try_begin: the repo is never marked dirty. + let guard = state + .encrypt_inflight + .try_begin(&repo.id) + .expect("push admits"); + assert_eq!( + state.encrypt_inflight.dirty(&repo.id), + Some(false), + "clean, no coalesce" + ); + + crate::api::repos::run_post_push_replication_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + server.url(), + true, + owner.clone(), + vec![obj1.clone()], + Some(vec![]), + HashSet::new(), + ) + .await; + + assert!( + state.db.is_pinned(&obj1).await.unwrap(), + "the single push's object is pinned" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the key is released after one pass" + ); + } + } } From c2fce67bae6f8ca67850de5a3b65581d2a2497d8 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 19:17:12 -0500 Subject: [PATCH 068/124] fix(node): wire GITLAWB_IPFS_MAX_REPOS_WALKED to the legacy-probe budget The knob was parsed and documented as the per-request /ipfs fan-out cap but never read in production: AppState seeded the legacy-probe budget from a fixed 256 constant, so setting the knob did nothing and an operator could still incur 256 acquire/probe operations per request. Seed ipfs_max_legacy_probes from the knob (default raised to 256 to preserve shipped behavior); leave the history-walk ceiling on its MAX_PIN_SOURCES+1 constant so a provenanced request is never truncated into a false 503. Updates the README and .env.example to the shipped behavior and adds cap-honored and plumbing tests. --- .env.example | 7 +-- README.md | 2 +- crates/gitlawb-node/src/config.rs | 57 +++++++++++++++++++++---- crates/gitlawb-node/src/main.rs | 5 ++- crates/gitlawb-node/src/state.rs | 12 ++++++ crates/gitlawb-node/src/test_support.rs | 49 +++++++++++++++++++++ 6 files changed, 118 insertions(+), 14 deletions(-) diff --git a/.env.example b/.env.example index 149ea8b1..1c390a1c 100644 --- a/.env.example +++ b/.env.example @@ -168,9 +168,10 @@ GITLAWB_MAX_CONCURRENT_IPFS_WALKS=32 # per-caller caps via GITLAWB_TRUSTED_PROXY; reject-before-insert bounded map). # Default 4. GITLAWB_IPFS_WALK_PER_SOURCE=4 -# Max repos walked per single /ipfs request, so one request cannot serialize a -# full-history walk over every repo carrying the CID. Default 64. -GITLAWB_IPFS_MAX_REPOS_WALKED=64 +# Max legacy (NULL-provenance) repos probed per single /ipfs request, bounding the +# scan-fallback fan-out (git cat-file per candidate repo) for an anonymous caller. A +# truncated scan sheds a retryable 503, never a false 404. Default 256. +GITLAWB_IPFS_MAX_REPOS_WALKED=256 # Max /ipfs/{cid} requests per client IP per hour (route flood brake, distinct # from the concurrency caps above). 0 disables. Default 600. GITLAWB_IPFS_RATE_LIMIT=600 diff --git a/README.md b/README.md index e9765876..a7e4425b 100644 --- a/README.md +++ b/README.md @@ -347,7 +347,7 @@ Important node settings: | `GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS` | Max seconds the storage-acquisition phase (Tigris HEAD/GET, push advisory-lock) of a served git op may run before the request is shed with a 503, separate from the git-run timeout. The concurrency permit is released on expiry so a stalled backend cannot pin the pool. Default 30. | | `GITLAWB_MAX_CONCURRENT_IPFS_WALKS` | Max concurrent `GET /ipfs/{cid}` visibility walks across all callers (own pool, disjoint from the served-git pools); over-cap sheds 503. Default 32. | | `GITLAWB_IPFS_WALK_PER_SOURCE` | Max concurrent `/ipfs` walks a single source IP may hold. Default 4. | -| `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max repos walked per `/ipfs/{cid}` request, bounding one request's fan-out. Default 64. | +| `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max legacy (NULL-provenance) repos probed per `/ipfs/{cid}` request, bounding the scan-fallback fan-out. A truncated scan returns a retryable 503, not a false 404. Default 256. | | `GITLAWB_IPFS_RATE_LIMIT` | Max `/ipfs/{cid}` requests per client IP per hour (route flood brake). 0 disables. Default 600. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index b3667024..a9b76067 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -372,17 +372,20 @@ pub struct Config { )] pub ipfs_walk_per_source: usize, - /// Upper bound on the number of candidate repos a single `/ipfs/{cid}` request - /// will walk before giving up (returning the opaque 404). The handler already - /// short-circuits the moment it serves the object, but a CID that is present in - /// (or path-gated out of) many repos could otherwise serialize one full-history - /// walk per repo inside a single held admission slot. Capping the count bounds - /// the worst-case work one request can pin its slot with. Must be between 1 and - /// 1_048_576. Default: 64. + /// Per-request ceiling on the number of legacy (NULL-provenance) repos the + /// `/ipfs/{cid}` resolver's scan fallback will PROBE (`acquire` + `git cat-file + /// -t`) before giving up. The provenance path targets one repo; the legacy scan, + /// absent this bound, fans one anonymous request out to O(repos) subprocess spawns + /// and cold-cache fetches for a CID enumerable from the public pins index (#173, + /// INV-10). A truncated scan surfaces as a retryable 503, never a false 404. Wired + /// into `AppState::ipfs_max_legacy_probes` at construction; the history-walk ceiling + /// stays constant (`MAX_HISTORY_WALKS_PER_REQUEST`) and is NOT governed by this knob + /// (a smaller value would falsely 503 a provenanced request). Must be between 1 and + /// 1_048_576. Default: 256. #[arg( long, env = "GITLAWB_IPFS_MAX_REPOS_WALKED", - default_value_t = 64, + default_value_t = crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST as usize, value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) )] pub ipfs_max_repos_walked: usize, @@ -534,7 +537,7 @@ mod tests { fn ipfs_max_repos_walked_defaults_and_rejects_out_of_range() { assert_eq!( Config::parse_from(["gitlawb-node"]).ipfs_max_repos_walked, - 64 + 256 ); assert_eq!( Config::parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "8"]) @@ -548,6 +551,42 @@ mod tests { ); } + /// The `GITLAWB_IPFS_MAX_REPOS_WALKED` knob must actually reach the legacy-probe + /// budget it advertises (R5, KTD5): production seeds `ipfs_max_legacy_probes` from + /// this helper, so the knob is a no-op unless the helper reflects it. RED while the + /// helper returns the hardcoded `MAX_LEGACY_PROBES_PER_REQUEST` (256 regardless of + /// the knob), GREEN once it reads the knob. + #[test] + fn ipfs_max_repos_walked_wires_the_legacy_probe_budget() { + use crate::state::AppState; + // Knob set to 1 → a one-probe legacy budget. + let one = Config::parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "1"]); + assert_eq!( + AppState::ipfs_legacy_probe_budget(&one), + 1, + "the knob must control the legacy-probe budget, not be ignored" + ); + // Unset knob preserves the shipped 256-probe behaviour. + let default = Config::parse_from(["gitlawb-node"]); + assert_eq!( + AppState::ipfs_legacy_probe_budget(&default), + 256, + "the default knob keeps the shipped 256-probe budget" + ); + assert_eq!( + AppState::ipfs_legacy_probe_budget(&default), + crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, + "the default budget equals the constant it replaced" + ); + // Ceiling guard: the knob never governs the history-walk ceiling, which must + // stay at MAX_PIN_SOURCES + 1 or a provenanced full source set false-503s. + assert!( + crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST + >= crate::db::MAX_PIN_SOURCES as u32 + 1, + "the history-walk ceiling is independent of the repos-walked knob" + ); + } + #[test] fn max_concurrent_reads_per_caller_defaults_and_rejects_out_of_range() { assert_eq!( diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 37190c76..0af288c2 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -375,7 +375,10 @@ async fn main() -> Result<()> { create_ip_rate_limiter, push_rate_limiter, ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, - ipfs_max_legacy_probes: crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, + // The legacy-probe budget is operator-tunable via GITLAWB_IPFS_MAX_REPOS_WALKED + // (R5); the history-walk ceiling above stays constant (a smaller value false-503s + // a provenanced request). Default 256 preserves the shipped behaviour. + ipfs_max_legacy_probes: AppState::ipfs_legacy_probe_budget(&config), ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, push_limiter_trust, sync_trigger_rate_limiter, diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 61bbec6a..bc57b2d9 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -242,6 +242,18 @@ impl AppState { pub fn is_shutting_down(&self) -> bool { *self.shutdown_tx.borrow() } + + /// Legacy-probe budget wired from the `GITLAWB_IPFS_MAX_REPOS_WALKED` operator + /// knob (R5, KTD5). The knob seeds `ipfs_max_legacy_probes` at construction so it + /// controls the per-request legacy (NULL-provenance) probe fan-out it advertises. + /// It deliberately does NOT feed `ipfs_max_history_walks`: that ceiling must stay + /// at `MAX_HISTORY_WALKS_PER_REQUEST` (`MAX_PIN_SOURCES + 1`) or a provenanced + /// request with a full source set is truncated into a false 503, and the knob's + /// range starts at 1. The knob is `usize`, the field `u32`; the range cap + /// (1_048_576) keeps the cast lossless. + pub(crate) fn ipfs_legacy_probe_budget(config: &crate::config::Config) -> u32 { + config.ipfs_max_repos_walked as u32 + } } /// Bounds the OUTSTANDING post-push encryption-task set by per-repo coalescing diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 20e325ac..30f63d83 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -3674,6 +3674,55 @@ mod tests { ); } + /// T2b (R5, KTD5): the `GITLAWB_IPFS_MAX_REPOS_WALKED` knob drives the legacy-probe + /// budget end to end. With the knob at 1 (fed through the same production helper the + /// state seeding uses) and two candidate repos that miss, the first repo spends the + /// single probe and the second is skipped at the cap → truncated → 503. If the knob + /// budget were not honoured (unbounded), both would probe, both miss, and the request + /// would be a definitive 404. Proves the wired knob=1 → exactly one probe path. + #[sqlx::test] + async fn ipfs_cid_repos_walked_knob_caps_legacy_probes(pool: PgPool) { + use clap::Parser; + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + // Seed the legacy-probe budget the way production does: from the operator knob. + let cfg = + crate::config::Config::parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "1"]); + state.ipfs_max_legacy_probes = AppState::ipfs_legacy_probe_budget(&cfg); + assert_eq!(state.ipfs_max_legacy_probes, 1, "knob=1 → one-probe budget"); + // The knob must not touch the history-walk ceiling (must stay MAX_PIN_SOURCES + 1). + assert_eq!( + state.ipfs_max_history_walks, + crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, + "the repos-walked knob leaves the history-walk ceiling untouched" + ); + + let _fx = seed_cid_repos(&slug, &short, &["k0", "k1"]); + for n in ["k0", "k1"] { + let repo = seed_repo(&owner_did, n); + state.db.create_repo(&repo).await.expect("seed repo"); + } + // A legacy pin whose oid is absent from every repo: the cap, not a hit, decides. + let bogus_oid = "0".repeat(64); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"absent-marker-knob").to_string(); + state + .db + .record_pinned_cid(&bogus_oid, &cid, None) + .await + .expect("record legacy pin"); + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::SERVICE_UNAVAILABLE, + "knob=1 caps the scan at one probe → incomplete search → retryable 503" + ); + } + /// T3 (F2): a walk-cap truncation must not false-404. Walk ceiling shrunk to 1; /// two public repos each carry a path-scoped rule over the object and deny anon. /// The 1st spends the single walk (deny), the 2nd is skipped at the cap — the From 0a1618f7fb81c8bac92721095035f5eb02b08c46 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 19:41:57 -0500 Subject: [PATCH 069/124] fix(node): split /ipfs work accounting off the route rate limiter The route middleware charged ipfs_rate_limiter once per request while the legacy scan and provenance walk charged the same bucket per probe/walk, so a one-probe request cost two tokens and GITLAWB_IPFS_RATE_LIMIT=1 admitted the request at the route then 429'd it at the pre-scan peek. Add a bounded ipfs_work_rate_limiter, move the in-scan, per-walk, and pre-scan charges onto it, and leave the route limiter as the pure once-per-request brake. The work-budget capacity derives from the route limit with a floor of one full legacy-probe budget per window (no new operator knob), so a default-config deep search never self-throttles. Adds it to the periodic sweep, updates all three AppState construction sites, and rewrites the contradictory limiter comments. --- crates/gitlawb-node/src/api/ipfs.rs | 36 +++-- crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/config.rs | 83 ++++++++++- crates/gitlawb-node/src/main.rs | 9 ++ crates/gitlawb-node/src/state.rs | 46 +++++- crates/gitlawb-node/src/test_support.rs | 182 ++++++++++++++++++++++-- 6 files changed, 322 insertions(+), 35 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 016e130d..c5fa35bc 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -38,11 +38,12 @@ use crate::state::AppState; use crate::visibility::{visibility_check, Decision}; /// Hard ceiling on the number of full-history reachability walks a single -/// `GET /ipfs/{cid}` request may spawn. The per-request `ipfs_rate_limiter` -/// check brakes *repeat* requests, but within one request the object can exist -/// under path-scoped rules in many repos, and each distinct repo pays its own -/// `spawn_blocking` walk (the memo only dedups the same repo). Without a ceiling -/// a single request fans out to O(repos) walks for one rate-limiter token — an +/// `GET /ipfs/{cid}` request may spawn. The route brake (`ipfs_rate_limiter`, charged +/// once per request by the middleware) caps request RATE, and the per-walk charge on +/// the separate `ipfs_work_rate_limiter` bounds the walk work across requests, but +/// within ONE request the object can exist under path-scoped rules in many repos, and +/// each distinct repo pays its own `spawn_blocking` walk (the memo only dedups the same +/// repo). Without a ceiling a single request fans out to O(repos) walks — an /// amplification sink (INV-10). Once this many walks have run, no further walk is /// spawned for the rest of the request: any remaining candidate that still needs /// a walk is skipped (and, with nothing else readable, the request falls through @@ -335,14 +336,18 @@ pub async fn get_by_cid( .await .map_err(AppError::Internal)?; if needs_scan { - // F3 (#173, INV-10/INV-15): peek the per-IP limiter WITHOUT consuming a token - // so an already-throttled source is shed BEFORE the O(repos) preload; the - // consuming per-probe charge inside gate_and_serve is left UNCHANGED (it is - // load-bearing for the across-request bound), so this adds no double-charge. + // F3 (#173, INV-10/INV-15): peek the per-IP WORK-budget limiter WITHOUT + // consuming a token so an already-throttled source is shed BEFORE the + // O(repos) preload; the consuming per-probe charge inside gate_and_serve is + // left UNCHANGED (it is load-bearing for the across-request bound), so this + // adds no double-charge. This peeks `ipfs_work_rate_limiter`, the SAME bucket + // the per-probe charge below debits — NOT the route limiter (`ipfs_rate_limiter`, + // charged once per request by the middleware): peeking the route bucket here + // would re-shed a request the route already admitted (R6, U5). if let Some(key) = crate::rate_limit::client_key(rctx.headers, rctx.peer, state.push_limiter_trust) { - if state.ipfs_rate_limiter.is_throttled(&key).await { + if state.ipfs_work_rate_limiter.is_throttled(&key).await { throttled = true; continue; } @@ -536,9 +541,10 @@ async fn gate_and_serve( // reset each request, leaving a NULL-provenance CID open to unbounded ACROSS- // request amplification: N requests spending N x budget cold `acquire` calls // against Tigris with zero limiter contact (#173, F3, jatmn). Charging the first - // probe makes those requests accumulate against the per-IP `ipfs_rate_limiter`, - // closing that path. The per-request cap below stays as the second bound (a - // single request's ceiling). A spent quota is the same non-fatal Throttled as the + // probe makes those requests accumulate against the per-IP `ipfs_work_rate_limiter` + // (the resolver's WORK bucket, separate from the once-per-request route brake + // `ipfs_rate_limiter` — R6, U5), closing that path. The per-request cap below stays + // as the second bound (a single request's ceiling). A spent quota is the same non-fatal Throttled as the // walk brake: keep scanning for a walk-free copy, and only a wholly-unservable // request becomes the 429. No resolvable key (a test oneshot with no peer/header) // skips the brake, as the walk brake does. The provenance path targets one repo @@ -553,7 +559,7 @@ async fn gate_and_serve( if let Some(key) = crate::rate_limit::client_key(ctx.headers, ctx.peer, state.push_limiter_trust) { - if !state.ipfs_rate_limiter.check(&key).await { + if !state.ipfs_work_rate_limiter.check(&key).await { return GateOutcome::Throttled; } } @@ -658,7 +664,7 @@ async fn gate_and_serve( if let Some(key) = crate::rate_limit::client_key(ctx.headers, ctx.peer, state.push_limiter_trust) { - if !state.ipfs_rate_limiter.check(&key).await { + if !state.ipfs_work_rate_limiter.check(&key).await { return GateOutcome::Throttled; } } diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index c03eb2e5..96e6f5ea 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -517,6 +517,7 @@ mod tests { create_ip_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), push_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_work_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, ipfs_max_legacy_probes: crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index a9b76067..5480beda 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -396,6 +396,11 @@ pub struct Config { /// concurrency cap above (a rate limit bounds request *rate*, the semaphore /// bounds concurrent slow holds — different axes). Keyed on the resolved client /// IP via `GITLAWB_TRUSTED_PROXY`. `0` disables. Default: 600. + /// + /// This is the pure once-per-request ROUTE brake. The resolver's internal + /// per-probe/per-walk WORK budget is a SEPARATE bucket whose capacity is DERIVED + /// from this value (`AppState::ipfs_work_budget`), not a knob of its own; `0` here + /// disables that derived bucket too. #[arg(long, env = "GITLAWB_IPFS_RATE_LIMIT", default_value_t = 600)] pub ipfs_rate_limit: usize, } @@ -581,12 +586,86 @@ mod tests { // Ceiling guard: the knob never governs the history-walk ceiling, which must // stay at MAX_PIN_SOURCES + 1 or a provenanced full source set false-503s. assert!( - crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST - >= crate::db::MAX_PIN_SOURCES as u32 + 1, + crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST > crate::db::MAX_PIN_SOURCES as u32, "the history-walk ceiling is independent of the repos-walked knob" ); } + /// The `/ipfs` work-budget capacity is DERIVED from the route limit (R6, KTD6), with + /// a hard floor of one full legacy search per window (the effective + /// `ipfs_max_legacy_probes`). This guards the derived default so a single + /// default-config deep search never self-throttles mid-scan and recreates the F6 + /// admit-then-429 for a legitimate caller. A `RateLimiter` sized to the derived + /// budget must admit the whole probe budget back to back. + #[test] + fn ipfs_work_budget_derives_from_route_limit_and_clears_the_probe_floor() { + use crate::state::AppState; + + // Default config: derived work budget = max(route 600, probe budget 256) = 600, + // comfortably above the 256-probe floor. + let default = Config::parse_from(["gitlawb-node"]); + let budget = AppState::ipfs_work_budget(&default); + assert_eq!(budget, 600, "default derives max(route 600, probe 256)"); + assert!( + budget >= AppState::ipfs_legacy_probe_budget(&default) as usize, + "the work budget must clear one full legacy search per window" + ); + + // Tight route limit (1): the floor lifts the work budget to the probe budget + // (256), NOT down to 1 — a single deep search still completes its full scan. + let tight = Config::parse_from(["gitlawb-node", "--ipfs-rate-limit", "1"]); + assert_eq!( + AppState::ipfs_work_budget(&tight), + 256, + "a tight route limit is floored at the 256-probe budget, not clamped to 1" + ); + + // Raised probe budget lifts the floor with it (the work budget tracks the + // effective probe budget, not the constant). + let raised = Config::parse_from([ + "gitlawb-node", + "--ipfs-rate-limit", + "10", + "--ipfs-max-repos-walked", + "1000", + ]); + assert_eq!( + AppState::ipfs_work_budget(&raised), + 1000, + "the floor tracks the operator-raised legacy-probe budget" + ); + + // 0 route limit disables the derived bucket too (a 0-capacity limiter admits all). + let disabled = Config::parse_from(["gitlawb-node", "--ipfs-rate-limit", "0"]); + assert_eq!( + AppState::ipfs_work_budget(&disabled), + 0, + "route limit 0 disables the derived work bucket alongside the route brake" + ); + + // Behavioral floor: a limiter sized to the derived (tight-route) budget admits + // the whole probe budget back to back for one source, then sheds the next. + let budget = AppState::ipfs_work_budget(&tight); + let limiter = + crate::rate_limit::RateLimiter::new(budget, std::time::Duration::from_secs(3600)); + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + rt.block_on(async { + for i in 0..budget { + assert!( + limiter.check("1.2.3.4").await, + "probe {i} of one full default-config scan must be admitted (no mid-scan throttle)" + ); + } + assert!( + !limiter.check("1.2.3.4").await, + "the probe past the derived budget is shed" + ); + }); + } + #[test] fn max_concurrent_reads_per_caller_defaults_and_rejects_out_of_range() { assert_eq!( diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 0af288c2..1ed45929 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -437,6 +437,15 @@ async fn main() -> Result<()> { std::time::Duration::from_secs(3600), 200_000, ), + // Separate WORK-budget bucket for the resolver's per-probe/per-walk charges (R6). + // Its capacity is DERIVED from the route limit (no new knob) and floored at the + // legacy-probe budget, so one full default-config legacy scan never self-throttles + // mid-request while the route brake above stays the pure once-per-request cap. + ipfs_work_rate_limiter: rate_limit::RateLimiter::new_bounded( + AppState::ipfs_work_budget(&config), + std::time::Duration::from_secs(3600), + 200_000, + ), git_bin: "git".to_string(), }; if config.ipfs_rate_limit == 0 { diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index bc57b2d9..51618b78 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -65,13 +65,29 @@ pub struct AppState { /// brake a push flood from a DID farm (one throwaway DID per repo), so the /// push path throttles on the resolved client IP instead. pub push_rate_limiter: RateLimiter, - /// Per-client-IP rate limiter for the `GET /ipfs/{cid}` full-history walk. - /// The route is anonymous and a valid tree CID (exposed by the public pins - /// index) makes each repeat request pay a fresh allowed-set walk (rev-list + - /// ls-tree per commit), memoized only per request — unbounded amplification - /// (INV-10). Braking the walk on the non-farmable source IP caps that cost - /// without touching cheap non-walk fetches. Keyed by `push_limiter_trust`. + /// Per-client-IP ROUTE brake for `GET /ipfs/{cid}`: charged ONCE per request by the + /// `rate_limit_by_ip` middleware (server.rs), never inside the handler. It bounds + /// request RATE (the "requests per hour" contract of `GITLAWB_IPFS_RATE_LIMIT`) on + /// the non-farmable source IP, so an anonymous flood of the public route is capped. + /// The per-probe/per-walk WORK accounting the resolver does WITHIN a request draws + /// from the SEPARATE `ipfs_work_rate_limiter` below — the two cannot share one bucket + /// or a single request that spends a route token and then its own probe token off the + /// same bucket is admitted at the route and falsely shed mid-request (#173 round-10, + /// R6). Keyed by `push_limiter_trust`. pub ipfs_rate_limiter: RateLimiter, + /// Per-client-IP WORK-budget limiter for the `GET /ipfs/{cid}` resolver's internal + /// fan-out: charged per legacy (NULL-provenance) PROBE (`acquire` + `cat-file`) and + /// per provenance-path WALK, and peeked non-consuming before the O(repos) legacy + /// preload. A legacy CID from the public pins index otherwise lets one request drive + /// O(repos) subprocess spawns and cold Tigris fetches, and repeat requests amplify + /// that across requests with zero limiter contact (INV-10, F3). Charging the work to + /// the non-farmable source IP bounds it. A bucket DISTINCT from the route brake above: + /// one request legitimately spends many work tokens (a full legacy scan is up to + /// `ipfs_max_legacy_probes` probes), so it must not double as the once-per-request + /// route bucket. Capacity is DERIVED from the route limit (`AppState::ipfs_work_budget`, + /// no separate operator knob), floored at the legacy-probe budget so a single default- + /// config deep search never self-throttles mid-scan. Keyed by `push_limiter_trust`. + pub ipfs_work_rate_limiter: RateLimiter, /// Per-request ceiling on full-history reachability walks the CID resolver /// may spawn (default `api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST`). A field, /// not a bare const, so tests can shrink it to exercise the cap cheaply; @@ -218,6 +234,7 @@ impl AppState { self.create_ip_rate_limiter.cleanup().await; self.push_rate_limiter.cleanup().await; self.ipfs_rate_limiter.cleanup().await; + self.ipfs_work_rate_limiter.cleanup().await; self.sync_trigger_rate_limiter.cleanup().await; self.peer_write_rate_limiter.cleanup().await; } @@ -254,6 +271,23 @@ impl AppState { pub(crate) fn ipfs_legacy_probe_budget(config: &crate::config::Config) -> u32 { config.ipfs_max_repos_walked as u32 } + + /// Work-budget capacity for [`ipfs_work_rate_limiter`](Self#structfield.ipfs_work_rate_limiter) + /// (R6, KTD6), DERIVED from the route limit rather than a new operator knob. The route + /// limiter (`ipfs_rate_limiter`) charges once per request; this separate bucket absorbs + /// the resolver's per-probe/per-walk work charges so both the route "requests per hour" + /// contract and the amplification bound hold. Floor: at least one full legacy search per + /// window — the effective `ipfs_max_legacy_probes` (the `GITLAWB_IPFS_MAX_REPOS_WALKED` + /// knob, U4) — so a single default-config deep search cannot self-throttle mid-scan and + /// recreate the F6 admit-then-429 for a legitimate caller. `GITLAWB_IPFS_RATE_LIMIT=0` + /// disables the route brake and this derived bucket alike (a 0-capacity limiter admits + /// everything). + pub(crate) fn ipfs_work_budget(config: &crate::config::Config) -> usize { + if config.ipfs_rate_limit == 0 { + return 0; + } + config.ipfs_rate_limit.max(config.ipfs_max_repos_walked) + } } /// Bounds the OUTSTANDING post-push encryption-task set by per-repo coalescing diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 30f63d83..01363abf 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -79,6 +79,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { create_ip_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), push_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_work_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, ipfs_max_legacy_probes: crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, @@ -3285,7 +3286,8 @@ mod tests { let slug = owner_did.replace([':', '/'], "_"); let short = owner_did.split(':').next_back().unwrap().to_string(); let mut state = test_state(pool).await; - state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; let fx = seed_cid_repos(&slug, &short, &["provthrottle"]); @@ -3455,7 +3457,8 @@ mod tests { let slug = owner_did.replace([':', '/'], "_"); let short = owner_did.split(':').next_back().unwrap().to_string(); let mut state = test_state(pool).await; - state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; let fx = seed_cid_repos(&slug, &short, &["fanout"]); @@ -3498,7 +3501,7 @@ mod tests { /// fresh batch of `acquire` + `cat-file` probes every request with zero limiter /// contact, unbounded anonymous amplification against Tigris. Charging every /// legacy probe from the first one makes those probes accumulate against the - /// per-IP `ipfs_rate_limiter` ACROSS requests. Four repos, none holding the CID, + /// per-IP `ipfs_work_rate_limiter` ACROSS requests. Four repos, none holding the CID, /// so a full scan probes all four; the per-IP budget is sized to exactly ONE such /// scan (4 tokens). req1 (a genuine absence) fully scans and 404s, spending the /// budget; req2 from the SAME IP is shed at the first probe → 429 (it never @@ -3515,7 +3518,8 @@ mod tests { let mut state = test_state(pool).await; // Budget = one full scan of the four seeded repos. A repeat scan from the same // IP then finds it spent. Keyed on XFF so `oneshot` can choose the source IP. - state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(4, Duration::from_secs(3600)); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(4, Duration::from_secs(3600)); state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; let names = ["a0", "a1", "a2", "a3"]; @@ -3578,7 +3582,8 @@ mod tests { let short = owner_did.split(':').next_back().unwrap().to_string(); let mut state = test_state(pool).await; // Budget 1, keyed on XFF so `oneshot` can choose the source IP. - state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; let _fx = seed_cid_repos(&slug, &short, &["r0"]); @@ -5298,7 +5303,8 @@ mod tests { // size the per-IP budget to admit exactly one full scan (2 probes). A repeat // scan from the same IP then finds the bucket spent. Keyed on the rightmost // X-Forwarded-For hop so the test can choose a source IP under `oneshot`. - state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(2, Duration::from_secs(3600)); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(2, Duration::from_secs(3600)); state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; let fx = seed_cid_repos(&slug, &short, &["walklimit"]); @@ -5504,7 +5510,8 @@ mod tests { // Budget = one full two-repo scan (2 probes), keyed on the rightmost XFF hop // so `oneshot` can choose a source IP (no socket peer). A repeat scan from the // same IP then finds the budget spent. - state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(2, Duration::from_secs(3600)); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(2, Duration::from_secs(3600)); state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; // Identical secret-blob content in both bare clones → one CID resolves to @@ -5578,10 +5585,11 @@ mod tests { } /// INV-10 amplification bound: a single `GET /ipfs/{cid}` must not fan out an - /// unbounded number of full-history walks. The per-request `ipfs_rate_limiter` - /// check only brakes REPEAT requests (it fires once per request); within one - /// request the same object can exist under path-scoped rules in many repos, - /// each paying its own walk. `MAX_HISTORY_WALKS_PER_REQUEST` caps that fan-out. + /// unbounded number of full-history walks. The route brake (`ipfs_rate_limiter`) + /// fires once per request and the per-walk `ipfs_work_rate_limiter` charge bounds + /// walk work across requests, but within ONE request the same object can exist under + /// path-scoped rules in many repos, each paying its own walk. + /// `MAX_HISTORY_WALKS_PER_REQUEST` caps that fan-out. /// /// Load-bearing witness (#173, F4): a readable public copy (no path rule → /// served via the no-walk path, exactly like @@ -5784,7 +5792,8 @@ mod tests { let short = owner_did.split(':').next_back().unwrap().to_string(); let mut state = test_state(pool).await; - state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; let fx = seed_cid_repos(&slug, &short, &["w0", "w1"]); @@ -5851,6 +5860,155 @@ mod tests { ); } + /// U5 (R6, KTD6), the observed defect: the `/ipfs` route rate limit and the + /// resolver's per-probe WORK budget are SEPARATE buckets, so a single request with + /// one probe COMPLETES even at route limit = 1. Through the production router the + /// `rate_limit_by_ip` middleware charges `ipfs_rate_limiter` once (its 1-slot bucket + /// is now full); the handler's legacy pre-scan peek and per-probe charge then draw + /// from `ipfs_work_rate_limiter`, a different bucket, so the walk-free public copy + /// still serves 200. RED before the split (both charges on `ipfs_rate_limiter`): the + /// middleware fills the one slot, the pre-scan peek reads it throttled, nothing is + /// servable → 429 on the FIRST request. Trust None so the middleware and the handler + /// resolve the same `ConnectInfo` peer IP. + #[sqlx::test] + async fn ipfs_route_limit_1_still_serves_one_probe(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(600, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // Public, no-rule legacy pin (NULL provenance) → the resolver takes the scan + // fallback and serves walk-free (exactly one probe). + let fx = seed_cid_repos(&slug, &short, &["routeone"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("routeone.git"); + let repo = seed_repo(&owner_did, "routeone"); + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + let router = crate::server::build_router(state); + let peer: std::net::SocketAddr = "203.0.113.7:5000".parse().unwrap(); + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut() + .insert(axum::extract::ConnectInfo(peer)); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::OK, + "a single /ipfs request with one probe must serve even at route limit = 1 \ + (the route brake and the resolver's work budget are separate buckets)" + ); + } + + /// U5 (R6): the two buckets are independent — the WORK budget can be exhausted + /// (429) WITHOUT draining the ROUTE bucket. Through the production router, route + /// generous (5) but work tight (1): one request drives two legacy probes, so the + /// second probe finds the work bucket spent → 429 (the route middleware admitted it). + /// The route bucket, charged once by the middleware, still has room afterward — the + /// work charges never touched it, so it admits four more direct checks. + #[sqlx::test] + async fn ipfs_work_exhaustion_leaves_route_bucket_intact(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(5, Duration::from_secs(3600)); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // A legacy pin absent from every repo so the scan probes both seeded repos: two + // probes, work budget 1 → the second probe is shed → 429. + let names = ["we0", "we1"]; + let _fx = seed_cid_repos(&slug, &short, &names); + for n in names { + state + .db + .create_repo(&seed_repo(&owner_did, n)) + .await + .expect("seed repo"); + } + let bogus_oid = "0".repeat(64); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"work-exhaustion").to_string(); + state + .db + .record_pinned_cid(&bogus_oid, &cid, None) + .await + .expect("legacy pin"); + + let route_bucket = state.ipfs_rate_limiter.clone(); + let peer_ip = "203.0.113.8"; + let peer: std::net::SocketAddr = format!("{peer_ip}:5000").parse().unwrap(); + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut() + .insert(axum::extract::ConnectInfo(peer)); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::TOO_MANY_REQUESTS, + "a request whose probes exceed the work budget is shed 429 (work bucket), \ + not blocked at the route (route bucket generous)" + ); + // The route bucket recorded only the single request the middleware charged; the + // work charges did not drain it. Sized 5, one used by the request → four left. + for i in 0..4 { + assert!( + route_bucket.check(peer_ip).await, + "route check {i} must still admit — work charges never drained the route bucket" + ); + } + } + + /// U5 (R6): the periodic cleanup task sweeps the NEW work-budget limiter too, not + /// only the route limiter and its siblings. Mirrors + /// `sweep_rate_limiters_includes_ipfs_limiter`: drive `sweep_rate_limiters` and + /// assert the work limiter's expired entry is evicted. Dropping the + /// `ipfs_work_rate_limiter.cleanup()` call from that method leaves the entry in place + /// (`tracked_keys` stays 1): the RED proof the sweep covers it. + #[sqlx::test] + async fn sweep_rate_limiters_includes_ipfs_work_limiter(pool: PgPool) { + let mut state = test_state(pool).await; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(5, Duration::from_millis(50)); + + assert!( + state.ipfs_work_rate_limiter.check("1.2.3.4").await, + "record a hit on the work limiter" + ); + assert_eq!( + state.ipfs_work_rate_limiter.tracked_keys().await, + 1, + "the source-IP key is tracked before the sweep" + ); + + tokio::time::sleep(Duration::from_millis(60)).await; + state.sweep_rate_limiters().await; + + assert_eq!( + state.ipfs_work_rate_limiter.tracked_keys().await, + 0, + "the periodic sweep must evict the work limiter's expired entries" + ); + } + // --------------------------------------------------------------------------- // Issue #120 — repo-scoped read surfaces visibility gate // --------------------------------------------------------------------------- From b774c435a63691c65473082208b771a790eeab9e Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 19:50:23 -0500 Subject: [PATCH 070/124] fix(node): retry pin-source and pinned-CID recording on transient errors The three warn-only record sites in the detached post-push task dropped a transient DB failure silently, leaving a permanently incomplete pin-source set that makes the resolver 404 a valid public copy. Wrap them in a bounded retry (inserts are idempotent via ON CONFLICT DO NOTHING); on exhaustion the warn still fires so behavior degrades to today's. Documents the residual crash/outage window that only a reconciliation sweep retires. --- crates/gitlawb-node/src/ipfs_pin.rs | 135 +++++++++++++++++++++++++++- 1 file changed, 132 insertions(+), 3 deletions(-) diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index e70d8f7b..1a37005d 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -9,6 +9,44 @@ use anyhow::Result; use gitlawb_core::cid::Cid; +use std::time::Duration; + +/// Attempts (including the first) for a transient DB-record retry. +const PIN_RECORD_ATTEMPTS: u32 = 3; +/// Backoff between DB-record retry attempts. +const PIN_RECORD_BACKOFF: Duration = Duration::from_millis(50); + +/// Run an idempotent DB-record operation with a bounded retry so a sub-second +/// transient error does not silently leave the pin-source set permanently +/// incomplete. The resolver treats a nonempty below-cap source set as complete, +/// so a dropped `record_pin_source`/`record_pinned_cid` makes `GET /ipfs/{cid}` +/// 404 a valid public copy. Every wrapped insert is idempotent (`ON CONFLICT DO +/// NOTHING` / provenance-preserving upsert), so re-running is safe. On exhausted +/// attempts the last error is returned and the caller keeps its warn — behavior +/// degrades to the pre-retry state, not worse. Process death mid-retry or a DB +/// outage outlasting the backoff horizon leaves the same residual hole (no +/// persisted marker to reconcile from at startup), retired only by a future +/// reconciliation sweep. Runs inside the already-detached post-push task, so the +/// backoff adds no push latency. +async fn retry_db_record(mut op: F) -> Result<()> +where + F: FnMut() -> Fut, + Fut: std::future::Future>, +{ + let mut attempt = 1; + loop { + match op().await { + Ok(()) => return Ok(()), + Err(e) => { + if attempt >= PIN_RECORD_ATTEMPTS { + return Err(e); + } + tokio::time::sleep(PIN_RECORD_BACKOFF).await; + attempt += 1; + } + } + } +} /// Pin a single git object to the local IPFS/Kubo node. /// @@ -134,7 +172,7 @@ pub async fn pin_new_objects( // and without it `GET /ipfs/{cid}` only ever knows the first pinner, so a // shared object first pinned from a private/quarantined repo 404s even // when this repo would serve it. Bounded per object (MAX_PIN_SOURCES). - if let Err(e) = db.record_pin_source(&sha, repo_id).await { + if let Err(e) = retry_db_record(|| db.record_pin_source(&sha, repo_id)).await { tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); } continue; @@ -167,12 +205,14 @@ pub async fn pin_new_objects( // verifies them against the requested CID, so the raw CID is the correct // key. Mirrors the pinata twin, which already records the raw CID. let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes(&data).to_string(); - if let Err(e) = db.record_pinned_cid(&sha, &raw_cid, Some(repo_id)).await { + if let Err(e) = + retry_db_record(|| db.record_pinned_cid(&sha, &raw_cid, Some(repo_id))).await + { tracing::warn!(sha = %sha, err = %e, "failed to record pinned CID in DB"); } // F1 (#173 round 8): also record the first pinner in pin_repo_sources so // every source (first and subsequent) is tried uniformly by the resolver. - if let Err(e) = db.record_pin_source(&sha, repo_id).await { + if let Err(e) = retry_db_record(|| db.record_pin_source(&sha, repo_id)).await { tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); } // Return the provider Hash (not the resolver key), mirroring the pinata @@ -190,3 +230,92 @@ pub async fn pin_new_objects( pinned } + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::Cell; + + // The retry helper is the load-bearing unit: it converts a sub-second + // transient DB error at the three warn-only record sites into a landed row, + // instead of a permanently incomplete pin-source set. These drive the helper + // directly against a controlled closure (the record sites take a concrete + // `&Db` over a `PgPool`, so a failing-first wrapper cannot slot in without + // changing signatures — see U6 seam note). + + #[tokio::test] + async fn retry_lands_after_transient_failures() { + let calls = Cell::new(0u32); + let result = retry_db_record(|| { + let n = calls.get() + 1; + calls.set(n); + async move { + if n < PIN_RECORD_ATTEMPTS { + Err(anyhow::anyhow!("transient failure on attempt {n}")) + } else { + Ok(()) + } + } + }) + .await; + + assert!( + result.is_ok(), + "retry lands the row after transient failures" + ); + assert_eq!( + calls.get(), + PIN_RECORD_ATTEMPTS, + "op is retried until it succeeds" + ); + } + + #[tokio::test] + async fn retry_returns_last_err_after_exhaustion() { + let calls = Cell::new(0u32); + let result = retry_db_record(|| { + let n = calls.get() + 1; + calls.set(n); + async move { Err::<(), _>(anyhow::anyhow!("attempt {n} failed")) } + }) + .await; + + let err = result.expect_err("all attempts fail so the last error surfaces"); + assert_eq!( + calls.get(), + PIN_RECORD_ATTEMPTS, + "attempts are bounded to the cap" + ); + assert_eq!( + err.to_string(), + "attempt 3 failed", + "the LAST error is returned, not the first" + ); + } + + // Happy path against a real DB: a single-attempt success lands the row, and a + // redundant call is idempotent (`ON CONFLICT DO NOTHING`), so the source set + // holds exactly one row. + #[sqlx::test] + async fn retry_records_pin_source_once(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + let sha = "a".repeat(64); + let repo_id = "repo-retry-1"; + + retry_db_record(|| db.record_pin_source(&sha, repo_id)) + .await + .expect("happy-path record succeeds in one attempt"); + retry_db_record(|| db.record_pin_source(&sha, repo_id)) + .await + .expect("a redundant record is idempotent"); + + let sources = db.pin_sources_for_oid(&sha).await.unwrap(); + assert_eq!( + sources, + vec![repo_id.to_string()], + "exactly one source row lands under ON CONFLICT DO NOTHING" + ); + } +} From 45e269b2ae5bcc328204f6179c9987bcfa0a7911 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 20:11:42 -0500 Subject: [PATCH 071/124] fix(node): opportunistically repair legacy provider-CID pins Before this PR the pin path stored the provider CID (Kubo dag-pb / Pinata) in pinned_cids.cid, which the new resolver recomputes-and-404s, while /api/v1/ipfs/pins still advertises the stale value. Repair a legacy row in the already-pinned skip branch: gate on a cheap stored-CID codec check (only a non-raw-codec CID reads bytes, so raw rows keep the DB-only skip cost), recompute the raw CID, and rewrite it while stashing the old value in a new v14 legacy_provider_cid column (distinct from pinata_cid, which gates the Pinata pin-skip). Rows whose bytes are gone stay withheld. Documents that the deferred one-shot sweep, not this opportunistic path, fully retires the window. --- README.md | 2 + crates/gitlawb-core/src/cid.rs | 45 ++++ crates/gitlawb-node/src/db/mod.rs | 57 ++++ crates/gitlawb-node/src/ipfs_pin.rs | 76 ++++++ crates/gitlawb-node/src/test_support.rs | 329 ++++++++++++++++++++++++ 5 files changed, 509 insertions(+) diff --git a/README.md b/README.md index a7e4425b..80f71cb7 100644 --- a/README.md +++ b/README.md @@ -355,6 +355,8 @@ Important node settings: Production note: change the default Postgres password before exposing a node publicly. +Legacy-pin window: releases before the CID-resolver work stored the provider CID (Kubo dag-pb / Pinata) as a pinned object's resolver key. The `/ipfs/{cid}` resolver now recomputes the raw-content CID from the object bytes and refuses to serve a key that does not match, so `GET /api/v1/ipfs/pins` can still advertise an unrepaired legacy CID that 404s. Such a row is repaired opportunistically the next time a push carries the object again (its key is rewritten to the raw CID, the old value kept in `legacy_provider_cid`), but git negotiation omits objects the node already has, so most legacy rows never re-enter a push delta. A deferred one-shot startup sweep, not this opportunistic path, is what fully retires the advertise-then-404 window. Rows whose object bytes are gone stay withheld. + --- ## Optional node staking diff --git a/crates/gitlawb-core/src/cid.rs b/crates/gitlawb-core/src/cid.rs index b7993cc4..2071d478 100644 --- a/crates/gitlawb-core/src/cid.rs +++ b/crates/gitlawb-core/src/cid.rs @@ -64,6 +64,19 @@ impl Cid { } } +/// True when `s` parses as a CIDv1 with the raw codec — the exact shape +/// [`Cid::from_git_object_bytes`] produces and the `/ipfs` resolver looks up. +/// A legacy provider CID (Kubo dag-pb, Pinata CIDv0) parses to a different +/// version or codec and returns `false`, marking it an opportunistic-repair +/// candidate. Decidable from the string alone (no object bytes), so the pin path +/// can gate the byte-read/recompute cost on it and leave non-legacy rows at the +/// existing DB-only skip cost. An unparseable string is non-canonical (`false`). +pub fn is_raw_cidv1(s: &str) -> bool { + s.parse::>() + .map(|c| c.version() == cid::Version::V1 && c.codec() == RAW) + .unwrap_or(false) +} + impl fmt::Display for Cid { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.0) @@ -177,6 +190,38 @@ mod tests { assert!(result.is_err()); } + #[test] + fn is_raw_cidv1_classifies_codec_from_string() { + // The canonical resolver key: CIDv1 + raw codec → not a repair candidate. + let raw = Cid::from_git_object_bytes(b"blob 5\0hello"); + assert!( + is_raw_cidv1(raw.as_str()), + "from_git_object_bytes output is CIDv1/raw" + ); + + // A CIDv0 (Pinata dag-pb legacy shape) → repair candidate. + assert!( + !is_raw_cidv1("QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG"), + "a CIDv0 dag-pb value is a legacy-repair candidate" + ); + + // A CIDv1 with the dag-pb codec (the Kubo above-block-size root) over the + // same multihash → still a repair candidate (codec, not just version). + let parsed = raw.as_str().parse::>().unwrap(); + const DAG_PB: u64 = 0x70; + let dagpb = CidGeneric::<64>::new_v1(DAG_PB, *parsed.hash()).to_string(); + assert!( + !is_raw_cidv1(&dagpb), + "a CIDv1 dag-pb value is a legacy-repair candidate" + ); + + // Garbage is non-canonical. + assert!( + !is_raw_cidv1("not-a-cid"), + "an unparseable string is non-canonical" + ); + } + #[test] fn sha256_hex_of_empty_input_is_well_known() { // SHA-256("") is a fixed constant; verifies the hasher is wired correctly. diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 8d723ffd..775e7c17 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -921,6 +921,22 @@ const MIGRATIONS: &[Migration] = &[ "CREATE INDEX IF NOT EXISTS idx_pin_repo_sources_sha ON pin_repo_sources(sha256_hex)", ], }, + Migration { + version: 14, + name: "pinned_cids_legacy_provider_cid", + stmts: &[ + // R8 (#173, jatmn round 10): the opportunistic legacy provider-CID repair + // rewrites `pinned_cids.cid` from a stored PROVIDER CID (Kubo dag-pb / + // Pinata CIDv0) to the raw-content resolver key and stashes the OLD value + // here, so the rewrite is auditable and the row's legacy origin survives. + // Distinct from `pinata_cid` on purpose: `has_pinata_cid` gates the Pinata + // pin-skip, so parking a Kubo-legacy CID there would make Pinata forever + // skip re-pinning that object. NEW versioned migration (never appended to an + // applied block, INV-7) so a node past v13 actually gets the column. + // Nullable: only a repaired row sets it. + "ALTER TABLE pinned_cids ADD COLUMN IF NOT EXISTS legacy_provider_cid TEXT", + ], + }, ]; /// Max distinct source repos recorded per pinned object (F1, #173 jatmn round 8). @@ -2287,6 +2303,47 @@ impl Db { Ok(()) } + /// The resolver key currently stored for a pinned object (`pinned_cids.cid`), + /// or `None` for an unpinned oid. The opportunistic legacy-repair path reads + /// it to decide candidacy from the codec of the string alone (no object bytes) + /// before it recomputes anything. + pub async fn cid_for_oid(&self, sha256_hex: &str) -> Result> { + let row = sqlx::query("SELECT cid FROM pinned_cids WHERE sha256_hex = $1") + .bind(sha256_hex) + .fetch_optional(&self.pool) + .await?; + Ok(row.map(|r| r.get::("cid"))) + } + + /// Rewrite a legacy provider-CID row to the raw-content resolver key, stashing + /// the old provider value in `legacy_provider_cid` (#173 R8, KTD8). Before this + /// branch the pin path stored the PROVIDER CID (Kubo dag-pb / Pinata CIDv0) in + /// `cid`; the `/ipfs` resolver recomputes the raw CID and 404s a mismatched key + /// even though `list_pinned_cids` still advertises it. The `WHERE cid = + /// $old_provider_cid` guard makes a concurrent double-repair a no-op (the second + /// writer sees the already-rewritten key and matches nothing) and never touches + /// a row keyed on a different value. Stashed in `legacy_provider_cid`, NOT + /// `pinata_cid`: the latter gates the Pinata pin-skip (`has_pinata_cid`), so a + /// Kubo-legacy CID parked there would make Pinata permanently skip the object. + pub async fn repair_legacy_provider_cid( + &self, + sha256_hex: &str, + raw_cid: &str, + old_provider_cid: &str, + ) -> Result<()> { + sqlx::query( + "UPDATE pinned_cids + SET cid = $2, legacy_provider_cid = $3 + WHERE sha256_hex = $1 AND cid = $3", + ) + .bind(sha256_hex) + .bind(raw_cid) + .bind(old_provider_cid) + .execute(&self.pool) + .await?; + Ok(()) + } + /// The repository a pinned object was recorded from (`pinned_cids.repo_id`), /// or `None` for a legacy pin (recorded before provenance existed) or an /// unpinned oid. `GET /ipfs/{cid}` uses this to gate+serve the ONE source diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 1a37005d..3cf5610c 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -48,6 +48,74 @@ where } } +/// Opportunistically repair a legacy provider-CID row on the already-pinned skip +/// path (#173 R8, KTD8). Releases before this branch stored the PROVIDER CID +/// (Kubo dag-pb / Pinata CIDv0) in `pinned_cids.cid`; the `/ipfs` resolver +/// recomputes the raw CID from object bytes and 404s any row whose key does not +/// match, yet `list_pinned_cids` still advertises the stored key — so a client +/// gets a CID the resolver deliberately withholds. When a re-push carries the +/// object again, rewrite the key to the raw CID and stash the old provider value +/// in `legacy_provider_cid`. +/// +/// COST GATE: candidacy is decided from the stored key's codec alone — a +/// CIDv1/raw key is already the resolver key and reads NO bytes, keeping the +/// steady-state skip cost DB-only. Only a legacy-codec row reads the object to +/// recompute. A row whose bytes are gone stays withheld (no destructive rewrite). +async fn repair_legacy_provider_cid( + repo_path: &std::path::Path, + sha: &str, + db: &crate::db::Db, +) -> Result<()> { + let stored = match db.cid_for_oid(sha).await? { + Some(c) => c, + None => return Ok(()), + }; + // Cost gate: a canonical raw CIDv1 key is already correct — never read bytes. + if gitlawb_core::cid::is_raw_cidv1(&stored) { + return Ok(()); + } + // Legacy-codec row: read the object to recompute. Counted so a test can prove + // the gate above spares non-legacy rows this read. + #[cfg(test)] + note_legacy_repair_read(); + let data = match crate::git::store::read_object(repo_path, sha)? { + Some((_ty, bytes)) => bytes, + // Bytes gone: the row stays withheld, never destructively rewritten. + None => return Ok(()), + }; + let raw = Cid::from_git_object_bytes(&data).to_string(); + if raw == stored { + return Ok(()); + } + db.repair_legacy_provider_cid(sha, &raw, &stored).await +} + +// Test-only cost-gate counter (R8, U7): how many times the opportunistic repair +// read an object's bytes on the skip path. The codec gate must spare a CIDv1/raw +// row this read; the counter is the both-ways guard (removing the gate reads the +// raw row and increments it). Same thread_local discipline as the serve-path +// oversize counter — the pin tests await `pin_new_objects` on a current-thread +// runtime, so the increment and the assertion share one thread. +#[cfg(test)] +thread_local! { + static LEGACY_REPAIR_READS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn reset_legacy_repair_reads() { + LEGACY_REPAIR_READS.with(|c| c.set(0)); +} + +#[cfg(test)] +pub(crate) fn legacy_repair_reads() -> usize { + LEGACY_REPAIR_READS.with(|c| c.get()) +} + +#[cfg(test)] +fn note_legacy_repair_read() { + LEGACY_REPAIR_READS.with(|c| c.set(c.get() + 1)); +} + /// Pin a single git object to the local IPFS/Kubo node. /// /// - `ipfs_api`: base URL of the Kubo HTTP API, e.g. `http://127.0.0.1:5001`. @@ -175,6 +243,14 @@ pub async fn pin_new_objects( if let Err(e) = retry_db_record(|| db.record_pin_source(&sha, repo_id)).await { tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); } + // R8 (#173 round 10): opportunistically repair a legacy provider-CID + // row (Kubo dag-pb / Pinata) to the raw-content resolver key on this + // re-push. Cost-gated on the stored key's codec — a non-legacy row + // reads no bytes. Warn-only: a failure leaves the row as-is for a + // later re-push or the deferred one-shot sweep. + if let Err(e) = repair_legacy_provider_cid(repo_path, &sha, db).await { + tracing::warn!(sha = %sha, err = %e, "failed to repair legacy provider CID"); + } continue; } Ok(false) => {} diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 01363abf..06b216c1 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -3271,6 +3271,335 @@ mod tests { ); } + /// Build a legacy provider CID (CIDv1 dag-pb — the Kubo above-block-size root + /// shape, and codec-equivalent to the Pinata CIDv0 legacy key for the cost + /// gate) over the object's own multihash. Non-raw codec, so `is_raw_cidv1` + /// flags it a repair candidate, and a different string from the raw key, so a + /// repair rewrites it. The existing `ipfs_cid_legacy_provider_cid_row_not_served` + /// fixture seeds a raw-codec decoy (an integrity negative the cost gate treats + /// as non-legacy on purpose); this produces the genuine dag-pb legacy shape the + /// repair path targets. Uses only the `cid` crate (already a node dep). + fn legacy_dagpb_cid(raw_cid: &str) -> String { + const DAG_PB: u64 = 0x70; + let parsed = raw_cid + .parse::>() + .expect("the raw CID parses"); + cid::CidGeneric::<64>::new_v1(DAG_PB, *parsed.hash()).to_string() + } + + /// #173 R8 (jatmn round 10, U7 — load-bearing): a legacy row keyed on a PROVIDER + /// CID (Kubo dag-pb / Pinata) is opportunistically rewritten to the raw-content + /// key on a re-push whose pack carries the object, stashing the old value in + /// `legacy_provider_cid`. The advertised key 404s while the row is legacy (the + /// resolver recomputes the raw CID and the stored key does not match) and serves + /// after repair. RED before the skip-branch repair lands (the raw key 404s post + /// pin). Also asserts the repair leaves `pinata_cid` NULL (scenario 3) and that + /// the retired provider CID still refuses to serve (scenario 6, integrity). + #[sqlx::test] + async fn ipfs_cid_legacy_provider_cid_repaired_on_repush(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["provsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("provsrc.git"); + let repo = seed_repo(&owner_did, "provsrc"); // public, no rule + state.db.create_repo(&repo).await.expect("seed repo"); + + // The canonical raw key the resolver accepts once the row is repaired. + let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes( + &crate::git::store::read_object(&bare, &fx.public_oid) + .unwrap() + .unwrap() + .1, + ) + .to_string(); + // The key stored today: a genuine legacy dag-pb provider CID. + let provider_cid = legacy_dagpb_cid(&raw_cid); + assert_ne!( + provider_cid, raw_cid, + "the provider CID differs from the raw resolver key" + ); + + // Legacy-shape row: cid = the PROVIDER CID (raw SQL — the helpers store the + // raw CID). The object itself is public and servable. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + ) + .bind(&fx.public_oid) + .bind(&provider_cid) + .bind("2020-01-01T00:00:00Z") + .bind(&repo.id) + .execute(&pool) + .await + .unwrap(); + + // RED baseline: the raw key a correct client sends 404s while the row is legacy. + let (st_before, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&raw_cid)) + .await + .unwrap(), + ) + .await; + assert_ne!( + st_before, + StatusCode::OK, + "the raw key 404s while the row is keyed on the provider CID" + ); + + // Re-push carries the object again: `pin_new_objects` hits the already-pinned + // skip branch and repairs the row. The `/add` mock must NOT fire — the object + // is already on IPFS, never re-pinned. + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyshouldnothappen"}"#) + .expect(0) + .create_async() + .await; + crate::ipfs_pin::pin_new_objects( + &server.url(), + &bare, + vec![fx.public_oid.clone()], + &state.db, + &repo.id, + ) + .await; + m.assert_async().await; + + // GREEN: the key is repaired to the raw CID and the old value is stashed. + let (stored_cid, stashed): (String, Option) = sqlx::query_as( + "SELECT cid, legacy_provider_cid FROM pinned_cids WHERE sha256_hex = $1", + ) + .bind(&fx.public_oid) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + stored_cid, raw_cid, + "the key is repaired to the raw-content CID" + ); + assert_eq!( + stashed.as_deref(), + Some(provider_cid.as_str()), + "the old provider CID is stashed in legacy_provider_cid" + ); + + // The advertised (raw) key now serves 200. + let (st_after, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&raw_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st_after, + StatusCode::OK, + "the repaired raw key serves after the re-push" + ); + assert!(body.contains("public bytes"), "the object's bytes serve"); + + // Scenario 3: repair never wrote `pinata_cid`, so the Pinata pin-skip gate + // (`has_pinata_cid`) is untouched and Pinata still pins the object. + assert!( + !state.db.has_pinata_cid(&fx.public_oid).await.unwrap(), + "repair leaves pinata_cid NULL" + ); + + // Scenario 6 (integrity negative): the retired provider CID still 404s — no + // serve-path alias for a CID the bytes do not hash to. + let (st_old, body_old) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&provider_cid)) + .await + .unwrap(), + ) + .await; + assert_ne!( + st_old, + StatusCode::OK, + "the retired provider CID must not serve after repair" + ); + assert!( + !body_old.contains("public bytes"), + "no bytes egress under the retired provider CID" + ); + } + + /// #173 R8 (U7 cost gate): a well-formed CIDv1/raw already-pinned row triggers NO + /// object read on the skip path — the codec check decides candidacy from the + /// stored string alone, so a non-legacy row keeps the DB-only skip cost. Also + /// covers the small-object equivalence: a small legacy object Kubo pins under the + /// raw key (raw-leaves) is already CIDv1/raw and needs no repair. The read counter + /// is the both-ways guard: removing the codec gate reads the raw row and trips it. + #[sqlx::test] + async fn ipfs_cid_repair_codec_gate_skips_raw_row(pool: PgPool) { + let state = test_state(pool).await; + let fx = seed_cid_repos("codecgate", "cg", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("codecgate") + .join("pinsrc.git"); + + // A correct raw-CID row (steady state), recorded via the production helper. + let raw_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + assert!( + gitlawb_core::cid::is_raw_cidv1(&raw_cid), + "the helper records a CIDv1/raw key" + ); + + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"x"}"#) + .expect(0) + .create_async() + .await; + + crate::ipfs_pin::reset_legacy_repair_reads(); + crate::ipfs_pin::pin_new_objects( + &server.url(), + &bare, + vec![fx.public_oid.clone()], + &state.db, + "repoCG", + ) + .await; + m.assert_async().await; + + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 0, + "a CIDv1/raw row triggers no object read on the skip path (cost gate)" + ); + assert_eq!( + state + .db + .cid_for_oid(&fx.public_oid) + .await + .unwrap() + .as_deref(), + Some(raw_cid.as_str()), + "the raw row is left as-is" + ); + } + + /// #173 R8 (U7): a legacy row whose object bytes are gone stays withheld — the + /// repair never destructively rewrites it, so the row is preserved for a future + /// re-push or the deferred one-shot sweep. + #[sqlx::test] + async fn ipfs_cid_repair_unrepairable_row_stays_withheld(pool: PgPool) { + let state = test_state(pool.clone()).await; + let _fx = seed_cid_repos("unrep", "ur", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("unrep") + .join("pinsrc.git"); + + // A legacy dag-pb row for an oid whose bytes are NOT in this bare repo. + let phantom_oid = "b".repeat(64); + let raw_cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"bytes that live nowhere").to_string(); + let provider_cid = legacy_dagpb_cid(&raw_cid); + sqlx::query("INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) VALUES ($1, $2, $3)") + .bind(&phantom_oid) + .bind(&provider_cid) + .bind("2020-01-01T00:00:00Z") + .execute(&pool) + .await + .unwrap(); + + let mut server = mockito::Server::new_async().await; + server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"x"}"#) + .expect(0) + .create_async() + .await; + + // Skip-branch runs (is_pinned true) but read_object returns None (bytes gone), + // so the repair returns without touching the row. + crate::ipfs_pin::pin_new_objects( + &server.url(), + &bare, + vec![phantom_oid.clone()], + &state.db, + "repoUR", + ) + .await; + + let (stored, stashed): (String, Option) = sqlx::query_as( + "SELECT cid, legacy_provider_cid FROM pinned_cids WHERE sha256_hex = $1", + ) + .bind(&phantom_oid) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + stored, provider_cid, + "an unrepairable row keeps its provider CID (no destructive rewrite)" + ); + assert_eq!( + stashed, None, + "no legacy_provider_cid is stashed when the bytes are gone" + ); + } + + /// #173 R8 (U7, INV-7 upgrade path): a node already at the prior-max schema (v13) + /// gets `pinned_cids.legacy_provider_cid` from the NEW v14 migration. Simulate the + /// pre-v14 node by dropping the column and un-applying v14, then re-migrate and + /// assert a repair round-trips through the column. RED before the v14 migration + /// exists (the column is never re-added → the repair UPDATE errors). + #[sqlx::test] + async fn pinned_cids_legacy_provider_cid_upgrade_path(pool: PgPool) { + let state = test_state(pool.clone()).await; + + // Pre-v14 shape: drop the column and forget v14 was applied. + sqlx::query("ALTER TABLE pinned_cids DROP COLUMN IF EXISTS legacy_provider_cid") + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version = 14") + .execute(&pool) + .await + .unwrap(); + + // Upgrade: re-run migrations → v14 re-adds the column. + state.db.run_migrations().await.expect("migrate to v14"); + + // A repair round-trips through the v14 column. + state + .db + .record_pinned_cid("upg_oid", "QmProviderLegacy", None) + .await + .unwrap(); + state + .db + .repair_legacy_provider_cid("upg_oid", "bRawContentKey", "QmProviderLegacy") + .await + .unwrap(); + let (cid, stashed): (String, Option) = sqlx::query_as( + "SELECT cid, legacy_provider_cid FROM pinned_cids WHERE sha256_hex = 'upg_oid'", + ) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(cid, "bRawContentKey", "v14 lets the repair rewrite the key"); + assert_eq!( + stashed.as_deref(), + Some("QmProviderLegacy"), + "the v14 legacy_provider_cid column is present after upgrade" + ); + } + /// #173 (provenance-path throttle): a walk-requiring provenanced candidate whose /// per-IP walk quota is spent returns 429 (the provenance arm's Throttled outcome, /// then the fall-through). quota=1, keyed on XFF. The first reader request runs the From fba3c6fda3a70f3cf7b9a1b6260cbbe263540ba5 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 20:27:35 -0500 Subject: [PATCH 072/124] fix(review): share one deadline across the /ipfs size+content reads Code review found each candidate's size and content reads were each granted a full git_service_timeout, so a single served candidate could hold the /ipfs walk permit for 2x the timeout. Share one deadline across both stages (mirrors build_filtered_pack) so the read holds admission for one timeout total, and correct the admission-bound comment: the worst case is O(candidates x timeout) bounded by the walk concurrency cap, not a single deadline. --- crates/gitlawb-node/src/api/ipfs.rs | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index c5fa35bc..624a1b27 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -196,9 +196,13 @@ pub async fn get_by_cid( // instant the handler future is torn down while a spawn_blocking git probe/walk/read // is still alive (the disconnect-spam cap bypass this closes, the /ipfs half of #174 // P1-a). Every git child on this pipeline is duration-bounded (the run_bounded_git - // probe/read twins + the bounded walk), so the detached task cannot hold admission - // past ~git_service_timeout_secs. The client key is already captured (`source_key`) - // before the spawn; the detached task has no request extractors. + // probe/read twins + the bounded walk), and each candidate's size+content read shares + // one deadline; so admission is bounded per candidate. The legacy scan can still + // iterate up to `ipfs_max_legacy_probes` candidates serially, so the worst-case hold + // is O(candidates x git_service_timeout_secs), NOT a single timeout — what actually + // bounds cancel-spam is the walk concurrency cap (global + per-source), not the + // per-child deadline. The client key is already captured (`source_key`) before the + // spawn; the detached task has no request extractors. let admission = crate::git::smart_http::AdmissionGuard::new(ipfs_walk_permit, ipfs_caller_permit); let serve: tokio::task::JoinHandle> = tokio::spawn(async move { @@ -759,9 +763,14 @@ async fn gate_and_serve( // deadline rather than left to pin the held /ipfs walk permit (#173 round-10, KTD2). // No outer timeout, mirroring the bounded walk; a `GitServiceTimeout` from either // twin surfaces as `ServedRead::ReadErr` -> truncated (retryable 503). - let read_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + // ONE deadline spans the size and content reads, so a single served candidate + // holds the /ipfs walk permit for at most `git_service_timeout_secs` total, not + // one full timeout per stage (mirrors `build_filtered_pack`'s shared deadline). + let read_deadline = std::time::Instant::now() + + std::time::Duration::from_secs(state.config.git_service_timeout_secs); let read = tokio::task::spawn_blocking(move || -> ServedRead { - match store::object_size_bounded(&git_bin, &read_repo, &read_sha, read_timeout) { + let size_budget = read_deadline.saturating_duration_since(std::time::Instant::now()); + match store::object_size_bounded(&git_bin, &read_repo, &read_sha, size_budget) { Ok(Some(size)) if size > max_bytes => return ServedRead::TooLarge(size), Ok(Some(_)) => {} // git ran and reported no such object (or an unparseable size): genuine @@ -771,12 +780,13 @@ async fn gate_and_serve( // infra/timeout failure, not a not-found. Err(e) => return ServedRead::ReadErr(e.to_string()), } + let content_budget = read_deadline.saturating_duration_since(std::time::Instant::now()); let content = match store::read_object_content_bounded( &git_bin, &read_repo, &read_sha, &read_type, - read_timeout, + content_budget, ) { Ok(c) => c, Err(e) => return ServedRead::ReadErr(e.to_string()), From 62a99270a935324956dcb99abbc620e3ed19750b Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 21:06:10 -0500 Subject: [PATCH 073/124] fix(node): bound the post-push pin and repair reads so a wedge cannot pin the coalescing key A cross-model adversarial pass found that run_post_push_replication holds the per-repo coalescing key until requeue_or_release, but pin_new_objects reached it only after two unbounded git reads: the U7 legacy-repair read and the pre-existing pin read, both plain Command::output with no teardown. A wedged cat-file on a stuck backend hung the task forever, so the key was held until process death and later pushes only marked it dirty without spawning a replacement (the same class this PR closed on the /ipfs serve path). Add read_object_bounded (the bounded twins under one shared deadline) and use it at both sites; a timeout skips that object and lets the task proceed to requeue_or_release. Adds a wedge-reap regression proven load-bearing two ways. --- crates/gitlawb-node/src/api/repos.rs | 2 + crates/gitlawb-node/src/git/store.rs | 36 +++++++ crates/gitlawb-node/src/ipfs_pin.rs | 45 +++++--- crates/gitlawb-node/src/test_support.rs | 131 ++++++++++++++++++++++++ 4 files changed, 201 insertions(+), 13 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index c9a2b3dd..08075e4e 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -885,6 +885,8 @@ async fn run_post_push_replication( crate::ipfs_pin::pin_new_objects( &ctx.ipfs_api, &ctx.disk_path, + &ctx.git_bin, + ctx.timeout, object_list.clone(), &ctx.db, &ctx.repo_id, diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index a79d596a..899394ee 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -407,6 +407,42 @@ pub fn read_object(repo_path: &Path, sha256_hex: &str) -> Result Result)>> { + let deadline = std::time::Instant::now() + timeout; + let obj_type = match object_type_bounded( + git_bin, + repo_path, + sha256_hex, + deadline.saturating_duration_since(std::time::Instant::now()), + )? { + Some(t) => t, + None => return Ok(None), + }; + let content = read_object_content_bounded( + git_bin, + repo_path, + sha256_hex, + &obj_type, + deadline.saturating_duration_since(std::time::Instant::now()), + )?; + Ok(Some((obj_type, content))) +} + /// Get the diff between two branches: changes on source_branch not in target_branch. pub fn branch_diff(repo_path: &Path, target_branch: &str, source_branch: &str) -> Result { let output = Command::new("git") diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 3cf5610c..3d2f13df 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -63,6 +63,8 @@ where /// recompute. A row whose bytes are gone stays withheld (no destructive rewrite). async fn repair_legacy_provider_cid( repo_path: &std::path::Path, + git_bin: &str, + git_timeout: Duration, sha: &str, db: &crate::db::Db, ) -> Result<()> { @@ -78,10 +80,18 @@ async fn repair_legacy_provider_cid( // the gate above spares non-legacy rows this read. #[cfg(test)] note_legacy_repair_read(); - let data = match crate::git::store::read_object(repo_path, sha)? { - Some((_ty, bytes)) => bytes, + let data = match crate::git::store::read_object_bounded(git_bin, repo_path, sha, git_timeout) { + Ok(Some((_ty, bytes))) => bytes, // Bytes gone: the row stays withheld, never destructively rewritten. - None => return Ok(()), + Ok(None) => return Ok(()), + // A wedged/D-state `git cat-file` (timeout/infra): the repair is opportunistic + // and best-effort, so skip it and return Ok so the pin task PROCEEDS to + // requeue_or_release rather than hanging the coalescing key until process death + // (grok F2, #173). A later re-push or the deferred sweep retries the repair. + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "skipping legacy provider-CID repair: bounded object read failed"); + return Ok(()); + } }; let raw = Cid::from_git_object_bytes(&data).to_string(); if raw == stored { @@ -204,6 +214,8 @@ pub async fn cat(ipfs_api: &str, cid: &str) -> Result> { pub async fn pin_new_objects( ipfs_api: &str, repo_path: &std::path::Path, + git_bin: &str, + git_timeout: Duration, object_list: Vec, db: &crate::db::Db, repo_id: &str, @@ -248,7 +260,9 @@ pub async fn pin_new_objects( // re-push. Cost-gated on the stored key's codec — a non-legacy row // reads no bytes. Warn-only: a failure leaves the row as-is for a // later re-push or the deferred one-shot sweep. - if let Err(e) = repair_legacy_provider_cid(repo_path, &sha, db).await { + if let Err(e) = + repair_legacy_provider_cid(repo_path, git_bin, git_timeout, &sha, db).await + { tracing::warn!(sha = %sha, err = %e, "failed to repair legacy provider CID"); } continue; @@ -260,15 +274,20 @@ pub async fn pin_new_objects( } } - // Read raw object content - let data = match crate::git::store::read_object(repo_path, &sha) { - Ok(Some((_obj_type, bytes))) => bytes, - Ok(None) => continue, - Err(e) => { - tracing::warn!(sha = %sha, err = %e, "failed to read git object for pinning"); - continue; - } - }; + // Read raw object content under a bounded read so a wedged/D-state `git + // cat-file` (stuck NFS/Tigris backend) is reaped at `git_timeout` instead of + // hanging pin_new_objects forever — which would pin the post-push coalescing + // key until process death (grok F2, #173). On Err the object is simply not + // pinned this pass; a later pass/push retries. + let data = + match crate::git::store::read_object_bounded(git_bin, repo_path, &sha, git_timeout) { + Ok(Some((_obj_type, bytes))) => bytes, + Ok(None) => continue, + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "failed to read git object for pinning"); + continue; + } + }; // Pin to IPFS match pin_git_object(ipfs_api, &sha, &data).await { diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 06b216c1..0d0ae2e8 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -2430,6 +2430,8 @@ mod tests { crate::ipfs_pin::pin_new_objects( &server.url(), &pub_bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), vec![fx.public_oid.clone()], &state.db, &pub_repo.id, @@ -2553,6 +2555,8 @@ mod tests { crate::ipfs_pin::pin_new_objects( &server.url(), &pub_bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), vec![fx.public_oid.clone()], &state.db, &pub_repo.id, @@ -3120,6 +3124,8 @@ mod tests { let pinned = crate::ipfs_pin::pin_new_objects( &server.url(), &bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), vec![fx.public_oid.clone()], &state.db, "repoZ", @@ -3142,6 +3148,123 @@ mod tests { ); } + /// #173 (grok F2): the post-push pin read is BOUNDED, so a wedged/D-state + /// `git cat-file` (stuck NFS/Tigris backend) is reaped at `git_timeout` and + /// `pin_new_objects` RETURNS — reaching `requeue_or_release` in production — + /// instead of hanging forever and pinning the per-repo coalescing key until + /// process death. A fake `git` whose `cat-file` records its pid then sleeps far + /// past a SHORT 1s timeout stands in for the wedged backend; the `run_bounded_git` + /// watchdog (SIGTERM -> grace -> SIGKILL of the process group) must reap it well + /// before its 8s natural exit, and the call must return with nothing pinned. + /// + /// REVERT PROOF (RED): swap `read_object_bounded` back to the bare + /// `store::read_object` at the pin read and the wedged child is STILL RUNNING at + /// the mid-flight liveness poll below (unbounded `Command::output` cannot be + /// reaped at the deadline) — the reap assertion fails. + #[cfg(unix)] + #[sqlx::test] + async fn pin_new_objects_reaps_wedged_read_at_deadline(pool: PgPool) { + use std::time::Duration; + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + // Fake `git`: `cat-file` records its own pid then sleeps 8s (>> the 1s + // deadline) so the read is genuinely wedged; the watchdog is what must end it. + let tmp = tempfile::TempDir::new().unwrap(); + let pidfile = tmp.path().join("catfile.pid"); + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + cat-file) echo $$ > \"{}\"; sleep 8 ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + pidfile.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + let repo = tmp.path().to_path_buf(); + let git = git_path.to_str().unwrap().to_string(); + // A never-pinned OID so the call reaches the object-read stage (not the + // already-pinned skip path). + let oid = "f".repeat(64); + // Non-empty ipfs_api so `pin_new_objects` does not early-return; the wedged + // read is reaped and the OID skipped before any `/add`, so this URL is unused. + let ipfs_api = "http://127.0.0.1:1".to_string(); + + // `pin_new_objects` must run on THIS runtime so its `is_pinned` DB call keeps + // the sqlx pool on its home runtime. The bounded read is a synchronous blocking + // call, so the reap poll runs on a separate OS thread (independent of tokio): it + // captures the wedged child's pid, waits past the deadline, records whether it + // was reaped, then SIGKILLs defensively so even a true infinite hang cannot leak + // an orphan or stall the awaited call. + let pidfile_poll = pidfile.clone(); + let poll = std::thread::spawn(move || -> (Option, bool) { + let alive = |pid: i32| unsafe { libc::kill(pid, 0) == 0 }; + let mut pid = None; + for _ in 0..500 { + if let Some(p) = std::fs::read_to_string(&pidfile_poll) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + pid = Some(p); + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + let pid = match pid { + Some(p) => p, + None => return (None, false), + }; + // Past the 1s deadline + SIGTERM grace but well before the 8s natural exit: + // the bounded read must already have reaped the wedged group. The unbounded + // `store::read_object` leaves it running here — the load-bearing RED. + std::thread::sleep(Duration::from_secs(3)); + let reaped = !alive(pid); + unsafe { + libc::kill(pid, libc::SIGKILL); + } + (Some(pid), reaped) + }); + + // The call must RETURN — reaching `requeue_or_release` in production — rather + // than hang on the 8s sleep. The poll thread's defensive SIGKILL guarantees the + // read completes even in the unbounded RED case, so this observes a bounded + // return either way; the reap assertion below is what separates RED from GREEN. + let pinned = tokio::time::timeout( + Duration::from_secs(6), + crate::ipfs_pin::pin_new_objects( + &ipfs_api, + &repo, + &git, + Duration::from_secs(1), + vec![oid], + &db, + "repoWedge", + ), + ) + .await + .expect("pin_new_objects must return within the bound, not hang on the wedged read"); + + let (pid, reaped) = poll.join().expect("poll thread joins"); + pid.expect("the fake cat-file must have spawned and recorded its pid"); + assert!( + reaped, + "the post-push pin read must reap the wedged cat-file child at the deadline, \ + not leave it running (which would pin the coalescing key until process death)" + ); + assert!( + pinned.is_empty(), + "a wedged read pins nothing this pass; a later pass/push retries" + ); + } + /// #173 (jatmn, F2): a legacy pin with NULL provenance backfills its source /// via `backfill_pin_provenance`, and the `AND repo_id IS NULL` guard preserves /// first-pinner-owns (a non-NULL provenance is left untouched). @@ -3248,6 +3371,8 @@ mod tests { let pinned = crate::ipfs_pin::pin_new_objects( &server.url(), &bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), vec![fx.public_oid.clone()], &state.db, "repoBF", @@ -3367,6 +3492,8 @@ mod tests { crate::ipfs_pin::pin_new_objects( &server.url(), &bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), vec![fx.public_oid.clone()], &state.db, &repo.id, @@ -3468,6 +3595,8 @@ mod tests { crate::ipfs_pin::pin_new_objects( &server.url(), &bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), vec![fx.public_oid.clone()], &state.db, "repoCG", @@ -3530,6 +3659,8 @@ mod tests { crate::ipfs_pin::pin_new_objects( &server.url(), &bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), vec![phantom_oid.clone()], &state.db, "repoUR", From 9a70d75d135decb7b21b2420ada161821afce1dd Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 27 Jul 2026 13:22:45 -0500 Subject: [PATCH 074/124] fix(node): own the repo write lock on one connection so cancellation cannot leak it `acquire_write` took `pg_try_advisory_lock` through the shared pool, then awaited the Tigris exists/download path before constructing `RepoWriteGuard`. Two bugs fell out of that. The `tokio::time::timeout` this branch added around the call can drop the future inside that window, after the lock is held and before the only unlock (`RepoWriteGuard::release`) exists, wedging every later push to the repo. And because a session advisory lock belongs to the connection that took it, running the lock and the unlock through the pool lets them land on different connections: the unlock returns false and leaks, while a competing acquire that draws the holding connection re-enters the lock and two writers run against the same repo concurrently. `RepoStore` now owns a dedicated lock pool built with an `after_release` hook that runs `pg_advisory_unlock_all()`. `acquire_write` checks out one connection, takes the lock on it, and moves it into the guard, which unlocks on that same connection. sqlx's `PoolConnection::drop` spawns `return_to_pool`, which runs the hook, so a connection dropped by cancellation still clears its locks; the unlock is asynchronous with respect to the drop, which the tests account for. The pool is separate from the main query pool because a push holds its connection for the whole receive-pack and `db_max_connections` (20) sits below `max_concurrent_git_pushes` (32). Seven tests cover the cancellation window, the same-session release, and the serialization the pool-mismatch bug was breaking. Both the cancellation regression and the concurrent-writer case were observed red before the fix, and removing the `after_release` hook or moving the unlock back to the pool turns them red again. --- crates/gitlawb-node/src/api/repos.rs | 5 +- crates/gitlawb-node/src/git/repo_store.rs | 452 +++++++++++++++++++++- crates/gitlawb-node/src/main.rs | 16 +- 3 files changed, 454 insertions(+), 19 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 08075e4e..b817318d 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1490,7 +1490,10 @@ pub async fn git_receive_pack( // exhausted Postgres pool (so the 60-count never advances) — and the write permit // is held the whole time, draining the pool (#174 P1-2). The outer // `tokio::time::timeout` cancels a mid-sleep/mid-`fetch_one` future, so it bounds - // both the loop and a hung iteration without any repo_store.rs change (KTD3). The + // both the loop and a hung iteration. Cancelling here is only safe because + // acquire_write holds its advisory lock on a connection from a pool whose + // `after_release` hook unlocks (#173): the dropped future used to leave the lock + // held with no guard alive to release it, wedging later pushes to that repo. The // permit is a handler local here (moved into the AdmissionGuard only after this), // so the early return on timeout drops it and frees the slot; shed a bounded 503. let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index a5c367e9..16c0378b 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -11,9 +11,12 @@ use std::collections::HashSet; use std::path::{Path, PathBuf}; use std::sync::Arc; +use std::time::Duration; use anyhow::{Context, Result}; -use sqlx::PgPool; +use sqlx::pool::PoolConnection; +use sqlx::postgres::PgPoolOptions; +use sqlx::{PgPool, Postgres}; use tokio::sync::Mutex; use tracing::{debug, info, warn}; @@ -25,30 +28,54 @@ use super::tigris::TigrisClient; pub struct RepoStore { repos_dir: PathBuf, tigris: Option, - /// Shared Postgres pool for advisory locks. - pool: PgPool, + /// Dedicated Postgres pool for repo write advisory locks, built by + /// `build_lock_pool` (see there for why it is separate and why it carries an + /// `after_release` hook). Never use this for ordinary queries. + lock_pool: PgPool, /// Tracks repos already confirmed to exist in Tigris — avoids redundant /// HEAD checks and background uploads for repos we've already migrated. migrated: Arc>>, + /// Test-only stall injected at the head of `acquire_write`'s Tigris phase, + /// i.e. AFTER the advisory lock is taken and BEFORE the guard exists. That + /// window is exactly where the outer `tokio::time::timeout` in + /// `api/repos.rs` can drop the future (#173). `TigrisClient` takes its + /// endpoint from process-wide AWS env vars and has no injectable seam, so + /// this flag is the smallest way to hold a real `acquire_write` open in that + /// window and cancel it there. + #[cfg(test)] + tigris_stall: Option, } impl RepoStore { + /// Derives its own lock pool from `pool`, so callers that only have the main + /// pool (tests, `for_testing` sites in other modules) still get the + /// `after_release` semantics `acquire_write` depends on. #[cfg(test)] pub fn for_testing(repos_dir: PathBuf, pool: PgPool) -> Self { - Self { + Self::new( repos_dir, - tigris: None, - pool, - migrated: Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new())), - } + None, + build_lock_pool(&pool, 8, Duration::from_secs(5)), + ) + } + + /// Test-only: see `tigris_stall`. + #[cfg(test)] + pub fn with_tigris_stall(mut self, stall: Duration) -> Self { + self.tigris_stall = Some(stall); + self } - pub fn new(repos_dir: PathBuf, tigris: Option, pool: PgPool) -> Self { + /// `lock_pool` must come from `build_lock_pool`; a plain pool leaks advisory + /// locks on cancellation. + pub fn new(repos_dir: PathBuf, tigris: Option, lock_pool: PgPool) -> Self { Self { repos_dir, tigris, - pool, + lock_pool, migrated: Arc::new(Mutex::new(HashSet::new())), + #[cfg(test)] + tigris_stall: None, } } @@ -157,13 +184,34 @@ impl RepoStore { let (owner_slug, local_path) = self.local_path(owner_did, repo_name)?; let lock_key = advisory_lock_key(&owner_slug, repo_name); + // Check out ONE connection from the lock pool and keep it for the whole + // lock lifetime. Two reasons, both bugs we hit with `fetch_one(&pool)`: + // + // * A session-level advisory lock belongs to the CONNECTION that took + // it. Running the lock and the unlock through the pool lets them land + // on different connections, so `pg_advisory_unlock` silently returns + // false and the lock leaks, while a competing acquire that happens to + // draw the holding connection re-enters the lock and pushes to the + // same repo run concurrently. + // * Cancellation. `api/repos.rs` bounds this call with + // `tokio::time::timeout`; when it fires during the Tigris phase below + // the future is dropped after the lock was taken and before + // `RepoWriteGuard` (the only caller of `pg_advisory_unlock`) exists. + // Dropping this connection instead runs the pool's `after_release` + // hook, which clears the lock (#173). + let mut lock_conn = self + .lock_pool + .acquire() + .await + .context("checking out a lock-pool connection")?; + // Acquire Postgres advisory lock with retry using pg_try_advisory_lock // to avoid blocking indefinitely on stale locks from crashed connections. let mut acquired = false; for attempt in 0..60 { let row: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") .bind(lock_key) - .fetch_one(&self.pool) + .fetch_one(&mut *lock_conn) .await .context("trying advisory lock")?; if row.0 { @@ -178,6 +226,11 @@ impl RepoStore { anyhow::bail!("could not acquire advisory lock after 60s — possible stale lock for {owner_slug}/{repo_name}"); } + #[cfg(test)] + if let Some(stall) = self.tigris_stall { + tokio::time::sleep(stall).await; + } + // Always download the latest from Tigris before writing. // Local disk may be stale if another machine pushed since our last access. if let Some(ref tigris) = self.tigris { @@ -202,7 +255,7 @@ impl RepoStore { repo_name: repo_name.to_string(), local_path, lock_key, - pool: self.pool.clone(), + lock_conn, tigris: self.tigris.clone(), }) } @@ -354,7 +407,12 @@ pub struct RepoWriteGuard { repo_name: String, pub local_path: PathBuf, lock_key: i64, - pool: PgPool, + /// The lock-pool connection that TOOK the advisory lock. It must be the one + /// that releases it (session locks are owned by their connection), and + /// holding it here is also what makes a guard dropped without `release` + /// safe: the drop returns the connection through `after_release`, which + /// clears the lock. + lock_conn: PoolConnection, tigris: Option, } @@ -369,7 +427,7 @@ impl RepoWriteGuard { /// half-applied or otherwise inconsistent repo would propagate corruption to /// Tigris (and to every node that later downloads it). The lock is always /// released regardless, to avoid stale locks blocking future writes. - pub async fn release(self, success: bool) { + pub async fn release(mut self, success: bool) { // Upload to Tigris only on success. if success { if let Some(ref tigris) = self.tigris { @@ -384,14 +442,56 @@ impl RepoWriteGuard { warn!(repo = %self.repo_name, "write failed — skipping tigris upload to avoid propagating an inconsistent repo"); } - // Release advisory lock + // Release the advisory lock on the connection that took it. Anything else + // (a fresh `&pool` checkout) is a no-op that returns false: Postgres + // scopes a session lock to its owning connection. let _ = sqlx::query("SELECT pg_advisory_unlock($1)") .bind(self.lock_key) - .execute(&self.pool) + .execute(&mut *self.lock_conn) .await; + // Dropping `self` returns the connection to the lock pool, where + // `after_release` sweeps anything the unlock above missed. } } +/// Build the dedicated advisory-lock pool a `RepoStore` runs its write locks on. +/// Connect options are cloned off an existing pool so callers need not re-parse +/// the database URL; the pool is lazy, so no connection is opened here. +/// +/// Two properties, both load-bearing: +/// +/// * The `after_release` hook runs `pg_advisory_unlock_all()` before a +/// connection goes back into the pool. sqlx's `PoolConnection::drop` spawns +/// `return_to_pool()`, which invokes this hook, so a connection dropped by +/// CANCELLATION still clears its locks. That is what keeps an `acquire_write` +/// killed mid-Tigris by the caller's `tokio::time::timeout` from leaking a +/// lock and wedging every later push to that repo (#173). Note the hook runs +/// from that spawned task, so the unlock is asynchronous with respect to the +/// drop: the lock clears shortly after the connection goes away, not +/// synchronously with it. +/// * It is a SEPARATE pool from the main query pool, not a slice of it. A push +/// holds its lock connection for the whole receive-pack, and +/// `db_max_connections` (default 20) is well below +/// `max_concurrent_git_pushes` (default 32), so drawing these from the main +/// pool would starve every other query during a push burst. +/// +/// `acquire_timeout` bounds the wait when every lock-pool connection is busy, so +/// exhaustion surfaces as a clean error rather than an unbounded hang. +pub fn build_lock_pool(source: &PgPool, max_connections: u32, acquire_timeout: Duration) -> PgPool { + PgPoolOptions::new() + .max_connections(max_connections) + .acquire_timeout(acquire_timeout) + .after_release(|conn, _meta| { + Box::pin(async move { + sqlx::query("SELECT pg_advisory_unlock_all()") + .execute(&mut *conn) + .await?; + Ok(true) + }) + }) + .connect_lazy_with((*source.connect_options()).clone()) +} + /// Compute a stable i64 hash for a Postgres advisory lock key. fn advisory_lock_key(owner_slug: &str, repo_name: &str) -> i64 { use std::hash::{Hash, Hasher}; @@ -404,6 +504,326 @@ fn advisory_lock_key(owner_slug: &str, repo_name: &str) -> i64 { #[cfg(test)] mod tests { use super::*; + use std::time::Duration; + + // ── advisory-lock test helpers (#173 U1) ─────────────────────────────── + + /// Postgres advisory locks live in a CLUSTER-wide space, not a per-database + /// one, so two `#[sqlx::test]` cases running against their own temporary + /// databases still share the key space. Every lock test therefore mints its + /// own key instead of reusing a fixed constant. + fn unique_lock_key() -> i64 { + use std::sync::atomic::{AtomicI64, Ordering}; + static NEXT: AtomicI64 = AtomicI64::new(0); + let n = NEXT.fetch_add(1, Ordering::Relaxed); + ((std::process::id() as i64) << 24) | (n & 0xff_ffff) + } + + /// A plain pool (no `after_release` hook, no idle timeout) at the same + /// database as the `#[sqlx::test]` pool. Two separate reasons these tests + /// cannot just use the pool the harness hands them: + /// + /// 1. Observing lock state has to happen from a session that is definitely + /// not the one under test. Session advisory locks are re-entrant, so + /// `pg_try_advisory_lock` on the very connection that already holds the key + /// returns true, and a same-pool probe silently reports a leaked lock free. + /// 2. The harness pool sets `idle_timeout(1s)`, so a connection returned to it + /// is closed about a second later and Postgres drops every lock that + /// session held. That would mask exactly the leak these tests exist to + /// catch, so the store under test runs on one of these too. + fn sibling_pool(pool: &PgPool, max_connections: u32) -> PgPool { + sqlx::postgres::PgPoolOptions::new() + .max_connections(max_connections) + .connect_lazy_with((*pool.connect_options()).clone()) + } + + /// Probe the lock from a connection that is NOT the one under test. Session + /// advisory locks are re-entrant within their own session, so a check from the + /// holding connection would pass vacuously and prove nothing. + async fn lock_is_free_elsewhere(pool: &PgPool, key: i64) -> bool { + let mut probe = pool.acquire().await.expect("probe connection"); + let taken: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *probe) + .await + .expect("probe try-lock"); + if taken.0 { + sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *probe) + .await + .expect("probe unlock"); + } + taken.0 + } + + /// `after_release` runs from the task sqlx spawns in `PoolConnection::drop`, + /// so the unlock is ASYNCHRONOUS with respect to the drop. Callers must poll + /// rather than assume the lock is gone the instant the connection goes away. + async fn wait_until_free(pool: &PgPool, key: i64, within: Duration) -> bool { + let deadline = std::time::Instant::now() + within; + loop { + if lock_is_free_elsewhere(pool, key).await { + return true; + } + if std::time::Instant::now() >= deadline { + return false; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + } + + // ── DESIGN GATE ──────────────────────────────────────────────────────── + // The whole cancellation-safety design rests on one sqlx behaviour: + // `PoolConnection::drop` spawns `return_to_pool()`, which invokes the pool's + // `after_release` hook before the connection is reused. If that holds, a + // connection dropped by cancellation still runs `pg_advisory_unlock_all()` + // and the lock cannot leak. This test proves it by execution, through the + // production `build_lock_pool` so that stripping the hook there turns it red. + + #[sqlx::test] + async fn dropped_pool_connection_runs_after_release_and_clears_locks(pool: PgPool) { + let key = unique_lock_key(); + let lock_pool = build_lock_pool(&pool, 4, Duration::from_secs(5)); + + { + let mut conn = lock_pool.acquire().await.expect("lock-pool connection"); + let taken: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *conn) + .await + .expect("try-lock"); + assert!(taken.0, "first try-lock must succeed"); + assert!( + !lock_is_free_elsewhere(&pool, key).await, + "lock must be observably HELD from another session while the connection lives" + ); + // Drop WITHOUT calling pg_advisory_unlock: this models cancellation. + } + + assert!( + wait_until_free(&pool, key, Duration::from_secs(5)).await, + "after_release must clear the advisory lock of a dropped connection" + ); + } + + // ── acquire_write cancellation safety (#173 U1) ──────────────────────── + + /// The reviewer's named regression. `api/repos.rs` wraps `acquire_write` in a + /// `tokio::time::timeout`; when that fires during the Tigris phase the future + /// is dropped after the advisory lock was taken and before `RepoWriteGuard` + /// (the only thing that unlocks) exists. The lock then leaks and every later + /// push to the same repo spins the 60-attempt / 60s ceiling and fails. + #[sqlx::test] + async fn cancelled_acquire_write_mid_tigris_does_not_leak_the_lock(pool: PgPool) { + let repos_dir = PathBuf::from("/tmp/gitlawb-test-repos"); + let owner = "did:key:z6MkCancelMidTigris"; + let repo = "cancel-mid-tigris"; + + let store_pool = sibling_pool(&pool, 8); + let stalling = RepoStore::for_testing(repos_dir.clone(), store_pool.clone()) + .with_tigris_stall(Duration::from_secs(30)); + let cancelled = tokio::time::timeout( + Duration::from_millis(500), + stalling.acquire_write(owner, repo), + ) + .await; + assert!( + cancelled.is_err(), + "the acquire must still be inside the Tigris phase when the timeout fires" + ); + + // Observed from an independent session, so the check cannot be satisfied + // by re-entrancy on whichever pooled connection happens to be handed back. + let probe = sibling_pool(&pool, 2); + let key = advisory_lock_key(&owner.replace([':', '/'], "_"), repo); + assert!( + wait_until_free(&probe, key, Duration::from_secs(5)).await, + "a cancelled acquire_write must leave no advisory lock held" + ); + + // A subsequent acquire for the SAME repo must succeed promptly. Before the + // fix it blocks on the leaked lock until the 60-attempt ceiling. + let store = RepoStore::for_testing(repos_dir, store_pool); + let guard = tokio::time::timeout(Duration::from_secs(5), store.acquire_write(owner, repo)) + .await + .expect("second acquire_write must not block on a leaked lock") + .expect("second acquire_write must succeed"); + guard.release(false).await; + } + + /// Cancellation BEFORE the lock is taken must leave nothing behind: no lock, + /// and no lock-pool connection stranded. The lock pool here holds exactly one + /// connection, so a stranded one would make the follow-up acquire time out + /// waiting for a checkout. + #[sqlx::test] + async fn cancelled_acquire_write_before_the_lock_leaves_nothing_held(pool: PgPool) { + let probe = sibling_pool(&pool, 2); + let owner = "did:key:z6MkCancelEarly"; + let repo = "cancel-early"; + let key = advisory_lock_key(&owner.replace([':', '/'], "_"), repo); + + let store = RepoStore::new( + PathBuf::from("/tmp/gitlawb-test-repos"), + None, + build_lock_pool(&pool, 1, Duration::from_secs(3)), + ); + + // A zero deadline polls the future once, which gets it no further than the + // first await (the pool checkout / the first try-lock round trip), so it is + // cancelled before any lock can be taken. + let cancelled = + tokio::time::timeout(Duration::ZERO, store.acquire_write(owner, repo)).await; + assert!(cancelled.is_err(), "the acquire must be cancelled"); + + assert!( + lock_is_free_elsewhere(&probe, key).await, + "no lock may be held when the acquire never got that far" + ); + + // The single lock-pool connection must be back: if cancellation stranded + // it, this checkout blocks until the 3s acquire timeout and fails. + let guard = tokio::time::timeout(Duration::from_secs(2), store.acquire_write(owner, repo)) + .await + .expect("the lock-pool connection must have been returned") + .expect("acquire after cancellation"); + guard.release(false).await; + } + + /// Lock-pool exhaustion is a bounded wait and a clean error, never a panic and + /// never an unbounded hang. + #[sqlx::test] + async fn lock_pool_exhaustion_is_a_bounded_error(pool: PgPool) { + let owner = "did:key:z6MkExhaustion"; + let store = RepoStore::new( + PathBuf::from("/tmp/gitlawb-test-repos"), + None, + build_lock_pool(&pool, 1, Duration::from_secs(2)), + ); + + let held = store + .acquire_write(owner, "exhaust-a") + .await + .expect("first acquire"); + + // Different repo, so this is not the advisory lock queueing: the only + // connection in the lock pool is checked out by `held`. + let started = std::time::Instant::now(); + let err = tokio::time::timeout( + Duration::from_secs(10), + store.acquire_write(owner, "exhaust-b"), + ) + .await + .expect("the wait must be bounded by the pool acquire timeout"); + let err = match err { + Ok(_) => panic!("an exhausted lock pool must surface an error, not a guard"), + Err(e) => e, + }; + assert!( + started.elapsed() < Duration::from_secs(6), + "the error must arrive on the acquire timeout, not after a long hang" + ); + assert!( + err.to_string().contains("lock-pool connection"), + "the error must name the lock-pool checkout, got: {err}" + ); + + held.release(false).await; + } + + /// Round trip: the lock is observably HELD between acquire and release, and + /// observably FREE after. Both checks run from an independent session; from + /// the holding session they would pass vacuously (session locks are + /// re-entrant) and would not notice an unlock that landed on the wrong + /// connection. + #[sqlx::test] + async fn acquire_write_holds_the_lock_until_release(pool: PgPool) { + let probe = sibling_pool(&pool, 2); + let owner = "did:key:z6MkRoundTrip"; + let repo = "round-trip"; + let key = advisory_lock_key(&owner.replace([':', '/'], "_"), repo); + + let store = RepoStore::for_testing(PathBuf::from("/tmp/gitlawb-test-repos"), pool.clone()); + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + assert!( + !lock_is_free_elsewhere(&probe, key).await, + "the lock must be held while the guard is alive" + ); + + // No polling here, deliberately. `release` must free the lock SYNCHRONOUSLY, + // which it can only do by unlocking on the connection that took it; a + // `pg_advisory_unlock` sent through the pool would land on some other + // session and return false. The `after_release` hook is a net for the + // cancellation path and fires from a spawned task well after this point, so + // it must not be what makes this assertion pass. + guard.release(true).await; + assert!( + lock_is_free_elsewhere(&probe, key).await, + "release must free the lock as seen from another session" + ); + } + + /// `release(false)` skips the Tigris upload but must still free the lock; a + /// failed write that kept the lock would wedge the repo. + #[sqlx::test] + async fn release_after_failed_write_still_frees_the_lock(pool: PgPool) { + let probe = sibling_pool(&pool, 2); + let owner = "did:key:z6MkFailedWrite"; + let repo = "failed-write"; + let key = advisory_lock_key(&owner.replace([':', '/'], "_"), repo); + + let store = RepoStore::for_testing(PathBuf::from("/tmp/gitlawb-test-repos"), pool.clone()); + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + guard.release(false).await; + + assert!( + wait_until_free(&probe, key, Duration::from_secs(5)).await, + "release(success = false) must still free the lock" + ); + } + + /// The lock is per repo: a second acquire for the SAME repo waits for the + /// first to release, while a different repo proceeds straight through. + #[sqlx::test] + async fn same_repo_acquires_serialize_and_different_repos_do_not(pool: PgPool) { + let repos_dir = PathBuf::from("/tmp/gitlawb-test-repos"); + let owner = "did:key:z6MkSerialize"; + let store = RepoStore::for_testing(repos_dir, pool.clone()); + + let first = store + .acquire_write(owner, "serialize-a") + .await + .expect("first acquire"); + + // Different repo: unaffected by the held lock. + let other = tokio::time::timeout( + Duration::from_secs(2), + store.acquire_write(owner, "serialize-b"), + ) + .await + .expect("a different repo must not wait on this lock") + .expect("acquire other repo"); + other.release(false).await; + + // Same repo: must not acquire while `first` is alive. + let contender = tokio::spawn({ + let store = store.clone(); + async move { store.acquire_write(owner, "serialize-a").await } + }); + tokio::time::sleep(Duration::from_millis(1500)).await; + assert!( + !contender.is_finished(), + "a second acquire for the same repo must block while the first guard lives" + ); + + first.release(false).await; + let second = tokio::time::timeout(Duration::from_secs(10), contender) + .await + .expect("contender must finish once the lock is free") + .expect("contender task") + .expect("contender acquire"); + second.release(false).await; + } // ── repo_name validation ─────────────────────────────────────────────── diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 1ed45929..fe18c706 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -279,8 +279,20 @@ async fn main() -> Result<()> { None }; - let repo_store = - git::repo_store::RepoStore::new(config.repos_dir.clone(), tigris, db.pool().clone()); + // Repo write locks run on their own pool, never the main query pool: each + // push holds its connection for the whole receive-pack, and + // db_max_connections (20) is below max_concurrent_git_pushes (32), so sharing + // would starve every other query under a push burst. Headroom above the push + // cap keeps a push from ever queueing here for a connection where it did not + // before. See build_lock_pool for the cancellation semantics (#173). + let lock_pool = git::repo_store::build_lock_pool( + db.pool(), + u32::try_from(config.max_concurrent_git_pushes) + .unwrap_or(u32::MAX) + .saturating_add(8), + std::time::Duration::from_secs(config.db_acquire_timeout_secs), + ); + let repo_store = git::repo_store::RepoStore::new(config.repos_dir.clone(), tigris, lock_pool); // Per-DID limiter for the creation endpoints. Keyed on the authenticated // DID (attacker-varied), so bound its key set to cap memory. From 8e6e2ad0fe2fa62960b2061df84d5a311bcbedb7 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 27 Jul 2026 13:52:44 -0500 Subject: [PATCH 075/124] fix(node): retry the requeue re-read so a coalesced push is not silently dropped `requeue_or_release` clears the dirty flag atomically with the decision to loop, which is correct and stays as it is. The problem was what the pass did next: a failed `get_repo_by_id` fell into an Err arm that yielded no rules, and `list_visibility_rules(...).ok()` collapsed its own error the same way. With no rules, `replication_withheld_set` returns `(false, None)`, the pass skipped its work entirely, and the loop exited having consumed the dirty bit that stood for the coalesced push. There is no reconciliation sweep to re-derive it, so that push's objects were never pinned and its withheld blobs never sealed. `requeue_refresh_state` now separates the three cases the old code collapsed. `Ok(Some)` proceeds. `Ok(None)` means the repo really is gone and releases without touching the retry budget. `Err` from either read is transient and is retried with doubling backoff up to three attempts. A rules-read error is no longer indistinguishable from a repo that has no rules. Exhausting the retries still drops the pass, which is the pre-existing residual, but it now logs at error level with the repo id and attempt count instead of disappearing behind a warning. Six tests cover the retried-then-lands path, the bounded-exhaustion path, repo-gone, the rules-read arm, freshness of the applied rules, and the coalescing property that had to survive. Reverting the Err arm, restoring the `.ok()`, raising the bound, or downgrading the error log each turns a test red. --- crates/gitlawb-node/src/api/repos.rs | 198 +++++++++- crates/gitlawb-node/src/test_support.rs | 501 ++++++++++++++++++++++++ 2 files changed, 685 insertions(+), 14 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index b817318d..35c49dd8 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -849,6 +849,177 @@ async fn requeue_full_scan_object_list( .await } +/// Test-only fault-injection seam for the requeue re-read (#173 U2). The defect this +/// unit fixes lives entirely on the `Err` arm of the two re-reads, which a real Postgres +/// pool will not produce on demand, so the two reads go through the wrappers below and +/// consult this table first. Keyed by `repo_id` (a fresh uuid per test) so tests running +/// in parallel in one process cannot see each other's injections, and it also records +/// the ATTEMPT counts the retry-bound assertions key on. +#[cfg(test)] +pub(crate) mod requeue_faults { + use std::collections::HashMap; + use std::sync::{Mutex, OnceLock}; + + #[derive(Default, Clone, Copy, Debug)] + pub(crate) struct Counters { + pub(crate) repo_read_failures_left: usize, + pub(crate) rules_read_failures_left: usize, + pub(crate) repo_read_attempts: usize, + pub(crate) rules_read_attempts: usize, + } + + fn table() -> &'static Mutex> { + static TABLE: OnceLock>> = OnceLock::new(); + TABLE.get_or_init(|| Mutex::new(HashMap::new())) + } + + /// Make the next `repo_read_failures` repo re-reads and the next + /// `rules_read_failures` rule re-reads for `repo_id` return `Err`, then succeed. + pub(crate) fn inject(repo_id: &str, repo_read_failures: usize, rules_read_failures: usize) { + table().lock().unwrap().insert( + repo_id.to_string(), + Counters { + repo_read_failures_left: repo_read_failures, + rules_read_failures_left: rules_read_failures, + ..Default::default() + }, + ); + } + + /// Observed attempt counts (and remaining injections) for `repo_id`. + pub(crate) fn counters(repo_id: &str) -> Counters { + table() + .lock() + .unwrap() + .get(repo_id) + .copied() + .unwrap_or_default() + } + + /// Production-path hook: count one repo re-read attempt, return whether it must fail. + pub(crate) fn take_repo_read(repo_id: &str) -> bool { + let mut map = table().lock().unwrap(); + let c = map.entry(repo_id.to_string()).or_default(); + c.repo_read_attempts += 1; + if c.repo_read_failures_left > 0 { + c.repo_read_failures_left -= 1; + return true; + } + false + } + + /// Production-path hook: count one rules re-read attempt, return whether it must fail. + pub(crate) fn take_rules_read(repo_id: &str) -> bool { + let mut map = table().lock().unwrap(); + let c = map.entry(repo_id.to_string()).or_default(); + c.rules_read_attempts += 1; + if c.rules_read_failures_left > 0 { + c.rules_read_failures_left -= 1; + return true; + } + false + } +} + +/// Attempts allowed for the requeue re-read before the task gives up (#173 U2). The +/// dirty flag that represented the coalesced push is already consumed by the atomic +/// check-and-clear at the tail and `EncryptInflightGuard::drop` removes the key, so the +/// flag cannot outlive the task and there is no reconciliation sweep to re-derive the +/// work: a transient read error must be RETRIED here or the push's pin/encrypt pass is +/// gone. The bound keeps a sustained outage from spinning forever; on exhaustion the +/// work is still lost (the pre-existing residual), but the give-up is logged at ERROR +/// so it is observable instead of silent. +const REQUEUE_REREAD_MAX_ATTEMPTS: usize = 3; + +/// Backoff before the next re-read attempt. Doubles per attempt. +const REQUEUE_REREAD_BACKOFF: std::time::Duration = std::time::Duration::from_millis(50); + +/// The outcome of the requeue's fresh state re-read, keeping the three cases the old +/// code collapsed into one distinct: a usable refresh, a repo that genuinely no longer +/// exists (terminal, and NOT a retry), and a transient read failure (retryable). +enum RequeueRefresh { + State { + rules: Vec, + is_public: bool, + owner_did: String, + }, + Gone, + Failed, +} + +/// Re-read repo state for a requeue pass, retrying transient read errors. +/// +/// Both reads are retryable and neither may be read as an absence: an `Err` from the +/// repo row is not "the repo is gone", and an `Err` from the rule list is not "this repo +/// has no rules" (the old `.ok()` made those indistinguishable, and a `None` rule set +/// makes `replication_withheld_set` return `None`, which skips the entire pass). Only +/// `Ok(None)` on the repo row is a terminal absence, and it consumes no retry budget. +async fn requeue_refresh_state(ctx: &PostPushReplication) -> RequeueRefresh { + let mut backoff = REQUEUE_REREAD_BACKOFF; + for attempt in 1..=REQUEUE_REREAD_MAX_ATTEMPTS { + let record = match requeue_get_repo(ctx).await { + Ok(Some(rec)) => rec, + Ok(None) => { + tracing::debug!(repo = %ctx.repo_id, "repo gone before requeue pass; releasing"); + return RequeueRefresh::Gone; + } + Err(e) => { + tracing::warn!( + repo = %ctx.repo_id, err = %e, attempt, + "requeue repo re-read failed; retrying" + ); + tokio::time::sleep(backoff).await; + backoff *= 2; + continue; + } + }; + match requeue_list_rules(ctx).await { + Ok(rules) => { + return RequeueRefresh::State { + rules, + is_public: record.is_public, + owner_did: record.owner_did, + } + } + Err(e) => { + tracing::warn!( + repo = %ctx.repo_id, err = %e, attempt, + "requeue visibility-rule re-read failed; retrying" + ); + tokio::time::sleep(backoff).await; + backoff *= 2; + } + } + } + tracing::error!( + repo = %ctx.repo_id, + attempts = REQUEUE_REREAD_MAX_ATTEMPTS, + "requeue re-read failed on every attempt; the coalesced push's pin/encrypt pass is \ + dropped (no reconciliation sweep re-derives it)" + ); + RequeueRefresh::Failed +} + +/// The requeue's repo re-read, behind the test-only fault seam above. +async fn requeue_get_repo(ctx: &PostPushReplication) -> anyhow::Result> { + #[cfg(test)] + if requeue_faults::take_repo_read(&ctx.repo_id) { + return Err(anyhow::anyhow!("injected repo re-read failure")); + } + ctx.db.get_repo_by_id(&ctx.repo_id).await +} + +/// The requeue's visibility-rule re-read, behind the test-only fault seam above. +async fn requeue_list_rules( + ctx: &PostPushReplication, +) -> anyhow::Result> { + #[cfg(test)] + if requeue_faults::take_rules_read(&ctx.repo_id) { + return Err(anyhow::anyhow!("injected visibility-rule re-read failure")); + } + ctx.db.list_visibility_rules(&ctx.repo_id).await +} + /// The detached post-push encryption + local-IPFS pin task, as a REQUEUE LOOP. /// /// Pass one uses the spawn-time captures (`first_*`) — the delta the push handler @@ -966,20 +1137,19 @@ async fn run_post_push_replication( // A push coalesced during this pass. Re-read repo state FRESH (never the stale // spawn-time captures) so a coalesced push that changed `.gitlawb` withholding // is walked under the new policy, then re-enumerate the pin set fail-closed. - let (r_rules, r_is_public, r_owner) = match ctx.db.get_repo_by_id(&ctx.repo_id).await { - Ok(Some(rec)) => ( - ctx.db.list_visibility_rules(&ctx.repo_id).await.ok(), - rec.is_public, - rec.owner_did, - ), - Ok(None) => { - tracing::debug!(repo = %ctx.repo_id, "repo gone before requeue pass; releasing"); - (None, false, String::new()) - } - Err(e) => { - tracing::warn!(repo = %ctx.repo_id, err = %e, "requeue repo re-read failed; skipping this pass's work"); - (None, false, String::new()) - } + // A read error here is retried rather than treated as "no state" (#173 U2): the + // dirty flag is already consumed, so skipping the pass would silently discard + // exactly the push this requeue exists to cover. + let (r_rules, r_is_public, r_owner) = match requeue_refresh_state(&ctx).await { + RequeueRefresh::State { + rules, + is_public, + owner_did, + } => (Some(rules), is_public, owner_did), + // The repo is gone (terminal) or the re-read never succeeded (already logged + // at ERROR). Either way there is no fresh state to act on, so exit; the guard + // Drop removes the key so the repo is never locked out of a future task. + RequeueRefresh::Gone | RequeueRefresh::Failed => break, }; let (_announce, r_withheld) = replication_withheld_set( r_rules.clone(), diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 0d0ae2e8..f97981b4 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -8323,5 +8323,506 @@ mod tests { "the key is released after one pass" ); } + + // ---- U2 (#173): a transient re-read failure must not discard the coalesced + // push's work ---- + // + // The requeue pass re-reads repo state fresh. Before this unit, the `Err` arm of + // either read (repo row, visibility rules) collapsed into "no rules", which made + // `replication_withheld_set` return `None`, which skipped the whole pass. The + // dirty flag was already consumed by the atomic check-and-clear at the tail and + // there is no reconciliation sweep, so the coalesced push's pin/encrypt work was + // lost silently. These tests drive that `Err` arm through the fault seam in + // `api::repos::requeue_faults` (a real pool will not fail on demand) and assert + // on the WORK PERFORMED, not on control flow. + mod u2_reread_retry { + use super::*; + use crate::api::repos::requeue_faults; + + /// Process-wide tracing capture so a test can assert the give-up is logged at + /// ERROR. A global default subscriber can only be installed once per process, + /// so it is shared by every test here and assertions filter on the repo id, + /// which is a fresh uuid per test. + mod logcap { + use std::sync::{Arc, Mutex, OnceLock}; + use tracing::{Event, Level, Subscriber}; + use tracing_subscriber::layer::{Context, Layer}; + use tracing_subscriber::prelude::*; + + type Lines = Arc>>; + + fn lines() -> &'static Lines { + static LINES: OnceLock = OnceLock::new(); + LINES.get_or_init(|| Arc::new(Mutex::new(Vec::new()))) + } + + struct Capture; + impl Layer for Capture { + fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) { + struct V(String); + impl tracing::field::Visit for V { + fn record_debug( + &mut self, + field: &tracing::field::Field, + value: &dyn std::fmt::Debug, + ) { + self.0.push_str(&format!(" {}={:?}", field.name(), value)); + } + } + let mut v = V(String::new()); + event.record(&mut v); + lines() + .lock() + .unwrap() + .push((*event.metadata().level(), v.0)); + } + } + + pub(super) fn install() { + static ONCE: OnceLock<()> = OnceLock::new(); + ONCE.get_or_init(|| { + let _ = tracing::subscriber::set_global_default( + tracing_subscriber::registry().with(Capture), + ); + }); + } + + pub(super) fn errors_containing(needle: &str) -> Vec { + lines() + .lock() + .unwrap() + .iter() + .filter(|(lvl, msg)| *lvl == Level::ERROR && msg.contains(needle)) + .map(|(_, msg)| msg.clone()) + .collect() + } + } + + /// SCENARIO 1. The repo re-read fails once, then succeeds: the requeue pass + /// must still RUN, under the refreshed state, and pin the coalesced push's + /// object. RED before the fix (the single `Err` yielded `(None, false, "")`, + /// the pass was skipped, and the already-consumed dirty flag meant the work + /// was gone for good). + #[sqlx::test] + async fn u2_transient_repo_reread_failure_is_retried_and_work_lands(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u2-retry"); + state.db.create_repo(&repo).await.expect("seed repo"); + let git_repo = init_repo(); + let obj1 = commit(&git_repo.path, "a.txt", "one\n"); + // The coalesced push B adds obj2, absent from push A's spawn captures. + let obj2 = commit(&git_repo.path, "b.txt", "two\n"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // One transient repo re-read failure, then the real DB answers. + requeue_faults::inject(&repo.id, 1, 0); + + let guard = state + .encrypt_inflight + .try_begin(&repo.id) + .expect("push A admits"); + assert!( + state.encrypt_inflight.try_begin(&repo.id).is_none(), + "push B coalesces while A is in flight" + ); + + crate::api::repos::run_post_push_replication_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + server.url(), + true, + owner.clone(), + vec![obj1.clone()], + Some(vec![]), + HashSet::new(), + ) + .await; + + assert!( + state.db.is_pinned(&obj2).await.unwrap(), + "the coalesced push's object is pinned after the retried re-read (RED \ + before this unit: the Err arm dropped the pass and the work with it)" + ); + let c = requeue_faults::counters(&repo.id); + assert_eq!( + c.repo_read_attempts, 2, + "the failed re-read is retried exactly once before it succeeds" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is released once the task exits" + ); + } + + /// SCENARIO 2. Every re-read attempt fails: the loop must give up on a BOUND + /// (asserted as a literal, so raising or removing the bound goes RED) and log + /// the give-up at ERROR so the residual loss is observable rather than silent. + #[sqlx::test] + async fn u2_sustained_repo_reread_failure_is_bounded_and_logged(pool: PgPool) { + logcap::install(); + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u2-bounded"); + state.db.create_repo(&repo).await.expect("seed repo"); + let git_repo = init_repo(); + let obj1 = commit(&git_repo.path, "a.txt", "one\n"); + let obj2 = commit(&git_repo.path, "b.txt", "two\n"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // Far more failures than the bound allows: the outage never clears. + requeue_faults::inject(&repo.id, 10_000, 0); + + let guard = state + .encrypt_inflight + .try_begin(&repo.id) + .expect("push A admits"); + assert!( + state.encrypt_inflight.try_begin(&repo.id).is_none(), + "push B coalesces" + ); + + crate::api::repos::run_post_push_replication_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + server.url(), + true, + owner.clone(), + vec![obj1.clone()], + Some(vec![]), + HashSet::new(), + ) + .await; + + let c = requeue_faults::counters(&repo.id); + assert_eq!( + c.repo_read_attempts, 3, + "the re-read is bounded at 3 attempts; unbounded retry or a raised \ + bound must fail here" + ); + assert!( + !state.db.is_pinned(&obj2).await.unwrap(), + "with the read never succeeding there is nothing fresh to act on" + ); + let errs = logcap::errors_containing(&repo.id); + assert!( + !errs.is_empty(), + "the exhausted requeue re-read is logged at ERROR with the repo id, so \ + the residual work loss is observable; captured: {errs:?}" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is still released on the give-up path" + ); + } + + /// SCENARIO 3. `Ok(None)` (the repo was deleted during the in-flight window) + /// is NOT a transient failure: it must release immediately without burning the + /// retry budget. The repo id is never inserted, so the re-read legitimately + /// returns `Ok(None)`. + #[sqlx::test] + async fn u2_repo_gone_releases_without_consuming_retries(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let missing_id = uuid::Uuid::new_v4().to_string(); + let git_repo = init_repo(); + let _obj1 = commit(&git_repo.path, "a.txt", "one\n"); + + let server = mockito::Server::new_async().await; + + requeue_faults::inject(&missing_id, 0, 0); + + let guard = state + .encrypt_inflight + .try_begin(&missing_id) + .expect("push A admits"); + assert!( + state.encrypt_inflight.try_begin(&missing_id).is_none(), + "push B coalesces" + ); + + // Empty object list: pass one touches no pin rows for a repo that is gone. + crate::api::repos::run_post_push_replication_for_test( + &state, + guard, + git_repo.path.clone(), + missing_id.clone(), + server.url(), + true, + owner.clone(), + vec![], + Some(vec![]), + HashSet::new(), + ) + .await; + + let c = requeue_faults::counters(&missing_id); + assert_eq!( + c.repo_read_attempts, 1, + "a deleted repo is a terminal answer, never retried" + ); + assert_eq!( + c.rules_read_attempts, 0, + "no rules read is attempted once the repo row is gone" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is released cleanly" + ); + } + + /// SCENARIO 4. A failed visibility-rule read is transient, never an empty + /// policy. RED before the fix, where `.ok()` made "the rules read failed" and + /// "this repo has no rules" the same value: the withheld blob was then neither + /// sealed nor covered, because a `None` rule set skips the pass entirely. + #[sqlx::test] + async fn u2_transient_rules_read_failure_is_retried_not_read_as_empty(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let reader = new_did(); + let repo = seed_repo(&owner, "u2-rules"); + state.db.create_repo(&repo).await.expect("seed repo"); + let git_repo = init_repo(); + let pub_oid = commit(&git_repo.path, "public/a.txt", "public\n"); + let secret_oid = commit(&git_repo.path, "secret/b.txt", "TOP SECRET\n"); + + // The coalesced push B is what added the path-scoped rule. + state + .db + .set_visibility_rule( + &repo.id, + "/secret/**", + VisibilityMode::B, + &[reader], + &owner, + ) + .await + .expect("set rule"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // The repo row reads fine; the RULES read is the one that blips. + requeue_faults::inject(&repo.id, 0, 1); + + let guard = state + .encrypt_inflight + .try_begin(&repo.id) + .expect("push A admits"); + assert!( + state.encrypt_inflight.try_begin(&repo.id).is_none(), + "push B coalesces" + ); + + // Push A's captures are stale: no rule, nothing withheld. + crate::api::repos::run_post_push_replication_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + server.url(), + true, + owner.clone(), + vec![pub_oid.clone()], + Some(vec![]), + HashSet::new(), + ) + .await; + + assert!( + state + .db + .encrypted_blob_recipients_tag(&repo.id, &secret_oid) + .await + .unwrap() + .is_some(), + "the withheld blob is sealed under the RETRIED rule set (RED with \ + list_visibility_rules(..).ok(): an empty policy seals nothing)" + ); + let c = requeue_faults::counters(&repo.id); + assert_eq!( + c.rules_read_attempts, 2, + "the failed rules read is retried, not collapsed into an empty rule set" + ); + assert!( + !state.db.is_pinned(&secret_oid).await.unwrap(), + "the withheld blob is never pinned in the clear by the requeue" + ); + } + + /// SCENARIO 5. The fault-free control for scenario 4: the rules applied by the + /// requeue are the COALESCED push's fresh ones, never the spawn-time capture, + /// and the retry path does not perturb that (exactly one read of each). + #[sqlx::test] + async fn u2_requeue_applies_fresh_rules_not_spawn_captures(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let reader = new_did(); + let repo = seed_repo(&owner, "u2-fresh"); + state.db.create_repo(&repo).await.expect("seed repo"); + let git_repo = init_repo(); + let pub_oid = commit(&git_repo.path, "public/a.txt", "public\n"); + let secret_oid = commit(&git_repo.path, "secret/b.txt", "TOP SECRET\n"); + state + .db + .set_visibility_rule( + &repo.id, + "/secret/**", + VisibilityMode::B, + &[reader], + &owner, + ) + .await + .expect("set rule"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + requeue_faults::inject(&repo.id, 0, 0); + + let guard = state + .encrypt_inflight + .try_begin(&repo.id) + .expect("push A admits"); + assert!( + state.encrypt_inflight.try_begin(&repo.id).is_none(), + "push B coalesces" + ); + + crate::api::repos::run_post_push_replication_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + server.url(), + true, + owner.clone(), + vec![pub_oid.clone()], + Some(vec![]), + HashSet::new(), + ) + .await; + + let c = requeue_faults::counters(&repo.id); + assert_eq!( + (c.repo_read_attempts, c.rules_read_attempts), + (1, 1), + "a healthy DB is read exactly once per requeue pass" + ); + assert!( + state.db.is_pinned(&pub_oid).await.unwrap(), + "the visible object is pinned under the fresh rules" + ); + assert!( + !state.db.is_pinned(&secret_oid).await.unwrap(), + "the freshly-read rule withholds the secret blob (the spawn-time \ + capture had no rules at all)" + ); + } + + /// SCENARIO 6. Regression guard on the property the fix must not disturb: the + /// tail check-and-clear is atomic, so a push coalescing during it is still + /// covered by exactly one more pass, and the key is released after. + #[sqlx::test] + async fn u2_coalesced_push_still_covered_by_exactly_one_requeue_pass(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u2-coalesce"); + state.db.create_repo(&repo).await.expect("seed repo"); + let git_repo = init_repo(); + let obj1 = commit(&git_repo.path, "a.txt", "one\n"); + let obj2 = commit(&git_repo.path, "b.txt", "two\n"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + requeue_faults::inject(&repo.id, 0, 0); + + let guard = state + .encrypt_inflight + .try_begin(&repo.id) + .expect("push A admits"); + // Push B lands during the in-flight window: dirty flag set. + assert!( + state.encrypt_inflight.try_begin(&repo.id).is_none(), + "push B coalesces" + ); + assert_eq!( + state.encrypt_inflight.dirty(&repo.id), + Some(true), + "the coalesced push marked the repo dirty" + ); + + crate::api::repos::run_post_push_replication_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + server.url(), + true, + owner.clone(), + vec![obj1.clone()], + Some(vec![]), + HashSet::new(), + ) + .await; + + assert_eq!( + requeue_faults::counters(&repo.id).repo_read_attempts, + 1, + "one coalesced push means exactly one requeue pass, no re-spin" + ); + assert!( + state.db.is_pinned(&obj1).await.unwrap(), + "push A's object is pinned" + ); + assert!( + state.db.is_pinned(&obj2).await.unwrap(), + "the coalesced push's object is covered by the requeue pass" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the key is released once the task is clean" + ); + } + } } } From fccdfc1820b32d7baf82a5895bb2ab76e9b7343d Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 27 Jul 2026 14:40:03 -0500 Subject: [PATCH 076/124] fix(node): record a failed pin-source write so the resolver keeps its fallback The resolver treated a non-empty, below-cap pin-source set as proof that every source had been recorded, and skipped the bounded scan on that basis. But `record_pin_source` is best effort at every call site: the ipfs_pin sites were warn-only once their retries exhausted, and both pinata sites had no retry at all. An object first pinned from a private repo and later pushed from a public one whose source write failed keeps a set naming only the private source, so `GET /ipfs/{cid}` returned 404 for an object the public repo would have served. Migration v15 adds `pinned_cids.pin_sources_incomplete`, NOT NULL DEFAULT FALSE so every existing row reads as complete. A record that fails outright sets it; a later successful record clears it, inside the same transaction as the insert so the two paths cannot drift. `needs_scan` now ORs the marker in, short-circuiting after the existing empty and at-cap checks so it costs nothing on the serve path. Pinata's two bare calls now share `retry_db_record` with ipfs_pin. The ipfs_pin first-pin path additionally records the CID and its source in one transaction; Pinata's remains two retried calls. INV-10 is preserved: a complete set still tail-404s with no preload, asserted on the existing preload counter. The fallback is not an authorization bypass, and the test for that asserts the scan actually ran before checking the denial, so the denial cannot pass vacuously. Dropping the marker term from the gate, forcing it true, or removing the clear each turns a test red. The marker is per-object rather than per-(object, repo): a successful record from one repo clears a marker set by another's failure. That is the conservative direction, since the marker only ever adds the fallback back. --- crates/gitlawb-node/src/api/ipfs.rs | 24 +- crates/gitlawb-node/src/db/mod.rs | 119 +++++- crates/gitlawb-node/src/ipfs_pin.rs | 36 +- crates/gitlawb-node/src/pinata.rs | 31 +- crates/gitlawb-node/src/test_support.rs | 542 ++++++++++++++++++++++++ 5 files changed, 728 insertions(+), 24 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 624a1b27..ba669027 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -329,15 +329,31 @@ pub async fn get_by_cid( // (e.g. a later PUBLIC pinner buried by 16 attacker sources — the // pin-source griefing hole). The scan gates every repo through the // real per-caller gate, so it finds that copy. - // A non-empty, non-full set is COMPLETE (every recorded source was just tried), so - // skip the scan and let the tail 404 — ordinary denials never fan out to O(repos) - // (INV-10 / F3). The at_cap query runs only on a provenance MISS (we return above - // on Served), so it never costs the serve path. + // - marked -> a `record_pin_source` for this object failed outright (U3, #173). + // `record_pin_source` is best effort at every pin call site, so a + // non-empty below-cap set is NOT self-evidently complete: an object + // first pinned from a PRIVATE repo and later pushed from a PUBLIC + // one whose record failed names only the private source. The + // durable `pin_sources_incomplete` marker is the node's own record + // that a source is missing, so the fallback stays available for + // exactly those objects instead of 404ing a servable public copy. + // Only a set with NONE of these three signals is treated as complete (every + // recorded source was just tried), so it skips the scan and lets the tail 404, and + // ordinary denials never fan out to O(repos) (INV-10 / F3). Both extra queries run + // only on a provenance MISS (we return above on Served), so neither costs the serve + // path, and the fallback is not an authorization bypass: the scan gates every repo + // through the SAME per-caller gate, so a caller who may not read the object is + // still denied. let needs_scan = sources.is_empty() || state .db .pin_sources_at_cap(sha256_hex) .await + .map_err(AppError::Internal)? + || state + .db + .pin_sources_incomplete(sha256_hex) + .await .map_err(AppError::Internal)?; if needs_scan { // F3 (#173, INV-10/INV-15): peek the per-IP WORK-budget limiter WITHOUT diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 775e7c17..c2a6cb60 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -937,6 +937,25 @@ const MIGRATIONS: &[Migration] = &[ "ALTER TABLE pinned_cids ADD COLUMN IF NOT EXISTS legacy_provider_cid TEXT", ], }, + Migration { + version: 15, + name: "pinned_cids_sources_incomplete", + stmts: &[ + // U3 (#173): `record_pin_source` is BEST EFFORT at every call site, so a + // non-empty, below-cap source set is not proof that every source was + // recorded. An object first pinned from a private repo and later pushed + // from a PUBLIC repo whose record failed keeps a set naming only the + // private source, and the resolver used to call that set complete and 404 + // an object the public repo would serve. Record the miss DURABLY here so + // `GET /ipfs/{cid}` keeps the bounded scan fallback for exactly those + // objects. Not inferable from row counts or timestamps: neither can tell + // "no other source exists" from "a source failed to record", which is the + // whole distinction. NEW versioned migration (never appended to an applied + // block, INV-7). NOT NULL DEFAULT FALSE so every pre-existing row reads as + // complete and ordinary denials stay off the O(repos) path (INV-10). + "ALTER TABLE pinned_cids ADD COLUMN IF NOT EXISTS pin_sources_incomplete BOOLEAN NOT NULL DEFAULT FALSE", + ], + }, ]; /// Max distinct source repos recorded per pinned object (F1, #173 jatmn round 8). @@ -2282,6 +2301,11 @@ impl Db { /// gets it filled the next time the object is re-pinned with a known source. /// `cid`/`pinned_at` are left untouched on conflict. `repo_id` is `None` only /// for a legacy pin with no known source; those fall back to the resolver's scan. + /// + /// The production first-pin path now goes through [`Self::record_pinned_cid_with_source`] + /// (U3, #173) so the pin and its source land atomically; this remains the seam for + /// seeding legacy, source-less rows in tests. + #[cfg_attr(not(test), allow(dead_code))] pub async fn record_pinned_cid( &self, sha256_hex: &str, @@ -2384,7 +2408,17 @@ impl Db { /// (`pin_sources_for_oid`) caps the ADDITIONAL sources at `MAX_PIN_SOURCES` (always /// keeping the first-pinner), so the INV-10 bound on serve-time work holds at /// `O(MAX_PIN_SOURCES + 1)` regardless of a table overshoot. + /// + /// A successful record also CLEARS the `pin_sources_incomplete` marker for the + /// object, in the SAME transaction as the insert (U3, #173), so the clear cannot + /// drift across the four call sites or land without the row it describes. The + /// marker is per-object, not per-(object, repo): a record from repo B clears a + /// marker set by a failed record from repo A, which can re-hide A's hole until A + /// pushes again. That is the deliberate cost of a single boolean, and it fails in + /// the safe direction relative to today (the marker only ever ADDS the fallback, + /// never removes a source the resolver already tries). pub async fn record_pin_source(&self, sha256_hex: &str, repo_id: &str) -> Result<()> { + let mut tx = self.pool.begin().await?; sqlx::query( "INSERT INTO pin_repo_sources (sha256_hex, repo_id) SELECT $1, $2 @@ -2394,11 +2428,94 @@ impl Db { .bind(sha256_hex) .bind(repo_id) .bind(MAX_PIN_SOURCES) - .execute(&self.pool) + .execute(&mut *tx) .await?; + sqlx::query( + "UPDATE pinned_cids SET pin_sources_incomplete = FALSE + WHERE sha256_hex = $1 AND pin_sources_incomplete", + ) + .bind(sha256_hex) + .execute(&mut *tx) + .await?; + tx.commit().await?; Ok(()) } + /// Record a first pin and its source ATOMICALLY (U3, #173). The first-pin path + /// used to run `record_pinned_cid` and `record_pin_source` as two independent + /// best-effort calls, so the pin could land while its source did not, leaving a + /// source set that is silently missing its own first pinner. One transaction + /// removes that window entirely: either both rows land or neither does, and a + /// total failure leaves the object unpinned so the next push retries it. + pub async fn record_pinned_cid_with_source( + &self, + sha256_hex: &str, + cid: &str, + repo_id: &str, + ) -> Result<()> { + let mut tx = self.pool.begin().await?; + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) + VALUES ($1, $2, $3, $4) + ON CONFLICT(sha256_hex) DO UPDATE SET + repo_id = COALESCE(pinned_cids.repo_id, EXCLUDED.repo_id)", + ) + .bind(sha256_hex) + .bind(cid) + .bind(Utc::now().to_rfc3339()) + .bind(repo_id) + .execute(&mut *tx) + .await?; + sqlx::query( + "INSERT INTO pin_repo_sources (sha256_hex, repo_id) + SELECT $1, $2 + WHERE (SELECT count(*) FROM pin_repo_sources WHERE sha256_hex = $1) < $3 + ON CONFLICT DO NOTHING", + ) + .bind(sha256_hex) + .bind(repo_id) + .bind(MAX_PIN_SOURCES) + .execute(&mut *tx) + .await?; + sqlx::query( + "UPDATE pinned_cids SET pin_sources_incomplete = FALSE + WHERE sha256_hex = $1 AND pin_sources_incomplete", + ) + .bind(sha256_hex) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Mark this object's pin-source set as KNOWN INCOMPLETE (U3, #173). Called when a + /// `record_pin_source` exhausts its retries, which is the only moment the node + /// knows a source it meant to record is missing. `GET /ipfs/{cid}` reads it to keep + /// the bounded scan fallback for that object, so a public copy that would serve is + /// no longer 404'd. A no-op when no `pinned_cids` row exists (the first-pin path is + /// transactional, so there is no half-recorded pin to describe). + pub async fn mark_pin_sources_incomplete(&self, sha256_hex: &str) -> Result<()> { + sqlx::query("UPDATE pinned_cids SET pin_sources_incomplete = TRUE WHERE sha256_hex = $1") + .bind(sha256_hex) + .execute(&self.pool) + .await?; + Ok(()) + } + + /// Whether this object's pin-source set is KNOWN INCOMPLETE (U3, #173): a + /// `record_pin_source` for it failed outright and no later record has repaired the + /// set. `false` for an unpinned oid and for every row predating the column, so the + /// common path is unchanged and an ordinary denial never fans out (INV-10). + pub async fn pin_sources_incomplete(&self, sha256_hex: &str) -> Result { + let flag: Option = sqlx::query_scalar( + "SELECT pin_sources_incomplete FROM pinned_cids WHERE sha256_hex = $1", + ) + .bind(sha256_hex) + .fetch_optional(&self.pool) + .await?; + Ok(flag.unwrap_or(false)) + } + /// Every source repository recorded for a pinned object (F1, #173 jatmn round 8): /// the union of the first-pinner `pinned_cids.repo_id` and the `pin_repo_sources` /// rows, deduped and ordered for a deterministic resolver walk. diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 3d2f13df..0e58e27e 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -22,13 +22,12 @@ const PIN_RECORD_BACKOFF: Duration = Duration::from_millis(50); /// so a dropped `record_pin_source`/`record_pinned_cid` makes `GET /ipfs/{cid}` /// 404 a valid public copy. Every wrapped insert is idempotent (`ON CONFLICT DO /// NOTHING` / provenance-preserving upsert), so re-running is safe. On exhausted -/// attempts the last error is returned and the caller keeps its warn — behavior -/// degrades to the pre-retry state, not worse. Process death mid-retry or a DB -/// outage outlasting the backoff horizon leaves the same residual hole (no -/// persisted marker to reconcile from at startup), retired only by a future -/// reconciliation sweep. Runs inside the already-detached post-push task, so the -/// backoff adds no push latency. -async fn retry_db_record(mut op: F) -> Result<()> +/// attempts the last error is returned and the caller records the durable +/// `pin_sources_incomplete` marker (U3, #173), which is what keeps the resolver's +/// bounded scan fallback available for that object instead of 404ing a public copy. +/// Shared with the `pinata.rs` twin so both pin paths retry identically. Runs +/// inside the already-detached post-push task, so the backoff adds no push latency. +pub(crate) async fn retry_db_record(mut op: F) -> Result<()> where F: FnMut() -> Fut, Fut: std::future::Future>, @@ -254,6 +253,14 @@ pub async fn pin_new_objects( // when this repo would serve it. Bounded per object (MAX_PIN_SOURCES). if let Err(e) = retry_db_record(|| db.record_pin_source(&sha, repo_id)).await { tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); + // U3 (#173): the retries are spent and this repo is NOT in the source + // set, so the set is known incomplete. Persist that, or the resolver + // reads a non-empty below-cap set as COMPLETE and 404s an object this + // repo would serve. Warn-only in turn: if the marker write also fails + // the object degrades to the pre-U3 behavior, never worse. + if let Err(e) = db.mark_pin_sources_incomplete(&sha).await { + tracing::warn!(sha = %sha, err = %e, "failed to mark pin sources incomplete"); + } } // R8 (#173 round 10): opportunistically repair a legacy provider-CID // row (Kubo dag-pb / Pinata) to the raw-content resolver key on this @@ -300,16 +307,19 @@ pub async fn pin_new_objects( // verifies them against the requested CID, so the raw CID is the correct // key. Mirrors the pinata twin, which already records the raw CID. let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes(&data).to_string(); + // F1 (#173 round 8): the first pinner is recorded in pin_repo_sources too, + // so every source (first and subsequent) is tried uniformly by the + // resolver. U3 (#173): the pin and its source go down in ONE transaction. + // As two independent best-effort calls this path could land the pin while + // dropping its own source, producing a source set silently missing its + // first pinner; atomically there is no such window, and a total failure + // leaves the object unpinned so the next push retries the whole thing. if let Err(e) = - retry_db_record(|| db.record_pinned_cid(&sha, &raw_cid, Some(repo_id))).await + retry_db_record(|| db.record_pinned_cid_with_source(&sha, &raw_cid, repo_id)) + .await { tracing::warn!(sha = %sha, err = %e, "failed to record pinned CID in DB"); } - // F1 (#173 round 8): also record the first pinner in pin_repo_sources so - // every source (first and subsequent) is tried uniformly by the resolver. - if let Err(e) = retry_db_record(|| db.record_pin_source(&sha, repo_id)).await { - tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); - } // Return the provider Hash (not the resolver key), mirroring the pinata // twin's contract: the DB `cid` is the raw resolver key (recorded above), // the returned value is the provider CID. Here the return is consumed only diff --git a/crates/gitlawb-node/src/pinata.rs b/crates/gitlawb-node/src/pinata.rs index 4d6b9704..e49f9205 100644 --- a/crates/gitlawb-node/src/pinata.rs +++ b/crates/gitlawb-node/src/pinata.rs @@ -114,9 +114,17 @@ pub async fn pin_new_objects( } // F1 (#173 round 8): record this repo as an additional source for the // already-pinned object (mirrors the ipfs_pin skip-branch insert) so the - // resolver can serve a shared object from any pin-path source. - if let Err(e) = db.record_pin_source(&sha, repo_id).await { + // resolver can serve a shared object from any pin-path source. U3 (#173): + // retried through the SHARED helper (this was a bare call, so a single + // transient error dropped the source outright) and, on exhaustion, marked + // durably so the resolver keeps the bounded scan fallback for the object. + if let Err(e) = + crate::ipfs_pin::retry_db_record(|| db.record_pin_source(&sha, repo_id)).await + { tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); + if let Err(e) = db.mark_pin_sources_incomplete(&sha).await { + tracing::warn!(sha = %sha, err = %e, "failed to mark pin sources incomplete"); + } } continue; } @@ -143,15 +151,26 @@ pub async fn pin_new_objects( // dag-pb/UnixFS, so its returned CID does not hash the raw content and // must not become an alias `/ipfs/{cid}` serves raw git bytes for (#173). let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes(&data).to_string(); - if let Err(e) = db - .record_pinata_cid(&sha, &raw_cid, &cid, Some(repo_id)) - .await + // U3 (#173): both records go through the shared retry helper, at parity + // with the ipfs_pin twin. These were bare calls, so one transient DB error + // permanently dropped a pin source. + if let Err(e) = crate::ipfs_pin::retry_db_record(|| { + db.record_pinata_cid(&sha, &raw_cid, &cid, Some(repo_id)) + }) + .await { tracing::warn!(sha = %sha, err = %e, "failed to record pinata_cid in DB"); } // F1 (#173 round 8): also record the first pinner in pin_repo_sources. - if let Err(e) = db.record_pin_source(&sha, repo_id).await { + // U3: an exhausted retry marks the set incomplete so the resolver keeps + // the scan fallback rather than 404ing a copy it could serve. + if let Err(e) = + crate::ipfs_pin::retry_db_record(|| db.record_pin_source(&sha, repo_id)).await + { tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); + if let Err(e) = db.mark_pin_sources_incomplete(&sha).await { + tracing::warn!(sha = %sha, err = %e, "failed to mark pin sources incomplete"); + } } pinned.push((sha, cid)); } diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index f97981b4..fd90e46a 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -2690,6 +2690,548 @@ mod tests { ); } + // ── U3 (#173): durable pin-source incompleteness marker ────────────────── + // + // `record_pin_source` is best effort at every call site, so a non-empty, + // below-cap source set is NOT proof of completeness: an object first pinned + // from a PRIVATE repo and later pushed from a PUBLIC repo whose record failed + // has a set that names only the private source. The resolver used to treat + // that set as complete and 404 an object the public repo would serve. The + // pinned_cids.pin_sources_incomplete marker records the miss durably so the + // bounded scan fallback still runs. These tests drive both arms: the marker + // set (fallback runs, object serves, denial still denies) and the marker + // clear (ordinary denials stay off the O(repos) path, INV-10). + + /// Make `record_pin_source` fail for the duration of `body` by moving the + /// `pin_repo_sources` table out from under it, the closest honest stand-in for + /// the transient DB error the retry wrapper is there to absorb. Every other + /// pin-path query keeps working, so only the source record (and its retries) + /// fails, which is exactly the partial-record shape the finding turns on. + async fn with_pin_sources_broken(pool: &PgPool, body: F) -> T + where + F: FnOnce() -> Fut, + Fut: std::future::Future, + { + sqlx::query("ALTER TABLE pin_repo_sources RENAME TO pin_repo_sources_hidden") + .execute(pool) + .await + .expect("hide pin_repo_sources"); + let out = body().await; + sqlx::query("ALTER TABLE pin_repo_sources_hidden RENAME TO pin_repo_sources") + .execute(pool) + .await + .expect("restore pin_repo_sources"); + out + } + + /// Pin `oid` from `repo_id` through the real ipfs_pin path with a mock Kubo that + /// must NOT be called (the object is already pinned, so this drives the + /// skip-branch `record_pin_source` and nothing else). + async fn repin_via_skip_branch( + state: &AppState, + bare: &std::path::Path, + oid: &str, + repo_id: &str, + ) { + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyshouldnothappen"}"#) + .expect(0) + .create_async() + .await; + crate::ipfs_pin::pin_new_objects( + &server.url(), + bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + vec![oid.to_string()], + &state.db, + repo_id, + ) + .await; + m.assert_async().await; + } + + /// U3 scenario 1 (#173, the finding's exact case): an object first pinned from a + /// PRIVATE repo, then pushed from a PUBLIC repo whose `record_pin_source` + /// exhausts its retries. The source set is non-empty and below cap, so the old + /// gate called it COMPLETE and 404'd an object the public repo would happily + /// serve. With the durable marker the bounded scan fallback still runs and the + /// public copy serves. RED before the marker (404); GREEN after (200). + #[sqlx::test] + async fn ipfs_cid_incomplete_source_set_falls_back_to_scan(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["u3priv", "u3pub"]); + let priv_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3priv.git"); + let pub_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3pub.git"); + + // Private first-pinner owns the only recorded source. + let mut priv_repo = seed_repo(&owner_did, "u3priv"); + priv_repo.is_public = false; + state + .db + .create_repo(&priv_repo) + .await + .expect("seed private"); + let cid = pin_cid_for_repo(&priv_bare, &fx.public_oid, &state.db, &priv_repo.id).await; + + // The PUBLIC repo holds the same object, but its source record never lands. + let pub_repo = seed_repo(&owner_did, "u3pub"); // public, no rule + state.db.create_repo(&pub_repo).await.expect("seed public"); + with_pin_sources_broken(&pool, || { + repin_via_skip_branch(&state, &pub_bare, &fx.public_oid, &pub_repo.id) + }) + .await; + + // The recorded set still names only the private repo, and it is below cap. + assert_eq!( + state.db.pin_sources_for_oid(&fx.public_oid).await.unwrap(), + vec![priv_repo.id.clone()], + "the public source really did fail to record" + ); + assert!( + !state.db.pin_sources_at_cap(&fx.public_oid).await.unwrap(), + "the set is below cap, so at_cap cannot be what triggers the fallback" + ); + + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "a KNOWN-incomplete source set must keep the scan fallback so the public copy serves" + ); + assert!( + body.contains("public bytes"), + "the served body is the public object's bytes" + ); + } + + /// U3 scenario 2 (#173, INV-10 guard): the marker must not turn ORDINARY denials + /// into an O(repos) fan-out. With the marker false, a non-empty below-cap source + /// set and a provenance miss, the request must 404 WITHOUT the scan preload ever + /// running. The preload counter is the both-ways proof: forcing the marker true + /// unconditionally turns this red (count 1), which is what keeps the assertion + /// from being vacuous. + #[sqlx::test] + async fn ipfs_cid_complete_source_set_never_preloads(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["u3only"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3only.git"); + + // One PRIVATE source, recorded cleanly: the set is complete and below cap. + let mut priv_repo = seed_repo(&owner_did, "u3only"); + priv_repo.is_public = false; + state + .db + .create_repo(&priv_repo) + .await + .expect("seed private"); + let cid = pin_cid_for_repo(&bare, &fx.secret_oid, &state.db, &priv_repo.id).await; + state + .db + .record_pin_source(&fx.secret_oid, &priv_repo.id) + .await + .expect("record source"); + assert!( + !state + .db + .pin_sources_incomplete(&fx.secret_oid) + .await + .unwrap(), + "a clean record leaves the set marked complete" + ); + assert!( + !state.db.pin_sources_at_cap(&fx.secret_oid).await.unwrap(), + "the set is below cap, so at_cap cannot be what drives the gate" + ); + + crate::api::ipfs::reset_preload_queries(); + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "an anonymous caller denied by the only recorded source gets the opaque 404" + ); + assert!( + !body.contains("TOP SECRET"), + "the 404 body must not leak the withheld object" + ); + assert_eq!( + crate::api::ipfs::preload_queries(), + 0, + "an ordinary denial against a COMPLETE source set must never run the O(repos) preload (INV-10)" + ); + } + + /// U3 scenario 3 (#173): the marker is not permanent. Once a later + /// `record_pin_source` for the object succeeds, nothing is missing, so the marker + /// clears and the scan stops being triggered. BOTH sources here are private, so the + /// provenance walk MISSES and the request actually reaches the `needs_scan` gate: + /// with a marker left stuck the gate arms the O(repos) preload for an ordinary + /// denial forever. Drop the clear and both halves go red (marker still true, preload + /// 1). A public second source would make the preload half vacuous, because the + /// provenance path serves and returns before the gate is ever evaluated. + #[sqlx::test] + async fn ipfs_cid_marker_clears_on_a_later_successful_record(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["u3cfirst", "u3csecond"]); + let first_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3cfirst.git"); + let second_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3csecond.git"); + + let mut first_repo = seed_repo(&owner_did, "u3cfirst"); + first_repo.is_public = false; + state.db.create_repo(&first_repo).await.expect("seed first"); + let cid = pin_cid_for_repo(&first_bare, &fx.secret_oid, &state.db, &first_repo.id).await; + let mut second_repo = seed_repo(&owner_did, "u3csecond"); + second_repo.is_public = false; + state + .db + .create_repo(&second_repo) + .await + .expect("seed second"); + + // First push from the second repo: the source record fails, so the set is marked. + with_pin_sources_broken(&pool, || { + repin_via_skip_branch(&state, &second_bare, &fx.secret_oid, &second_repo.id) + }) + .await; + assert!( + state + .db + .pin_sources_incomplete(&fx.secret_oid) + .await + .unwrap(), + "the exhausted record marked the set incomplete" + ); + + // A later push from the same repo records cleanly, so nothing is missing. + repin_via_skip_branch(&state, &second_bare, &fx.secret_oid, &second_repo.id).await; + assert!( + !state + .db + .pin_sources_incomplete(&fx.secret_oid) + .await + .unwrap(), + "a successful record clears the marker" + ); + assert_eq!( + state + .db + .pin_sources_for_oid(&fx.secret_oid) + .await + .unwrap() + .len(), + 2, + "the repaired set really does name both sources" + ); + + crate::api::ipfs::reset_preload_queries(); + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "both sources are private, so the anonymous caller is denied" + ); + assert!( + !body.contains("TOP SECRET"), + "the 404 body must not leak the withheld object" + ); + assert_eq!( + crate::api::ipfs::preload_queries(), + 0, + "a repaired source set stops triggering the scan: the denial is back off the O(repos) path" + ); + } + + /// U3 scenario 4 (#173): the marker tracks the record's OUTCOME, not the attempt. + /// An exhausted retry sets it; a first-attempt success never does. Without the + /// second arm the first could be satisfied by marking unconditionally. + #[sqlx::test] + async fn pin_sources_incomplete_marks_only_exhausted_records(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["u3mark"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3mark.git"); + let repo = seed_repo(&owner_did, "u3mark"); + state.db.create_repo(&repo).await.expect("seed repo"); + let _ = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo.id).await; + + // Arm A: a first-attempt success must leave the marker alone. + repin_via_skip_branch(&state, &bare, &fx.public_oid, &repo.id).await; + assert!( + !state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "a record that lands on the first attempt never marks the set incomplete" + ); + + // Arm B: an exhausted retry marks it. + with_pin_sources_broken(&pool, || { + repin_via_skip_branch(&state, &bare, &fx.public_oid, &repo.id) + }) + .await; + assert!( + state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "an exhausted record marks the set incomplete" + ); + + // An unpinned oid has no row and must read as complete, never as missing. + assert!( + !state + .db + .pin_sources_incomplete(&"f".repeat(64)) + .await + .unwrap(), + "an unpinned oid reads complete, so an unknown CID cannot arm the fallback" + ); + } + + /// U3 scenario 5 (#173): the Pinata pin path had BARE `record_pin_source` calls, so + /// one transient DB error dropped a source permanently. It now shares the ipfs_pin + /// retry helper and marks/clears the same marker. The elapsed-time assertion is the + /// retry proof: a bare call returns immediately, whereas the wrapper sleeps + /// `PIN_RECORD_BACKOFF` between each of `PIN_RECORD_ATTEMPTS` tries. + #[sqlx::test] + async fn pinata_pin_path_retries_and_marks_incomplete(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["u3pinata"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3pinata.git"); + let repo = seed_repo(&owner_did, "u3pinata"); + state.db.create_repo(&repo).await.expect("seed repo"); + + // Already carries a pinata_cid, so pin_new_objects takes the skip branch and the + // only DB write under test is the source record. + let (_ty, raw) = crate::git::store::read_object(&bare, &fx.public_oid) + .unwrap() + .expect("object readable"); + let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes(&raw).to_string(); + state + .db + .record_pinata_cid(&fx.public_oid, &raw_cid, "QmProvider", Some(&repo.id)) + .await + .expect("seed pinata pin"); + + let client = reqwest::Client::new(); + let run = |db_broken: bool| { + let client = client.clone(); + let bare = bare.clone(); + let oid = fx.public_oid.clone(); + let repo_id = repo.id.clone(); + let state = &state; + async move { + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Any) + .with_status(200) + .with_body(r#"{"data":{"cid":"QmShouldNotHappen"}}"#) + .expect(0) + .create_async() + .await; + let started = std::time::Instant::now(); + crate::pinata::pin_new_objects( + &client, + &server.url(), + "test-jwt", + &bare, + vec![oid], + &state.db, + &repo_id, + ) + .await; + m.assert_async().await; // the upload is skipped: DB-only path + let _ = db_broken; + started.elapsed() + } + }; + + // Failing arm: retried (so it sleeps the full backoff horizon) and marked. + let elapsed = with_pin_sources_broken(&pool, || run(true)).await; + assert!( + elapsed >= std::time::Duration::from_millis(100), + "the pinata source record now RETRIES (bare call returns at once, got {elapsed:?})" + ); + assert!( + state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "an exhausted pinata record marks the set incomplete, same as the ipfs_pin path" + ); + + // Recovery arm: a later successful pinata record clears it, same as ipfs_pin. + run(false).await; + assert!( + !state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "a successful pinata record clears the marker" + ); + assert_eq!( + state.db.pin_sources_for_oid(&fx.public_oid).await.unwrap(), + vec![repo.id.clone()], + "the recovered record actually landed the source row" + ); + } + + /// U3 scenario 6 (#173, authorization): the marker arms a FALLBACK, never a bypass. + /// With the set marked incomplete and the object living only in a repo the caller + /// may not read, the scan gates every repo through the same per-caller gate, so the + /// caller is still denied and no bytes leak. + #[sqlx::test] + async fn ipfs_cid_marked_incomplete_still_denies_unauthorized_caller(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["u3deny"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3deny.git"); + let mut priv_repo = seed_repo(&owner_did, "u3deny"); + priv_repo.is_public = false; + state + .db + .create_repo(&priv_repo) + .await + .expect("seed private"); + let cid = pin_cid_for_repo(&bare, &fx.secret_oid, &state.db, &priv_repo.id).await; + + // The set is marked incomplete, so the fallback scan definitely runs. + with_pin_sources_broken(&pool, || { + repin_via_skip_branch(&state, &bare, &fx.secret_oid, &priv_repo.id) + }) + .await; + assert!( + state + .db + .pin_sources_incomplete(&fx.secret_oid) + .await + .unwrap(), + "the marker is set, so the scan fallback is armed for this object" + ); + + crate::api::ipfs::reset_preload_queries(); + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + crate::api::ipfs::preload_queries(), + 1, + "the fallback really did run (otherwise the denial below proves nothing)" + ); + assert_eq!( + st, + StatusCode::NOT_FOUND, + "the fallback scan gates every repo, so an unauthorized caller is still denied" + ); + assert!( + !body.contains("TOP SECRET"), + "the denial must not leak the withheld object's bytes" + ); + } + + /// U3 scenario 7 (#173, INV-7 upgrade path): a node already past v14 gets + /// `pinned_cids.pin_sources_incomplete` from the NEW v15 migration, re-running the + /// migrations is idempotent, and a row written before the column existed reads as + /// COMPLETE (so an upgrade cannot arm the O(repos) fallback for every legacy pin). + #[sqlx::test] + async fn pinned_cids_sources_incomplete_upgrade_path(pool: PgPool) { + let state = test_state(pool.clone()).await; + + // Pre-v15 shape: drop the column and forget v15 was applied. + sqlx::query("ALTER TABLE pinned_cids DROP COLUMN IF EXISTS pin_sources_incomplete") + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version = 15") + .execute(&pool) + .await + .unwrap(); + sqlx::query("INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) VALUES ($1, $2, $3)") + .bind("preu3oid") + .bind("preu3cid") + .bind(chrono::Utc::now().to_rfc3339()) + .execute(&pool) + .await + .unwrap(); + + state.db.run_migrations().await.expect("re-migrate"); + state + .db + .run_migrations() + .await + .expect("migrations are idempotent: a second run succeeds"); + + assert!( + !state.db.pin_sources_incomplete("preu3oid").await.unwrap(), + "a row predating the column reads COMPLETE, so the upgrade arms no fallback" + ); + state + .db + .mark_pin_sources_incomplete("preu3oid") + .await + .expect("mark after upgrade"); + assert!( + state.db.pin_sources_incomplete("preu3oid").await.unwrap(), + "the v15 column is present and writable after the upgrade" + ); + } + /// #173 (jatmn round 8, F2 — load-bearing): a legacy `pinned_cids` row keyed on a /// PROVIDER CID (Pinata/Kubo dag-pb — every release before this branch stored the /// provider CID as the resolver key, not the raw-content CID) must NOT serve raw git From 73429c7a56efb6a201a906f2d1b7b1e3dfd6631b Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:53:26 -0500 Subject: [PATCH 077/124] fix(node): sweep legacy provider-CID pins instead of waiting for a re-push `repair_legacy_provider_cid` had exactly one trigger: the already-pinned skip branch inside `pin_new_objects`, which a push only reaches when it re-carries the object. Normal git negotiation omits objects the node already has, so on an upgraded node that push generally never arrives. Meanwhile `list_pinned_cids` kept advertising the stored provider CID that this branch's `/ipfs/{cid}` deliberately withholds, so the node handed out CIDs it would then refuse to serve, and previously pinned data stayed unresolvable. Two halves, both needed. A detached background sweep walks `pinned_cids` in bounded batches with an inter-batch delay, resolves each row's repo from its recorded provenance, and reuses the existing repair (and its cost gate, so a row already keyed on a raw CIDv1 reads no bytes). Migration v16 adds a single-row cursor table so a restart mid-walk resumes rather than rewinding; the cursor advances on every row read, before any skip, so an unrepairable row cannot wedge the walk. A row whose bytes are gone, whose provenance is NULL, or whose repo is not on local disk is left exactly as it is. And `list_pinned_cids` now omits rows whose key is not a raw CIDv1, so the window before the sweep catches up advertises nothing the resolver withholds. The sweep resolves paths directly rather than through `repo_store.acquire`, which would pull cold repos back from Tigris and turn a repair pass into a bulk restore. Defaults are deliberately quiet: 64 rows per batch, 60s between batches. Ten tests cover the repair, the bytes-gone and unrepairable arms, the cost gate, the batch bound, cursor resumption, both states of one row through the advertise filter, and the degenerate empty and no-legacy-rows tables. Removing the filter, the batch bound, the cursor advance, the bytes-gone early return, or the inter-batch sleep each turns a test red. One pre-existing test used placeholder resolver keys that the new filter drops; its values are now real raw CIDv1s with every assertion unchanged. --- crates/gitlawb-node/src/config.rs | 62 +++ crates/gitlawb-node/src/db/mod.rs | 91 ++++ crates/gitlawb-node/src/ipfs_pin.rs | 160 ++++++ crates/gitlawb-node/src/main.rs | 37 ++ crates/gitlawb-node/src/test_support.rs | 638 +++++++++++++++++++++++- 5 files changed, 984 insertions(+), 4 deletions(-) diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 5480beda..aff27d1e 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -403,6 +403,39 @@ pub struct Config { /// disables that derived bucket too. #[arg(long, env = "GITLAWB_IPFS_RATE_LIMIT", default_value_t = 600)] pub ipfs_rate_limit: usize, + + /// Rows the legacy provider-CID repair sweep reads per batch (U4, #173). + /// + /// The sweep walks every `pinned_cids` row on the node once, repairing rows that + /// releases before this branch keyed on a PROVIDER CID (Kubo dag-pb / Pinata CIDv0) + /// instead of the raw-content resolver key. This bounds one batch, so the sweep can + /// never turn into a single unbounded table scan competing with request traffic. + /// Conservative on purpose: paired with the inter-batch delay below the default is + /// ~64 rows per minute, which finishes a large pin set in hours of idle background + /// work rather than one expensive burst. Must be between 1 and 100_000. + #[arg( + long, + env = "GITLAWB_PIN_REPAIR_SWEEP_BATCH", + default_value_t = 64, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=100_000) + )] + pub pin_repair_sweep_batch: i64, + + /// Seconds the legacy provider-CID repair sweep sleeps between batches (U4, #173). + /// + /// Each batch costs an indexed range scan plus, for the legacy rows in it, a + /// `git cat-file` per row. The delay is what keeps that off the DB's and the disk's + /// critical path: the sweep is repairing rows that have been unresolvable since the + /// upgrade, so finishing slowly is fine and finishing fast at the cost of live + /// traffic is not. `0` disables the pause (test and one-off operational use only). + /// Must be between 0 and 86_400. + #[arg( + long, + env = "GITLAWB_PIN_REPAIR_SWEEP_DELAY_SECS", + default_value_t = 60, + value_parser = clap::builder::RangedU64ValueParser::::new().range(0..=86_400) + )] + pub pin_repair_sweep_delay_secs: u64, } impl Config { @@ -523,6 +556,35 @@ mod tests { ); } + /// U4 (#173): the repair sweep's bounds are conservative by default and a batch of + /// 0 (a sweep that walks nothing and never terminates) is a CLI error, not a + /// runtime hang. The delay does accept 0, for tests and one-off operational runs. + #[test] + fn pin_repair_sweep_knobs_default_conservatively() { + let c = Config::parse_from(["gitlawb-node"]); + assert_eq!(c.pin_repair_sweep_batch, 64); + assert_eq!(c.pin_repair_sweep_delay_secs, 60); + + assert!(Config::try_parse_from(["gitlawb-node", "--pin-repair-sweep-batch", "0"]).is_err()); + assert!( + Config::try_parse_from(["gitlawb-node", "--pin-repair-sweep-batch", "100001"]).is_err() + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--pin-repair-sweep-batch", "8"]) + .pin_repair_sweep_batch, + 8 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--pin-repair-sweep-delay-secs", "0"]) + .pin_repair_sweep_delay_secs, + 0 + ); + assert!( + Config::try_parse_from(["gitlawb-node", "--pin-repair-sweep-delay-secs", "86401"]) + .is_err() + ); + } + #[test] fn ipfs_walk_per_source_defaults_and_rejects_out_of_range() { assert_eq!(Config::parse_from(["gitlawb-node"]).ipfs_walk_per_source, 4); diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index c2a6cb60..4b107da3 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -956,6 +956,26 @@ const MIGRATIONS: &[Migration] = &[ "ALTER TABLE pinned_cids ADD COLUMN IF NOT EXISTS pin_sources_incomplete BOOLEAN NOT NULL DEFAULT FALSE", ], }, + Migration { + version: 16, + name: "pin_repair_sweep_cursor", + stmts: &[ + // U4 (#173): the legacy provider-CID repair sweep walks `pinned_cids` in + // bounded batches over an ordered `sha256_hex` cursor. The cursor has to be + // DURABLE, or a restart rewinds the walk to the start of the table and an + // upgraded node with a large pin set never finishes repairing it. One row + // (`id = 1`, enforced by the CHECK) rather than a key-value table: there is + // exactly one sweep and no second consumer, and a real constraint beats a + // convention nobody can enforce. NEW versioned migration (never appended to + // an applied block, INV-7). No default row is inserted: an absent row is the + // "never swept" state, which the empty-string cursor start already means, so + // there is no first-run special case to get wrong. + "CREATE TABLE IF NOT EXISTS pin_repair_sweep ( + id INTEGER NOT NULL PRIMARY KEY CHECK (id = 1), + cursor TEXT NOT NULL + )", + ], + }, ]; /// Max distinct source repos recorded per pinned object (F1, #173 jatmn round 8). @@ -2368,6 +2388,65 @@ impl Db { Ok(()) } + /// One ordered batch of `pinned_cids` rows strictly after `cursor`, for the U4 + /// legacy provider-CID repair sweep. Returns `(sha256_hex, cid)` ordered by + /// `sha256_hex` (the table's primary key, so the walk rides the PK index) and + /// capped at `limit` rows, which is what BOUNDS the sweep: one pass can never read + /// more than a batch, however large the pin set is. + /// + /// Deliberately NOT filtered to legacy rows in SQL. "Is this a raw CIDv1" is a + /// multibase+codec decode (`is_raw_cidv1`), which Postgres cannot express, and a + /// prefix-match approximation would silently mis-classify keys under a different + /// multihash. The caller applies the real predicate, so `limit` bounds rows READ + /// (the DB cost), not rows repaired. + pub async fn pinned_cids_after( + &self, + cursor: &str, + limit: i64, + ) -> Result> { + let rows = sqlx::query( + "SELECT sha256_hex, cid FROM pinned_cids + WHERE sha256_hex > $1 + ORDER BY sha256_hex + LIMIT $2", + ) + .bind(cursor) + .bind(limit) + .fetch_all(&self.pool) + .await?; + Ok(rows + .into_iter() + .map(|r| (r.get::("sha256_hex"), r.get::("cid"))) + .collect()) + } + + /// Where the U4 repair sweep's walk left off, or `""` before it has ever run. + /// Empty string sorts below every hex oid, so a first run and a rewound run are + /// the same code path (`sha256_hex > ''` is the whole table). + pub async fn pin_repair_cursor(&self) -> Result { + let row = sqlx::query("SELECT cursor FROM pin_repair_sweep WHERE id = 1") + .fetch_optional(&self.pool) + .await?; + Ok(row + .map(|r| r.get::("cursor")) + .unwrap_or_default()) + } + + /// Persist the sweep's walk position. Written after every batch, so a restart + /// resumes rather than re-walking the table from the beginning. A rewrite is a + /// plain upsert: the sweep is the single writer, and re-repairing an + /// already-repaired row is a no-op anyway (the codec cost gate spares it). + pub async fn set_pin_repair_cursor(&self, cursor: &str) -> Result<()> { + sqlx::query( + "INSERT INTO pin_repair_sweep (id, cursor) VALUES (1, $1) + ON CONFLICT (id) DO UPDATE SET cursor = EXCLUDED.cursor", + ) + .bind(cursor) + .execute(&self.pool) + .await?; + Ok(()) + } + /// The repository a pinned object was recorded from (`pinned_cids.repo_id`), /// or `None` for a legacy pin (recorded before provenance existed) or an /// unpinned oid. `GET /ipfs/{cid}` uses this to gate+serve the ONE source @@ -2642,6 +2721,17 @@ impl Db { Ok(row.map(|r| r.get("recipients_tag"))) } + /// Every pinned object this node ADVERTISES (`GET /api/v1/ipfs/pins`). + /// + /// U4 (#173): rows still keyed on a legacy PROVIDER CID (Kubo dag-pb / Pinata + /// CIDv0, written by releases before this branch) are withheld from the listing. + /// The `/ipfs/{cid}` resolver recomputes the raw-content CID from the object bytes + /// and refuses any row whose stored key does not match, so advertising the legacy + /// key hands a client a CID this node deliberately will not serve. The background + /// repair sweep rewrites those rows to the raw key, and each one reappears here the + /// moment it is repaired. Filtering is done in Rust because the raw-CIDv1 test is a + /// multibase+codec decode (`is_raw_cidv1`), not something SQL can express; it is the + /// SAME predicate the repair path uses as its cost gate, so the two cannot drift. pub async fn list_pinned_cids(&self) -> Result> { let rows = sqlx::query( "SELECT sha256_hex, cid, pinned_at, pinata_cid FROM pinned_cids ORDER BY pinned_at DESC", @@ -2650,6 +2740,7 @@ impl Db { .await?; Ok(rows .into_iter() + .filter(|r| gitlawb_core::cid::is_raw_cidv1(r.get::<&str, _>("cid"))) .map(|r| PinnedCidRecord { sha256_hex: r.get("sha256_hex"), cid: r.get("cid"), diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 0e58e27e..1f0cc65a 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -99,6 +99,166 @@ async fn repair_legacy_provider_cid( db.repair_legacy_provider_cid(sha, &raw, &stored).await } +/// What one sweep pass (or a whole sweep run) did. `scanned` counts `pinned_cids` +/// rows READ, which is the quantity the batch size bounds; `repaired` counts rows +/// whose key was actually rewritten to the raw CID. +#[derive(Debug, Default, PartialEq, Eq)] +pub(crate) struct SweepStats { + pub scanned: usize, + pub repaired: usize, + pub passes: usize, +} + +/// One bounded pass of the U4 sweep: read at most `batch` `pinned_cids` rows after +/// the persisted cursor, repair the legacy ones, and persist the new cursor. +/// +/// The batch is what bounds the pass. It caps rows READ, not rows repaired, because +/// the legacy predicate is a codec decode SQL cannot express; a table of raw rows +/// therefore costs one indexed range scan per pass and nothing else. +/// +/// The cursor advances to the LAST row read whatever happened to each row, including +/// rows that were skipped as unrepairable. A cursor that only advanced on success +/// would re-read the same unrepairable row on every pass and the sweep would never +/// reach the rows behind it. +async fn sweep_pass( + repos_dir: &std::path::Path, + git_bin: &str, + git_timeout: Duration, + batch: i64, + db: &crate::db::Db, +) -> Result { + let cursor = db.pin_repair_cursor().await?; + let rows = db.pinned_cids_after(&cursor, batch).await?; + let scanned = rows.len(); + let mut repaired = 0usize; + let mut last = cursor; + + for (sha, stored) in rows { + // Advance FIRST: every path below this line may skip the row, and none of them + // may wedge the walk (scenario 7). + last = sha.clone(); + // Same cost gate as the skip-path repair: a canonical raw CIDv1 key is already + // the resolver key, so it reads no bytes and resolves no repo. + if gitlawb_core::cid::is_raw_cidv1(&stored) { + continue; + } + // Resolve the row's repo from its recorded provenance (first-pinner plus the + // bounded additional source set). An empty set is a pin recorded before + // provenance existed: nothing to read the bytes from, so skip it. + let sources = match db.pin_sources_for_oid(&sha).await { + Ok(s) => s, + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "sweep: failed to read pin sources"); + continue; + } + }; + for repo_id in sources { + let repo = match db.get_repo_by_id(&repo_id).await { + Ok(Some(r)) => r, + // The repo row is gone: a later source may still hold the bytes. + Ok(None) => continue, + Err(e) => { + tracing::warn!(repo_id = %repo_id, err = %e, "sweep: failed to read repo"); + continue; + } + }; + // Derive the LOCAL disk path rather than going through `repo_store.acquire`. + // The sweep is opportunistic background maintenance over every pinned row on + // the node, so it must never pull a cold repo back from remote storage: that + // would turn a repair pass into a bulk restore. A repo that is not on local + // disk simply reads no bytes here and stays withheld for a later pass or a + // re-push, which is the same non-destructive outcome as missing bytes. + let repo_path = + crate::git::store::repo_disk_path(repos_dir, &repo.owner_did, &repo.name); + if let Err(e) = + repair_legacy_provider_cid(&repo_path, git_bin, git_timeout, &sha, db).await + { + tracing::warn!(sha = %sha, err = %e, "sweep: legacy provider-CID repair failed"); + continue; + } + // `repair_legacy_provider_cid` is best-effort and silent about which of its + // outcomes it took (bytes gone, read failed, rewritten), so read the key back + // to decide whether to stop trying sources. A no-op re-read on an + // already-repaired row is one indexed lookup. + match db.cid_for_oid(&sha).await { + Ok(Some(c)) if gitlawb_core::cid::is_raw_cidv1(&c) => { + repaired += 1; + break; + } + Ok(_) => continue, + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "sweep: failed to re-read repaired key"); + continue; + } + } + } + } + + db.set_pin_repair_cursor(&last).await?; + Ok(SweepStats { + scanned, + repaired, + passes: 1, + }) +} + +/// Test seam for a single bounded pass (scenarios 4 and 5 drive passes by hand to +/// observe the batch bound and the restart-resumes-from-cursor behavior). +#[cfg_attr(not(test), allow(dead_code))] +pub(crate) async fn sweep_legacy_provider_cids_once( + repos_dir: &std::path::Path, + git_bin: &str, + git_timeout: Duration, + batch: i64, + db: &crate::db::Db, +) -> Result { + sweep_pass(repos_dir, git_bin, git_timeout, batch, db).await +} + +/// U4 (#173): the one-shot legacy provider-CID migration sweep. +/// +/// Releases before this branch stored the PROVIDER CID (Kubo dag-pb / Pinata CIDv0) +/// in `pinned_cids.cid`. This branch's `/ipfs/{cid}` resolver recomputes the raw +/// content CID and withholds any row whose key does not match, so those rows are +/// unresolvable. The opportunistic repair on the already-pinned skip path only fires +/// when a later push re-carries the object, and normal git negotiation omits objects +/// the node already has, so on an upgraded node that push generally never comes. This +/// walks the table instead. +/// +/// Runs until a pass comes back short of a full batch, which is the end of the table. +/// Sleeps `delay` between full batches so it cannot monopolize the DB, and persists +/// its cursor every pass so a restart continues instead of rewinding. Errors reading +/// or repairing an individual row are warn-and-skip; only a failure of the batch query +/// or the cursor write ends the run, and a later run picks up from the stored cursor. +pub(crate) async fn sweep_legacy_provider_cids( + repos_dir: &std::path::Path, + git_bin: &str, + git_timeout: Duration, + batch: i64, + delay: Duration, + db: &crate::db::Db, +) -> SweepStats { + let mut totals = SweepStats::default(); + loop { + let pass = match sweep_pass(repos_dir, git_bin, git_timeout, batch, db).await { + Ok(p) => p, + Err(e) => { + tracing::warn!(err = %e, "legacy provider-CID sweep pass failed; stopping"); + return totals; + } + }; + totals.scanned += pass.scanned; + totals.repaired += pass.repaired; + totals.passes += 1; + // A short batch means the ordered walk reached the end of the table. Stop here + // rather than after an extra empty pass, and do NOT sleep on the way out. + if (pass.scanned as i64) < batch { + return totals; + } + tokio::time::sleep(delay).await; + } +} + // Test-only cost-gate counter (R8, U7): how many times the opportunistic repair // read an object's bytes on the skip path. The codec gate must spare a CIDv1/raw // row this read; the counter is the both-ways guard (removing the gate reads the diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index fe18c706..648d7428 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -520,6 +520,43 @@ async fn main() -> Result<()> { }); } + // U4 (#173): one-shot legacy provider-CID repair sweep. Releases before this + // version stored the PROVIDER CID (Kubo dag-pb / Pinata CIDv0) in `pinned_cids.cid`, + // and this version's `/ipfs/{cid}` resolver withholds any row whose stored key is + // not the raw-content CID. The opportunistic repair on the pin path only fires when + // a push re-carries the object, which normal git negotiation makes it not do, so + // those rows need a walk. DETACHED, never on the boot path: the server below starts + // and serves while this runs, and the sweep's own batch bound plus inter-batch delay + // keep it off the DB's critical path. Its cursor is durable, so a restart mid-walk + // resumes instead of rewinding. + { + let db = state.db.clone(); + let repos_dir = config.repos_dir.clone(); + let git_bin = state.git_bin.clone(); + let git_timeout = std::time::Duration::from_secs(config.git_service_timeout_secs); + let batch = config.pin_repair_sweep_batch; + let delay = std::time::Duration::from_secs(config.pin_repair_sweep_delay_secs); + let mut shutdown_rx = state.subscribe_shutdown(); + tokio::spawn(async move { + tokio::select! { + stats = ipfs_pin::sweep_legacy_provider_cids( + &repos_dir, &git_bin, git_timeout, batch, delay, &db, + ) => { + if stats.repaired > 0 { + tracing::info!( + scanned = stats.scanned, + repaired = stats.repaired, + "legacy provider-CID sweep finished" + ); + } + } + // Shutdown mid-walk simply drops the run; the persisted cursor means the + // next boot picks up where this one stopped. + _ = shutdown_rx.changed() => {} + } + }); + } + let router = server::build_router(state.clone()); // Re-register the socket bound at startup — same fd, so there was never a // moment with the port closed between the degraded and full servers. diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index fd90e46a..47428248 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -3523,11 +3523,17 @@ mod tests { async fn record_pinata_cid_stores_and_coalesces_provenance(pool: PgPool) { let state = test_state(pool).await; + // Real raw-CIDv1 resolver keys, as the pin paths write them: `list_pinned_cids` + // withholds any row keyed on a non-raw (legacy provider) value (U4, #173), so a + // placeholder string here would be filtered out and make the assertions vacuous. + let raw1 = gitlawb_core::cid::Cid::from_git_object_bytes(b"pinata raw 1").to_string(); + let local2 = gitlawb_core::cid::Cid::from_git_object_bytes(b"local raw 2").to_string(); + // A new row created via the pinata path carries provenance, and stores the // raw CID in `cid` with the provider CID in `pinata_cid`. state .db - .record_pinata_cid("po1", "rawcid1", "pcid1", Some("repoA")) + .record_pinata_cid("po1", &raw1, "pcid1", Some("repoA")) .await .unwrap(); assert_eq!( @@ -3542,7 +3548,7 @@ mod tests { .into_iter() .find(|r| r.sha256_hex == "po1") .expect("po1 row exists"); - assert_eq!(po1.cid, "rawcid1", "resolver-key cid is the raw CID"); + assert_eq!(po1.cid, raw1, "resolver-key cid is the raw CID"); assert_eq!( po1.pinata_cid.as_deref(), Some("pcid1"), @@ -3553,7 +3559,7 @@ mod tests { // prior local pin's `cid` is left untouched (not overwritten by the raw arg). state .db - .record_pinned_cid("po2", "localcid2", None) + .record_pinned_cid("po2", &local2, None) .await .unwrap(); state @@ -3575,7 +3581,7 @@ mod tests { .find(|r| r.sha256_hex == "po2") .expect("po2 row exists"); assert_eq!( - po2.cid, "localcid2", + po2.cid, local2, "on conflict the prior local pin's cid is left untouched" ); @@ -4273,6 +4279,630 @@ mod tests { ); } + // ---- #173 U4: legacy provider-CID migration sweep ---- + + /// Seed a legacy PROVIDER-CID `pinned_cids` row for `oid` (the pre-branch shape: + /// `cid` holds the Kubo dag-pb / Pinata key, not the raw-content resolver key). + /// Returns `(raw_cid, provider_cid)`. Raw SQL because every production helper + /// stores the already-correct raw key. + async fn seed_legacy_pin( + pool: &PgPool, + bare: &std::path::Path, + oid: &str, + repo_id: Option<&str>, + ) -> (String, String) { + let (_ty, bytes) = crate::git::store::read_object(bare, oid) + .expect("read object bytes") + .expect("object exists in the bare repo"); + let raw = gitlawb_core::cid::Cid::from_git_object_bytes(&bytes).to_string(); + let provider = legacy_dagpb_cid(&raw); + assert_ne!(provider, raw, "the legacy key differs from the raw key"); + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + ) + .bind(oid) + .bind(&provider) + .bind("2020-01-01T00:00:00Z") + .bind(repo_id) + .execute(pool) + .await + .unwrap(); + (raw, provider) + } + + /// The `pinned_cids.cid` currently stored for an oid, unfiltered (unlike + /// `list_pinned_cids`, which withholds unrepaired legacy rows). + async fn stored_pin(pool: &PgPool, oid: &str) -> (String, Option) { + sqlx::query_as("SELECT cid, legacy_provider_cid FROM pinned_cids WHERE sha256_hex = $1") + .bind(oid) + .fetch_one(pool) + .await + .unwrap() + } + + /// U4 (#173, INV-7 upgrade path): a node already at the prior-max schema (v15) gets + /// the `pin_repair_sweep` cursor table from the NEW v16 migration. Simulate the + /// pre-v16 node by dropping the table and un-applying v16, then re-migrate and + /// assert the cursor round-trips. RED before the v16 migration exists (the table is + /// never recreated, so the cursor read errors). + #[sqlx::test] + async fn pin_repair_sweep_cursor_upgrade_path(pool: PgPool) { + let state = test_state(pool.clone()).await; + + // Pre-v16 shape: drop the table and forget v16 was applied. + sqlx::query("DROP TABLE IF EXISTS pin_repair_sweep") + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version = 16") + .execute(&pool) + .await + .unwrap(); + + state.db.run_migrations().await.expect("migrate to v16"); + + // Absent row reads as the "never swept" start, and a write round-trips. + assert_eq!( + state.db.pin_repair_cursor().await.unwrap(), + "", + "a node that has never swept starts at the beginning of the table" + ); + state.db.set_pin_repair_cursor("abc").await.unwrap(); + state.db.set_pin_repair_cursor("def").await.unwrap(); + assert_eq!( + state.db.pin_repair_cursor().await.unwrap(), + "def", + "the v16 cursor table persists the walk position across writes" + ); + let rows: i64 = sqlx::query_scalar("SELECT count(*) FROM pin_repair_sweep") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(rows, 1, "the cursor is a single row, not an append log"); + } + + /// U4 scenario 1 (#173): a legacy provider-CID row with intact object bytes is + /// repaired to the raw-content resolver key by the SWEEP alone, with the old value + /// stashed in `legacy_provider_cid`. No push, no re-pin: this is the whole point of + /// U4, because normal git negotiation omits objects the node already has, so the + /// skip-branch repair's re-push trigger generally never fires on an upgraded node. + /// RED before the sweep is implemented (the row keeps its provider key). + #[sqlx::test] + async fn sweep_repairs_legacy_row_without_a_push(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["swsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("swsrc.git"); + let repo = seed_repo(&owner_did, "swsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + let (raw_cid, provider_cid) = + seed_legacy_pin(&pool, &bare, &fx.public_oid, Some(&repo.id)).await; + + let stats = crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + ) + .await; + assert_eq!(stats.repaired, 1, "the sweep repairs the one legacy row"); + + let (stored, stashed) = stored_pin(&pool, &fx.public_oid).await; + assert_eq!( + stored, raw_cid, + "the key is rewritten to the raw-content CID" + ); + assert_eq!( + stashed.as_deref(), + Some(provider_cid.as_str()), + "the old provider CID is stashed in legacy_provider_cid" + ); + + // End to end: the repaired key is now advertised AND serves. + assert!( + state + .db + .list_pinned_cids() + .await + .unwrap() + .iter() + .any(|r| r.cid == raw_cid), + "the repaired row is advertised" + ); + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&raw_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "the repaired raw key serves"); + assert!(body.contains("public bytes"), "the object's bytes serve"); + } + + /// U4 scenario 2 (#173): a legacy row whose object bytes are gone is left exactly + /// as it is by the sweep: never rewritten, never deleted. The row stays withheld + /// until the bytes come back, which is the non-destructive contract the skip-branch + /// repair already holds. + #[sqlx::test] + async fn sweep_leaves_a_bytes_gone_row_untouched(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let _fx = seed_cid_repos(&slug, &short, &["gonesrc"]); + let repo = seed_repo(&owner_did, "gonesrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + // An oid whose bytes are NOT in the repo, but whose provenance resolves fine. + let phantom_oid = "d".repeat(64); + let raw_cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"bytes that live nowhere").to_string(); + let provider_cid = legacy_dagpb_cid(&raw_cid); + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + ) + .bind(&phantom_oid) + .bind(&provider_cid) + .bind("2020-01-01T00:00:00Z") + .bind(&repo.id) + .execute(&pool) + .await + .unwrap(); + + let stats = crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + ) + .await; + assert_eq!(stats.repaired, 0, "an unrepairable row is not repaired"); + + let (stored, stashed) = stored_pin(&pool, &phantom_oid).await; + assert_eq!( + stored, provider_cid, + "the bytes-gone row keeps its provider CID (no destructive rewrite)" + ); + assert_eq!(stashed, None, "nothing is stashed when the bytes are gone"); + let count: i64 = sqlx::query_scalar("SELECT count(*) FROM pinned_cids") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(count, 1, "the row is not deleted"); + } + + /// U4 scenario 3 (#173): the sweep inherits `repair_legacy_provider_cid`'s cost + /// gate, so a row already keyed on a raw CIDv1 is NEVER read for bytes. The + /// test-only `legacy_repair_reads` counter is the both-ways guard: dropping the + /// codec gate reads the raw row and trips it off zero. + #[sqlx::test] + async fn sweep_never_reads_bytes_for_a_raw_cidv1_row(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["rawsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("rawsrc.git"); + let repo = seed_repo(&owner_did, "rawsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + let raw_cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo.id).await; + assert!( + gitlawb_core::cid::is_raw_cidv1(&raw_cid), + "the seeded row is already the canonical resolver key" + ); + + crate::ipfs_pin::reset_legacy_repair_reads(); + let stats = crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + ) + .await; + assert_eq!(stats.scanned, 1, "the sweep walked the row"); + assert_eq!(stats.repaired, 0, "a raw row needs no repair"); + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 0, + "a raw-CIDv1 row is never read for bytes (cost gate)" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + raw_cid, + "the raw row is left as-is" + ); + } + + /// U4 scenario 4 (#173, BOUND): one pass reads at most `batch` rows, so it repairs + /// at most `batch` of them. The exact count is asserted, so raising or removing the + /// bound fails. This is what keeps the sweep from monopolizing the DB on a node + /// with a large `pinned_cids` table. + #[sqlx::test] + async fn sweep_one_pass_is_bounded_by_the_batch_size(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["batchsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("batchsrc.git"); + let repo = seed_repo(&owner_did, "batchsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + // Five legacy rows, batch of two. + for oid in [ + &fx.public_oid, + &fx.secret_oid, + &fx.public_tree_oid, + &fx.secret_tree_oid, + &fx.commit_oid, + ] { + seed_legacy_pin(&pool, &bare, oid, Some(&repo.id)).await; + } + + let stats = crate::ipfs_pin::sweep_legacy_provider_cids_once( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 2, + &state.db, + ) + .await + .expect("one pass runs"); + assert_eq!(stats.scanned, 2, "one pass reads exactly the batch size"); + assert_eq!(stats.repaired, 2, "one pass repairs at most the batch size"); + + let repaired: i64 = sqlx::query_scalar( + "SELECT count(*) FROM pinned_cids WHERE legacy_provider_cid IS NOT NULL", + ) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(repaired, 2, "exactly two of the five rows were rewritten"); + } + + /// U4 scenario 5 (#173, RESUMPTION): the walk cursor persists, so a sweep + /// interrupted mid-table continues from where it stopped instead of restarting. + /// Two bounded passes are driven by hand (the restart), and the second pass is + /// asserted to repair the NEXT two rows in cursor order, not the first two again. + /// The read counter proves the already-repaired rows are not re-read. + #[sqlx::test] + async fn sweep_resumes_from_the_persisted_cursor(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["resumesrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("resumesrc.git"); + let repo = seed_repo(&owner_did, "resumesrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + let mut oids = vec![ + fx.public_oid.clone(), + fx.secret_oid.clone(), + fx.public_tree_oid.clone(), + fx.secret_tree_oid.clone(), + ]; + for oid in &oids { + seed_legacy_pin(&pool, &bare, oid, Some(&repo.id)).await; + } + // The cursor is an ordered walk over the `pinned_cids` primary key. + oids.sort(); + + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let pass1 = crate::ipfs_pin::sweep_legacy_provider_cids_once( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 2, + &state.db, + ) + .await + .expect("pass 1 runs"); + assert_eq!(pass1.repaired, 2, "pass 1 repairs the first two rows"); + + // The restart: a second pass over the SAME state must continue, not rewind. + crate::ipfs_pin::reset_legacy_repair_reads(); + let pass2 = crate::ipfs_pin::sweep_legacy_provider_cids_once( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 2, + &state.db, + ) + .await + .expect("pass 2 runs"); + assert_eq!(pass2.repaired, 2, "pass 2 repairs the NEXT two rows"); + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 2, + "pass 2 reads bytes only for the two rows it repaired; the already-repaired \ + rows are not re-read" + ); + for oid in &oids { + let (_cid, stashed) = stored_pin(&pool, oid).await; + assert!( + stashed.is_some(), + "every row is repaired after two resumed passes" + ); + } + } + + /// U4 scenario 7 (#173, cursor liveness): a row that cannot be repaired (NULL + /// provenance, or a provenance whose repo row is gone) is skipped AND the cursor + /// still advances past it. With `batch = 1` the two unrepairable rows sort first, + /// so a cursor that failed to advance would re-read the same row forever and never + /// reach the repairable row behind them. The outer timeout turns that into a + /// FAILURE rather than a hung suite. + #[sqlx::test] + async fn sweep_advances_past_unrepairable_rows(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["skipsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("skipsrc.git"); + let repo = seed_repo(&owner_did, "skipsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + // Two blockers that sort ahead of any real 64-hex oid: one with NULL + // provenance, one naming a repo row that no longer exists. + let null_prov_oid = "0".repeat(64); + let ghost_repo_oid = format!("{}1", "0".repeat(63)); + seed_legacy_pin(&pool, &bare, &fx.public_oid, Some(&repo.id)).await; + for (oid, prov) in [ + (&null_prov_oid, None), + (&ghost_repo_oid, Some("repo-that-is-gone")), + ] { + let raw = gitlawb_core::cid::Cid::from_git_object_bytes(oid.as_bytes()).to_string(); + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + ) + .bind(oid) + .bind(legacy_dagpb_cid(&raw)) + .bind("2020-01-01T00:00:00Z") + .bind(prov) + .execute(&pool) + .await + .unwrap(); + } + assert!( + null_prov_oid < fx.public_oid && ghost_repo_oid < fx.public_oid, + "the blockers really do sort ahead of the repairable row" + ); + + let stats = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 1, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the sweep terminates instead of looping on an unrepairable row"); + + assert_eq!( + stats.repaired, 1, + "the sweep advanced past both blockers and repaired the row behind them" + ); + assert!( + stored_pin(&pool, &fx.public_oid).await.1.is_some(), + "the row behind the blockers is the one that got repaired" + ); + for oid in [&null_prov_oid, &ghost_repo_oid] { + assert_eq!( + stored_pin(&pool, oid).await.1, + None, + "an unrepairable row is left untouched" + ); + } + } + + /// U4 scenario 8 (#173, degenerate states): an empty `pinned_cids` table and a + /// table with zero legacy rows both complete cleanly, with no repair and no read. + #[sqlx::test] + async fn sweep_completes_on_degenerate_tables(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + + // Empty table. + crate::ipfs_pin::reset_legacy_repair_reads(); + let empty = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 4, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the sweep terminates on an empty table"); + assert_eq!( + (empty.scanned, empty.repaired), + (0, 0), + "an empty table is a clean no-op" + ); + + // Zero legacy rows: every row already carries the canonical raw key. + let fx = seed_cid_repos(&slug, &short, &["degensrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("degensrc.git"); + let repo = seed_repo(&owner_did, "degensrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + for oid in [&fx.public_oid, &fx.secret_oid, &fx.commit_oid] { + pin_cid_for_repo(&bare, oid, &state.db, &repo.id).await; + } + + let clean = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 4, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the sweep terminates on a table with no legacy rows"); + assert_eq!(clean.scanned, 3, "every row is walked"); + assert_eq!(clean.repaired, 0, "nothing needs repair"); + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 0, + "no object bytes are read when no row is legacy" + ); + } + + /// U4 (#173, BOUND): the inter-batch delay is real, observed by wall clock. Five + /// rows at a batch of two means two full batches and a trailing partial one, so the + /// run sleeps twice. Without the sleep the whole run is sub-millisecond DB work and + /// a node's `pinned_cids` table gets walked as fast as Postgres will answer. + #[sqlx::test] + async fn sweep_sleeps_between_batches(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["delaysrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("delaysrc.git"); + let repo = seed_repo(&owner_did, "delaysrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + for oid in [ + &fx.public_oid, + &fx.secret_oid, + &fx.public_tree_oid, + &fx.secret_tree_oid, + &fx.commit_oid, + ] { + pin_cid_for_repo(&bare, oid, &state.db, &repo.id).await; + } + + let delay = std::time::Duration::from_millis(150); + let started = std::time::Instant::now(); + let stats = crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 2, + delay, + &state.db, + ) + .await; + let elapsed = started.elapsed(); + + assert_eq!( + stats.passes, 3, + "five rows at a batch of two is three passes" + ); + assert!( + elapsed >= delay * 2, + "the run sleeps once between each pair of full batches: {elapsed:?} < {:?}", + delay * 2 + ); + } + + /// U4 scenario 6 (#173): `list_pinned_cids` never advertises a key the `/ipfs` + /// resolver would withhold. The resolver recomputes the raw CIDv1 from the object + /// bytes and 404s any row keyed on a legacy PROVIDER CID, so advertising that key + /// hands clients a CID this node deliberately refuses. Both states of ONE row are + /// asserted (omitted while legacy, present once repaired) so the test cannot pass + /// by accident. RED before the `is_raw_cidv1` filter lands: the legacy row is + /// advertised. + #[sqlx::test] + async fn list_pinned_cids_omits_unrepaired_legacy_row(pool: PgPool) { + let state = test_state(pool).await; + + let raw_cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"u4 advertise bytes").to_string(); + let provider_cid = legacy_dagpb_cid(&raw_cid); + let oid = "c".repeat(64); + state + .db + .record_pinned_cid(&oid, &provider_cid, None) + .await + .unwrap(); + + let listed = state.db.list_pinned_cids().await.unwrap(); + assert!( + !listed.iter().any(|r| r.sha256_hex == oid), + "an unrepaired legacy provider-CID row is not advertised" + ); + + // Same row, repaired: it comes back, keyed on the raw CID the resolver serves. + state + .db + .repair_legacy_provider_cid(&oid, &raw_cid, &provider_cid) + .await + .unwrap(); + let listed = state.db.list_pinned_cids().await.unwrap(); + let rec = listed + .iter() + .find(|r| r.sha256_hex == oid) + .expect("the repaired row is advertised again"); + assert_eq!( + rec.cid, raw_cid, + "the advertised key is the raw-content resolver key" + ); + } + /// #173 (provenance-path throttle): a walk-requiring provenanced candidate whose /// per-IP walk quota is spent returns 429 (the provenance arm's Throttled outcome, /// then the fall-through). quota=1, keyed on XFF. The first reader request runs the From 1c82060c2bbd4459dec3bc2d17dbd65eb996e8fc Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:47:56 -0500 Subject: [PATCH 078/124] fix(node): stop the lock pool from being occupied by spinners and racing the reaper Three regressions from the previous four commits, each demonstrated before being fixed. The advisory-lock retry loop checked its connection out before spinning, so a caller that lost the lock held a pool connection through all sixty one-second attempts. The old pool-per-query shape returned it between iterations, so a spinner occupied nothing. That matters more than the pool sizing suggested: `acquire_write` has three non-push callers (two in issues, one in pulls) that hold no concurrency permit, and the issue routes carry no rate limit, so any self-minted DID could fire enough concurrent closes at one repo to strand the pool and fail authenticated pushes on every repo. The checkout now happens per attempt and a losing attempt returns the connection before sleeping, so only the winner retains one. Lock-pool exhaustion also carries its own error now and sheds 503 with Retry-After instead of surfacing as a 500. Holding the lock on one connection then meant a client disconnect released it early: the handler future dies mid-receive-pack, the guard drops without `release`, and `after_release` frees the lock while the detached reaper is still giving the process group its SIGTERM grace. A second writer was admitted 90ms after the drop, well inside that window, which is the invariant smart_http already documents. The write guard now rides the AdmissionGuard into the reaper, the same seam the admission permits already use, so the lock outlives the group. The success path still reclaims the guard and releases it synchronously, and a dropped guard performs no Tigris upload, so an interrupted push cannot publish a half-applied repo. The requeue loop's exit on an exhausted re-read discarded a push that coalesced during the retry window: `requeue_or_release` had cleared the dirty bit without marking the guard released, so `Drop` removed the key and no pass was ever attempted. It now falls through with nothing to replicate and lets the tail's atomic check-and-clear decide. A lap only happens when a push actually coalesced and each pays a full bounded re-read, so a sustained outage still terminates, which is tested with ten thousand injected faults under a watchdog rather than argued. Also clamps the lock pool's derived size, which previously followed max_concurrent_git_pushes up to its million-connection ceiling, and documents the node's total connection budget on that knob. --- crates/gitlawb-node/src/api/repos.rs | 462 +++++++++++++++++++++- crates/gitlawb-node/src/config.rs | 11 + crates/gitlawb-node/src/git/repo_store.rs | 257 ++++++++++-- crates/gitlawb-node/src/git/smart_http.rs | 32 +- crates/gitlawb-node/src/main.rs | 65 ++- crates/gitlawb-node/src/test_support.rs | 192 +++++++++ 6 files changed, 969 insertions(+), 50 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 35c49dd8..c8e9f2a6 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1146,10 +1146,25 @@ async fn run_post_push_replication( is_public, owner_did, } => (Some(rules), is_public, owner_did), - // The repo is gone (terminal) or the re-read never succeeded (already logged - // at ERROR). Either way there is no fresh state to act on, so exit; the guard - // Drop removes the key so the repo is never locked out of a future task. - RequeueRefresh::Gone | RequeueRefresh::Failed => break, + // The repo is gone: terminal, and there will never be fresh state to act on. + // Exit; the guard Drop removes the key so the repo is never locked out of a + // future task. + RequeueRefresh::Gone => break, + // The re-read never succeeded (already logged at ERROR). Do NOT exit here + // (#173 F3): `requeue_or_release` cleared the dirty bit without marking the + // guard released, so a push that coalesced during the ~350ms retry window is + // recorded only in a flag that `EncryptInflightGuard::drop` then removes, + // with no pass ever attempted and no reconciliation sweep to re-derive it. + // Fall through with nothing to replicate and let the TAIL's atomic + // check-and-clear make the call: dirty -> one more lap (which re-reads + // afresh), clean -> exit. Bounded under a sustained outage, because a lap + // only happens when a push actually coalesced and each one pays a full + // bounded re-read (REQUEUE_REREAD_MAX_ATTEMPTS with backoff), so this cannot + // hot-spin. + RequeueRefresh::Failed => { + withheld = None; + continue; + } }; let (_announce, r_withheld) = replication_withheld_set( r_rules.clone(), @@ -1225,6 +1240,23 @@ pub(crate) async fn run_post_push_replication_for_test( .await; } +/// Map an `acquire_write` failure to the right `AppError`. An exhausted repo write-lock +/// POOL is a capacity signal, not a broken repo, so it sheds 503 + Retry-After the same +/// way the admission caps around it do; it used to fall into the generic git 500, which +/// tells the client nothing about retrying (#173 F1). Anything else stays a git error. +fn acquire_write_app_error(err: &anyhow::Error, repo: &str) -> AppError { + if err + .downcast_ref::() + .is_some() + { + tracing::warn!(repo = %repo, err = %err, "write-lock pool exhausted; shedding with 503"); + AppError::Overloaded("git write locks at capacity, retry shortly".into()) + } else { + tracing::error!(repo = %repo, err = %err, "acquire_write failed"); + AppError::Git(err.to_string()) + } +} + /// Map an error from a `smart_http` git service call to the right `AppError`: /// [`smart_http::GitServiceTimeout`] to 504, a malformed client request to 400, /// anything else to a 500 git error. Pure (no logging) so it is unit-testable; @@ -1678,10 +1710,7 @@ pub async fn git_receive_pack( tracing::warn!(repo = %name, "acquire_write timed out; shedding with 503"); AppError::Overloaded("git service acquisition timed out, retry shortly".into()) })? - .map_err(|e| { - tracing::error!(repo = %name, err = %e, "acquire_write failed"); - AppError::Git(e.to_string()) - })?; + .map_err(|e| acquire_write_app_error(&e, name))?; let disk_path = guard.path().to_path_buf(); tracing::debug!(repo = %name, path = %disk_path.display(), "running git receive-pack"); let body_len = body.len(); @@ -1691,7 +1720,24 @@ pub async fn git_receive_pack( // instant a disconnect drops this future while the detached reaper runs (#174 P1-a). // The handler keeps no copy. This is independent of the write-lock `guard.release` // below: admission tracks the git process lifetime, the write lock tracks the repo. - let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + // + // The WRITE LOCK rides the same seam (#173 F2). `guard.release(..)` below is only + // reached if `receive_pack` returns, so on a client disconnect the guard would drop + // with the future and the lock pool's `after_release` hook would free the advisory + // lock immediately, while `KillGroupOnDrop`'s detached reaper is still giving the + // group its ~2s SIGTERM grace. A second push admitted in that window puts two + // `git receive-pack` groups on one repo, which is exactly what the timeout path + // reaps to prevent ("a caller releasing a write lock can't race them"). Sharing the + // guard rather than moving it outright is what lets the SUCCESS path still reclaim + // it for the Tigris upload: the copy retained here can only DELAY release, never + // perform it early, because the handler reaches the take below only after + // `receive_pack` has returned (group reaped or disarmed). On the disconnect path + // this copy dies with the future and the reaper's copy is last, so the lock frees + // after the reap with no upload, which is the release(success = false) semantics an + // interrupted push must have. + let guard = std::sync::Arc::new(std::sync::Mutex::new(Some(guard))); + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit) + .with_hold(std::sync::Arc::clone(&guard)); let receive_result = smart_http::receive_pack( &state.git_bin, &disk_path, @@ -1704,7 +1750,12 @@ pub async fn git_receive_pack( // Always release the advisory lock — even on error — to prevent stale locks // from blocking subsequent pushes. Only upload to Tigris when the push // succeeded; uploading a half-applied repo would propagate corruption. - guard.release(receive_result.is_ok()).await; + let reclaimed = guard + .lock() + .expect("repo write-lock mutex poisoned") + .take() + .expect("the write lock is only taken here, and only once"); + reclaimed.release(receive_result.is_ok()).await; let result = receive_result.map_err(|e| { let app = git_service_app_error(&e); @@ -4684,6 +4735,397 @@ mod tests { ); } + /// Reproduce `repo_store::advisory_lock_key` (private there) so a test can probe the + /// exact key `acquire_write` derives. + #[cfg(unix)] + fn write_lock_key(owner_slug: &str, repo_name: &str) -> i64 { + use std::hash::{Hash, Hasher}; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + owner_slug.hash(&mut hasher); + repo_name.hash(&mut hasher); + hasher.finish() as i64 + } + + #[cfg(unix)] + fn pid_alive(pid: i32) -> bool { + // SAFETY: kill(2) with signal 0 only probes; it takes integers and borrows no + // Rust memory. + unsafe { libc::kill(pid, 0) == 0 } + } + + /// SIGKILL the recorded pids if the test unwinds, so a RED run leaks no orphan. + #[cfg(unix)] + struct KillOnPanic(Vec); + #[cfg(unix)] + impl Drop for KillOnPanic { + fn drop(&mut self) { + for pid in &self.0 { + // SAFETY: as above. + unsafe { + libc::kill(*pid, libc::SIGKILL); + } + } + } + } + + /// Is the repo write lock takeable from an INDEPENDENT session right now? Session + /// advisory locks are re-entrant within their own session, so this must not run on + /// any connection the code under test might be using. + #[cfg(unix)] + async fn write_lock_is_takeable(pool: &sqlx::PgPool, key: i64) -> bool { + let mut probe = pool.acquire().await.expect("probe connection"); + let taken: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *probe) + .await + .expect("probe try-lock"); + if taken.0 { + sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *probe) + .await + .expect("probe unlock"); + } + taken.0 + } + + /// #173 F2 (RED-before/GREEN-after): on a CLIENT DISCONNECT the repo write lock must + /// stay held until the receive-pack process group is confirmed reaped. + /// + /// The handler's `guard.release(..)` line is only reached if `receive_pack` returns. + /// When the request future is dropped mid-push the guard drops instead, and (since + /// #173 U1 gave the lock pool an `after_release` hook) that FREES the advisory lock + /// immediately, while `KillGroupOnDrop`'s detached reaper is still giving the group + /// its ~2s SIGTERM grace. A second `acquire_write` admitted inside that window puts + /// two `git receive-pack` groups on one repo. `smart_http.rs` states the invariant + /// the other way round on the timeout path: "a caller releasing a write lock can't + /// race them". + /// + /// Real seam, not a stand-in: the production `git_receive_pack` handler, a fake git + /// whose descendant IGNORES SIGTERM (so the group genuinely survives the grace and + /// the window is ~2s wide, not a scheduling artifact), and the lock probed from an + /// independent session. RED before the fix: the lock is takeable while the group is + /// still alive. GREEN after: takeable only once the group is gone. + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_disconnect_holds_the_write_lock_until_the_group_is_reaped( + pool: sqlx::PgPool, + ) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let owner = "z6disc"; + let name = "dc1"; + let repos_dir = tempfile::TempDir::new().unwrap(); + let tmp = tempfile::TempDir::new().unwrap(); + let descfile = tmp.path().join("desc.pid"); + // The leader dies on the group SIGTERM; its descendant traps SIGTERM and loops + // (bounded at ~30s so a RED run leaks nothing permanent), so the group is only + // gone once the reaper escalates to SIGKILL. The descendant inherits the stdout + // pipe, which keeps drive_git_child's read_to_end pending until we drop. + let body = format!( + "#!/bin/sh\n\ + sh -c 'trap \"\" TERM; echo $$ > \"{}\"; i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done' &\n\ + wait\n", + descfile.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + + let mut state = crate::test_support::test_state(pool.clone()).await; + state.git_bin = git_bin; + state.repo_store = crate::git::repo_store::RepoStore::new( + repos_dir.path().to_path_buf(), + None, + crate::git::repo_store::build_lock_pool(&pool, 4, std::time::Duration::from_secs(5)), + ); + let mut cfg = (*state.config).clone(); + // Long enough that the git-service timeout is never what ends this push; the + // disconnect is. + cfg.git_service_timeout_secs = 600; + state.config = std::sync::Arc::new(cfg); + state + .db + .upsert_mirror_repo(owner, name, "/tmp/z6disc-dc1", None, false) + .await + .unwrap(); + + // The mirror row stores the short owner as owner_did, so the slug is the owner. + let key = write_lock_key(&owner.replace([':', '/'], "_"), name); + // Probe from a pool that is NOT the store's lock pool and NOT the harness pool. + let probe = sqlx::postgres::PgPoolOptions::new() + .max_connections(2) + .connect_lazy_with((*pool.connect_options()).clone()); + + assert!( + write_lock_is_takeable(&probe, key).await, + "the write lock must be free before the push" + ); + + // Drive the handler a slice at a time until the fake git's SIGTERM-ignoring + // descendant records its pid, i.e. receive-pack is genuinely running under the + // write lock. `Ok(_)` means the handler returned early; stop polling then, since + // re-polling a completed future panics. + // + // Retried on a miss for the same reason `smart_http`'s disconnect tests retry: + // under `cargo test` fork-storm load a freshly written fake `git` can transiently + // fail to exec (ETXTBSY, a concurrent worker forked while its write fd was open), + // which leaves no pid. A losing attempt's future is dropped, which reaps whatever + // spawned and releases its write lock, so retries do not leak. The winning + // attempt's future is kept PENDING: dropping it below is the disconnect under test. + const SPAWN_ATTEMPTS: u64 = 12; + let (fut, desc) = { + let mut attempt = 0u64; + loop { + attempt += 1; + let _ = std::fs::remove_file(&descfile); + let mut fut = Box::pin(git_receive_pack( + State(state.clone()), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid( + "did:key:z6MkDisconnectWriteLockProofDidAAAAAAAA".to_string(), + )), + crate::rate_limit::PeerAddr(Some( + "203.0.113.81:5000".parse::().unwrap(), + )), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + )); + let mut found: Option = None; + for _ in 0..500 { + let finished = + tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut) + .await + .is_ok(); + if let Some(p) = std::fs::read_to_string(&descfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + found = Some(p); + break; + } + if finished { + break; + } + } + match found { + Some(p) => break (fut, p), + None => { + drop(fut); + assert!( + attempt < SPAWN_ATTEMPTS, + "the push never reached receive-pack after {SPAWN_ATTEMPTS} \ + attempts (persistent failure, not a transient runner miss)" + ); + tokio::time::sleep(std::time::Duration::from_millis(100 * attempt)).await; + } + } + } + }; + let _cleanup = KillOnPanic(vec![desc]); + assert!( + pid_alive(desc), + "the receive-pack group must be running before the disconnect" + ); + assert!( + !write_lock_is_takeable(&probe, key).await, + "the write lock must be held while receive-pack runs" + ); + + // Client disconnect: drop the request future mid-receive-pack. + drop(fut); + + let mut takeable_while_group_alive = false; + let mut freed_after_reap = false; + for _ in 0..800 { + let takeable = write_lock_is_takeable(&probe, key).await; + let group_alive = pid_alive(desc); + if takeable && group_alive { + takeable_while_group_alive = true; + } + if takeable && !group_alive { + freed_after_reap = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + // Clean up regardless so a RED run leaves no orphan behind. + // SAFETY: kill(2) takes integers only. + unsafe { + libc::kill(desc, libc::SIGKILL); + } + assert!( + !takeable_while_group_alive, + "the repo write lock was takeable while a receive-pack group was still alive \ + on that repo: a second push can enter and two git receive-pack groups run \ + against one repo (#173 F2)" + ); + assert!( + freed_after_reap, + "the write lock must be released once the disconnected push's group is reaped" + ); + } + + /// #173 F2, the other half: carrying the write lock through the admission seam must + /// NOT cost the success path its `release(true)`. A push that completes normally has + /// to reclaim the lock and release it explicitly (that is what performs the Tigris + /// upload), synchronously, not leave it to the pool's `after_release` net. The lock + /// is probed immediately after the handler returns, with no polling, so a fix that + /// only ever dropped the guard would fail here. + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_success_reclaims_and_releases_the_write_lock(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let owner = "z6succ"; + let name = "sc1"; + let repos_dir = tempfile::TempDir::new().unwrap(); + let tmp = tempfile::TempDir::new().unwrap(); + // A receive-pack that succeeds. It DRAINS stdin first: exiting while the handler + // is still writing the request body would EPIPE that write, which + // `drive_git_child` surfaces as an error after a successful exit status, making + // the push fail for a reason that has nothing to do with the lock under test. + let git_bin = write_fake_git(tmp.path(), "#!/bin/sh\ncat >/dev/null\nexit 0\n"); + + let mut state = crate::test_support::test_state(pool.clone()).await; + state.git_bin = git_bin; + state.repo_store = crate::git::repo_store::RepoStore::new( + repos_dir.path().to_path_buf(), + None, + crate::git::repo_store::build_lock_pool(&pool, 4, std::time::Duration::from_secs(5)), + ); + state + .db + .upsert_mirror_repo(owner, name, "/tmp/z6succ-sc1", None, false) + .await + .unwrap(); + + let key = write_lock_key(&owner.replace([':', '/'], "_"), name); + let probe = sqlx::postgres::PgPoolOptions::new() + .max_connections(2) + .connect_lazy_with((*pool.connect_options()).clone()); + + // Retried ONLY on the ETXTBSY exec race a freshly written fake `git` hits under + // fork-storm load (a concurrent test worker forked while its write fd was open). + // Narrow on purpose: any other failure still fails the assertion below loudly. + const SPAWN_ATTEMPTS: u64 = 12; + let mut result = None; + for attempt in 1..=SPAWN_ATTEMPTS { + let outcome = tokio::time::timeout( + std::time::Duration::from_secs(30), + git_receive_pack( + State(state.clone()), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid( + "did:key:z6MkPushSuccessReleaseProofDidAAAAAAAA".to_string(), + )), + crate::rate_limit::PeerAddr(Some( + "203.0.113.83:5000".parse::().unwrap(), + )), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ), + ) + .await + .expect("the push must return"); + let exec_race = + matches!(&outcome, Err(AppError::Git(m)) if m.contains("Text file busy")); + if exec_race && attempt < SPAWN_ATTEMPTS { + tokio::time::sleep(std::time::Duration::from_millis(100 * attempt)).await; + continue; + } + result = Some(outcome); + break; + } + let result = result.expect("one attempt must have produced an outcome"); + assert!( + result.is_ok(), + "the fake receive-pack succeeds, so the handler must too; got {result:?}" + ); + + // No polling: `release` unlocks on the connection that took the lock, so the + // lock is free the instant the handler returns. Falling back to the async + // `after_release` net would not satisfy this. + assert!( + write_lock_is_takeable(&probe, key).await, + "a completed push must reclaim its write lock and release it synchronously" + ); + } + + /// #173 F1 (RED-before/GREEN-after): an exhausted repo write-lock POOL is a capacity + /// signal, so the push must shed 503 + Retry-After (Overloaded) like every other + /// admission path here, not report a 500 git error. Both directions: the shed with + /// the single lock-pool connection occupied by a guard on a DIFFERENT repo (so this + /// is pool capacity, not advisory-lock contention), and the must-not case once that + /// connection is back. Before the fix `acquire_write`'s checkout failure fell into + /// the generic `AppError::Git` arm (500, no Retry-After). + #[sqlx::test] + async fn receive_pack_lock_pool_exhaustion_sheds_503_not_500(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let owner = "z6lockpool"; + let name = "lp1"; + let mut state = crate::test_support::test_state(pool.clone()).await; + // One lock-pool connection, short checkout timeout so the exhaustion surfaces + // promptly rather than at the handler's own acquire deadline. + state.repo_store = crate::git::repo_store::RepoStore::new( + std::path::PathBuf::from("/tmp/gitlawb-lockpool-shed"), + None, + crate::git::repo_store::build_lock_pool(&pool, 1, std::time::Duration::from_secs(1)), + ); + state + .db + .upsert_mirror_repo(owner, name, "/tmp/z6lockpool-lp1", None, false) + .await + .unwrap(); + + let did = "did:key:z6MkLockPoolShedProofDidAAAAAAAAAAAAAAAAAA"; + let peer: SocketAddr = "203.0.113.71:5000".parse().unwrap(); + + // Occupy the only lock-pool connection with a write on an UNRELATED repo. + let held = state + .repo_store + .acquire_write(owner, "other-repo") + .await + .expect("the first write takes the only lock-pool connection"); + + let shed = git_receive_pack( + State(state.clone()), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(peer)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await; + assert!( + matches!(shed, Err(AppError::Overloaded(_))), + "an exhausted lock pool must shed 503 + Retry-After, not a 500 git error; \ + got {shed:?}" + ); + + // MUST-NOT: with the pool free again, the push is not shed as capacity (it fails + // later on the nonexistent on-disk repo, which is a git error, not Overloaded). + held.release(false).await; + let admitted = git_receive_pack( + State(state.clone()), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some("203.0.113.72:5000".parse().unwrap())), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await; + assert!( + !matches!(admitted, Err(AppError::Overloaded(_))), + "with the lock pool free, a push must not be shed as capacity; got {admitted:?}" + ); + } + /// #174 U5 (P1-e, RED-before/GREEN-after): the post-push encryption walk acquires a /// `git_encrypt_semaphore` permit before running, so completed pushes cannot spawn /// unbounded concurrent full-history walks. With the pool exhausted the gated walk diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index aff27d1e..0729e04a 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -307,6 +307,17 @@ pub struct Config { /// Default: 32. Must be between 1 and 1_048_576 (the ceiling keeps the value /// under tokio's `Semaphore` permit limit so an oversized value is a clean CLI /// error rather than a boot-time panic). + /// + /// CONNECTION BUDGET. A push holds a Postgres connection from the node's separate + /// advisory-lock pool for the whole receive-pack, and that pool is sized from this + /// knob (this value + 8, clamped to 64 in `main.rs`). The node's total ceiling is + /// therefore `db_max_connections` (default 20) + the lock pool (default 40), i.e. + /// 60 by default, and at most `db_max_connections` + 64. Size BOTH against the + /// database server's `max_connections`: `db_max_connections`' own doc predates the + /// lock pool and no longer covers most of the node's connections. The +8 headroom + /// is shared with the three non-push `acquire_write` callers (`api/issues.rs` x2, + /// `api/pulls.rs`). Raising this knob past the clamp does NOT buy more lock-pool + /// connections; pushes beyond it wait briefly and then shed a 503 + Retry-After. #[arg( long, env = "GITLAWB_MAX_CONCURRENT_GIT_PUSHES", diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 16c0378b..226b50f2 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -184,47 +184,60 @@ impl RepoStore { let (owner_slug, local_path) = self.local_path(owner_did, repo_name)?; let lock_key = advisory_lock_key(&owner_slug, repo_name); - // Check out ONE connection from the lock pool and keep it for the whole - // lock lifetime. Two reasons, both bugs we hit with `fetch_one(&pool)`: + // Acquire the Postgres advisory lock with retry, using pg_try_advisory_lock so a + // stale lock from a crashed connection can't block us indefinitely. // - // * A session-level advisory lock belongs to the CONNECTION that took - // it. Running the lock and the unlock through the pool lets them land - // on different connections, so `pg_advisory_unlock` silently returns - // false and the lock leaks, while a competing acquire that happens to - // draw the holding connection re-enters the lock and pushes to the - // same repo run concurrently. - // * Cancellation. `api/repos.rs` bounds this call with - // `tokio::time::timeout`; when it fires during the Tigris phase below - // the future is dropped after the lock was taken and before - // `RepoWriteGuard` (the only caller of `pg_advisory_unlock`) exists. - // Dropping this connection instead runs the pool's `after_release` - // hook, which clears the lock (#173). - let mut lock_conn = self - .lock_pool - .acquire() - .await - .context("checking out a lock-pool connection")?; - - // Acquire Postgres advisory lock with retry using pg_try_advisory_lock - // to avoid blocking indefinitely on stale locks from crashed connections. - let mut acquired = false; + // The connection is checked out INSIDE the loop and RETURNED before each sleep. + // Only the connection that actually took the lock is retained. Two constraints + // pull in opposite directions here, and this is what satisfies both: + // + // * Session ownership. A session-level advisory lock belongs to the CONNECTION + // that took it, so the lock and its `pg_advisory_unlock` must run on the same + // one. Running them through the pool (`fetch_one(&self.pool)`) lets them land + // on different connections: the unlock silently returns false and the lock + // leaks, while a competing acquire that happens to draw the holding + // connection re-enters the lock and two pushes to one repo run concurrently. + // Hence: keep the connection that WON. + // * Occupancy. Holding a connection across the ~60 one-second sleeps would let + // one spinning acquire park a lock-pool connection for a minute. That is not + // just a push-path concern: `api/issues.rs` and `api/pulls.rs` reach + // acquire_write holding no concurrency permit at all, so a caller could park + // the whole pool and starve authenticated pushes on every repo (#173 F1). + // Hence: return the connection when we LOSE, before sleeping. + // + // Returning a losing connection is safe with respect to the cancellation design: + // `after_release` runs `pg_advisory_unlock_all()`, a no-op on a connection that + // took nothing, so it cannot disturb a lock held by any other connection + // (proven by `returning_an_unlocked_connection_does_not_clear_another_connections_lock`). + // + // Cancellation safety is unchanged: the future can only be dropped while a + // connection is checked out, and dropping it runs the same `after_release` hook, + // which clears whatever lock it had just taken (#173 U1). + let mut lock_conn = None; for attempt in 0..60 { + let mut conn = self.lock_pool.acquire().await.map_err(|e| { + anyhow::Error::new(LockPoolBusy) + .context(format!("checking out a lock-pool connection: {e}")) + })?; let row: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") .bind(lock_key) - .fetch_one(&mut *lock_conn) + .fetch_one(&mut *conn) .await .context("trying advisory lock")?; if row.0 { - acquired = true; + lock_conn = Some(conn); break; } + // Lost the race: give the connection back so a spinning acquire occupies + // nothing while it waits. + drop(conn); if attempt < 59 { tokio::time::sleep(std::time::Duration::from_secs(1)).await; } } - if !acquired { + let Some(lock_conn) = lock_conn else { anyhow::bail!("could not acquire advisory lock after 60s — possible stale lock for {owner_slug}/{repo_name}"); - } + }; #[cfg(test)] if let Some(stall) = self.tigris_stall { @@ -400,6 +413,19 @@ fn validate_repo_name(repo_name: &str) -> Result<()> { Ok(()) } +/// Error marker for "no lock-pool connection was available in time". +/// +/// Carried through the `anyhow` chain (like [`smart_http::GitServiceTimeout`]) so the +/// HTTP handler can `downcast_ref` it and shed a 503 + Retry-After instead of the +/// generic 500 a git error maps to: an exhausted lock pool is a CAPACITY signal, and +/// telling the client to retry shortly is the same shed semantics the surrounding +/// admission code already uses (#173 F1). +/// +/// [`smart_http::GitServiceTimeout`]: crate::git::smart_http::GitServiceTimeout +#[derive(Debug, thiserror::Error)] +#[error("no lock-pool connection available")] +pub struct LockPoolBusy; + /// Guard returned by `acquire_write()`. Holds the Postgres advisory lock and /// uploads to Tigris + releases the lock on `release()`. pub struct RepoWriteGuard { @@ -731,6 +757,183 @@ mod tests { held.release(false).await; } + /// #173 F1 (RED-before/GREEN-after). A contended `acquire_write` spins for up to + /// 60 one-second attempts. It must not OCCUPY a lock-pool connection for that whole + /// spin: `acquire_write` has non-push callers (`api/issues.rs`, `api/pulls.rs`) that + /// hold no concurrency permit, so any self-minted did:key could otherwise park a + /// connection per call and starve authenticated pushes on EVERY repo. + /// + /// Lock pool of exactly 2, two spinners. Pre-fix (checkout hoisted above the retry + /// loop) they pin both connections for the full spin and an UNCONTENDED acquire on a + /// third repo dies on the pool acquire timeout. Post-fix each spinner returns its + /// connection before sleeping, so it occupies ~0 and the uncontended acquire sails + /// through. + #[sqlx::test] + async fn a_spinning_acquire_write_does_not_occupy_a_lock_pool_connection(pool: PgPool) { + let owner = "did:key:z6MkSpinOccupancy"; + let owner_slug = owner.replace([':', '/'], "_"); + let store = RepoStore::new( + PathBuf::from("/tmp/gitlawb-test-repos"), + None, + build_lock_pool(&pool, 2, Duration::from_secs(2)), + ); + + // An independent session holds both contended keys, so the spinners' try-locks + // return false on every iteration and they stay in the retry loop. + let holder = sibling_pool(&pool, 2); + let mut held_conn = holder.acquire().await.expect("holder connection"); + for repo in ["spin-a", "spin-b"] { + let taken: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(advisory_lock_key(&owner_slug, repo)) + .fetch_one(&mut *held_conn) + .await + .expect("holder try-lock"); + assert!(taken.0, "the holder must own {repo}'s key"); + } + + let mut spinners = Vec::new(); + for repo in ["spin-a", "spin-b"] { + let store = store.clone(); + spinners.push(tokio::spawn(async move { + store.acquire_write(owner, repo).await + })); + } + // Let both reach the spin (each has done at least one failed try-lock by now). + tokio::time::sleep(Duration::from_millis(500)).await; + + let started = std::time::Instant::now(); + let uncontended = tokio::time::timeout( + Duration::from_secs(10), + store.acquire_write(owner, "spin-free"), + ) + .await + .expect("the uncontended acquire must return, not hang"); + let elapsed = started.elapsed(); + let free_guard = uncontended.unwrap_or_else(|e| { + panic!( + "an UNCONTENDED acquire_write on a DIFFERENT repo must not be starved by \ + spinners holding the lock pool; got: {e}" + ) + }); + assert!( + elapsed < Duration::from_secs(2), + "the uncontended acquire must not queue behind the spinners for the pool \ + acquire timeout; took {elapsed:?}" + ); + free_guard.release(false).await; + + // The drop-and-retake cycle must still END in a real, exclusive lock: free + // spin-a's key and the spinner that was cycling connections must take it. + sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(advisory_lock_key(&owner_slug, "spin-a")) + .execute(&mut *held_conn) + .await + .expect("release spin-a"); + let winner = tokio::time::timeout(Duration::from_secs(15), spinners.remove(0)) + .await + .expect("the spinner must finish once its key frees") + .expect("spinner task") + .expect("the spinner must acquire once the key frees"); + let probe = sibling_pool(&pool, 2); + assert!( + !lock_is_free_elsewhere(&probe, advisory_lock_key(&owner_slug, "spin-a")).await, + "the lock a spinner finally took must be observably held from another session" + ); + winner.release(false).await; + + for s in spinners { + s.abort(); + } + sqlx::query("SELECT pg_advisory_unlock_all()") + .execute(&mut *held_conn) + .await + .expect("release the remaining holder lock"); + } + + /// #173 F1, the property the fix rests on: returning a lock-pool connection that + /// holds NOTHING runs `after_release`'s `pg_advisory_unlock_all()`, which is a no-op + /// and must not disturb a lock held on a DIFFERENT connection of the same pool. + /// Session advisory locks are per connection, so this is by construction, but the + /// spin fix depends on it, so it is proven by execution rather than assumed. + #[sqlx::test] + async fn returning_an_unlocked_connection_does_not_clear_another_connections_lock( + pool: PgPool, + ) { + let owner = "did:key:z6MkNoOpUnlockAll"; + let repo = "noop-unlock"; + let key = advisory_lock_key(&owner.replace([':', '/'], "_"), repo); + let probe = sibling_pool(&pool, 2); + let lock_pool = build_lock_pool(&pool, 4, Duration::from_secs(5)); + let store = RepoStore::new( + PathBuf::from("/tmp/gitlawb-test-repos"), + None, + lock_pool.clone(), + ); + + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + + // Churn the pool: check out and drop connections that hold no lock, exactly what + // a spinning acquire now does between attempts. Each return fires + // pg_advisory_unlock_all() on that connection. + for _ in 0..10 { + let mut conn = lock_pool.acquire().await.expect("churn checkout"); + let _: (i32,) = sqlx::query_as("SELECT 1") + .fetch_one(&mut *conn) + .await + .expect("churn query"); + drop(conn); + tokio::time::sleep(Duration::from_millis(10)).await; + } + + assert!( + !lock_is_free_elsewhere(&probe, key).await, + "a held write lock must survive other lock-pool connections being returned" + ); + guard.release(true).await; + assert!( + lock_is_free_elsewhere(&probe, key).await, + "release must still free the lock after the churn" + ); + } + + /// #173 F1: lock-pool exhaustion is a DISTINCT error the handler can shed as a 503, + /// not a generic git 500. Both directions: an exhausted pool downcasts to + /// [`LockPoolBusy`], and an unrelated failure (a rejected repo name) does not. + #[sqlx::test] + async fn lock_pool_exhaustion_is_a_distinct_downcastable_error(pool: PgPool) { + let owner = "did:key:z6MkBusyDowncast"; + let store = RepoStore::new( + PathBuf::from("/tmp/gitlawb-test-repos"), + None, + build_lock_pool(&pool, 1, Duration::from_secs(1)), + ); + let held = store + .acquire_write(owner, "busy-a") + .await + .expect("first acquire"); + + let err = match store.acquire_write(owner, "busy-b").await { + Ok(_) => panic!("an exhausted lock pool must error, not hand back a guard"), + Err(e) => e, + }; + assert!( + err.downcast_ref::().is_some(), + "lock-pool exhaustion must be downcastable so the handler sheds 503, got: {err}" + ); + + // MUST-NOT: an ordinary rejection is not a capacity signal. + let other = match store.acquire_write(owner, "../escape").await { + Ok(_) => panic!("a traversal repo name must be rejected"), + Err(e) => e, + }; + assert!( + other.downcast_ref::().is_none(), + "a validation failure must not masquerade as lock-pool capacity, got: {other}" + ); + + held.release(false).await; + } + /// Round trip: the lock is observably HELD between acquire and release, and /// observably FREE after. Both checks run from an independent session; from /// the holding session they would pass vacuously (session locks are diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index 50145652..bbaf319c 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -14,13 +14,13 @@ use tokio::process::Command; /// the work they admitted, so admission is released only when that work is truly /// done — not the instant the handler future drops on a client disconnect. /// -/// A move-only wrapper: no methods beyond construction and `Drop`. The handler -/// MOVEs its permits in and keeps no copy (a retained copy would drop early and -/// release admission the moment the future is dropped, defeating the guard). It is -/// threaded into `drive_git_child`, whose [`KillGroupOnDrop`] moves it into the -/// detached reaper on disconnect, so both permits drop only after the process group -/// is confirmed reaped (`kill(-pgid,0)==ESRCH`) rather than while the group is still -/// alive holding PIDs past the concurrency cap (#174 P1-a, plain-spawn residual). +/// A drop-only wrapper: nothing here inspects what it holds. The handler MOVEs its +/// permits in and keeps no copy (a retained copy would drop early and release admission +/// the moment the future is dropped, defeating the guard). It is threaded into +/// `drive_git_child`, whose [`KillGroupOnDrop`] moves it into the detached reaper on +/// disconnect, so both permits drop only after the process group is confirmed reaped +/// (`kill(-pgid,0)==ESRCH`) rather than while the group is still alive holding PIDs past +/// the concurrency cap (#174 P1-a, plain-spawn residual). /// /// The `be0cdd6` path-scoped upload-pack walk already applies this discipline by /// moving its permits into the `spawn_blocking`; this generalizes it to the plain @@ -31,6 +31,8 @@ pub struct AdmissionGuard { // 'static` so the guard can move into the detached reaper task. _global: Option>, _caller: Option>, + // Any further work-scoped hold that must outlive the process group; see `with_hold`. + _hold: Option>, } impl AdmissionGuard { @@ -40,8 +42,24 @@ impl AdmissionGuard { Self { _global: Some(Box::new(global)), _caller: caller.map(|c| Box::new(c) as Box), + _hold: None, } } + + /// Attach a further hold that must not be released until the process group is + /// reaped, and ride it through the same seam as the permits. + /// + /// The push handler uses this for the repo WRITE LOCK (#173 F2). Its + /// `guard.release(..)` line is only reached if `receive_pack` returns, so on a client + /// disconnect the lock used to be freed by the dropped future while the detached + /// reaper was still giving the group its SIGTERM grace, admitting a second + /// `receive-pack` on the same repo. Carrying the lock here holds it until the group + /// is ESRCH-confirmed gone, which is the same invariant the timeout path already + /// keeps ("a caller releasing a write lock can't race them", `reap_group_on_timeout`). + pub fn with_hold(mut self, hold: impl Send + 'static) -> Self { + self._hold = Some(Box::new(hold)); + self + } } /// Handle `GET /:owner/:repo/info/refs?service=git-upload-pack` diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 648d7428..d4f4e63d 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -58,6 +58,32 @@ struct DbStartupStatus { next_retry_secs: AtomicU64, } +/// Hard ceiling on the advisory-lock pool's `max_connections`. +/// +/// `max_concurrent_git_pushes` is validated all the way up to 1_048_576, and the lock +/// pool used to derive its size straight from that knob, so raising the push cap +/// silently raised the node's Postgres connection ceiling with no CLI error and no +/// relation to the server's own `max_connections` (#173 F4). The node's total budget is +/// now bounded: `db_max_connections` (default 20) + at most this. +const LOCK_POOL_MAX_CONNECTIONS: u32 = 64; + +/// Connections the lock pool keeps above the push cap. Covers the three non-push +/// `acquire_write` callers (`api/issues.rs` x2, `api/pulls.rs`), which hold no +/// concurrency permit, so a push never queues here for a connection where it did not +/// before. +const LOCK_POOL_PUSH_HEADROOM: u8 = 8; + +/// Size the advisory-lock pool for a given push cap: the cap plus +/// [`LOCK_POOL_PUSH_HEADROOM`], clamped to [`LOCK_POOL_MAX_CONNECTIONS`]. Past the +/// clamp a push may wait for a lock-pool connection, which is a bounded wait that sheds +/// a clean 503 (see `LockPoolBusy`), not an unbounded hang. +fn lock_pool_size(max_concurrent_git_pushes: usize) -> u32 { + u32::try_from(max_concurrent_git_pushes) + .unwrap_or(u32::MAX) + .saturating_add(u32::from(LOCK_POOL_PUSH_HEADROOM)) + .min(LOCK_POOL_MAX_CONNECTIONS) +} + #[tokio::main] async fn main() -> Result<()> { tracing_subscriber::fmt() @@ -282,14 +308,11 @@ async fn main() -> Result<()> { // Repo write locks run on their own pool, never the main query pool: each // push holds its connection for the whole receive-pack, and // db_max_connections (20) is below max_concurrent_git_pushes (32), so sharing - // would starve every other query under a push burst. Headroom above the push - // cap keeps a push from ever queueing here for a connection where it did not - // before. See build_lock_pool for the cancellation semantics (#173). + // would starve every other query under a push burst. See build_lock_pool for + // the cancellation semantics (#173). let lock_pool = git::repo_store::build_lock_pool( db.pool(), - u32::try_from(config.max_concurrent_git_pushes) - .unwrap_or(u32::MAX) - .saturating_add(8), + lock_pool_size(config.max_concurrent_git_pushes), std::time::Duration::from_secs(config.db_acquire_timeout_secs), ); let repo_store = git::repo_store::RepoStore::new(config.repos_dir.clone(), tigris, lock_pool); @@ -1139,6 +1162,36 @@ fn load_or_create_keypair(config: &Config) -> Result { } } +#[cfg(test)] +mod lock_pool_sizing_tests { + use super::{lock_pool_size, LOCK_POOL_MAX_CONNECTIONS, LOCK_POOL_PUSH_HEADROOM}; + + /// The default push cap gets its cap plus headroom, so no push ever queues for a + /// lock-pool connection where it did not before. + #[test] + fn default_push_cap_gets_headroom_over_the_cap() { + assert_eq!(lock_pool_size(32), 32 + u32::from(LOCK_POOL_PUSH_HEADROOM)); + assert_eq!(lock_pool_size(1), 1 + u32::from(LOCK_POOL_PUSH_HEADROOM)); + } + + /// #173 F4: `max_concurrent_git_pushes` is validated all the way to 1_048_576, so an + /// operator raising it used to raise the node's Postgres connection ceiling with it, + /// silently and without bound. The lock pool is CLAMPED instead. + #[test] + fn an_oversized_push_cap_is_clamped_not_propagated() { + assert_eq!(lock_pool_size(1_048_576), LOCK_POOL_MAX_CONNECTIONS); + assert_eq!(lock_pool_size(usize::MAX), LOCK_POOL_MAX_CONNECTIONS); + // The largest cap that still fits under the clamp keeps its full headroom. + let widest = (LOCK_POOL_MAX_CONNECTIONS - u32::from(LOCK_POOL_PUSH_HEADROOM)) as usize; + assert_eq!(lock_pool_size(widest), LOCK_POOL_MAX_CONNECTIONS); + assert_eq!( + lock_pool_size(widest - 1), + LOCK_POOL_MAX_CONNECTIONS - 1, + "values below the clamp must not be rounded up to it" + ); + } +} + #[cfg(test)] mod gossip_ssrf_tests { use super::ping_peer_health; diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 47428248..303aed00 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -9995,6 +9995,198 @@ mod tests { "the key is released once the task is clean" ); } + + /// Wait for the tail's atomic check-and-clear to consume the pending dirty + /// bit (`Some(true)` -> `Some(false)`), which is the exact instant the task + /// enters `requeue_refresh_state`'s retry window. Deterministic, so the + /// coalescing push below lands INSIDE that window rather than on a sleep + /// guess. `None` means the key is already gone (the task exited), which the + /// caller reports as its own failure. + async fn wait_for_refresh_window( + inflight: &crate::state::EncryptInflight, + repo_id: &str, + ) -> bool { + for _ in 0..5_000 { + match inflight.dirty(repo_id) { + Some(false) => return true, + None => return false, + Some(true) => {} + } + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + } + false + } + + /// SCENARIO 7 (#173 F3, RED-before/GREEN-after). A push that coalesces WHILE + /// the re-read is retrying must not be thrown away when that re-read finally + /// gives up. `requeue_or_release` returning true cleared the dirty bit without + /// marking the guard released, so a `break` on `Failed` let + /// `EncryptInflightGuard::drop` remove the key outright and push C's pass was + /// never attempted, a silent drop with no reconciliation sweep behind it. + /// + /// Exactly `REQUEUE_REREAD_MAX_ATTEMPTS` injected repo-read faults, so the + /// first refresh exhausts its budget and the DB is healthy for the next one. + /// Push C coalesces inside that window. RED with `Failed => break`: obj_c is + /// never pinned. GREEN when `Failed` falls through and lets the tail decide. + #[sqlx::test] + async fn u2_failed_reread_keeps_a_push_that_coalesced_during_the_window(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u2-window"); + state.db.create_repo(&repo).await.expect("seed repo"); + let git_repo = init_repo(); + let obj_a = commit(&git_repo.path, "a.txt", "one\n"); + let obj_c = commit(&git_repo.path, "c.txt", "three\n"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // Exactly the bound: the FIRST refresh burns all three attempts and gives + // up; every later refresh sees a healthy DB. + requeue_faults::inject(&repo.id, 3, 0); + + let guard = state + .encrypt_inflight + .try_begin(&repo.id) + .expect("push A admits"); + assert!( + state.encrypt_inflight.try_begin(&repo.id).is_none(), + "push B coalesces while A is in flight" + ); + + // Push C lands during the retry window, after the tail already consumed + // push B's dirty bit. + let inflight = state.encrypt_inflight.clone(); + let repo_id = repo.id.clone(); + let coalesced = tokio::spawn(async move { + if !wait_for_refresh_window(&inflight, &repo_id).await { + return false; + } + inflight.try_begin(&repo_id).is_none() + }); + + crate::api::repos::run_post_push_replication_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + server.url(), + true, + owner.clone(), + vec![obj_a.clone()], + Some(vec![]), + HashSet::new(), + ) + .await; + + assert!( + coalesced.await.expect("coalescing task"), + "push C must have coalesced inside the retry window for this test to \ + mean anything" + ); + assert!( + state.db.is_pinned(&obj_c).await.unwrap(), + "the push that coalesced during the retry window must still get a pass \ + once the DB recovers (RED with `Failed => break`: the dirty bit was \ + already consumed, so the pass was dropped with nothing to re-derive it)" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is released once the task exits" + ); + } + + /// SCENARIO 8 (#173 F3, the sustained-outage guard on the fall-through). Falling + /// through on `Failed` means `requeue_or_release` runs again, so a DB that never + /// recovers must still TERMINATE rather than spin. It does: an extra lap only + /// happens when a push actually coalesced, and each lap pays a full bounded + /// re-read (3 attempts with backoff). One coalescing push during the window buys + /// exactly one extra lap: 6 repo-read attempts, then exit. + #[sqlx::test] + async fn u2_sustained_failure_with_a_coalesce_terminates_after_one_more_lap( + pool: PgPool, + ) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u2-sustained-window"); + state.db.create_repo(&repo).await.expect("seed repo"); + let git_repo = init_repo(); + let obj_a = commit(&git_repo.path, "a.txt", "one\n"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // The outage never clears. + requeue_faults::inject(&repo.id, 10_000, 0); + + let guard = state + .encrypt_inflight + .try_begin(&repo.id) + .expect("push A admits"); + assert!( + state.encrypt_inflight.try_begin(&repo.id).is_none(), + "push B coalesces" + ); + + let inflight = state.encrypt_inflight.clone(); + let repo_id = repo.id.clone(); + let coalesced = tokio::spawn(async move { + if !wait_for_refresh_window(&inflight, &repo_id).await { + return false; + } + inflight.try_begin(&repo_id).is_none() + }); + + // The watchdog is the real assertion: a fall-through that re-spins without + // the dirty gate would never return here. + tokio::time::timeout( + std::time::Duration::from_secs(60), + crate::api::repos::run_post_push_replication_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + server.url(), + true, + owner.clone(), + vec![obj_a.clone()], + Some(vec![]), + HashSet::new(), + ), + ) + .await + .expect( + "the task must terminate under a sustained outage; a fall-through that \ + does not gate on the dirty bit spins forever", + ); + + assert!( + coalesced.await.expect("coalescing task"), + "push C must have coalesced inside the retry window" + ); + assert_eq!( + requeue_faults::counters(&repo.id).repo_read_attempts, + 6, + "one coalescing push buys exactly one more bounded re-read lap \ + (3 + 3 attempts), never an unbounded retry" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is released on the give-up path" + ); + } } } } From c519d54fc1da249f8ba1c4faa6a2ac6404c5c6a9 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 27 Jul 2026 18:29:01 -0500 Subject: [PATCH 079/124] fix(node): keep the incompleteness marker until a source is really recorded, and let the sweep retry Four follow-ups from an adversarial pass over the previous commits. The marker clear ran unconditionally inside `record_pin_source`'s transaction, but its INSERT is `ON CONFLICT DO NOTHING` guarded by the cap, so it records nothing when the repo is already a source. That is the common case, not a rare one: the pin path calls `record_pin_source` for every already-pinned object, and on a requeue pass that is the whole-repo enumeration. So a re-push from the repo that was already recorded cleared a marker set by a different repo's failure and the public copy went back to 404ing. The clear is now gated on the insert actually adding a row. A genuine record from a third repo still clears the marker, which needs a per-(oid, repo) table to close and is documented rather than papered over. The sweep advanced its cursor before every skip, which stops a wedge but made a transient skip permanent: once the walk reached the end of the table the cursor parked there and no later boot read another row. On a Tigris-backed node the common skip is "repo is not on local disk", so the sweep would repair whatever happened to be warm at boot and then never run again, and the new advertise filter turned those rows from "advertised but 404" into "never advertised, never repaired". Repairs now report whether a skip was retryable or terminal, and a run that skipped anything retryable rewinds the cursor once, after the walk, so the next run re-walks. A permanently unrepairable row is terminal and does not trigger a rewind, so this cannot become a hot loop. The sweep also reached its repo paths through the raw join helper, skipping the three-layer traversal barrier `RepoStore::local_path` applies. A test with a repo row named `../../escapee` showed it reading and repairing bytes from outside `repos_dir`. That barrier is now extracted and shared, so the two callers cannot drift. And the repair's synchronous `git cat-file` moved under `spawn_blocking`, matching the existing treatment of the withheld walk, so a wedged read no longer parks a worker thread for the full service timeout. Finally the resolver's two source-set queries ran ahead of the work-budget peek that exists to shed a spent caller before doing work. The peek now runs first. A throttled caller whose set turns out complete gets 429 where it used to get 404, which is the honest answer since its search never ran, and it removes an oracle that let a throttled caller tell a complete source set from an incomplete one. --- crates/gitlawb-node/src/api/ipfs.rs | 88 +++-- crates/gitlawb-node/src/db/mod.rs | 74 ++-- crates/gitlawb-node/src/git/repo_store.rs | 79 ++-- crates/gitlawb-node/src/ipfs_pin.rs | 153 ++++++-- crates/gitlawb-node/src/test_support.rs | 428 ++++++++++++++++++++++ 5 files changed, 711 insertions(+), 111 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index ba669027..e3116973 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -340,38 +340,52 @@ pub async fn get_by_cid( // Only a set with NONE of these three signals is treated as complete (every // recorded source was just tried), so it skips the scan and lets the tail 404, and // ordinary denials never fan out to O(repos) (INV-10 / F3). Both extra queries run - // only on a provenance MISS (we return above on Served), so neither costs the serve - // path, and the fallback is not an authorization bypass: the scan gates every repo + // only on a provenance MISS (we return above on Served) by a caller that still has + // work budget, so neither costs the serve path nor a shed caller, and the fallback + // is not an authorization bypass: the scan gates every repo // through the SAME per-caller gate, so a caller who may not read the object is // still denied. - let needs_scan = sources.is_empty() - || state + // + // F3 (#173, INV-10/INV-15): peek the per-IP WORK-budget limiter WITHOUT + // consuming a token so an already-throttled source is shed BEFORE the + // O(repos) preload; the consuming per-probe charge inside gate_and_serve is + // left UNCHANGED (it is load-bearing for the across-request bound), so this + // adds no double-charge. This peeks `ipfs_work_rate_limiter`, the SAME bucket + // the per-probe charge below debits — NOT the route limiter (`ipfs_rate_limiter`, + // charged once per request by the middleware): peeking the route bucket here + // would re-shed a request the route already admitted (R6, U5). + // + // The peek runs BEFORE the two marker queries (#173 round 11, F5): shedding is + // the whole point of a peek, so a spent-budget caller should not pay two + // lookups per request first. It stays AFTER the provenance walk, so no caller + // who could have been served is shed. The one caller this moves: a spent-budget + // caller whose source set turns out COMPLETE now takes the 429 tail instead of + // the 404 tail. That is the honest answer (its search never ran), and it drops + // an oracle, since the old order let a throttled caller tell a complete source + // set from an incomplete one by 404 vs 429. + if let Some(key) = + crate::rate_limit::client_key(rctx.headers, rctx.peer, state.push_limiter_trust) + { + if state.ipfs_work_rate_limiter.is_throttled(&key).await { + throttled = true; + continue; + } + } + let needs_scan = sources.is_empty() || { + #[cfg(test)] + bump_marker_queries(); + state .db .pin_sources_at_cap(sha256_hex) .await .map_err(AppError::Internal)? - || state - .db - .pin_sources_incomplete(sha256_hex) - .await - .map_err(AppError::Internal)?; + || state + .db + .pin_sources_incomplete(sha256_hex) + .await + .map_err(AppError::Internal)? + }; if needs_scan { - // F3 (#173, INV-10/INV-15): peek the per-IP WORK-budget limiter WITHOUT - // consuming a token so an already-throttled source is shed BEFORE the - // O(repos) preload; the consuming per-probe charge inside gate_and_serve is - // left UNCHANGED (it is load-bearing for the across-request bound), so this - // adds no double-charge. This peeks `ipfs_work_rate_limiter`, the SAME bucket - // the per-probe charge below debits — NOT the route limiter (`ipfs_rate_limiter`, - // charged once per request by the middleware): peeking the route bucket here - // would re-shed a request the route already admitted (R6, U5). - if let Some(key) = - crate::rate_limit::client_key(rctx.headers, rctx.peer, state.push_limiter_trust) - { - if state.ipfs_work_rate_limiter.is_throttled(&key).await { - throttled = true; - continue; - } - } // Load the scan context once, lazily (shared across oid candidates). if scan_ctx.is_none() { #[cfg(test)] @@ -910,6 +924,30 @@ fn bump_preload_queries() { PRELOAD_QUERIES.with(|c| c.set(c.get() + 1)); } +// Test-only cost counter (F5, #173 round 11): how many times the fallback gate ran the +// `pin_sources_at_cap` / `pin_sources_incomplete` pair. The work-budget peek sits ahead +// of them, so an already-throttled caller leaves this at 0; putting the peek back after +// the pair turns that assertion red. Same thread_local discipline as the preload counter. +#[cfg(test)] +thread_local! { + static MARKER_QUERIES: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn reset_marker_queries() { + MARKER_QUERIES.with(|c| c.set(0)); +} + +#[cfg(test)] +pub(crate) fn marker_queries() -> usize { + MARKER_QUERIES.with(|c| c.get()) +} + +#[cfg(test)] +fn bump_marker_queries() { + MARKER_QUERIES.with(|c| c.set(c.get() + 1)); +} + // Test-only INV-10 cost counter (F6, U6/U7): how many times the serve path withheld an // object because it exceeded `ipfs_max_served_object_bytes`. The bounded read must reject // an oversized object rather than buffer it on the worker; the counter is the both-ways diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 4b107da3..75f279d7 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -2488,17 +2488,26 @@ impl Db { /// keeping the first-pinner), so the INV-10 bound on serve-time work holds at /// `O(MAX_PIN_SOURCES + 1)` regardless of a table overshoot. /// - /// A successful record also CLEARS the `pin_sources_incomplete` marker for the - /// object, in the SAME transaction as the insert (U3, #173), so the clear cannot - /// drift across the four call sites or land without the row it describes. The - /// marker is per-object, not per-(object, repo): a record from repo B clears a - /// marker set by a failed record from repo A, which can re-hide A's hole until A - /// pushes again. That is the deliberate cost of a single boolean, and it fails in - /// the safe direction relative to today (the marker only ever ADDS the fallback, - /// never removes a source the resolver already tries). + /// A record that ACTUALLY ADDS a source row also CLEARS the + /// `pin_sources_incomplete` marker for the object, in the SAME transaction as the + /// insert (U3, #173), so the clear cannot drift across the four call sites or land + /// without the row it describes. + /// + /// The clear is gated on `rows_affected() > 0` because the INSERT is a no-op in two + /// ordinary cases: the `(oid, repo)` pair already exists (`ON CONFLICT DO NOTHING`) + /// and the source set is at cap (the count guard). The skip path calls this for + /// EVERY already-pinned object, and on a requeue pass that list is the whole-repo + /// enumeration, so an unconditional clear meant the next coalesced push from a repo + /// already in the set wiped the marker for every object in the repo without + /// recording anything (round 11 regression). The residual, which the gate does not + /// close: the marker is per-object, not per-(object, repo), so a GENUINE record from + /// a third repo C still clears a marker that repo A's failed record set. That is the + /// deliberate cost of a single boolean; closing it needs a per-(oid, repo) marker + /// table, and it fails in the safe direction (the marker only ever ADDS the scan + /// fallback, never removes a source the resolver already tries). pub async fn record_pin_source(&self, sha256_hex: &str, repo_id: &str) -> Result<()> { let mut tx = self.pool.begin().await?; - sqlx::query( + let inserted = sqlx::query( "INSERT INTO pin_repo_sources (sha256_hex, repo_id) SELECT $1, $2 WHERE (SELECT count(*) FROM pin_repo_sources WHERE sha256_hex = $1) < $3 @@ -2508,14 +2517,17 @@ impl Db { .bind(repo_id) .bind(MAX_PIN_SOURCES) .execute(&mut *tx) - .await?; - sqlx::query( - "UPDATE pinned_cids SET pin_sources_incomplete = FALSE - WHERE sha256_hex = $1 AND pin_sources_incomplete", - ) - .bind(sha256_hex) - .execute(&mut *tx) - .await?; + .await? + .rows_affected(); + if inserted > 0 { + sqlx::query( + "UPDATE pinned_cids SET pin_sources_incomplete = FALSE + WHERE sha256_hex = $1 AND pin_sources_incomplete", + ) + .bind(sha256_hex) + .execute(&mut *tx) + .await?; + } tx.commit().await?; Ok(()) } @@ -2526,6 +2538,13 @@ impl Db { /// source set that is silently missing its own first pinner. One transaction /// removes that window entirely: either both rows land or neither does, and a /// total failure leaves the object unpinned so the next push retries it. + /// + /// The marker clear carries the same `rows_affected` gate as `record_pin_source`. + /// It is not load-bearing here: this path runs only when `is_pinned` said no row + /// exists, and `mark_pin_sources_incomplete` is a no-op without a `pinned_cids` row, + /// so there is no marker to wrongly clear. The gate is kept for the one window that + /// is not covered by that argument, a concurrent pinner landing the row between the + /// `is_pinned` check and this upsert, and so the two clears cannot drift apart. pub async fn record_pinned_cid_with_source( &self, sha256_hex: &str, @@ -2545,7 +2564,7 @@ impl Db { .bind(repo_id) .execute(&mut *tx) .await?; - sqlx::query( + let inserted = sqlx::query( "INSERT INTO pin_repo_sources (sha256_hex, repo_id) SELECT $1, $2 WHERE (SELECT count(*) FROM pin_repo_sources WHERE sha256_hex = $1) < $3 @@ -2555,14 +2574,17 @@ impl Db { .bind(repo_id) .bind(MAX_PIN_SOURCES) .execute(&mut *tx) - .await?; - sqlx::query( - "UPDATE pinned_cids SET pin_sources_incomplete = FALSE - WHERE sha256_hex = $1 AND pin_sources_incomplete", - ) - .bind(sha256_hex) - .execute(&mut *tx) - .await?; + .await? + .rows_affected(); + if inserted > 0 { + sqlx::query( + "UPDATE pinned_cids SET pin_sources_incomplete = FALSE + WHERE sha256_hex = $1 AND pin_sources_incomplete", + ) + .bind(sha256_hex) + .execute(&mut *tx) + .await?; + } tx.commit().await?; Ok(()) } diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 226b50f2..ebdd7757 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -323,40 +323,61 @@ impl RepoStore { /// segment is rejected. This is the CodeQL-recognised barrier /// pattern for `rust/path-injection`. fn local_path(&self, owner_did: &str, repo_name: &str) -> Result<(String, PathBuf)> { - validate_path_components(owner_did, repo_name)?; - let owner_slug = owner_did.replace([':', '/'], "_"); - let local_path = self - .repos_dir - .join(&owner_slug) - .join(format!("{repo_name}.git")); - - if !local_path.starts_with(&self.repos_dir) { - anyhow::bail!( - "computed repo path escaped repos_dir: {}", - local_path.display() - ); - } + let local_path = validated_repo_disk_path(&self.repos_dir, owner_did, repo_name)?; + Ok((owner_slug, local_path)) + } +} - // Explicit component walk — sanitisation barrier that static analysers - // (CodeQL `rust/path-injection`) recognise. The path must be composed - // entirely of Normal segments after the root prefix; any ParentDir or - // CurDir component is a traversal attempt. - for component in local_path.components() { - use std::path::Component; - match component { - Component::Prefix(_) | Component::RootDir | Component::Normal(_) => {} - Component::ParentDir => { - anyhow::bail!("path contains parent-directory component"); - } - Component::CurDir => { - anyhow::bail!("path contains current-directory component"); - } +/// The three-layer validated form of `store::repo_disk_path`, with NO Tigris fetch and +/// no `RepoStore` (#173 round 11, F3). Extracted from `RepoStore::local_path` so a +/// second caller that must not pull a cold repo, the U4 legacy provider-CID sweep, gets +/// the same barrier instead of the raw join. `local_path` is now a thin wrapper over +/// this, so the two cannot drift. +/// +/// Three-layer defence against path traversal: +/// 1. Strict allowlist on `owner_did` and `repo_name` (no `..`, slashes, +/// null bytes, leading dots; length-bounded). +/// 2. The joined path must remain rooted at `repos_dir`. +/// 3. Every component of the joined path must be `Component::Normal` +/// (or the prefix/root from `repos_dir`); any `ParentDir`/`CurDir` +/// segment is rejected. This is the CodeQL-recognised barrier +/// pattern for `rust/path-injection`. +pub(crate) fn validated_repo_disk_path( + repos_dir: &Path, + owner_did: &str, + repo_name: &str, +) -> Result { + validate_path_components(owner_did, repo_name)?; + + let owner_slug = owner_did.replace([':', '/'], "_"); + let local_path = repos_dir.join(&owner_slug).join(format!("{repo_name}.git")); + + if !local_path.starts_with(repos_dir) { + anyhow::bail!( + "computed repo path escaped repos_dir: {}", + local_path.display() + ); + } + + // Explicit component walk — sanitisation barrier that static analysers + // (CodeQL `rust/path-injection`) recognise. The path must be composed + // entirely of Normal segments after the root prefix; any ParentDir or + // CurDir component is a traversal attempt. + for component in local_path.components() { + use std::path::Component; + match component { + Component::Prefix(_) | Component::RootDir | Component::Normal(_) => {} + Component::ParentDir => { + anyhow::bail!("path contains parent-directory component"); + } + Component::CurDir => { + anyhow::bail!("path contains current-directory component"); } } - - Ok((owner_slug, local_path)) } + + Ok(local_path) } /// Strict allowlist validator for `owner_did` and `repo_name`. diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 1f0cc65a..dd9b8545 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -66,37 +66,77 @@ async fn repair_legacy_provider_cid( git_timeout: Duration, sha: &str, db: &crate::db::Db, -) -> Result<()> { +) -> Result { let stored = match db.cid_for_oid(sha).await? { Some(c) => c, - None => return Ok(()), + None => return Ok(RepairOutcome::Settled), }; // Cost gate: a canonical raw CIDv1 key is already correct — never read bytes. if gitlawb_core::cid::is_raw_cidv1(&stored) { - return Ok(()); + return Ok(RepairOutcome::Settled); } // Legacy-codec row: read the object to recompute. Counted so a test can prove // the gate above spares non-legacy rows this read. #[cfg(test)] note_legacy_repair_read(); - let data = match crate::git::store::read_object_bounded(git_bin, repo_path, sha, git_timeout) { - Ok(Some((_ty, bytes))) => bytes, - // Bytes gone: the row stays withheld, never destructively rewritten. - Ok(None) => return Ok(()), + // `read_object_bounded` is SYNCHRONOUS `git cat-file`, and its budget is + // `git_service_timeout_secs` (600 by default), so running it inline parks a tokio + // worker for as long as git takes: one wedged read on the sweep's first pass at boot + // holds a worker for ten minutes, per legacy row. Push it to the blocking pool, the + // same shape `replication_withheld_set` uses in api/repos.rs (#173 round 11, F4). + // Both callers of this function are async, so neither changes shape. The read-counter + // increment above stays on THIS thread so the thread_local cost-gate assertion holds. + let read = { + let repo_path = repo_path.to_path_buf(); + let git_bin = git_bin.to_string(); + let sha = sha.to_string(); + tokio::task::spawn_blocking(move || { + crate::git::store::read_object_bounded(&git_bin, &repo_path, &sha, git_timeout) + }) + .await + }; + let data = match read { + Ok(Ok(Some((_ty, bytes)))) => bytes, + // Bytes gone: the row stays withheld, never destructively rewritten. Nothing a + // later pass changes, so this is a TERMINAL outcome for the sweep's re-walk gate. + Ok(Ok(None)) => return Ok(RepairOutcome::Settled), // A wedged/D-state `git cat-file` (timeout/infra): the repair is opportunistic // and best-effort, so skip it and return Ok so the pin task PROCEEDS to // requeue_or_release rather than hanging the coalescing key until process death // (grok F2, #173). A later re-push or the deferred sweep retries the repair. - Err(e) => { + Ok(Err(e)) => { tracing::warn!(sha = %sha, err = %e, "skipping legacy provider-CID repair: bounded object read failed"); - return Ok(()); + return Ok(RepairOutcome::Retryable); + } + // The blocking task panicked or was cancelled: same best-effort treatment, and + // worth another walk because it says nothing about the row itself. + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "skipping legacy provider-CID repair: object read task failed"); + return Ok(RepairOutcome::Retryable); } }; let raw = Cid::from_git_object_bytes(&data).to_string(); if raw == stored { - return Ok(()); + return Ok(RepairOutcome::Settled); } - db.repair_legacy_provider_cid(sha, &raw, &stored).await + db.repair_legacy_provider_cid(sha, &raw, &stored).await?; + Ok(RepairOutcome::Repaired) +} + +/// What one opportunistic repair did with a row, so the sweep can tell a skip a later +/// run could fix from one nothing will (U4 re-walk, #173 round 11). The push skip path +/// ignores the value: it repairs whatever the push happens to carry and a failure there +/// is already warn-only. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum RepairOutcome { + /// Nothing to do, or nothing a re-walk would change: the stored key is already the + /// raw resolver key, the recomputed key matches it, or the object's bytes are gone. + Settled, + /// The bounded object read failed (a wedged `git cat-file`, an unreadable repo). + /// The bytes may be readable later, so the row is worth walking again. + Retryable, + /// The row's key was rewritten to the raw-content CID. + Repaired, } /// What one sweep pass (or a whole sweep run) did. `scanned` counts `pinned_cids` @@ -107,6 +147,12 @@ pub(crate) struct SweepStats { pub scanned: usize, pub repaired: usize, pub passes: usize, + /// Rows left unrepaired for a reason a LATER run could fix (the source repo is not + /// on this node's local disk, a DB read failed, a bounded object read failed). A + /// nonzero count is what makes the run rewind its cursor instead of parking it at + /// the end of the table forever. Rows that are unrepairable in principle (no + /// provenance, the repo row is gone, the bytes are gone) are NOT counted here. + pub retryable_skips: usize, } /// One bounded pass of the U4 sweep: read at most `batch` `pinned_cids` rows after @@ -131,6 +177,7 @@ async fn sweep_pass( let rows = db.pinned_cids_after(&cursor, batch).await?; let scanned = rows.len(); let mut repaired = 0usize; + let mut retryable_skips = 0usize; let mut last = cursor; for (sha, stored) in rows { @@ -149,16 +196,24 @@ async fn sweep_pass( Ok(s) => s, Err(e) => { tracing::warn!(sha = %sha, err = %e, "sweep: failed to read pin sources"); + // A DB read error says nothing about the row, so a later run retries it. + retryable_skips += 1; continue; } }; + // Whether this row ended the source walk repaired, and whether anything it hit + // along the way was a transient obstacle rather than a permanent one. + let mut row_repaired = false; + let mut row_retryable = false; for repo_id in sources { let repo = match db.get_repo_by_id(&repo_id).await { Ok(Some(r)) => r, - // The repo row is gone: a later source may still hold the bytes. + // The repo row is gone: a later source may still hold the bytes. A + // deleted repo does not come back, so this is not a retryable skip. Ok(None) => continue, Err(e) => { tracing::warn!(repo_id = %repo_id, err = %e, "sweep: failed to read repo"); + row_retryable = true; continue; } }; @@ -166,32 +221,50 @@ async fn sweep_pass( // The sweep is opportunistic background maintenance over every pinned row on // the node, so it must never pull a cold repo back from remote storage: that // would turn a repair pass into a bulk restore. A repo that is not on local - // disk simply reads no bytes here and stays withheld for a later pass or a - // re-push, which is the same non-destructive outcome as missing bytes. - let repo_path = - crate::git::store::repo_disk_path(repos_dir, &repo.owner_did, &repo.name); - if let Err(e) = - repair_legacy_provider_cid(&repo_path, git_bin, git_timeout, &sha, db).await - { - tracing::warn!(sha = %sha, err = %e, "sweep: legacy provider-CID repair failed"); + // disk simply reads no bytes here and stays withheld, but it IS a retryable + // skip: on a Tigris-backed node the repo is cold now and warm later, and + // without the re-walk that row would never be repaired by anything. + // The path goes through the repo store's VALIDATED resolver (allowlisted + // components, rooted at repos_dir, no ParentDir/CurDir segment), not the raw + // join: the sweep is a second caller of that path logic and gets the same + // barrier the acquire path has (#173 round 11, F3). It is the non-fetching + // variant, so the no-cold-pull property above is untouched. + let repo_path = match crate::git::repo_store::validated_repo_disk_path( + repos_dir, + &repo.owner_did, + &repo.name, + ) { + Ok(p) => p, + // An unsafe name is not something a later run fixes, so it is terminal. + Err(e) => { + tracing::warn!(repo_id = %repo_id, err = %e, "sweep: rejected unsafe repo path"); + continue; + } + }; + if !repo_path.is_dir() { + row_retryable = true; continue; } - // `repair_legacy_provider_cid` is best-effort and silent about which of its - // outcomes it took (bytes gone, read failed, rewritten), so read the key back - // to decide whether to stop trying sources. A no-op re-read on an - // already-repaired row is one indexed lookup. - match db.cid_for_oid(&sha).await { - Ok(Some(c)) if gitlawb_core::cid::is_raw_cidv1(&c) => { + match repair_legacy_provider_cid(&repo_path, git_bin, git_timeout, &sha, db).await { + Ok(RepairOutcome::Repaired) => { repaired += 1; + row_repaired = true; break; } - Ok(_) => continue, + // The bytes could not be read from this source right now: try the next + // source, and if none of them works, walk the row again on a later run. + Ok(RepairOutcome::Retryable) => row_retryable = true, + // Nothing to repair from this source and nothing a re-walk changes. + Ok(RepairOutcome::Settled) => {} Err(e) => { - tracing::warn!(sha = %sha, err = %e, "sweep: failed to re-read repaired key"); - continue; + tracing::warn!(sha = %sha, err = %e, "sweep: legacy provider-CID repair failed"); + row_retryable = true; } } } + if !row_repaired && row_retryable { + retryable_skips += 1; + } } db.set_pin_repair_cursor(&last).await?; @@ -199,6 +272,7 @@ async fn sweep_pass( scanned, repaired, passes: 1, + retryable_skips, }) } @@ -230,6 +304,16 @@ pub(crate) async fn sweep_legacy_provider_cids_once( /// its cursor every pass so a restart continues instead of rewinding. Errors reading /// or repairing an individual row are warn-and-skip; only a failure of the batch query /// or the cursor write ends the run, and a later run picks up from the stored cursor. +/// +/// A run that skipped at least one RETRYABLE row rewinds the cursor to the start of the +/// table on its way out (#173 round 11). Without that the cursor parked at the maximum +/// `sha256_hex` for good: every later boot read zero rows, so a row skipped for a +/// transient reason (its repo cold on a Tigris-backed node, a DB or object read error) +/// was skipped permanently, unadvertised and unresolvable with nothing left to fix it. +/// The rewind is a per-RUN decision made after the walk has already finished, never +/// mid-walk, so it cannot spin: the cost is one extra ordered scan on the next run, and +/// a row that is unrepairable in principle (bytes gone, provenance gone) does not count +/// as retryable, so a node holding one does not re-walk on every boot forever. pub(crate) async fn sweep_legacy_provider_cids( repos_dir: &std::path::Path, git_bin: &str, @@ -244,19 +328,26 @@ pub(crate) async fn sweep_legacy_provider_cids( Ok(p) => p, Err(e) => { tracing::warn!(err = %e, "legacy provider-CID sweep pass failed; stopping"); - return totals; + break; } }; totals.scanned += pass.scanned; totals.repaired += pass.repaired; + totals.retryable_skips += pass.retryable_skips; totals.passes += 1; // A short batch means the ordered walk reached the end of the table. Stop here // rather than after an extra empty pass, and do NOT sleep on the way out. if (pass.scanned as i64) < batch { - return totals; + break; } tokio::time::sleep(delay).await; } + if totals.retryable_skips > 0 { + if let Err(e) = db.set_pin_repair_cursor("").await { + tracing::warn!(err = %e, "failed to rewind the legacy provider-CID sweep cursor"); + } + } + totals } // Test-only cost-gate counter (R8, U7): how many times the opportunistic repair diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 303aed00..440a7cd1 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -2973,6 +2973,148 @@ mod tests { ); } + /// F5 (#173 round 11): the work-budget peek sheds an already-throttled caller BEFORE + /// the two marker queries, so a spent-budget source stops paying two lookups per + /// request for a scan it will never be allowed to run. The source set here is + /// non-empty and complete, which is the case that used to reach the queries anyway. + /// The counter is the both-ways guard: moving the peek back below the pair reads 1. + #[sqlx::test] + async fn ipfs_cid_throttled_caller_sheds_before_the_marker_queries(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + // One PRIVATE source, recorded cleanly: the set is non-empty, below cap and + // unmarked, so nothing but the peek can keep the request off the queries. + let fx = seed_cid_repos(&slug, &short, &["f5only"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("f5only.git"); + let mut repo = seed_repo(&owner_did, "f5only"); + repo.is_public = false; + state.db.create_repo(&repo).await.expect("seed private"); + let cid = pin_cid_for_repo(&bare, &fx.secret_oid, &state.db, &repo.id).await; + state + .db + .record_pin_source(&fx.secret_oid, &repo.id) + .await + .expect("record source"); + + // Spend the caller's whole work budget before the request. + assert!( + state.ipfs_work_rate_limiter.check("9.9.9.9").await, + "the budget starts with room" + ); + + crate::api::ipfs::reset_marker_queries(); + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&cid, "9.9.9.9")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::TOO_MANY_REQUESTS, + "a spent-budget caller is shed at the peek" + ); + assert!( + !body.contains("TOP SECRET"), + "the shed response must not leak the withheld object" + ); + assert_eq!( + crate::api::ipfs::marker_queries(), + 0, + "a shed caller pays neither marker query" + ); + } + + /// U3 scenario 6 (#173, regression): a record that inserts NOTHING must not clear + /// the marker. `record_pin_source` is called for EVERY already-pinned object on the + /// skip path, and on a requeue pass that is the whole-repo enumeration, so the next + /// coalesced push from a repo ALREADY in the source set re-runs the insert as a + /// no-op. Clearing on that no-op re-hides the hole a different repo's failed record + /// recorded: the public copy stops being scanned for and 404s again. The assertion + /// is the SERVE outcome, not the column, so it still bites if the resolver ever + /// stops consulting the marker. RED before the rows_affected gate (404); GREEN after. + #[sqlx::test] + async fn ipfs_cid_noop_record_must_not_clear_the_marker(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["u3npriv", "u3npub"]); + let priv_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3npriv.git"); + let pub_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3npub.git"); + + // Repo A (private) is the first pinner AND is already recorded as a source, so a + // later record from A is a pure no-op insert. + let mut priv_repo = seed_repo(&owner_did, "u3npriv"); + priv_repo.is_public = false; + state + .db + .create_repo(&priv_repo) + .await + .expect("seed private"); + let cid = pin_cid_for_repo(&priv_bare, &fx.public_oid, &state.db, &priv_repo.id).await; + state + .db + .record_pin_source(&fx.public_oid, &priv_repo.id) + .await + .expect("record the first pinner as a source"); + + // Repo B (public) holds the same object, but its source record never lands, so + // the node marks the set known-incomplete. + let pub_repo = seed_repo(&owner_did, "u3npub"); + state.db.create_repo(&pub_repo).await.expect("seed public"); + with_pin_sources_broken(&pool, || { + repin_via_skip_branch(&state, &pub_bare, &fx.public_oid, &pub_repo.id) + }) + .await; + assert!( + state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "the exhausted record marked the set incomplete" + ); + + // A pushes again. The insert affects zero rows (A is already a source), so it + // recorded nothing and must not claim the set is complete. + repin_via_skip_branch(&state, &priv_bare, &fx.public_oid, &priv_repo.id).await; + assert_eq!( + state.db.pin_sources_for_oid(&fx.public_oid).await.unwrap(), + vec![priv_repo.id.clone()], + "the re-push really did add no source" + ); + + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "a no-op record must not clear the marker: the public copy still has to serve" + ); + assert!( + body.contains("public bytes"), + "the served body is the public object's bytes" + ); + } + /// U3 scenario 4 (#173): the marker tracks the record's OUTCOME, not the attempt. /// An exhausted retry sets it; a first-attempt success never does. Without the /// second arm the first could be satisfied by marking unconditionally. @@ -4740,6 +4882,292 @@ mod tests { } } + /// U4 scenario 9 (#173, regression): a row skipped for a TRANSIENT reason is + /// retried by a later run. The sweep never pulls a cold repo back from remote + /// storage, so on a Tigris-backed node a repo that is not on local disk at boot + /// contributes nothing to the pass. With the cursor parked at the end of the table + /// that row was skipped FOREVER: every later boot read zero rows and the row stayed + /// unadvertised and unresolvable with nothing left to repair it. Here the repo is + /// off disk for the first run and back for the second, so only a re-walk repairs it. + /// RED before the transient-skip cursor reset (the second run scans nothing). + #[sqlx::test] + async fn sweep_rewalks_after_a_transient_skip(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + + let fx = seed_cid_repos(&slug, &short, &["coldsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("coldsrc.git"); + let repo = seed_repo(&owner_did, "coldsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + let (raw_cid, _provider) = + seed_legacy_pin(&pool, &bare, &fx.public_oid, Some(&repo.id)).await; + + // The repo is COLD: its provenance resolves, but the bytes are not on this + // node's disk right now, exactly the state the sweep refuses to fix by pulling. + let stashed_away = bare.with_extension("git.away"); + let _ = std::fs::remove_dir_all(&stashed_away); + std::fs::rename(&bare, &stashed_away).expect("take the repo off local disk"); + + let first = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the first run terminates"); + assert_eq!( + (first.scanned, first.repaired), + (1, 0), + "the cold repo's row is walked but cannot be repaired yet" + ); + + // The repo is warm again (a later boot, a fetch, an operator restore). + std::fs::rename(&stashed_away, &bare).expect("put the repo back on local disk"); + + let second = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the second run terminates"); + assert_eq!( + second.repaired, 1, + "a later run re-walks the transiently skipped row and repairs it" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + raw_cid, + "the row now carries the raw-content resolver key" + ); + assert!( + state + .db + .list_pinned_cids() + .await + .unwrap() + .iter() + .any(|r| r.cid == raw_cid), + "the repaired row is advertised again" + ); + } + + /// U4 scenario 10 (#173, the other arm of scenario 9): a PERMANENTLY unrepairable + /// row must not make the sweep re-walk forever. Bytes that are genuinely gone are a + /// terminal skip, so the cursor stays parked and a later run reads nothing. Without + /// that split the transient-skip reset of scenario 9 turns every boot on such a node + /// into a full table walk. Both runs are timeout-bounded, so a hot loop FAILS here. + #[sqlx::test] + async fn sweep_does_not_rewalk_for_a_terminal_skip(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + + // The repo IS on local disk; the object's bytes are not in it and never will be. + let _fx = seed_cid_repos(&slug, &short, &["termsrc"]); + let repo = seed_repo(&owner_did, "termsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + let phantom_oid = "e".repeat(64); + let raw_cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"bytes that live nowhere").to_string(); + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + ) + .bind(&phantom_oid) + .bind(legacy_dagpb_cid(&raw_cid)) + .bind("2020-01-01T00:00:00Z") + .bind(&repo.id) + .execute(&pool) + .await + .unwrap(); + + let first = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the first run terminates"); + assert_eq!( + (first.scanned, first.repaired), + (1, 0), + "the row is walked and cannot be repaired" + ); + + let second = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the second run terminates"); + assert_eq!( + (second.scanned, second.repaired, second.passes), + (0, 0, 1), + "a terminal skip leaves the cursor parked: the next run re-reads nothing" + ); + } + + /// U4 scenario 11 (#173, path barrier): the sweep resolves a source repo's disk path + /// through the SAME validated logic the repo store uses, so a repo row whose name + /// carries `..` reads nothing. Names are validated at creation today, so this is a + /// defence-in-depth barrier on a second caller of the raw path helper rather than a + /// live exploit. The escapee repo really does hold the object's bytes, so before the + /// barrier the sweep happily read them from outside `repos_dir` and repaired the row. + /// RED before routing through the validated path (repaired 1). + #[sqlx::test] + async fn sweep_refuses_a_source_path_that_escapes_repos_dir(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + // The bytes live at /tmp/{slug}/escapee.git, OUTSIDE the repos_dir below. + let fx = seed_cid_repos(&slug, &short, &["escapee"]); + let escapee_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("escapee.git"); + let repos_dir = std::path::PathBuf::from("/tmp").join(&slug).join("root"); + std::fs::create_dir_all(repos_dir.join(&slug)).expect("create the repos_dir tree"); + + // A repo row whose name walks back out of repos_dir: repos_dir/{slug}/../../escapee.git + let mut repo = seed_repo(&owner_did, "../../escapee"); + repo.disk_path = escapee_bare.display().to_string(); + state.db.create_repo(&repo).await.expect("seed repo"); + let (_raw_cid, provider_cid) = + seed_legacy_pin(&pool, &escapee_bare, &fx.public_oid, Some(&repo.id)).await; + assert!( + crate::git::store::repo_disk_path(&repos_dir, &owner_did, &repo.name).exists(), + "the unvalidated helper really does resolve to the escapee repo" + ); + + let stats = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + &repos_dir, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the sweep terminates"); + + assert_eq!( + stats.repaired, 0, + "a repo path that escapes repos_dir must never be read" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + provider_cid, + "the row is untouched because its bytes were never read" + ); + } + + /// U4 scenario 12 (#173, F4): the repair's object read is SYNCHRONOUS `git cat-file`, + /// so running it inline parks the async worker for as long as git takes, up to the + /// whole `git_service_timeout_secs` budget on a wedged read, and the sweep does this + /// per legacy row starting at boot. A slow git stand-in makes that observable: a + /// concurrent 20ms ticker cannot tick at all while the only worker thread is blocked, + /// and ticks freely once the read is on the blocking pool. RED before the + /// `spawn_blocking` (0 ticks). + #[sqlx::test] + async fn repair_object_read_does_not_block_the_async_worker(pool: PgPool) { + use gitlawb_core::identity::Keypair; + use std::sync::atomic::{AtomicUsize, Ordering}; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["slowsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("slowsrc.git"); + let repo = seed_repo(&owner_did, "slowsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + seed_legacy_pin(&pool, &bare, &fx.public_oid, Some(&repo.id)).await; + + // A git that takes 300ms per invocation (the read makes two: type, then content). + let slow_git = std::env::temp_dir().join(format!("gl-slow-git-{short}")); + std::fs::write(&slow_git, "#!/bin/sh\nsleep 0.3\nexec git \"$@\"\n").expect("write shim"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&slow_git, std::fs::Permissions::from_mode(0o755)) + .expect("chmod shim"); + } + + let ticks = std::sync::Arc::new(AtomicUsize::new(0)); + let ticker = { + let ticks = ticks.clone(); + tokio::spawn(async move { + loop { + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + ticks.fetch_add(1, Ordering::Relaxed); + } + }) + }; + + let stats = crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + slow_git.to_str().unwrap(), + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + ) + .await; + ticker.abort(); + + assert_eq!(stats.repaired, 1, "the slow git still repairs the row"); + assert!( + ticks.load(Ordering::Relaxed) >= 5, + "the runtime kept running other tasks during the blocking git read (ticks: {})", + ticks.load(Ordering::Relaxed) + ); + } + /// U4 scenario 8 (#173, degenerate states): an empty `pinned_cids` table and a /// table with zero legacy rows both complete cleanly, with no repair and no read. #[sqlx::test] From 03aee267e609831fb3debc97b35f87d0eeed9aa5 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 27 Jul 2026 21:14:55 -0500 Subject: [PATCH 080/124] fix(node): shed 503 from the issue and PR write paths when the lock pool is exhausted The lock-pool work gave `acquire_write` a distinct exhaustion error and mapped it to 503 with Retry-After, but only on the push handler. The three other callers, both issue writes and the PR merge, still turned it into a generic git error and returned 500. Those are exactly the callers that hold no concurrency permit, so they are the likeliest to meet an exhausted pool in the first place. All three now go through the push handler's helper rather than repeating the mapping, so the log level and the message cannot drift between the four sites. Each has a test that occupies the single connection of a one-slot pool with a guard on a different repo, so the failure is pool capacity rather than advisory-lock contention, and asserts status 503 plus retry-after through IntoResponse. Reverting the mapping at one site turns only that site's test red. Worth knowing for review, and left alone deliberately: close_issue takes the write guard before its author check, so this shed is reachable by any authenticated caller whether or not they could close the issue. That ordering predates this change and reordering the handler is not this commit's business, but it does mean the 503 discloses that the repo's write pool is busy. --- crates/gitlawb-node/src/api/issues.rs | 174 +++++++++++++++++++++++++- crates/gitlawb-node/src/api/pulls.rs | 118 ++++++++++++++++- crates/gitlawb-node/src/api/repos.rs | 6 +- 3 files changed, 294 insertions(+), 4 deletions(-) diff --git a/crates/gitlawb-node/src/api/issues.rs b/crates/gitlawb-node/src/api/issues.rs index 17acf9af..0eacfa72 100644 --- a/crates/gitlawb-node/src/api/issues.rs +++ b/crates/gitlawb-node/src/api/issues.rs @@ -61,11 +61,14 @@ pub async fn create_issue( let json_str = serde_json::to_string(&issue) .map_err(|e| AppError::BadRequest(format!("serialization error: {e}")))?; + // Shed 503 + Retry-After on an exhausted write-lock POOL instead of a generic + // git 500 (#173 F1). This path holds no admission permit, so it reaches the pool + // unthrottled; reuse the push handler's mapping so the two cannot drift. let guard = state .repo_store .acquire_write(&record.owner_did, &record.name) .await - .map_err(|e| AppError::Git(e.to_string()))?; + .map_err(|e| crate::api::repos::acquire_write_app_error(&e, &repo))?; let disk_path = guard.path().to_path_buf(); let create_result = git_issues::create_issue(&disk_path, &issue_id, &json_str); @@ -229,11 +232,13 @@ pub async fn close_issue( .await? .ok_or_else(|| AppError::RepoNotFound(format!("{owner}/{repo}")))?; + // Same capacity shed as create_issue above (#173 F1): an exhausted write-lock + // pool is a 503 + Retry-After, not a 500 git error. let guard = state .repo_store .acquire_write(&record.owner_did, &record.name) .await - .map_err(|e| AppError::Git(e.to_string()))?; + .map_err(|e| crate::api::repos::acquire_write_app_error(&e, &repo))?; let disk_path = guard.path().to_path_buf(); // Owner OR issue author may close. The author lives in the issue's git-JSON @@ -279,3 +284,168 @@ pub async fn close_issue( Ok(Json(issue)) } + +/// #173 F1 follow-up: the two issue write paths reach `acquire_write` holding NO +/// admission permit (unlike the push handler, which is capped by the git-push +/// semaphore), so they are the callers most likely to meet an exhausted write-lock +/// POOL under load. An exhausted pool is a capacity signal, so both must shed +/// 503 + Retry-After (`AppError::Overloaded`) the way the push handler does, not +/// report the generic 500 git error that says nothing about retrying. +#[cfg(test)] +mod lock_pool_shed_tests { + use super::*; + use axum::response::IntoResponse; + use sqlx::PgPool; + + fn seed_repo(owner_did: &str, name: &str) -> crate::db::RepoRecord { + let now = Utc::now(); + crate::db::RepoRecord { + id: Uuid::new_v4().to_string(), + name: name.to_string(), + owner_did: owner_did.to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + created_at: now, + updated_at: now, + disk_path: format!("/tmp/{name}"), + forked_from: None, + machine_id: None, + } + } + + /// State whose repo store draws write locks from a ONE-connection pool with a + /// short checkout timeout, so a single held guard exhausts it promptly rather + /// than at the pool default. + async fn one_connection_lock_pool_state(pool: &PgPool) -> AppState { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.repo_store = crate::git::repo_store::RepoStore::new( + std::path::PathBuf::from("/tmp/gitlawb-issues-lockpool"), + None, + crate::git::repo_store::build_lock_pool(pool, 1, std::time::Duration::from_secs(1)), + ); + state + } + + /// The shed must be a real 503 carrying Retry-After, not just an internal enum + /// variant: assert on the rendered response so a remapping of `Overloaded` is + /// caught here too. + fn assert_sheds_503_with_retry_after(err: AppError, what: &str) { + let resp = err.into_response(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "{what}: an exhausted write-lock pool must shed 503, not a 500 git error" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .map(|v| v.to_str().unwrap()), + Some("1"), + "{what}: a capacity shed must tell the client when to retry" + ); + } + + /// RED-before/GREEN-after for `create_issue`. Both directions: the shed while the + /// only lock-pool connection is held by a guard on a DIFFERENT repo (so this is + /// pool capacity, not advisory-lock contention on this repo), and the must-not + /// case once that connection is back. + #[sqlx::test] + async fn create_issue_lock_pool_exhaustion_sheds_503_not_500(pool: PgPool) { + let owner = "did:key:zISSUECREATELOCKPOOLAAAAAAAAAAAAAAAAAAAA"; + let state = one_connection_lock_pool_state(&pool).await; + state + .db + .create_repo(&seed_repo(owner, "lp-create")) + .await + .expect("seed repo"); + + let held = state + .repo_store + .acquire_write(owner, "other-repo") + .await + .expect("the first write takes the only lock-pool connection"); + + let shed = create_issue( + State(state.clone()), + Extension(AuthenticatedDid(owner.to_string())), + Path((owner.to_string(), "lp-create".to_string())), + Json(CreateIssueRequest { + title: "t".to_string(), + body: None, + signed_payload: None, + }), + ) + .await; + let err = shed.expect_err("an exhausted lock pool must fail the call"); + assert_sheds_503_with_retry_after(err, "create_issue"); + + // MUST-NOT: with the pool free again the call is not shed as capacity (it + // fails later on the nonexistent on-disk repo, which is a git 500). + held.release(false).await; + let admitted = create_issue( + State(state.clone()), + Extension(AuthenticatedDid(owner.to_string())), + Path((owner.to_string(), "lp-create".to_string())), + Json(CreateIssueRequest { + title: "t".to_string(), + body: None, + signed_payload: None, + }), + ) + .await; + assert!( + !matches!(admitted, Err(AppError::Overloaded(_))), + "with the lock pool free, create_issue must not be shed as capacity; got {:?}", + admitted.err() + ); + } + + /// RED-before/GREEN-after for `close_issue`, same two directions. + #[sqlx::test] + async fn close_issue_lock_pool_exhaustion_sheds_503_not_500(pool: PgPool) { + let owner = "did:key:zISSUECLOSELOCKPOOLBBBBBBBBBBBBBBBBBBBBB"; + let state = one_connection_lock_pool_state(&pool).await; + state + .db + .create_repo(&seed_repo(owner, "lp-close")) + .await + .expect("seed repo"); + + let held = state + .repo_store + .acquire_write(owner, "other-repo") + .await + .expect("the first write takes the only lock-pool connection"); + + let shed = close_issue( + State(state.clone()), + Extension(AuthenticatedDid(owner.to_string())), + Path(( + owner.to_string(), + "lp-close".to_string(), + "deadbeef".to_string(), + )), + ) + .await; + let err = shed.expect_err("an exhausted lock pool must fail the call"); + assert_sheds_503_with_retry_after(err, "close_issue"); + + held.release(false).await; + let admitted = close_issue( + State(state.clone()), + Extension(AuthenticatedDid(owner.to_string())), + Path(( + owner.to_string(), + "lp-close".to_string(), + "deadbeef".to_string(), + )), + ) + .await; + assert!( + !matches!(admitted, Err(AppError::Overloaded(_))), + "with the lock pool free, close_issue must not be shed as capacity; got {:?}", + admitted.err() + ); + } +} diff --git a/crates/gitlawb-node/src/api/pulls.rs b/crates/gitlawb-node/src/api/pulls.rs index 26be6109..6255ef24 100644 --- a/crates/gitlawb-node/src/api/pulls.rs +++ b/crates/gitlawb-node/src/api/pulls.rs @@ -209,11 +209,14 @@ pub async fn merge_pr( return Err(AppError::BadRequest(format!("PR is already {}", pr.status))); } + // Shed 503 + Retry-After on an exhausted write-lock POOL instead of a generic + // git 500 (#173 F1). Merging holds no admission permit, so it reaches the pool + // unthrottled; reuse the push handler's mapping so the two cannot drift. let guard = state .repo_store .acquire_write(&record.owner_did, &record.name) .await - .map_err(|e| AppError::Git(e.to_string()))?; + .map_err(|e| crate::api::repos::acquire_write_app_error(&e, &name))?; let disk_path = guard.path().to_path_buf(); let merger_did = auth.0; let merge_result = store::merge_branch( @@ -424,3 +427,116 @@ pub async fn list_comments( let comments = state.db.list_pr_comments(&pr.id).await?; Ok(Json(serde_json::json!({ "comments": comments }))) } + +/// #173 F1 follow-up: `merge_pr` reaches `acquire_write` holding NO admission permit +/// (unlike the push handler, which is capped by the git-push semaphore), so it is one +/// of the callers most likely to meet an exhausted write-lock POOL under load. An +/// exhausted pool is a capacity signal, so the merge must shed 503 + Retry-After +/// (`AppError::Overloaded`) the way the push handler does, not report the generic +/// 500 git error that says nothing about retrying. +#[cfg(test)] +mod lock_pool_shed_tests { + use super::*; + use axum::response::IntoResponse; + use sqlx::PgPool; + + fn seed_repo(owner_did: &str, name: &str) -> crate::db::RepoRecord { + let now = Utc::now(); + crate::db::RepoRecord { + id: Uuid::new_v4().to_string(), + name: name.to_string(), + owner_did: owner_did.to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + created_at: now, + updated_at: now, + disk_path: format!("/tmp/{name}"), + forked_from: None, + machine_id: None, + } + } + + /// RED-before/GREEN-after for `merge_pr`. Both directions: the shed while the only + /// lock-pool connection is held by a guard on a DIFFERENT repo (so this is pool + /// capacity, not advisory-lock contention on this repo), and the must-not case + /// once that connection is back. + #[sqlx::test] + async fn merge_pr_lock_pool_exhaustion_sheds_503_not_500(pool: PgPool) { + let owner = "did:key:zMERGELOCKPOOLOWNERAAAAAAAAAAAAAAAAAAAAA"; + let mut state = crate::test_support::test_state(pool.clone()).await; + // One lock-pool connection with a short checkout timeout, so a single held + // guard exhausts it promptly rather than at the pool default. + state.repo_store = crate::git::repo_store::RepoStore::new( + std::path::PathBuf::from("/tmp/gitlawb-pulls-lockpool"), + None, + crate::git::repo_store::build_lock_pool(&pool, 1, std::time::Duration::from_secs(1)), + ); + + let repo = seed_repo(owner, "lp-merge"); + let repo_id = repo.id.clone(); + state.db.create_repo(&repo).await.expect("seed repo"); + let now = Utc::now().to_rfc3339(); + state + .db + .create_pr(&PullRequest { + id: Uuid::new_v4().to_string(), + repo_id: repo_id.clone(), + number: 1, + title: "lp".to_string(), + body: None, + author_did: owner.to_string(), + source_branch: "feature".to_string(), + target_branch: "main".to_string(), + status: "open".to_string(), + merged_by_did: None, + merged_at: None, + created_at: now.clone(), + updated_at: now, + }) + .await + .expect("seed open PR"); + + let held = state + .repo_store + .acquire_write(owner, "other-repo") + .await + .expect("the first write takes the only lock-pool connection"); + + let shed = merge_pr( + State(state.clone()), + Extension(AuthenticatedDid(owner.to_string())), + Path((owner.to_string(), "lp-merge".to_string(), 1)), + ) + .await; + let err = shed.expect_err("an exhausted lock pool must fail the call"); + let resp = err.into_response(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "merge_pr: an exhausted write-lock pool must shed 503, not a 500 git error" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .map(|v| v.to_str().unwrap()), + Some("1"), + "merge_pr: a capacity shed must tell the client when to retry" + ); + + // MUST-NOT: with the pool free again the merge is not shed as capacity (it + // fails later on the nonexistent on-disk repo, which is a git 500). + held.release(false).await; + let admitted = merge_pr( + State(state.clone()), + Extension(AuthenticatedDid(owner.to_string())), + Path((owner.to_string(), "lp-merge".to_string(), 1)), + ) + .await; + assert!( + !matches!(admitted, Err(AppError::Overloaded(_))), + "with the lock pool free, merge_pr must not be shed as capacity; got {:?}", + admitted.err() + ); + } +} diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index c8e9f2a6..49ab1a96 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1244,7 +1244,11 @@ pub(crate) async fn run_post_push_replication_for_test( /// POOL is a capacity signal, not a broken repo, so it sheds 503 + Retry-After the same /// way the admission caps around it do; it used to fall into the generic git 500, which /// tells the client nothing about retrying (#173 F1). Anything else stays a git error. -fn acquire_write_app_error(err: &anyhow::Error, repo: &str) -> AppError { +/// +/// Shared with the non-push `acquire_write` callers (`api/issues.rs`, `api/pulls.rs`) +/// rather than copied: those hold no admission permit, so they meet an exhausted pool +/// first, and a second copy of this mapping would be free to drift from the push path. +pub(crate) fn acquire_write_app_error(err: &anyhow::Error, repo: &str) -> AppError { if err .downcast_ref::() .is_some() From 370a364b0f9f79b2b6d7905a02a25dd69dc81b12 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 27 Jul 2026 21:44:31 -0500 Subject: [PATCH 081/124] test(node): cover the sweep's boot wiring and prove an interrupted push never reaches the upload Two claims from earlier commits were argued rather than executed. Both are now observed. The legacy-CID sweep's spawn sat inline in main() where no test could reach it, so the batch size, the inter-batch delay, the detachment and the shutdown select were all reasoned. The block moves verbatim into `spawn_legacy_cid_sweep`, which a test calls directly. It seeds rows whose keys are already canonical, so the cost gate skips each and the persisted cursor reads out walk progress cleanly, then asserts the cursor stops at the configured batch boundary, the call returns without blocking, the task is still alive inside its delay, and shutdown ends it. Never spawning, reading the batch or delay from the wrong knob, or removing the shutdown branch each turns it red. What remains reasoned is the single call line inside main(), down from a twenty-six line block. The claim that a disconnect-interrupted push never uploads to Tigris was structural: the tests build the store with no Tigris client, so a counter inside the client arm could never move and would have proved nothing. The counter now sits at the upload decision, before the client is consulted, and the disconnect test asserts zero while the success test asserts exactly one. That positive control is what makes the zero meaningful, and it is itself load-bearing: releasing the success path with failure semantics drops it to zero, and adding a Drop impl that uploads raises the disconnect count to one. The counter observes reaching the upload site, not an S3 request. TigrisClient wraps a concrete SDK client with no injectable endpoint, and plumbing one through would be out of proportion to the risk here. --- crates/gitlawb-node/src/api/repos.rs | 25 +++ crates/gitlawb-node/src/git/repo_store.rs | 38 +++++ crates/gitlawb-node/src/main.rs | 184 +++++++++++++++++----- 3 files changed, 211 insertions(+), 36 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 49ab1a96..89203edb 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -4968,6 +4968,20 @@ mod tests { freed_after_reap, "the write lock must be released once the disconnected push's group is reaped" ); + // The other half of the disconnect invariant, and the reason the guard rides the + // reaper rather than being released there: an interrupted push must not publish a + // half-applied repo. The guard is gone by now (the lock above only frees when it + // is), so the upload site has had its whole chance to be reached. The positive + // control is `receive_pack_success_reclaims_and_releases_the_write_lock`, which + // observes the same counter at 1: without it, a zero here would pass on any build + // where an upload is simply impossible. + assert_eq!( + state.repo_store.tigris_upload_site_reached(), + 0, + "a push interrupted by a client disconnect must not reach the Tigris upload \ + site: publishing a half-applied repo propagates it to every node that later \ + downloads it (#173 F2)" + ); } /// #173 F2, the other half: carrying the write lock through the admission seam must @@ -5056,6 +5070,17 @@ mod tests { write_lock_is_takeable(&probe, key).await, "a completed push must reclaim its write lock and release it synchronously" ); + // POSITIVE CONTROL for the disconnect case's "no upload" assertion. A push that + // completed does reach the Tigris upload site, exactly once, so the zero the + // disconnect test observes is a real difference between the two paths rather than + // an artifact of tests running with no Tigris client configured. Exactly once, + // not at least once: a retried exec race releases with success = false and must + // not count. + assert_eq!( + state.repo_store.tigris_upload_site_reached(), + 1, + "a completed push must reach the Tigris upload site once" + ); } /// #173 F1 (RED-before/GREEN-after): an exhausted repo write-lock POOL is a capacity diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index ebdd7757..02d02efb 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -44,6 +44,21 @@ pub struct RepoStore { /// window and cancel it there. #[cfg(test)] tigris_stall: Option, + /// Test-only counter of how many times a write guard from this store REACHED the + /// Tigris upload site in `release` (the point past the `success` check, where a + /// configured client would be uploaded to). It counts the decision, not a network + /// call: `TigrisClient` takes its endpoint from process-wide AWS env vars and has no + /// injectable seam, so every test runs with `tigris: None` and a counter inside the + /// `Some` arm could never move. Reaching the site is the property under test anyway: + /// an interrupted push must not publish a half-applied repo, and the disconnect path + /// must therefore never get here (#173 F2). + /// + /// Per store rather than a process global, so cases running in parallel do not see + /// each other's uploads, and an `Arc` rather than a `thread_local` because the guard + /// is released from a detached task on another worker thread. Same test-only counter + /// idiom as `ipfs_pin::note_legacy_repair_read`. + #[cfg(test)] + upload_site_reached: Arc, } impl RepoStore { @@ -66,6 +81,14 @@ impl RepoStore { self } + /// Test-only: how many write guards from this store have reached the Tigris upload + /// site. See [`RepoStore::upload_site_reached`]. + #[cfg(test)] + pub fn tigris_upload_site_reached(&self) -> usize { + self.upload_site_reached + .load(std::sync::atomic::Ordering::SeqCst) + } + /// `lock_pool` must come from `build_lock_pool`; a plain pool leaks advisory /// locks on cancellation. pub fn new(repos_dir: PathBuf, tigris: Option, lock_pool: PgPool) -> Self { @@ -76,6 +99,8 @@ impl RepoStore { migrated: Arc::new(Mutex::new(HashSet::new())), #[cfg(test)] tigris_stall: None, + #[cfg(test)] + upload_site_reached: Arc::new(std::sync::atomic::AtomicUsize::new(0)), } } @@ -270,6 +295,8 @@ impl RepoStore { lock_key, lock_conn, tigris: self.tigris.clone(), + #[cfg(test)] + upload_site_reached: Arc::clone(&self.upload_site_reached), }) } @@ -461,6 +488,10 @@ pub struct RepoWriteGuard { /// clears the lock. lock_conn: PoolConnection, tigris: Option, + /// Shared with the store that handed this guard out; see + /// [`RepoStore::upload_site_reached`]. + #[cfg(test)] + upload_site_reached: Arc, } impl RepoWriteGuard { @@ -477,6 +508,13 @@ impl RepoWriteGuard { pub async fn release(mut self, success: bool) { // Upload to Tigris only on success. if success { + // The upload site, recorded for tests before the client is consulted: with + // no injectable seam on `TigrisClient` a counter inside the arm below could + // never move, and it is reaching this point at all that an interrupted push + // must not do (#173 F2). + #[cfg(test)] + self.upload_site_reached + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); if let Some(ref tigris) = self.tigris { if let Err(e) = tigris .upload(&self.owner_slug, &self.repo_name, &self.local_path) diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index d4f4e63d..2a3505a5 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -543,42 +543,7 @@ async fn main() -> Result<()> { }); } - // U4 (#173): one-shot legacy provider-CID repair sweep. Releases before this - // version stored the PROVIDER CID (Kubo dag-pb / Pinata CIDv0) in `pinned_cids.cid`, - // and this version's `/ipfs/{cid}` resolver withholds any row whose stored key is - // not the raw-content CID. The opportunistic repair on the pin path only fires when - // a push re-carries the object, which normal git negotiation makes it not do, so - // those rows need a walk. DETACHED, never on the boot path: the server below starts - // and serves while this runs, and the sweep's own batch bound plus inter-batch delay - // keep it off the DB's critical path. Its cursor is durable, so a restart mid-walk - // resumes instead of rewinding. - { - let db = state.db.clone(); - let repos_dir = config.repos_dir.clone(); - let git_bin = state.git_bin.clone(); - let git_timeout = std::time::Duration::from_secs(config.git_service_timeout_secs); - let batch = config.pin_repair_sweep_batch; - let delay = std::time::Duration::from_secs(config.pin_repair_sweep_delay_secs); - let mut shutdown_rx = state.subscribe_shutdown(); - tokio::spawn(async move { - tokio::select! { - stats = ipfs_pin::sweep_legacy_provider_cids( - &repos_dir, &git_bin, git_timeout, batch, delay, &db, - ) => { - if stats.repaired > 0 { - tracing::info!( - scanned = stats.scanned, - repaired = stats.repaired, - "legacy provider-CID sweep finished" - ); - } - } - // Shutdown mid-walk simply drops the run; the persisted cursor means the - // next boot picks up where this one stopped. - _ = shutdown_rx.changed() => {} - } - }); - } + let _legacy_cid_sweep = spawn_legacy_cid_sweep(&state, &config); let router = server::build_router(state.clone()); // Re-register the socket bound at startup — same fd, so there was never a @@ -702,6 +667,49 @@ async fn main() -> Result<()> { Ok(()) } +/// U4 (#173): spawn the one-shot legacy provider-CID repair sweep. Releases before this +/// version stored the PROVIDER CID (Kubo dag-pb / Pinata CIDv0) in `pinned_cids.cid`, +/// and this version's `/ipfs/{cid}` resolver withholds any row whose stored key is not +/// the raw-content CID. The opportunistic repair on the pin path only fires when a push +/// re-carries the object, which normal git negotiation makes it not do, so those rows +/// need a walk. DETACHED, never on the boot path: the caller keeps serving while this +/// runs, and the sweep's own batch bound plus inter-batch delay keep it off the DB's +/// critical path. Its cursor is durable, so a restart mid-walk resumes instead of +/// rewinding. +/// +/// A named function rather than an inline block in `main` so the WIRING has a seam a +/// test can call: that the task is spawned at all, that it reads its batch and delay +/// from the config knobs rather than some other field, that the caller is not blocked +/// on it, and that the shutdown watcher actually ends it mid-walk. The sweep's own +/// behavior is covered elsewhere; this is the boot-path half. +fn spawn_legacy_cid_sweep(state: &AppState, config: &Config) -> tokio::task::JoinHandle<()> { + let db = state.db.clone(); + let repos_dir = config.repos_dir.clone(); + let git_bin = state.git_bin.clone(); + let git_timeout = std::time::Duration::from_secs(config.git_service_timeout_secs); + let batch = config.pin_repair_sweep_batch; + let delay = std::time::Duration::from_secs(config.pin_repair_sweep_delay_secs); + let mut shutdown_rx = state.subscribe_shutdown(); + tokio::spawn(async move { + tokio::select! { + stats = ipfs_pin::sweep_legacy_provider_cids( + &repos_dir, &git_bin, git_timeout, batch, delay, &db, + ) => { + if stats.repaired > 0 { + tracing::info!( + scanned = stats.scanned, + repaired = stats.repaired, + "legacy provider-CID sweep finished" + ); + } + } + // Shutdown mid-walk simply drops the run; the persisted cursor means the + // next boot picks up where this one stopped. + _ = shutdown_rx.changed() => {} + } + }) +} + fn spawn_shutdown_signal(tx: watch::Sender) { tokio::spawn(async move { #[cfg(unix)] @@ -1162,6 +1170,110 @@ fn load_or_create_keypair(config: &Config) -> Result { } } +#[cfg(test)] +mod legacy_cid_sweep_wiring_tests { + use super::spawn_legacy_cid_sweep; + use sqlx::PgPool; + use std::time::Duration; + + /// Seed `count` `pinned_cids` rows whose keys are already canonical raw CIDv1, in a + /// known `sha256_hex` order. The sweep's own cost gate skips a raw-CIDv1 row without + /// reading bytes or resolving a repo, so each row is SCANNED (it advances the cursor) + /// and nothing else. That is what makes the cursor a clean readout of how far the + /// walk got, with no dependency on repos on disk. + async fn seed_scannable_rows(pool: &PgPool, count: usize) -> Vec { + let mut shas = Vec::new(); + for i in 1..=count { + let sha = format!("wire{i:02}"); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(sha.as_bytes()).to_string(); + assert!( + gitlawb_core::cid::is_raw_cidv1(&cid), + "the seeded key must hit the sweep's raw-CIDv1 skip, not a repair attempt" + ); + sqlx::query("INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) VALUES ($1, $2, $3)") + .bind(&sha) + .bind(&cid) + .bind("2020-01-01T00:00:00Z") + .execute(pool) + .await + .unwrap(); + shas.push(sha); + } + shas + } + + /// Poll the persisted sweep cursor until it reaches `want`, or give up. + async fn cursor_reaches(db: &crate::db::Db, want: &str, within: Duration) -> String { + let deadline = std::time::Instant::now() + within; + loop { + let c = db.pin_repair_cursor().await.unwrap(); + if c == want || std::time::Instant::now() >= deadline { + return c; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + } + + /// #173 U4, the BOOT-PATH half. The sweep's own logic (batching, cursor resumption, + /// terminal vs retryable skips) is covered in `test_support`; what this covers is the + /// wiring `main` performs, which nothing else executes: the task is spawned at all, + /// it takes its batch and delay from the two `pin_repair_sweep_*` knobs rather than + /// some other config field, the caller is not blocked on the walk, and the shutdown + /// watcher ends the run mid-walk. + /// + /// Six scannable rows, batch 2, delay 30s. One pass must land the cursor on exactly + /// the second row and the task must then still be alive in its inter-batch sleep, + /// which pins both knobs at once: a different batch stops at a different row, and a + /// delay that did not come from the knob either finishes the table or leaves the task + /// gone. Shutdown must then end it while four rows are still unwalked. + #[sqlx::test] + async fn the_boot_path_spawns_the_sweep_detached_with_its_configured_knobs(pool: PgPool) { + let state = crate::test_support::test_state(pool.clone()).await; + let shas = seed_scannable_rows(&pool, 6).await; + let repos_dir = tempfile::TempDir::new().unwrap(); + + let mut config = (*state.config).clone(); + config.repos_dir = repos_dir.path().to_path_buf(); + config.pin_repair_sweep_batch = 2; + // Far longer than this test runs, so a task still alive after the first pass can + // only be one that is honoring the configured inter-batch delay. + config.pin_repair_sweep_delay_secs = 30; + + let started = std::time::Instant::now(); + let handle = spawn_legacy_cid_sweep(&state, &config); + let spawn_cost = started.elapsed(); + + let cursor = cursor_reaches(&state.db, &shas[1], Duration::from_secs(10)).await; + assert_eq!( + cursor, shas[1], + "the spawned sweep must run and stop its first pass at the CONFIGURED batch \ + bound (2), leaving the cursor on the second row" + ); + assert!( + spawn_cost < Duration::from_secs(1), + "the sweep must be detached, not awaited on the boot path; the spawn took \ + {spawn_cost:?}" + ); + assert!( + !handle.is_finished(), + "with a 30s inter-batch delay the task must still be sleeping between passes, \ + not finished: a finished task means the delay was not the configured one" + ); + + state.shutdown(); + tokio::time::timeout(Duration::from_secs(10), handle) + .await + .expect("the shutdown watcher must end the sweep, and not after its 30s delay") + .expect("the sweep task must not panic"); + + assert_eq!( + state.db.pin_repair_cursor().await.unwrap(), + shas[1], + "shutdown must have ended the run MID-walk, with the remaining rows unwalked" + ); + } +} + #[cfg(test)] mod lock_pool_sizing_tests { use super::{lock_pool_size, LOCK_POOL_MAX_CONNECTIONS, LOCK_POOL_PUSH_HEADROOM}; From 34a92c78c787fafdf5b2c608c2d9e893240b6c93 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 27 Jul 2026 22:45:48 -0500 Subject: [PATCH 082/124] fix(node): take the per-source write permit before parking on the repo lease A push buffers its whole pack before the handler runs, then parks on the per-repo write lease for up to git_service_timeout_secs * 2 + 60, which is 1260s at defaults. Both admission permits were taken after that park, so nothing accounted for waiters: any number of requests could sit on one contended repo, each holding a fully buffered body, invisible to both semaphores. The per-source sub-cap now runs before the park. It already exists, is sized at a quarter of the push pool, and is keyed on the resolved source IP, so parked bodies are bounded at four per source without a new knob or a new cap to tune. The global permit deliberately stays below the lease. Moving it up is what f02c5e1 fixed: a lease-blocked waiter holding a global slot sheds pushes to every other repo node-wide. A per-source key does not have that shape, since exhausting it denies only the source doing the exhausting, so the two permits belong on opposite sides of the park and the comment there now says why. The load-bearing test is not that a capped source sheds, which any cap satisfies, including the global one this rejects. It is that a different source still parks and is served while the first is at its cap. Keying the permit on a constant leaves the shed test green and turns only that one red. Moving the global permit above the lease turns the f02c5e1 guard red. Bounded per source, not globally. A distributed set of sources can still each hold their four, and the unbounded-total term is the pre-auth buffering in #263 rather than the lease. --- crates/gitlawb-node/src/api/repos.rs | 449 +++++++++++++++++++++++++-- 1 file changed, 428 insertions(+), 21 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index f258a4bc..768496c2 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1541,8 +1541,9 @@ pub async fn git_receive_pack( let name = smart_http_repo_name(&repo)?; // Fast-path shed BEFORE the DB lookup when the write pool is saturated, so a push // flood on a full pool does not hit Postgres per request. Best-effort (racy) and - // NON-holding: the authoritative, held permit is taken after the per-repo lease below - // (so a lease-blocked waiter pins no write slot — #174 F3 review). This peek only + // NON-holding: the authoritative, held GLOBAL write permit is taken after the per-repo + // lease below (so a lease-blocked waiter pins no global write slot, #174 F3 review; + // the per-SOURCE sub-cap is taken above the lease, see there). This peek only // restores the cheap pre-DB shed the permit reorder would otherwise have lost. if state.git_write_semaphore.available_permits() == 0 { return Err(AppError::Overloaded( @@ -1637,27 +1638,31 @@ pub async fn git_receive_pack( // overlap is the ~4s disconnect/reap window, which the reaper-carried clone (a) covers. let lease_steal_after = std::time::Duration::from_secs(state.config.git_service_timeout_secs * 2 + 60); - let lease = state - .repo_write_leases - .acquire(&record.id, lease_steal_after) - .await; - // Admission permits are taken HERE, AFTER the per-repo lease and BEFORE acquire_write. - // Ordering is the fix (#174 P2 DoS): the lease is a block-and-wait serializer, so a - // second same-repo push can park on `acquire` above for up to steal_after. Taking the - // scarce write permits only once we own the lease means a lease-blocked waiter pins NO - // write-pool slot while it waits. Otherwise a few hostile sources could stack same-repo - // pushes, hold every global slot on zero-byte lease-waiters, and shed 503 on every push - // to every OTHER repo node-wide. Still before acquire_write, so the git op stays - // admission-gated (INV-10) and a saturated pool sheds 503 before spawning git. + // The two write admission permits are DELIBERATELY split around the lease park. // - // Per-source sub-cap first (#174 P1-d): one source IP cannot occupy the whole write - // pool via many slow pushes. Owner enforcement defaults off, so any valid did:key is - // accepted (auth != authz) and the push rate limiter bounds arrival RATE, not in-flight - // concurrency. Keyed on the resolved source IP, NEVER the signed DID (a DID farm defeats - // a DID key); no resolvable key -> global write pool only. Then the global write permit: - // pushes draw from the dedicated WRITE pool, separate from reads, and it is held for the - // whole op (moved into the AdmissionGuard below). + // Per-source sub-cap BEFORE the lease (#174 P1-d, and F1 review). One source IP + // cannot occupy the whole write pool via many slow pushes: owner enforcement defaults + // off, so any valid did:key is accepted (auth != authz) and the push rate limiter + // bounds arrival RATE, not in-flight concurrency. It is taken here, above the park, + // because `body: Bytes` means axum has ALREADY buffered the entire pack (up to + // max_pack_bytes) before this function runs, and `acquire` below is a block-and-wait + // serializer whose waiter set is unbounded (state.rs) and whose wait runs to + // steal_after (git_service_timeout_secs * 2 + 60 = 1260s at defaults). With both + // permits below the lease nothing accounted for parked waiters, so one source could + // stack unbounded buffered bodies on a contended repo. The sub-cap is + // (max_concurrent_git_pushes / 8).max(1) = 4 at defaults, so it bounds parked bodies + // at 4 per source. Keyed on the resolved source IP, NEVER the signed DID (a DID farm + // defeats a DID key); no resolvable key -> global write pool only. + // + // The GLOBAL write permit deliberately stays BELOW the lease (#174 P2 DoS, f02c5e1). + // That is a scarce node-wide resource: a lease-blocked waiter holding one lets a few + // hostile sources stack same-repo pushes, pin every global slot on zero-byte + // lease-waiters, and shed 503 on every push to every OTHER repo node-wide. A + // per-source key is not global, so exhausting it denies only the exhausting source. + // Self-inflicted denial is the property that makes the sub-cap safe above the park and + // the global permit unsafe there. Both still land before acquire_write, so the git op + // stays admission-gated (INV-10) and a saturated pool sheds 503 before spawning git. let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); let _caller_permit = acquire_read_caller_permit( &state.git_write_per_caller, @@ -1665,6 +1670,12 @@ pub async fn git_receive_pack( name, "receive-pack", )?; + + let lease = state + .repo_write_leases + .acquire(&record.id, lease_steal_after) + .await; + let _permit = git_permit(&state.git_write_semaphore)?; tracing::debug!(repo = %name, "acquiring write lock"); @@ -7083,4 +7094,400 @@ mod tests { assert_eq!(resp.status(), 200, "push B lands 200 after serialization"); assert!(b_ran.exists(), "push B ran its receive-pack once unblocked"); } + + /// Fake git for the U1 lease-park tests. The FIRST receive-pack invocation marks + /// that it reached the pack and hangs in a BOUNDED loop (so a RED run leaks no + /// permanent orphan); every later invocation marks that it got past the lease. + /// Returns `(git_bin, a_inpack marker, later_ran marker)`. + #[cfg(unix)] + fn f1_hanging_first_git( + tmp: &std::path::Path, + ) -> (String, std::path::PathBuf, std::path::PathBuf) { + let seq_a = tmp.join("f1_seq_a"); // first receive-pack wins this mkdir = push A + let a_inpack = tmp.join("f1_a.inpack"); + let later_ran = tmp.join("f1_later.ran"); + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + receive-pack)\n\ + cat >/dev/null 2>/dev/null\n\ + if mkdir \"{seq}\" 2>/dev/null; then\n\ + echo 1 > \"{ainp}\"\n\ + i=0; while [ $i -lt 100 ]; do sleep 0.1; i=$((i+1)); done\n\ + else\n\ + echo 1 > \"{later}\"\n\ + fi ;;\n\ + rev-parse) echo deadbeef ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + seq = seq_a.display(), + ainp = a_inpack.display(), + later = later_ran.display(), + ); + (write_fake_git(tmp, &body), a_inpack, later_ran) + } + + /// #174 F1 (U1, RED-before/GREEN-after): a source already at its per-source WRITE + /// sub-cap must SHED on a contended repo rather than park on the per-repo write + /// lease. `git_receive_pack` takes `body: Bytes`, so axum has already buffered the + /// whole pack (up to `max_pack_bytes`, 2 GB by default) before the handler runs, + /// and the lease park runs to `lease_steal_after` = `git_service_timeout_secs * 2 + /// + 60` = 1260s at defaults. The waiter set in `RepoWriteLeases::acquire` is + /// unbounded, so with BOTH admission permits taken after the lease nothing accounts + /// for parked bodies. Taking the per-caller write permit BEFORE the lease bounds + /// them at `(max_concurrent_git_pushes / 8).max(1)` = 4 per source. + /// + /// Load-bearing: push A (source X) holds the lease and X's only per-caller slot. + /// Pre-fix push B (also X) parks on the lease and does not return inside the test + /// window (RED). Post-fix it sheds on the per-caller cap immediately (GREEN). The + /// elapsed assertion is the point: an error returned after `steal_after` would be + /// no fix at all. + #[cfg(unix)] + #[sqlx::test] + async fn f1_capped_source_sheds_instead_of_parking_on_a_contended_lease(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let tmp = tempfile::TempDir::new().unwrap(); + let (git_bin, a_inpack, _later_ran) = f1_hanging_first_git(tmp.path()); + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f1cap", "c1", false).await; + // Per-source write cap of 1 so push A alone exhausts source X's budget. The + // global write pool stays generous (64), so nothing here sheds on it. + state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + let did = "did:key:z6MkF1CapPusherAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let src_x: SocketAddr = "203.0.113.61:5000".parse().unwrap(); + + // Push A: drive its handler future in slices until it reaches receive-pack. It + // now holds the per-repo lease AND source X's single per-caller write slot. + let mut fut_a = Box::pin(git_receive_pack( + State(state.clone()), + Path(("z6f1cap".to_string(), "c1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(src_x)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + )); + for _ in 0..1000 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut_a).await; + if a_inpack.exists() { + break; + } + } + assert!( + a_inpack.exists(), + "push A must reach receive-pack and hold the lease plus source X's write slot" + ); + + // Push B: SAME source, same repo. It must be turned away, not parked. + let started = std::time::Instant::now(); + let b = tokio::time::timeout( + std::time::Duration::from_secs(10), + git_receive_pack( + State(state.clone()), + Path(("z6f1cap".to_string(), "c1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(src_x)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ), + ) + .await; + let elapsed = started.elapsed(); + let b = b.expect( + "F1 RED: a source already at its per-source write cap PARKED on the contended \ + repo lease holding a fully buffered pack body. The per-caller write permit \ + must be taken BEFORE the lease so the request sheds instead of waiting out \ + steal_after (1260s at defaults)", + ); + assert!( + matches!(b, Err(AppError::Overloaded(_))), + "the capped source's second push must shed with Overloaded/503; got {b:?}" + ); + assert!( + elapsed < std::time::Duration::from_secs(5), + "the shed must return promptly, not after the lease park; took {elapsed:?}" + ); + + drop(fut_a); + } + + /// #174 F1 (U1) THE DISCRIMINATOR: while source X sits at its per-source write cap + /// on a contended repo, a push from a DIFFERENT source Y must still park on the + /// lease and be served once the lease frees. This is what distinguishes moving the + /// PER-CALLER permit above the lease from the rejected alternative of moving the + /// GLOBAL write permit (or any node-wide waiter cap) above it: exhausting a + /// per-source key denies only the exhausting source, so the denial is + /// self-inflicted. Key the permit on a constant instead of `read_caller_key`'s + /// resolved source IP and Y sheds too, which is exactly the node-wide shed + /// `f02c5e1` was written to prevent. + /// + /// Load-bearing on both halves: X's second push sheds promptly (RED pre-fix, where + /// it parks) AND Y's push stays parked rather than shedding (RED under the + /// constant-key mutation). + #[cfg(unix)] + #[sqlx::test] + async fn f1_capped_source_shed_leaves_a_different_source_parked_and_served(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let tmp = tempfile::TempDir::new().unwrap(); + let (git_bin, a_inpack, later_ran) = f1_hanging_first_git(tmp.path()); + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f1disc", "d1", false).await; + state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + let did = "did:key:z6MkF1DiscPusherAAAAAAAAAAAAAAAAAAAAAAAAA"; + let src_x: SocketAddr = "203.0.113.63:5000".parse().unwrap(); + let src_y: SocketAddr = "203.0.113.64:5000".parse().unwrap(); + + // Push A (source X) reaches receive-pack: holds the lease and X's only slot. + let mut fut_a = Box::pin(git_receive_pack( + State(state.clone()), + Path(("z6f1disc".to_string(), "d1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(src_x)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + )); + for _ in 0..1000 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut_a).await; + if a_inpack.exists() { + break; + } + } + assert!(a_inpack.exists(), "push A must reach receive-pack"); + + // Push C from source Y: it has its own budget, so it takes Y's permit and parks + // on the lease A holds. + let state_c = state.clone(); + let handle_c = tokio::spawn(async move { + git_receive_pack( + State(state_c), + Path(("z6f1disc".to_string(), "d1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(src_y)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await + }); + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + assert!( + !handle_c.is_finished(), + "F1 discriminator RED: a push from a DIFFERENT source was turned away while \ + source X was at its cap. The write sub-cap must be keyed on the resolved \ + source IP, so exhausting it denies only the exhausting source" + ); + + // Push B from source X: shed, promptly, while C stays parked. + let started = std::time::Instant::now(); + let b = tokio::time::timeout( + std::time::Duration::from_secs(10), + git_receive_pack( + State(state.clone()), + Path(("z6f1disc".to_string(), "d1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(src_x)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ), + ) + .await + .expect("the capped source must shed rather than park on the contended lease"); + assert!( + matches!(b, Err(AppError::Overloaded(_))), + "source X's second push must shed with Overloaded/503; got {b:?}" + ); + assert!( + started.elapsed() < std::time::Duration::from_secs(5), + "source X's shed must be prompt" + ); + assert!( + !handle_c.is_finished(), + "source Y must still be parked on the lease, unaffected by X hitting its cap" + ); + assert!( + !later_ran.exists(), + "no second receive-pack may run while A holds the lease" + ); + + // Free the lease: A disconnects. Source Y's parked push is served. + drop(fut_a); + let resp = tokio::time::timeout(std::time::Duration::from_secs(30), handle_c) + .await + .expect("push C must complete once A's group is reaped and the lease frees") + .expect("push C task must not panic") + .expect("push C must succeed"); + assert_eq!( + resp.status(), + 200, + "the unaffected source's parked push lands 200 once the lease frees" + ); + assert!( + later_ran.exists(), + "push C ran its receive-pack once the lease freed" + ); + } + + /// #174 F1 (U1) key shape: the write sub-cap is keyed on the SOURCE, not the repo. + /// A source at its cap is shed on EVERY repo (otherwise one source could hold four + /// buffered bodies per repo, and the bound would be `repos x cap x max_pack_bytes`), + /// while a different source is shed on none. Uncontended leases here: this pins the + /// key's shape, not the acquisition order. + #[sqlx::test] + async fn f1_write_cap_key_is_per_source_not_per_repo(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + let mut state = crate::test_support::test_state(pool).await; + state.git_write_semaphore = Arc::new(Semaphore::new(4)); + state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + for name in ["k1", "k2"] { + state + .db + .upsert_mirror_repo( + "z6f1key", + name, + &format!("/tmp/f1-key-{name}-nonexistent"), + None, + false, + ) + .await + .unwrap(); + } + + let did = "did:key:z6MkF1KeyPusherAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let capped: SocketAddr = "203.0.113.65:5000".parse().unwrap(); + let other: SocketAddr = "203.0.113.66:5000".parse().unwrap(); + let _slot = state + .git_write_per_caller + .try_acquire(&capped.ip().to_string()) + .expect("pin the capped source at its single write slot"); + + let push = |peer: SocketAddr, repo: &'static str| { + let st = state.clone(); + async move { + git_receive_pack( + State(st), + Path(("z6f1key".to_string(), repo.to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(peer)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await + } + }; + + for repo in ["k1", "k2"] { + let r = push(capped, repo).await; + assert!( + matches!(r, Err(AppError::Overloaded(_))), + "the capped source must shed on repo {repo} too: the sub-cap is per \ + source, not per repo; got {r:?}" + ); + } + let r = push(other, "k2").await; + assert!( + !matches!(r, Err(AppError::Overloaded(_))), + "a different source must not be shed on any repo while the capped source \ + holds its slot; got {r:?}" + ); + } + + /// #174 F1 (U1) fallback: a caller with no resolvable source key (no trusted + /// header, no peer address) takes no per-caller permit and is bounded by the global + /// write pool only, exactly as before the move. `acquire_read_caller_permit` + /// returns `Ok(None)` for a `None` key, so it must never 503 here. + #[sqlx::test] + async fn f1_write_cap_is_inert_without_a_resolvable_source_key(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::sync::Arc; + use tokio::sync::Semaphore; + + let mut state = crate::test_support::test_state(pool).await; + state.git_write_semaphore = Arc::new(Semaphore::new(4)); + state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 1); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6f1none", "n1", "/tmp/f1-none-nonexistent", None, false) + .await + .unwrap(); + // Saturate the limiter's only key slot as well, so a request that DID resolve a + // key would be shed. The keyless request must still pass. + let _slot = state + .git_write_per_caller + .try_acquire("203.0.113.67") + .expect("occupy the limiter's single key slot"); + + let r = git_receive_pack( + State(state.clone()), + Path(("z6f1none".to_string(), "n1".to_string())), + Extension(crate::auth::AuthenticatedDid( + "did:key:z6MkF1NoKeyPusherAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + )), + crate::rate_limit::PeerAddr(None), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await; + assert!( + !matches!(r, Err(AppError::Overloaded(_))), + "a caller with no resolvable source key must fall back to the global write \ + pool only, never shed on the per-caller cap; got {r:?}" + ); + } + + /// #174 F1 (U1) no-regression: moving the per-caller permit above the lease must + /// not leak it. Two SEQUENTIAL clean pushes from the SAME source with the sub-cap + /// set to 1 must both land 200; if the first push's permit outlived its request the + /// second would 503, breaking every repeat pusher. + #[cfg(unix)] + #[sqlx::test] + async fn f1_sequential_pushes_from_one_source_release_the_write_permit(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let tmp = tempfile::TempDir::new().unwrap(); + let body = "#!/bin/sh\ncase \"$1\" in\n receive-pack) cat >/dev/null 2>/dev/null ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f1seq", "s1", false).await; + state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + let did = "did:key:z6MkF1SeqPusherAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let src: SocketAddr = "203.0.113.68:5000".parse().unwrap(); + + for attempt in 1..=2 { + let resp = tokio::time::timeout( + std::time::Duration::from_secs(30), + git_receive_pack( + State(state.clone()), + Path(("z6f1seq".to_string(), "s1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(src)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ), + ) + .await + .expect("a clean push must not wedge") + .unwrap_or_else(|e| panic!("push {attempt} from the same source must succeed: {e:?}")); + assert_eq!(resp.status(), 200, "clean push {attempt} lands 200"); + } + assert_eq!( + state.git_write_per_caller.tracked_keys(), + 0, + "every per-source write permit must be released when its push completes" + ); + } } From 92a7e58b238efc80516218c34c3d70963e0e4cac Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 27 Jul 2026 23:02:15 -0500 Subject: [PATCH 083/124] fix(node): end the session when the advisory unlock errors, instead of pooling a locked connection `release` discarded the unlock's result and set `released` regardless, so an unlock that errored while its session stayed alive (a statement timeout, an admin cancel) returned a connection to the pool still holding the repo's advisory lock, with the Drop backstop already disarmed by that flag. The error is now captured. On failure the connection is taken and closed, which ends the session and makes Postgres drop the lock server-side, and only then is the guard marked released, because the lock went with the session rather than with the unlock. `close()` rather than `detach()` since we are already in an async fn and can wait for the teardown deterministically. Disposal is the only mechanism on this path. It cannot coexist with an armed Drop backstop: disposal needs the connection taken, and Drop returns early when there is none, while leaving the connection in place so Drop can use it just puts the locked connection back in the pool when that spawned task ends. The Drop doc now says so rather than claiming to cover a case it cannot. The cancellation path is untouched. It was already correct, since `released` is set after the await, and its test still passes unmodified. The new tests assert against `pg_locks` from a connection held out of the pool before the acquire, never by calling `acquire_write` a second time. That shortcut would pass whether or not the fix is present, because the pool can hand back the same connection and session advisory locks are re-entrant. The unlock error is injected by leaving the pinned connection in an aborted transaction, which errors the next statement while keeping the session and its lock alive, and the test asserts that precondition before releasing. Not covered: that the failure path still skips the Tigris upload. The test store builds with no Tigris client, so the upload site is unobservable here without a new seam, and that block is untouched by this change. --- crates/gitlawb-node/src/git/repo_store.rs | 204 +++++++++++++++++++++- 1 file changed, 197 insertions(+), 7 deletions(-) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index bd27a186..2bc15e0a 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -384,7 +384,9 @@ pub struct RepoWriteGuard { /// The pooled connection that took the advisory lock. Session-level pg /// advisory locks are connection-affine, so the guard pins that connection /// for its whole lifetime and unlocks on it (in `release`, or the `Drop` - /// backstop). `None` only after the connection has been taken to unlock. + /// backstop). `None` only after the connection has been taken, either to run + /// the detached unlock in `Drop` or to be closed when `release`'s unlock + /// errored (#174 F3b). conn: Option>, /// Set once the advisory lock has actually been taken. A guard dropped /// before the lock is held (or after `release`) performs no unlock. @@ -437,17 +439,49 @@ impl RepoWriteGuard { if self.locked { #[cfg(test)] let pre_unlock_gate = self.test_pre_unlock_gate.clone(); - if let Some(conn) = self.conn.as_deref_mut() { + let unlock = if let Some(conn) = self.conn.as_deref_mut() { // Test-only: park right before the unlock await so a test can drop // this future mid-unlock (connection owned, not yet released). #[cfg(test)] if let Some(gate) = pre_unlock_gate { gate.notified().await; } - let _ = sqlx::query("SELECT pg_advisory_unlock($1)") - .bind(self.lock_key) - .execute(&mut *conn) - .await; + Some( + sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(self.lock_key) + .execute(&mut *conn) + .await, + ) + } else { + None + }; + // An unlock that ERRORS is a different failure from a cancellation: the + // await resolved, so `Drop` is about to be made inert by `released` + // below, but the session is still alive and still holds the lock + // (statement timeout, admin cancel, aborted transaction). Returning that + // `PoolConnection` to the pool would hand the next caller a connection + // holding a lock nobody tracks (#174 F3b). Connection disposal is the + // single mechanism here, and it is why we do not instead try to keep the + // `Drop` backstop armed: disposal needs `conn.take()`, and `Drop` + // early-returns on `conn == None`. Ending the session is what frees the + // lock, so `released = true` still holds. + if let Some(Err(e)) = unlock { + warn!(repo = %self.repo_name, err = %e, + "advisory unlock failed, closing the connection so the session ends and postgres drops the lock"); + if let Some(conn) = self.conn.take() { + // `close()` over `detach()`: both consume the `PoolConnection` by + // value in sqlx 0.8.6, but we are in an async fn, so `close()` + // sends Terminate and waits for the socket to go down before + // `release` returns. `detach()` would only end the session + // whenever the returned `PgConnection` is dropped and its + // background close completes. If this future is cancelled during + // `close()`, the connection is dropped mid-close, which still + // tears the session down. + if let Err(e) = conn.close().await { + warn!(repo = %self.repo_name, err = %e, + "closing the write-lock connection failed, the session teardown still frees the lock server-side"); + } + } } } self.released = true; @@ -458,7 +492,10 @@ impl Drop for RepoWriteGuard { /// Cancellation-safe backstop: if the guard is dropped while still holding the /// advisory lock (a `tokio::time::timeout` cancelled `acquire_write`, or a /// handler future was dropped before `release`), unlock on the pinned - /// connection. `Drop` cannot await, so spawn a detached unlock — it runs on the + /// connection. This is NOT the backstop for an unlock that ran and returned an + /// error: that case is closed inside `release` by disposing of the connection, + /// because `Drop` early-returns on `conn == None` and the two mechanisms cannot + /// both apply (#174 F3b). `Drop` cannot await, so spawn a detached unlock — it runs on the /// same session (connection-affine). An off-runtime drop falls back to a log; /// the ~60s stale-lock retry loop in `acquire_write` reclaims it. On runtime /// SHUTDOWN the spawned unlock task may be dropped before it polls, so the unlock @@ -831,6 +868,159 @@ mod tests { again.release(true).await; } + // ── unlock error disposes the connection (#174 F3b, RED-before/GREEN-after) ─ + + /// Put the guard's pinned connection into a failed-transaction state, so the + /// next statement on it errors while the SESSION stays alive and keeps holding + /// the session-level advisory lock (those survive a transaction abort; only + /// `pg_advisory_xact_lock` would not). This is the smallest injection that + /// reproduces F3b's shape: `pg_advisory_unlock` returning `Err` on a live, + /// still-locking session. No production seam is needed because the tests live + /// in this module and can reach `conn` directly. + async fn poison_guard_connection(guard: &mut RepoWriteGuard) { + let conn = guard + .conn + .as_deref_mut() + .expect("guard holds its connection before release"); + sqlx::query("BEGIN") + .execute(&mut *conn) + .await + .expect("open a transaction on the guard connection"); + let poisoned = sqlx::query("SELECT 1 / 0").execute(&mut *conn).await; + assert!( + poisoned.is_err(), + "the poison statement must fail so the transaction is aborted" + ); + } + + /// F3b (P1): when `pg_advisory_unlock` ERRORS while the session is still alive + /// (statement timeout, admin cancel, aborted transaction), the lock must not + /// survive `release`. The old code discarded the error with `let _ =` and set + /// `released = true` anyway, so `Drop` early-returned and the `PoolConnection` + /// went back to the pool still holding the session lock. + /// + /// Observed from a SEPARATE connection held out of the pool before acquiring: + /// session advisory locks are re-entrant and counted, so probing from the + /// holding session (or via a fresh `acquire_write` that may be handed the same + /// connection) would report free whether or not the fix is present. + /// + /// Load-bearing: RED before the fix (lock still held → `pg_try_advisory_lock` + /// returns false), GREEN after (the errored connection is closed, so the + /// session ends and Postgres drops the lock). + #[sqlx::test] + async fn write_guard_release_with_failing_unlock_frees_the_lock(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone()); + let owner = "did:key:z6MkUnlockErrorProofFFFFFFFFFFFFFFFFFFFFFF"; + let name = "unlockerrtest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + // Distinct session for the probe, held out of the pool before acquiring. + let mut checker = pool.acquire().await.expect("checker connection"); + + let mut guard = store.acquire_write(owner, name).await.expect("acquire"); + poison_guard_connection(&mut guard).await; + + // Sanity: the poisoned session is still alive and still holds the lock, so + // the assertion below measures the release path and not a dead session. + let (free_before,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *checker) + .await + .unwrap(); + assert!( + !free_before, + "the poisoned session must still hold the lock before release" + ); + + guard.release(false).await; + // Give a background close a moment to reach the server. + tokio::time::sleep(std::time::Duration::from_millis(400)).await; + + let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *checker) + .await + .unwrap(); + assert!( + free, + "an errored pg_advisory_unlock must not leave the lock held: release \ + must dispose of the connection so the session ends" + ); + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } + + /// F3b: the connection that saw the unlock error must leave the pool entirely, + /// rather than being handed to the next caller while still holding the lock. + /// `PgPool::size()` counts the connections the pool owns, so closing the + /// errored one is observable as a drop in that count. + #[sqlx::test] + async fn write_guard_release_with_failing_unlock_does_not_return_the_connection( + pool: sqlx::PgPool, + ) { + let dir = tempfile::TempDir::new().unwrap(); + let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone()); + let owner = "did:key:z6MkUnlockErrorPoolProofGGGGGGGGGGGGGGGGGG"; + let name = "unlockerrpooltest"; + + let mut guard = store.acquire_write(owner, name).await.expect("acquire"); + poison_guard_connection(&mut guard).await; + let size_before = pool.size(); + assert!(size_before > 0, "the pool owns the guard's connection"); + + guard.release(false).await; + tokio::time::sleep(std::time::Duration::from_millis(400)).await; + + assert_eq!( + pool.size(), + size_before - 1, + "the connection that saw the unlock error must be closed, not returned \ + to the pool still holding the session lock" + ); + } + + /// F3b regression guard on the success path: a normal unlock keeps the + /// connection in the pool and marks the guard released, so the disposal branch + /// is confined to the error case. + #[sqlx::test] + async fn write_guard_release_success_keeps_the_connection_and_frees_the_lock( + pool: sqlx::PgPool, + ) { + let dir = tempfile::TempDir::new().unwrap(); + let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone()); + let owner = "did:key:z6MkUnlockOkProofHHHHHHHHHHHHHHHHHHHHHHHH"; + let name = "unlockoktest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + let mut checker = pool.acquire().await.expect("checker connection"); + let guard = store.acquire_write(owner, name).await.expect("acquire"); + let size_before = pool.size(); + + guard.release(false).await; + tokio::time::sleep(std::time::Duration::from_millis(400)).await; + + assert_eq!( + pool.size(), + size_before, + "a successful unlock must leave the connection in the pool" + ); + let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *checker) + .await + .unwrap(); + assert!(free, "the success path must still free the lock"); + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } + /// F4: releasing a guard that never took the lock (`locked == false`, the state /// acquire_write leaves after a failed acquisition) must not unlock or panic. #[sqlx::test] From 316bdc0cbb827849271a5290af04397f9e158f02 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 28 Jul 2026 00:05:04 -0500 Subject: [PATCH 084/124] fix(node): coalesce a repeat push before it parks on the encrypt pool, not after The post-push tail resolved the withheld set and materialized the object list before it ever consulted the per-repo coalescer, so rapid pushes to one repo each parked on the encrypt semaphore and repeated the scan before discovering they should have coalesced. The gate now runs first, keyed on the cheap listable-at-root predicate that `replication_withheld_set` already computes from rules alone before it touches the semaphore. Three things made this less mechanical than it looks, all of them ways a naive move would have shipped a regression. The withheld set had a second consumer. It also gated the Pinata pin and the branch-to-CID upsert, so a coalesced push that skipped the walk would have pinned nothing and recorded no CID, while a test asserting merely that the announce spawn ran would have stayed green. That gate moves to the announce predicate too. Coalescing must not return from the tail. The announce spawn sits below the coalescer in the same task, and returning early would drop gossip, the GraphQL broadcast, Arweave anchoring and peer notify, which are per-push and non-idempotent. A flag skips the encrypt section and falls through. The publish paths are gated on announce and iterate the ref updates, not the object list, so handing the tail the cheap predicate would have made a repo whose walk is failing start gossiping and anchoring, losing the fail-closed property. `pinata_object_list_for_refs` already recomputes the real set inside the pin permit, so it now returns that verdict alongside the list and the tail uses it. No additional walk is introduced. Moving the gate earlier also opens a window where a push can coalesce while the admitted one is still walking. If that walk then fails, the admitted task now still spawns the drain with an empty snapshot so those tips are consumed, rather than dropping the guard and discarding them. Note a released guard and a dropped guard are indistinguishable by the in-flight map, since Drop clears the key either way, so that case is asserted on drained work rather than on the map being empty. Each of the four is pinned by a mutation: walking anyway, returning early, leaving the Pinata gate on the withheld set, and dropping instead of draining each turn exactly one test red. --- crates/gitlawb-node/src/api/repos.rs | 1291 +++++++++++++++++++------- crates/gitlawb-node/src/state.rs | 13 + 2 files changed, 983 insertions(+), 321 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 768496c2..667baf9a 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1027,8 +1027,8 @@ async fn pinata_object_list_for_refs( owner_did: String, git_bin: String, timeout: std::time::Duration, -) -> Vec { - let (_announce, withheld) = replication_withheld_set( +) -> (bool, Vec) { + let (announce, withheld) = replication_withheld_set( encrypt_sem.clone(), rules_opt.clone(), &owner_did, @@ -1039,10 +1039,14 @@ async fn pinata_object_list_for_refs( ) .await; // Not announceable, or the withheld walk failed: replicate nothing (fail - // closed) — mirrors the receive-pack tail's `withheld == None` handling. + // closed), mirroring the receive-pack tail's `withheld == None` handling. The + // announce decision is returned with the list because this recomputation is the + // tail's only walk once coalescing runs ahead of it (#174 F2a): a repo whose + // walk is failing must still suppress gossip, the GraphQL broadcast, Arweave and + // peer-notify, and `announce` is false on exactly those arms. let withheld_set = match withheld { Some(w) => w, - None => return Vec::new(), + None => return (announce, Vec::new()), }; let new_tips: Vec = ref_updates .iter() @@ -1064,7 +1068,7 @@ async fn pinata_object_list_for_refs( false, ) .await; - if pin_set.full_scan { + let object_list = if pin_set.full_scan { fail_closed_full_scan_objects( encrypt_sem, disk_path, @@ -1078,7 +1082,8 @@ async fn pinata_object_list_for_refs( .await } else { crate::git::visibility_pack::replicable_objects(pin_set.candidates, &withheld_set) - } + }; + (announce, object_list) } /// The pin/encrypt pipeline shared by the snapshot iteration and the @@ -1851,16 +1856,82 @@ pub async fn git_receive_pack( // including its own always-spawned announce, so per-push announcements are never // coalesced away (the announce spawn stays out of the per-repo encrypt coalescing). let did = did.to_string(); - tokio::spawn(async move { - // Replication enforcement (Phase 2): decide once per push whether the public - // may read this repo at all and, if so, which blob OIDs must not leave the - // node. `withheld == None` means replicate nothing (private / mode A / - // undetermined): skip every pin so even commit and tree objects (which - // withheld_blob_oids never lists) stay local. `announce` gates the - // network-facing announcements. Fail closed: a private or undetermined repo - // never leaks. - let rules_opt = state.db.list_visibility_rules(&record.id).await.ok(); - let (announce, withheld) = replication_withheld_set( + tokio::spawn(post_receive_replication_tail( + state, + record, + ref_updates, + disk_path, + did, + )); + + Ok(result) +} + +/// The detached post-receive replication tail (#174 F2): everything a landed push +/// still owes after its git response has been returned: the replication decision, +/// the per-repo-coalesced pin/encrypt task, and this push's own Pinata + announce +/// task. Split out of `git_receive_pack` so the ordering the coalescing gate depends +/// on is directly testable; the handler spawns it and returns. +async fn post_receive_replication_tail( + state: AppState, + record: RepoRecord, + ref_updates: Vec, + disk_path: std::path::PathBuf, + did: String, +) { + // Replication enforcement (Phase 2): decide once per push whether the public + // may read this repo at all and, if so, which blob OIDs must not leave the + // node. `withheld == None` means this push pins nothing (private / mode A / + // undetermined, or a walk that failed): skip every pin so even commit and tree + // objects (which withheld_blob_oids never lists) stay local. Fail closed: a + // private or undetermined repo never leaks. The announce decision that gates + // the network-facing sends is taken separately, below. + let rules_opt = state.db.list_visibility_rules(&record.id).await.ok(); + + // #174 F2a: take the per-repo coalescing key BEFORE the walk, not after it. + // `replication_withheld_set` decides announceability from the rules snapshot + // alone and returns `(false, None)` before it touches the scan pool or spawns + // any git, so the same predicate can be evaluated here and used to gate + // `try_begin`. With the gate below the walk, rapid pushes to one repo each + // parked on `git_encrypt_semaphore` and re-ran the walk plus the object-list + // materialization before finding out they were going to coalesce; now a push + // that will coalesce does none of that. Not announceable is the same as + // before: nothing replicates, so no key is taken and no walk runs. + let announce_at_root = match &rules_opt { + Some(rules) => { + crate::visibility::listable_at_root(rules, record.is_public, &record.owner_did, None) + } + None => false, + }; + let mut coalesced = false; + let mut inflight = None; + if announce_at_root { + let tip_pairs: Vec<(String, String)> = ref_updates + .iter() + .map(|u| (u.old_sha.clone(), u.new_sha.clone())) + .collect(); + match state.encrypt_inflight.try_begin(&record.id, tip_pairs) { + crate::state::BeginOutcome::Coalesced => { + coalesced = true; + tracing::debug!( + repo = %record.id, + "post-push encryption task already in flight for this repo; coalesced \ + — this push's tip pairs are queued for that task's drain" + ); + } + crate::state::BeginOutcome::Admitted(guard) => inflight = Some(guard), + } + } + + // The walk feeds this push's own pin/encrypt snapshot, so it is skipped both + // when nothing may replicate and when this push coalesced (the in-flight + // task drains its tips). The walk's own announce decision is deliberately + // not kept here: it does not exist on the coalesced path, and the Pinata / + // announce tail below re-derives it (see `do_pinata_replication`). + let withheld = if coalesced || !announce_at_root { + None + } else { + replication_withheld_set( state.git_encrypt_semaphore.clone(), rules_opt.clone(), &record.owner_did, @@ -1869,193 +1940,202 @@ pub async fn git_receive_pack( state.git_bin.clone(), std::time::Duration::from_secs(state.config.git_service_timeout_secs), ) - .await; + .await + .1 + }; - // Resolve the per-push pin candidate set once, off the async worker, then - // filter to what may actually replicate. Delta path: the reachable-only - // `withheld` set suffices (delta objects are reachable). Full-scan path: the - // candidate set can include dangling blobs the withheld set never classified, - // so fail closed — replicate a blob only if it is reachable AND - // visibility-allowed (#99). Only computed when something will actually - // replicate; every degraded path logs rather than failing silently. - let object_list: Vec = if let Some(withheld_set) = withheld.clone() { - let new_tips: Vec = ref_updates - .iter() - .map(|u| u.new_sha.clone()) - .filter(|s| s != ZERO_SHA) - .collect(); - let old_tips: Vec = ref_updates - .iter() - .map(|u| u.old_sha.clone()) - .filter(|s| s != ZERO_SHA) - .collect(); - let pin_set = crate::git::push_delta::resolve_candidates_for_push( + // Resolve the per-push pin candidate set once, off the async worker, then + // filter to what may actually replicate. Delta path: the reachable-only + // `withheld` set suffices (delta objects are reachable). Full-scan path: the + // candidate set can include dangling blobs the withheld set never classified, + // so fail closed — replicate a blob only if it is reachable AND + // visibility-allowed (#99). Only computed when something will actually + // replicate; every degraded path logs rather than failing silently. + let object_list: Vec = if let Some(withheld_set) = withheld { + let new_tips: Vec = ref_updates + .iter() + .map(|u| u.new_sha.clone()) + .filter(|s| s != ZERO_SHA) + .collect(); + let old_tips: Vec = ref_updates + .iter() + .map(|u| u.old_sha.clone()) + .filter(|s| s != ZERO_SHA) + .collect(); + let pin_set = crate::git::push_delta::resolve_candidates_for_push( + state.git_encrypt_semaphore.clone(), + disk_path.clone(), + new_tips, + old_tips, + state.git_bin.clone(), + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + false, + ) + .await; + if pin_set.full_scan { + fail_closed_full_scan_objects( state.git_encrypt_semaphore.clone(), disk_path.clone(), - new_tips, - old_tips, + rules_opt.clone().unwrap_or_default(), + record.is_public, + record.owner_did.clone(), + pin_set.candidates, state.git_bin.clone(), std::time::Duration::from_secs(state.config.git_service_timeout_secs), - false, ) - .await; - if pin_set.full_scan { - fail_closed_full_scan_objects( - state.git_encrypt_semaphore.clone(), - disk_path.clone(), - rules_opt.clone().unwrap_or_default(), - record.is_public, - record.owner_did.clone(), - pin_set.candidates, - state.git_bin.clone(), - std::time::Duration::from_secs(state.config.git_service_timeout_secs), - ) - .await - } else { - crate::git::visibility_pack::replicable_objects(pin_set.candidates, &withheld_set) - } + .await } else { - Vec::new() - }; - - // Pin new git objects to the local IPFS node (no-op if ipfs_api is empty). - // Skipped entirely when the public cannot read the repo (withheld == None). - // - // Coalesce-and-requeue per repo (#174 P2-2 + F5): the spawned task's walks park - // on `git_encrypt_semaphore` (which DEFERS when the pool is full rather than - // dropping the recovery copy). To bound the OUTSTANDING task set, at most one - // task per repo is in flight; a push arriving while one is in flight does NOT - // spawn a duplicate — and is NOT dropped either. The in-flight task pins only - // its own pre-spawn object-list snapshot, so this push's (old, new) tip pairs - // are merged into the in-flight key's pending slot in the same critical section - // as the presence check, and the task loop-drains them (fresh rules, fail - // closed) before releasing the key. Without the requeue a coalesced push's pins - // and recovery copies would be silently absent until an unrelated later push - // (the F5 loss). The guard still releases the key on panic (Drop on unwind), so - // a crashed walk never permanently locks the repo out. - if withheld.is_some() { - let tip_pairs: Vec<(String, String)> = ref_updates - .iter() - .map(|u| (u.old_sha.clone(), u.new_sha.clone())) - .collect(); - match state.encrypt_inflight.try_begin(&record.id, tip_pairs) { - crate::state::BeginOutcome::Coalesced => { - tracing::debug!( - repo = %record.id, - "post-push encryption task already in flight for this repo; coalesced \ - — this push's tip pairs are queued for that task's drain" - ); - } - crate::state::BeginOutcome::Admitted(inflight_guard) => { - let ctx = EncryptTaskCtx { - ipfs_api: state.config.ipfs_api.clone(), - repo_path: disk_path.clone(), - db: state.db.clone(), - repo_id: record.id.clone(), - owner_did: record.owner_did.clone(), - repo_name: record.name.clone(), - irys_url: state.config.irys_url.clone(), - http_client: std::sync::Arc::clone(&state.http_client), - node_did: state.node_did.to_string(), - node_keypair: std::sync::Arc::clone(&state.node_keypair), - git_bin: state.git_bin.clone(), - git_timeout: std::time::Duration::from_secs( - state.config.git_service_timeout_secs, - ), - encrypt_sem: state.git_encrypt_semaphore.clone(), - pin_sem: state.pin_semaphore.clone(), - }; - tokio::spawn(run_encrypt_pin_task( - ctx, - inflight_guard, - object_list.clone(), - rules_opt.clone(), - record.is_public, - )); - } - } + crate::git::visibility_pack::replicable_objects(pin_set.candidates, &withheld_set) } + } else { + Vec::new() + }; - // Pin new git objects to Pinata, then record branch→CID and gossip. - // - // #174 P2-2 scope note: this SECOND detached spawn is deliberately NOT brought - // under the per-repo encryption coalescing above, because unlike the idempotent - // recovery-copy walk it does PER-PUSH, PER-REF work — branch→CID upserts, gossip - // publish, GraphQL subscription broadcast, Arweave anchoring, and peer notify, each - // keyed to THIS push's ref_updates. Coalescing (or shedding) it against an in-flight - // task for the same repo would DROP a later push's ref-update announcements (a - // correctness regression), not merely delay a duplicate. So the task stays one per - // push and every push's effects fire exactly once. - // - // #174 F2 / KTD-3: {bounded memory, no dropped effects, no handler latency} are - // jointly unsatisfiable by coalesce/shed/block, so instead of retaining the full - // object list we bound the thing that actually accumulates. The task captures only - // the small ref tuples and RE-DERIVES the object set inside the worker once a pin - // slot frees (see `pinata_object_list_for_refs`); the MB-scale OID list is never - // held by a parked task. - { - let pinata_jwt = state.config.pinata_jwt.clone(); - let pinata_upload_url = state.config.pinata_upload_url.clone(); - let repo_path_clone = disk_path.clone(); - let db_clone = state.db.clone(); - let http_client = Arc::clone(&state.http_client); - let node_did_str = state.node_did.to_string(); - let repo_slug = format!( - "{}/{}", - crate::db::normalize_owner_key(&record.owner_did), - record.name - ); - let ref_updates_clone = ref_updates - .iter() - .map(|u| (u.ref_name.clone(), u.old_sha.clone(), u.new_sha.clone())) - .collect::>(); - let p2p_handle = state.p2p.clone(); - let pusher_did_clone = did.to_string(); - let db_for_peers = state.db.clone(); - let ref_update_tx = state.ref_update_tx.clone(); - let irys_url = state.config.irys_url.clone(); - let owner_did_for_arweave = record.owner_did.clone(); - let self_public_url = state.config.public_url.clone(); - let node_keypair = Arc::clone(&state.node_keypair); - let do_pinata_replication = withheld.is_some(); - // #174 F2 / KTD-3: capture only the small inputs the re-derivation needs; the - // MB-scale object list is NOT moved in. `pinata_object_list_for_refs` recomputes - // it from these once a pin slot frees. rules/owner/is_public drive the fresh - // fail-closed withheld filter; encrypt_sem + git_bin + timeout keep the re-derive - // git children under the same INV-22 bounded, group-reaped scan admission. - let pinata_rules_opt = rules_opt.clone(); - let pinata_owner_did = record.owner_did.clone(); - let pinata_is_public = record.is_public; - let pinata_git_bin = state.git_bin.clone(); - let pinata_git_timeout = - std::time::Duration::from_secs(state.config.git_service_timeout_secs); - let pinata_encrypt_sem = state.git_encrypt_semaphore.clone(); - // Same global pin-admission bound as the IPFS loop (#174 F6): the Pinata pin - // loop holds a re-derived object-id list while pinning it, so it shares the cap. - // It DEFERS on a full pool rather than dropping the pin. - let pin_sem_pinata = state.pin_semaphore.clone(); - tokio::spawn(async move { - let pinned = if do_pinata_replication { - let _pin_permit = pin_sem_pinata - .acquire_owned() - .await - .expect("pin_semaphore is never closed"); - // Re-derive the object set now that a pin slot is free (#174 F2 / - // KTD-3). A parked task retained only `ref_updates_clone` (O(ref - // tuples)), never this list, so a slow Pinata backend cannot grow - // outstanding memory O(pushes x object-list). Fresh + fail-closed; - // each git child is INV-22 bounded and process-group reaped. - let object_list = pinata_object_list_for_refs( - pinata_encrypt_sem, - repo_path_clone.clone(), - &ref_updates_clone, - pinata_rules_opt, - pinata_is_public, - pinata_owner_did, - pinata_git_bin, - pinata_git_timeout, - ) - .await; + // Pin new git objects to the local IPFS node (no-op if ipfs_api is empty). + // Skipped entirely when the public cannot read the repo (no key was taken). + // + // Coalesce-and-requeue per repo (#174 P2-2 + F5): the spawned task's walks park + // on `git_encrypt_semaphore` (which DEFERS when the pool is full rather than + // dropping the recovery copy). To bound the OUTSTANDING task set, at most one + // task per repo is in flight; a push arriving while one is in flight does NOT + // spawn a duplicate — and is NOT dropped either. The in-flight task pins only + // its own pre-spawn object-list snapshot, so this push's (old, new) tip pairs + // are merged into the in-flight key's pending slot in the same critical section + // as the presence check, and the task loop-drains them (fresh rules, fail + // closed) before releasing the key. Without the requeue a coalesced push's pins + // and recovery copies would be silently absent until an unrelated later push + // (the F5 loss). The guard still releases the key on panic (Drop on unwind), so + // a crashed walk never permanently locks the repo out. + // + // #174 F2a: the key was taken above, so this is only the spawn. An admitted + // push ALWAYS spawns, including when its own walk failed and `object_list` is + // therefore empty: pushes can have coalesced into the pending slot while that + // walk ran, and the task's drain loop is what consumes them. Releasing or + // dropping the guard instead would discard that work with a warn (the F5 loss + // class again), and the two are indistinguishable from outside (Drop removes + // the key whenever the guard is still armed). + if let Some(inflight_guard) = inflight { + let ctx = EncryptTaskCtx { + ipfs_api: state.config.ipfs_api.clone(), + repo_path: disk_path.clone(), + db: state.db.clone(), + repo_id: record.id.clone(), + owner_did: record.owner_did.clone(), + repo_name: record.name.clone(), + irys_url: state.config.irys_url.clone(), + http_client: std::sync::Arc::clone(&state.http_client), + node_did: state.node_did.to_string(), + node_keypair: std::sync::Arc::clone(&state.node_keypair), + git_bin: state.git_bin.clone(), + git_timeout: std::time::Duration::from_secs(state.config.git_service_timeout_secs), + encrypt_sem: state.git_encrypt_semaphore.clone(), + pin_sem: state.pin_semaphore.clone(), + }; + tokio::spawn(run_encrypt_pin_task( + ctx, + inflight_guard, + object_list, + rules_opt.clone(), + record.is_public, + )); + } + + // Pin new git objects to Pinata, then record branch→CID and gossip. + // + // #174 P2-2 scope note: this SECOND detached spawn is deliberately NOT brought + // under the per-repo encryption coalescing above, because unlike the idempotent + // recovery-copy walk it does PER-PUSH, PER-REF work — branch→CID upserts, gossip + // publish, GraphQL subscription broadcast, Arweave anchoring, and peer notify, each + // keyed to THIS push's ref_updates. Coalescing (or shedding) it against an in-flight + // task for the same repo would DROP a later push's ref-update announcements (a + // correctness regression), not merely delay a duplicate. So the task stays one per + // push and every push's effects fire exactly once. + // + // #174 F2 / KTD-3: {bounded memory, no dropped effects, no handler latency} are + // jointly unsatisfiable by coalesce/shed/block, so instead of retaining the full + // object list we bound the thing that actually accumulates. The task captures only + // the small ref tuples and RE-DERIVES the object set inside the worker once a pin + // slot frees (see `pinata_object_list_for_refs`); the MB-scale OID list is never + // held by a parked task. + { + let pinata_jwt = state.config.pinata_jwt.clone(); + let pinata_upload_url = state.config.pinata_upload_url.clone(); + let repo_path_clone = disk_path.clone(); + let db_clone = state.db.clone(); + let http_client = Arc::clone(&state.http_client); + let node_did_str = state.node_did.to_string(); + let repo_slug = format!( + "{}/{}", + crate::db::normalize_owner_key(&record.owner_did), + record.name + ); + let ref_updates_clone = ref_updates + .iter() + .map(|u| (u.ref_name.clone(), u.old_sha.clone(), u.new_sha.clone())) + .collect::>(); + let p2p_handle = state.p2p.clone(); + let pusher_did_clone = did.to_string(); + let db_for_peers = state.db.clone(); + let ref_update_tx = state.ref_update_tx.clone(); + let irys_url = state.config.irys_url.clone(); + let owner_did_for_arweave = record.owner_did.clone(); + let self_public_url = state.config.public_url.clone(); + let node_keypair = Arc::clone(&state.node_keypair); + // #174 F2a: gated on the cheap announce predicate, not on `withheld`. + // `withheld` is None for a push that coalesced (it never walked), and + // this task's work is per-push and non-idempotent, so keying it on the + // walk result would silently stop pinning and stop recording a branch to + // CID mapping for every coalesced push. The fail-closed source for this + // path is now `pinata_object_list_for_refs`'s own recomputation of + // `replication_withheld_set` inside the pin permit: it returns an empty + // list AND announce=false when the walk fails or the repo may not + // replicate, so neither blobs nor announcements escape an unvetted push. + let do_pinata_replication = announce_at_root; + // #174 F2 / KTD-3: capture only the small inputs the re-derivation needs; the + // MB-scale object list is NOT moved in. `pinata_object_list_for_refs` recomputes + // it from these once a pin slot frees. rules/owner/is_public drive the fresh + // fail-closed withheld filter; encrypt_sem + git_bin + timeout keep the re-derive + // git children under the same INV-22 bounded, group-reaped scan admission. + let pinata_rules_opt = rules_opt.clone(); + let pinata_owner_did = record.owner_did.clone(); + let pinata_is_public = record.is_public; + let pinata_git_bin = state.git_bin.clone(); + let pinata_git_timeout = + std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let pinata_encrypt_sem = state.git_encrypt_semaphore.clone(); + // Same global pin-admission bound as the IPFS loop (#174 F6): the Pinata pin + // loop holds a re-derived object-id list while pinning it, so it shares the cap. + // It DEFERS on a full pool rather than dropping the pin. + let pin_sem_pinata = state.pin_semaphore.clone(); + tokio::spawn(async move { + // `announce` comes back from the re-derivation below rather than from + // the tail's own walk (#174 F2a): a coalesced push has no walk of its + // own, and this is the recomputation that fails closed for it. When + // the repo is not announceable at root there is no re-derivation and + // no announcement either, which is the same answer the walk gave. + let (announce, pinned) = if do_pinata_replication { + let _pin_permit = pin_sem_pinata + .acquire_owned() + .await + .expect("pin_semaphore is never closed"); + // Re-derive the object set now that a pin slot is free (#174 F2 / + // KTD-3). A parked task retained only `ref_updates_clone` (O(ref + // tuples)), never this list, so a slow Pinata backend cannot grow + // outstanding memory O(pushes x object-list). Fresh + fail-closed; + // each git child is INV-22 bounded and process-group reaped. + let (announce, object_list) = pinata_object_list_for_refs( + pinata_encrypt_sem, + repo_path_clone.clone(), + &ref_updates_clone, + pinata_rules_opt, + pinata_is_public, + pinata_owner_did, + pinata_git_bin, + pinata_git_timeout, + ) + .await; + ( + announce, crate::pinata::pin_new_objects( &http_client, &pinata_upload_url, @@ -2064,155 +2144,145 @@ pub async fn git_receive_pack( object_list, &db_clone, ) - .await - } else { - Vec::new() - }; + .await, + ) + } else { + (false, Vec::new()) + }; - if !pinned.is_empty() { - tracing::info!(count = pinned.len(), "pinned git objects to Pinata"); - } + if !pinned.is_empty() { + tracing::info!(count = pinned.len(), "pinned git objects to Pinata"); + } - // Build sha→cid map from pinned objects - let cid_map: std::collections::HashMap = - pinned.into_iter().collect(); + // Build sha→cid map from pinned objects + let cid_map: std::collections::HashMap = pinned.into_iter().collect(); - // Record branch→CID for each ref update and publish gossip - for (ref_name, old_sha, new_sha) in &ref_updates_clone { - let cid = cid_map.get(new_sha).map(|s| s.as_str()); - - if let Some(cid_str) = cid { - let _ = db_clone - .upsert_branch_cid( - &repo_slug, - ref_name, - new_sha, - cid_str, - &node_did_str, - ) - .await; - } + // Record branch→CID for each ref update and publish gossip + for (ref_name, old_sha, new_sha) in &ref_updates_clone { + let cid = cid_map.get(new_sha).map(|s| s.as_str()); - if announce { - if let Some(p2p) = &p2p_handle { - p2p.publish_ref_update(crate::p2p::RefUpdateEvent { - node_did: node_did_str.clone(), - pusher_did: pusher_did_clone.clone(), - repo: repo_slug.clone(), - ref_name: ref_name.clone(), - old_sha: old_sha.clone(), - new_sha: new_sha.clone(), - timestamp: chrono::Utc::now().to_rfc3339(), - cert_id: None, - cid: cid.map(|s| s.to_string()), - }) - .await; - } - } + if let Some(cid_str) = cid { + let _ = db_clone + .upsert_branch_cid(&repo_slug, ref_name, new_sha, cid_str, &node_did_str) + .await; } - // Broadcast ref update to GraphQL subscription listeners — one per ref. - // Gated on `announce`: /graphql/ws is unauthenticated (mounted after - // the optional_signature layer), and the subscription resolver has no - // caller to gate against, so only publicly-readable ref updates may - // reach anonymous subscribers. Mirrors the gossip (above) and Arweave - // (below) sends, which are already `announce`-gated. Without this a - // private-repo push would leak live ref metadata over the socket — - // the subscription analog of #112/#114. - let now_ts = chrono::Utc::now().to_rfc3339(); if announce { - for (ref_name, old_sha, new_sha) in &ref_updates_clone { - let _ = ref_update_tx.send(crate::state::RefUpdateBroadcast { + if let Some(p2p) = &p2p_handle { + p2p.publish_ref_update(crate::p2p::RefUpdateEvent { + node_did: node_did_str.clone(), + pusher_did: pusher_did_clone.clone(), repo: repo_slug.clone(), ref_name: ref_name.clone(), old_sha: old_sha.clone(), new_sha: new_sha.clone(), - pusher_did: pusher_did_clone.clone(), - node_did: node_did_str.clone(), - timestamp: now_ts.clone(), - }); + timestamp: chrono::Utc::now().to_rfc3339(), + cert_id: None, + cid: cid.map(|s| s.to_string()), + }) + .await; } } + } - // Arweave permanent anchoring — fire for each ref update. - // Suppressed for repos the public cannot read (public permanent ledger). - if announce && !irys_url.is_empty() { - for (ref_name, old_sha, new_sha) in &ref_updates_clone { - let cid = cid_map.get(new_sha).cloned(); - let anchor = crate::arweave::RefAnchor { - repo: repo_slug.clone(), - owner_did: owner_did_for_arweave.clone(), - ref_name: ref_name.clone(), - old_sha: old_sha.clone(), - new_sha: new_sha.clone(), - cid: cid.clone(), - timestamp: now_ts.clone(), - node_did: node_did_str.clone(), - }; - match crate::arweave::anchor_ref_update(&http_client, &irys_url, &anchor) - .await - { - Ok(tx_id) if !tx_id.is_empty() => { - let arweave_url = crate::arweave::arweave_url(&tx_id); - let _ = db_clone - .record_arweave_anchor(&crate::db::RecordAnchorInput { - repo: &repo_slug, - owner_did: &owner_did_for_arweave, - ref_name, - old_sha, - new_sha, - cid: cid.as_deref(), - irys_tx_id: &tx_id, - arweave_url: &arweave_url, - node_did: &node_did_str, - }) - .await; - } - Ok(_) => {} - Err(e) => { - tracing::warn!(repo=%repo_slug, err=%e, "Arweave anchor failed") - } + // Broadcast ref update to GraphQL subscription listeners — one per ref. + // Gated on `announce`: /graphql/ws is unauthenticated (mounted after + // the optional_signature layer), and the subscription resolver has no + // caller to gate against, so only publicly-readable ref updates may + // reach anonymous subscribers. Mirrors the gossip (above) and Arweave + // (below) sends, which are already `announce`-gated. Without this a + // private-repo push would leak live ref metadata over the socket — + // the subscription analog of #112/#114. + let now_ts = chrono::Utc::now().to_rfc3339(); + if announce { + for (ref_name, old_sha, new_sha) in &ref_updates_clone { + let _ = ref_update_tx.send(crate::state::RefUpdateBroadcast { + repo: repo_slug.clone(), + ref_name: ref_name.clone(), + old_sha: old_sha.clone(), + new_sha: new_sha.clone(), + pusher_did: pusher_did_clone.clone(), + node_did: node_did_str.clone(), + timestamp: now_ts.clone(), + }); + } + } + + // Arweave permanent anchoring — fire for each ref update. + // Suppressed for repos the public cannot read (public permanent ledger). + if announce && !irys_url.is_empty() { + for (ref_name, old_sha, new_sha) in &ref_updates_clone { + let cid = cid_map.get(new_sha).cloned(); + let anchor = crate::arweave::RefAnchor { + repo: repo_slug.clone(), + owner_did: owner_did_for_arweave.clone(), + ref_name: ref_name.clone(), + old_sha: old_sha.clone(), + new_sha: new_sha.clone(), + cid: cid.clone(), + timestamp: now_ts.clone(), + node_did: node_did_str.clone(), + }; + match crate::arweave::anchor_ref_update(&http_client, &irys_url, &anchor).await + { + Ok(tx_id) if !tx_id.is_empty() => { + let arweave_url = crate::arweave::arweave_url(&tx_id); + let _ = db_clone + .record_arweave_anchor(&crate::db::RecordAnchorInput { + repo: &repo_slug, + owner_did: &owner_did_for_arweave, + ref_name, + old_sha, + new_sha, + cid: cid.as_deref(), + irys_tx_id: &tx_id, + arweave_url: &arweave_url, + node_did: &node_did_str, + }) + .await; + } + Ok(_) => {} + Err(e) => { + tracing::warn!(repo=%repo_slug, err=%e, "Arweave anchor failed") } } } + } - // HTTP peer notification — notify all known peers to pull from us. - // This is the reliable fallback when Gossipsub p2p is not yet connected. - // Suppressed for repos the public cannot read. Runs last so a slow or - // unreachable peer cannot delay the local GraphQL broadcast or Arweave - // anchoring above; this is the lowest-priority best-effort step. - if announce { - if let Ok(peers) = db_for_peers.list_peers().await { - for peer in peers { - if peer.http_url.is_empty() { + // HTTP peer notification — notify all known peers to pull from us. + // This is the reliable fallback when Gossipsub p2p is not yet connected. + // Suppressed for repos the public cannot read. Runs last so a slow or + // unreachable peer cannot delay the local GraphQL broadcast or Arweave + // anchoring above; this is the lowest-priority best-effort step. + if announce { + if let Ok(peers) = db_for_peers.list_peers().await { + for peer in peers { + if peer.http_url.is_empty() { + continue; + } + let peer_url = peer.http_url.trim_end_matches('/'); + if let Some(self_url) = self_public_url.as_deref() { + if peer_url == self_url.trim_end_matches('/') { continue; } - let peer_url = peer.http_url.trim_end_matches('/'); - if let Some(self_url) = self_public_url.as_deref() { - if peer_url == self_url.trim_end_matches('/') { - continue; - } - } - let notify_url = format!("{peer_url}{SYNC_NOTIFY_PATH}"); - notify_peer_of_refs( - &http_client, - node_keypair.as_ref(), - &peer.did, - ¬ify_url, - &repo_slug, - &ref_updates_clone, - &node_did_str, - &pusher_did_clone, - ) - .await; } + let notify_url = format!("{peer_url}{SYNC_NOTIFY_PATH}"); + notify_peer_of_refs( + &http_client, + node_keypair.as_ref(), + &peer.did, + ¬ify_url, + &repo_slug, + &ref_updates_clone, + &node_did_str, + &pusher_did_clone, + ) + .await; } } - }); - } - }); - - Ok(result) + } + }); + } } /// GET /api/v1/repos/{owner}/{repo}/refs @@ -6365,6 +6435,7 @@ mod tests { timeout, ) .await + .1 .into_iter() .collect(); @@ -6434,7 +6505,7 @@ mod tests { means the re-derivation git is not deadline-bounded / group-reaped (RED)", ); assert!( - got.is_empty(), + got.1.is_empty(), "a hung git yields nothing this push (the reconciliation sweep backstops)" ); } @@ -7490,4 +7561,582 @@ mod tests { "every per-source write permit must be released when its push completes" ); } + // ---- #174 F2a: the coalescing gate runs BEFORE the withheld walk ---- + // + // These drive `post_receive_replication_tail` directly (the handler's detached + // tail, extracted so the ordering the gate depends on is observable) over a REAL + // git repo, with a logging git shim in front of the real binary. The shim's log + // is the seam: `ls-tree` lines are withheld-walk children, and a line naming a + // tip sha attributes a scan to the push that pushed it. + + /// A git shim that appends its argv to `log`, then delegates to the real git, so + /// the walks stay real while every child is observable. + #[cfg(unix)] + fn f2a_logging_git(dir: &std::path::Path, log: &std::path::Path) -> String { + write_fake_git( + dir, + &format!( + "#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"{}\"\nexec git \"$@\"\n", + log.display() + ), + ) + } + + fn f2a_log(log: &std::path::Path) -> String { + std::fs::read_to_string(log).unwrap_or_default() + } + + /// Withheld-walk children run so far. `ls-tree` is the walk's signature child + /// (`blob_paths` lists every reachable commit's tree); the delta scan and the + /// full-scan fallback use `rev-list` / `cat-file` instead. + fn f2a_walks(log: &std::path::Path) -> usize { + f2a_log(log) + .lines() + .filter(|l| l.starts_with("ls-tree")) + .count() + } + + /// A state whose git is the shim, plus a repo row (optionally path-scoped, so + /// the withheld walk actually runs rather than taking the no-rule shortcut). + /// The repo's on-disk path is passed to the tail directly, so no repo_store or + /// receive-pack plumbing is involved. + async fn f2a_state( + pool: sqlx::PgPool, + git_bin: &str, + owner: &str, + name: &str, + path_scoped: bool, + ) -> (AppState, crate::db::RepoRecord) { + let mut state = crate::test_support::test_state(pool).await; + state.git_bin = git_bin.to_string(); + state + .db + .upsert_mirror_repo(owner, name, "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + if path_scoped { + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkF2aReaderAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + } + (state, rec) + } + + fn f2a_update(ref_name: &str, new_sha: &str) -> Vec { + vec![RefUpdate { + old_sha: ZERO_SHA.to_string(), + new_sha: new_sha.to_string(), + ref_name: ref_name.to_string(), + }] + } + + const F2A_PUSHER: &str = "did:key:z6MkF2aPusherAAAAAAAAAAAAAAAAAAAAAAAAAA"; + + /// Scenario 1 (the finding). A second rapid push to the same repo coalesces + /// WITHOUT running the withheld walk. Asserted on the walk's git children, not + /// on the `Coalesced` outcome: with `try_begin` below the walk (the pre-fix + /// order) the second push still parks on the scan pool and re-walks, which is + /// exactly the accumulation jatmn found. + /// + /// The pin pool's only permit is held for the whole test, so both pushes' pin + /// tasks park before doing any git of their own: every `ls-tree` in the log is a + /// tail walk. + #[cfg(unix)] + #[sqlx::test] + async fn f2a_coalesced_push_does_not_run_the_withheld_walk(pool: sqlx::PgPool) { + let repo = tempfile::TempDir::new().unwrap(); + let bin = tempfile::TempDir::new().unwrap(); + u5_init_repo(repo.path()); + let c1 = u5_commit_file(repo.path(), "a.txt", "one\n"); + let c2 = u5_commit_file(repo.path(), "secret/s.txt", "two\n"); + let log = bin.path().join("git.log"); + let git_bin = f2a_logging_git(bin.path(), &log); + let (mut state, rec) = f2a_state(pool, &git_bin, "z6f2acoal", "c1", true).await; + state.pin_semaphore = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + let _held = state.pin_semaphore.clone().acquire_owned().await.unwrap(); + + post_receive_replication_tail( + state.clone(), + rec.clone(), + f2a_update("refs/heads/main", &c2), + repo.path().to_path_buf(), + F2A_PUSHER.to_string(), + ) + .await; + let after_first = f2a_walks(&log); + assert!( + after_first >= 1, + "the admitted push must have run the withheld walk; log:\n{}", + f2a_log(&log) + ); + assert_eq!( + state.encrypt_inflight.len(), + 1, + "the admitted push's task holds the repo key while it is parked on the pin pool" + ); + + post_receive_replication_tail( + state.clone(), + rec.clone(), + f2a_update("refs/heads/second", &c1), + repo.path().to_path_buf(), + F2A_PUSHER.to_string(), + ) + .await; + + assert_eq!( + f2a_walks(&log), + after_first, + "a push that coalesces must not run the withheld walk at all; log:\n{}", + f2a_log(&log) + ); + assert_eq!( + state.encrypt_inflight.pending_for(&rec.id), + Some(crate::state::PendingWork::Tips(vec![( + ZERO_SHA.to_string(), + c1.clone() + )])), + "the coalesced push's tip pairs are queued for the in-flight task's drain" + ); + } + /// Poll `cond` until it holds, with a bound so a regression fails the test + /// rather than hanging the suite. + async fn f2a_wait_for(mut cond: impl FnMut() -> bool, what: &str) { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); + while !cond() { + assert!( + std::time::Instant::now() < deadline, + "timed out waiting for {what}" + ); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + } + + /// A `rev-list --objects` line names the tips a DELTA scan was asked to resolve, + /// so it attributes that scan to one push's tips. The withheld walk's own + /// `rev-list --all` / `ls-tree` lines never carry a tip as an argument this way. + fn f2a_delta_scanned(log: &std::path::Path, tip: &str) -> bool { + f2a_log(log) + .lines() + .any(|l| l.starts_with("rev-list --objects") && l.contains(tip)) + } + + /// Scenario 2. The tip pairs a coalesced push queues are consumed by the + /// in-flight task's drain, which is what makes coalescing lossless. Asserted on + /// the drained WORK (the delta scan the drain runs for those tips), not on the + /// key going empty: an armed guard's Drop empties the key too, so `is_empty` + /// cannot tell a drain from a discard. + /// + /// The coalesced tips are injected through `try_begin` directly rather than by a + /// second tail. A second tail would spawn its own Pinata worker, which re-derives + /// from the SAME tips, and the two scans are indistinguishable in the git log; the + /// tail-to-`try_begin` half is covered by scenario 1's pending-slot assertion. + #[cfg(unix)] + #[sqlx::test] + async fn f2a_coalesced_tips_are_drained_by_the_inflight_task(pool: sqlx::PgPool) { + let repo = tempfile::TempDir::new().unwrap(); + let bin = tempfile::TempDir::new().unwrap(); + u5_init_repo(repo.path()); + u5_commit_file(repo.path(), "a.txt", "one\n"); + let c2 = u5_commit_file(repo.path(), "secret/s.txt", "two\n"); + let log = bin.path().join("git.log"); + let git_bin = f2a_logging_git(bin.path(), &log); + let (mut state, rec) = f2a_state(pool, &git_bin, "z6f2adrain", "d1", true).await; + state.pin_semaphore = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + let held = state.pin_semaphore.clone().acquire_owned().await.unwrap(); + + // Push A is admitted; its task then parks on the held pin pool, key retained. + post_receive_replication_tail( + state.clone(), + rec.clone(), + f2a_update("refs/heads/main", &c2), + repo.path().to_path_buf(), + F2A_PUSHER.to_string(), + ) + .await; + + // A later push lands while A's task is in flight: it coalesces, and its tip + // advances main to a commit no other push in this test names. + let c3 = u5_commit_file(repo.path(), "b.txt", "three\n"); + assert!( + matches!( + state + .encrypt_inflight + .try_begin(&rec.id, vec![(c2.clone(), c3.clone())]), + crate::state::BeginOutcome::Coalesced + ), + "a push arriving while a task is in flight must coalesce" + ); + assert!( + !f2a_delta_scanned(&log, &c3), + "nothing has scanned the coalesced tip yet" + ); + + drop(held); + f2a_wait_for( + || f2a_delta_scanned(&log, &c3), + "the in-flight task's drain to resolve the coalesced push's tips", + ) + .await; + } + + /// Scenario 3 (trap 4). A push coalesces WHILE the admitted push is walking, the + /// admitted walk then FAILS, and the coalesced work is still drained. Moving + /// `try_begin` above the walk opens this window, so the failed-walk arm must + /// still spawn the task (with an empty snapshot) rather than let the guard go: + /// dropping it discards the pending tips with a warn. + /// + /// The git shim fails the FIRST `rev-list --all` (the withheld walk's commit + /// enumeration) after signalling that the walk has started and waiting for the + /// test to inject the coalescing push, then behaves normally, so the drain that + /// follows is a real one. + #[cfg(unix)] + #[sqlx::test] + async fn f2a_walk_failure_still_drains_the_coalesced_work(pool: sqlx::PgPool) { + let repo = tempfile::TempDir::new().unwrap(); + let bin = tempfile::TempDir::new().unwrap(); + u5_init_repo(repo.path()); + u5_commit_file(repo.path(), "a.txt", "one\n"); + let c2 = u5_commit_file(repo.path(), "secret/s.txt", "two\n"); + let c3 = u5_commit_file(repo.path(), "b.txt", "three\n"); + let log = bin.path().join("git.log"); + let started = bin.path().join("walk.started"); + let go = bin.path().join("walk.go"); + let once = bin.path().join("walk.once"); + let git_bin = write_fake_git( + bin.path(), + &format!( + "#!/bin/sh\n\ + printf '%s\\n' \"$*\" >> \"{log}\"\n\ + case \"$*\" in\n\ + 'rev-list --all'*)\n\ + if [ ! -f \"{once}\" ]; then\n\ + : > \"{once}\"\n\ + : > \"{started}\"\n\ + while [ ! -f \"{go}\" ]; do sleep 0.05; done\n\ + exit 1\n\ + fi ;;\n\ + esac\n\ + exec git \"$@\"\n", + log = log.display(), + once = once.display(), + started = started.display(), + go = go.display(), + ), + ); + let (state, rec) = f2a_state(pool, &git_bin, "z6f2afail", "f1", true).await; + + let tail = tokio::spawn(post_receive_replication_tail( + state.clone(), + rec.clone(), + f2a_update("refs/heads/main", &c2), + repo.path().to_path_buf(), + F2A_PUSHER.to_string(), + )); + f2a_wait_for(|| started.exists(), "the admitted push's walk to start").await; + + // Mid-walk arrival: the key is already taken, so this push coalesces into the + // slot the walking task owns. (With the gate back below the walk it would be + // ADMITTED here instead, and this assertion is what catches that.) + assert!( + matches!( + state + .encrypt_inflight + .try_begin(&rec.id, vec![(c2.clone(), c3.clone())]), + crate::state::BeginOutcome::Coalesced + ), + "a push arriving mid-walk must coalesce, not start a second task" + ); + std::fs::write(&go, b"").unwrap(); + tail.await.unwrap(); + + f2a_wait_for( + || f2a_delta_scanned(&log, &c3), + "the failed walk's task to drain the coalesced push's tips", + ) + .await; + } + + /// Mount a Pinata upload endpoint that assigns every object the same CID, and + /// point the state at it. Returns the server (kept alive by the caller) and CID. + async fn f2a_pinata(state: &mut AppState) -> (mockito::ServerGuard, String) { + let cid = "bafyf2acoalescedmapping".to_string(); + let mut server = mockito::Server::new_async().await; + server + .mock("POST", "/") + .with_status(200) + .with_body(format!(r#"{{"data":{{"cid":"{cid}"}}}}"#)) + .expect_at_least(1) + .create_async() + .await; + let mut cfg = (*state.config).clone(); + cfg.pinata_jwt = "f2a-test-jwt".to_string(); + cfg.pinata_upload_url = server.url(); + state.config = std::sync::Arc::new(cfg); + (server, cid) + } + + /// Poll the branch to CID table until the push's mapping lands (the Pinata + /// worker is detached), bounded so a regression fails rather than hangs. + async fn f2a_wait_for_branch_cid( + db: &crate::db::Db, + slug: &str, + what: &str, + ) -> Vec { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); + loop { + let rows = db.list_branch_cids(slug).await.unwrap(); + if !rows.is_empty() { + return rows; + } + assert!( + std::time::Instant::now() < deadline, + "timed out waiting for {what}" + ); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + } + + fn f2a_slug(rec: &crate::db::RepoRecord) -> String { + format!( + "{}/{}", + crate::db::normalize_owner_key(&rec.owner_did), + rec.name + ) + } + + /// Scenario 4 (traps 1 and 2). A coalesced push still does its own per-push work: + /// it records a branch to CID mapping and broadcasts its own ref update. Both + /// would be lost by returning early on `Coalesced`, and the mapping alone would be + /// lost by leaving the Pinata gate on `withheld.is_some()` (a coalesced push never + /// walks, so its `withheld` is None) while a test that only checked "the spawn + /// ran" stayed green. + #[cfg(unix)] + #[sqlx::test] + async fn f2a_coalesced_push_still_pins_and_announces(pool: sqlx::PgPool) { + let repo = tempfile::TempDir::new().unwrap(); + let bin = tempfile::TempDir::new().unwrap(); + u5_init_repo(repo.path()); + let c1 = u5_commit_file(repo.path(), "a.txt", "one\n"); + let c2 = u5_commit_file(repo.path(), "secret/s.txt", "two\n"); + let log = bin.path().join("git.log"); + let git_bin = f2a_logging_git(bin.path(), &log); + let (mut state, rec) = f2a_state(pool, &git_bin, "z6f2apin", "p1", true).await; + let (_server, cid) = f2a_pinata(&mut state).await; + let mut updates = state.ref_update_tx.subscribe(); + + // A task for this repo is already in flight, so the push below coalesces. + let _inflight = match state.encrypt_inflight.try_begin(&rec.id, Vec::new()) { + crate::state::BeginOutcome::Admitted(g) => g, + crate::state::BeginOutcome::Coalesced => panic!("the first begin must admit"), + }; + + post_receive_replication_tail( + state.clone(), + rec.clone(), + vec![RefUpdate { + old_sha: c1.clone(), + new_sha: c2.clone(), + ref_name: "refs/heads/main".to_string(), + }], + repo.path().to_path_buf(), + F2A_PUSHER.to_string(), + ) + .await; + + let slug = f2a_slug(&rec); + let mapped = f2a_wait_for_branch_cid( + &state.db, + &slug, + "the coalesced push's branch to CID mapping", + ) + .await; + assert_eq!( + mapped.len(), + 1, + "one mapping, for the ref this push advanced" + ); + assert_eq!(mapped[0].ref_name, "refs/heads/main"); + assert_eq!(mapped[0].sha, c2, "mapped to the tip this push landed"); + assert_eq!(mapped[0].cid, cid); + + let broadcast = updates + .try_recv() + .expect("a coalesced push still fires its own announce"); + assert_eq!(broadcast.new_sha, c2); + assert_eq!(broadcast.ref_name, "refs/heads/main"); + } + + /// Scenario 5 (trap 3, fail-closed). On a repo whose withheld walk is failing, a + /// coalesced push must not publish. Before the gate moved, every push on such a + /// repo got `announce = false` from its own walk; a coalesced push has no walk, so + /// the announce decision now comes from the Pinata worker's recomputation, which + /// fails closed the same way. Asserted on the broadcast channel: nothing is sent. + #[cfg(unix)] + #[sqlx::test] + async fn f2a_coalesced_push_on_a_failing_walk_does_not_publish(pool: sqlx::PgPool) { + let repo = tempfile::TempDir::new().unwrap(); + let bin = tempfile::TempDir::new().unwrap(); + u5_init_repo(repo.path()); + let c1 = u5_commit_file(repo.path(), "a.txt", "one\n"); + let c2 = u5_commit_file(repo.path(), "secret/s.txt", "two\n"); + let log = bin.path().join("git.log"); + // Every withheld walk fails: the repo cannot be vetted, so it must neither + // replicate nor announce. + let git_bin = write_fake_git( + bin.path(), + &format!( + "#!/bin/sh\n\ + printf '%s\\n' \"$*\" >> \"{log}\"\n\ + case \"$*\" in 'rev-list --all'*) exit 1 ;; esac\n\ + exec git \"$@\"\n", + log = log.display(), + ), + ); + let (mut state, rec) = f2a_state(pool, &git_bin, "z6f2aclosed", "x1", true).await; + let (_server, _cid) = f2a_pinata(&mut state).await; + let mut updates = state.ref_update_tx.subscribe(); + + let _inflight = match state.encrypt_inflight.try_begin(&rec.id, Vec::new()) { + crate::state::BeginOutcome::Admitted(g) => g, + crate::state::BeginOutcome::Coalesced => panic!("the first begin must admit"), + }; + + post_receive_replication_tail( + state.clone(), + rec.clone(), + vec![RefUpdate { + old_sha: c1.clone(), + new_sha: c2.clone(), + ref_name: "refs/heads/main".to_string(), + }], + repo.path().to_path_buf(), + F2A_PUSHER.to_string(), + ) + .await; + + // The worker has reached its recomputation (and failed it) once the walk's + // commit enumeration shows up in the log; give the rest of the task a settle. + f2a_wait_for( + || { + f2a_log(&log) + .lines() + .any(|l| l.starts_with("rev-list --all")) + }, + "the Pinata worker's fail-closed recomputation", + ) + .await; + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + + assert!( + matches!( + updates.try_recv(), + Err(tokio::sync::broadcast::error::TryRecvError::Empty) + ), + "a push whose replication could not be vetted must not broadcast" + ); + assert!( + state + .db + .list_branch_cids(&f2a_slug(&rec)) + .await + .unwrap() + .is_empty(), + "and it must pin nothing, so it maps no CID" + ); + } + + /// Scenario 6. A repo the anonymous public cannot read at root takes no key, runs + /// no walk, and spawns no git at all: the cheap predicate answers before anything + /// is acquired, exactly as `replication_withheld_set`'s own early return did. + #[cfg(unix)] + #[sqlx::test] + async fn f2a_private_repo_takes_no_key_and_runs_no_git(pool: sqlx::PgPool) { + let repo = tempfile::TempDir::new().unwrap(); + let bin = tempfile::TempDir::new().unwrap(); + u5_init_repo(repo.path()); + let c1 = u5_commit_file(repo.path(), "a.txt", "one\n"); + let log = bin.path().join("git.log"); + let git_bin = f2a_logging_git(bin.path(), &log); + let (state, mut rec) = f2a_state(pool, &git_bin, "z6f2apriv", "v1", false).await; + rec.is_public = false; + + post_receive_replication_tail( + state.clone(), + rec.clone(), + f2a_update("refs/heads/main", &c1), + repo.path().to_path_buf(), + F2A_PUSHER.to_string(), + ) + .await; + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + + assert!( + state.encrypt_inflight.is_empty(), + "a repo that replicates nothing must not take the coalescing key" + ); + assert_eq!( + f2a_log(&log), + "", + "and it must spawn no git: no walk, no candidate scan, no re-derivation" + ); + } + + /// Scenario 7. The first push is unaffected: it is admitted, it runs the walk, and + /// it still does the full per-push work (pin, mapping, announce). + #[cfg(unix)] + #[sqlx::test] + async fn f2a_first_push_is_admitted_and_does_the_full_work(pool: sqlx::PgPool) { + let repo = tempfile::TempDir::new().unwrap(); + let bin = tempfile::TempDir::new().unwrap(); + u5_init_repo(repo.path()); + let c1 = u5_commit_file(repo.path(), "a.txt", "one\n"); + let c2 = u5_commit_file(repo.path(), "secret/s.txt", "two\n"); + let log = bin.path().join("git.log"); + let git_bin = f2a_logging_git(bin.path(), &log); + let (mut state, rec) = f2a_state(pool, &git_bin, "z6f2afirst", "f1", true).await; + let (_server, cid) = f2a_pinata(&mut state).await; + let mut updates = state.ref_update_tx.subscribe(); + + post_receive_replication_tail( + state.clone(), + rec.clone(), + vec![RefUpdate { + old_sha: c1.clone(), + new_sha: c2.clone(), + ref_name: "refs/heads/main".to_string(), + }], + repo.path().to_path_buf(), + F2A_PUSHER.to_string(), + ) + .await; + + assert!( + f2a_walks(&log) >= 1, + "the admitted push runs the withheld walk itself; log:\n{}", + f2a_log(&log) + ); + let slug = f2a_slug(&rec); + let mapped = f2a_wait_for_branch_cid( + &state.db, + &slug, + "the admitted push's branch to CID mapping", + ) + .await; + assert_eq!(mapped[0].sha, c2); + assert_eq!(mapped[0].cid, cid); + let broadcast = updates + .try_recv() + .expect("the admitted push fires its announce"); + assert_eq!(broadcast.new_sha, c2); + } } diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 213b8b35..654399c7 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -372,6 +372,19 @@ impl EncryptInflight { pub fn is_empty(&self) -> bool { self.len() == 0 } + + /// The work currently queued against a repo's in-flight task, or `None` when + /// no task holds the key. Test-only observability: it lets a test assert that + /// a coalesced push's tip pairs were actually recorded, rather than that the + /// outcome merely was `Coalesced`. + #[cfg(test)] + pub fn pending_for(&self, repo_id: &str) -> Option { + self.repos + .lock() + .expect("encrypt_inflight mutex poisoned") + .get(repo_id) + .cloned() + } } /// Merge a coalesced push's tip pairs into a repo's pending slot. FullScan absorbs From b71201ef406c57ef8e05de7045528b31d7fd0f61 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 28 Jul 2026 02:40:37 -0500 Subject: [PATCH 085/124] test(node): observe the write-cap shed as state, not as a wall-clock deadline The two F1 tests asserted "the capped source shed rather than parked" by racing a ten second timeout, and drove the holding push in ten millisecond slices. Under `cargo test --workspace` that is a race against the scheduler rather than a test: a reviewer saw two of five runs go red, and under load the pair stretched from 1.8s to 19.2s while the fake git holding the lease only lived for 10s, past which every assertion in both tests is meaningless. Shed versus park is a question of which path ran, so both are now read as state. A shed is `Err(Overloaded)`. A park is a second reference on the lease entry, exposed by a test-only `refs_for`; `acquire` takes that reference synchronously before the cancellable wait, so a waiter is visible the instant it reaches the lease, and it stays visible because the steal bound is 1260s. The tests spawn their pushes and poll until one of the two mutually exclusive states appears, so a starved machine iterates more rather than failing. Every wall-clock assertion is gone. The worst of them was a sleep followed by `!is_finished()`: a runtime that had not yet polled that task satisfied it vacuously, which means the constant-key mutation could have gone green under load, and the discriminator is the whole reason that test exists. All three mutations still go red, now in seconds rather than by timeout expiry: the per-caller acquire below the lease, the global permit above it, and the per-caller key replaced with a constant. Also fixes a third flake found on the way: the acquire-deadline test's follow-up push inherited the deliberately short two second acquire timeout, which has to cover a real advisory-lock round trip, so its `Overloaded` was indistinguishable from the drained-pool `Overloaded` the assertion was actually testing for. The sqlx `databases_pkey` collisions seen alongside this are machine residue, not a leak from these tests: sqlx derives the test database name from a hash of the test path and does cleanup-then-insert without a lock, so any two concurrent test binaries running the same path collide. The orphan count here held at 53 across every run. --- crates/gitlawb-node/src/api/repos.rs | 252 ++++++++++++++++++--------- crates/gitlawb-node/src/state.rs | 15 ++ 2 files changed, 185 insertions(+), 82 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 7d8bbe2a..62516148 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -5090,6 +5090,14 @@ mod tests { .execute(&mut *lock_conn) .await .expect("release the advisory lock"); + // The follow-up asserts an ADMISSION property (the freed write slot is usable), + // so it must not inherit the 2s acquire deadline the shed above is testing. That + // budget covers a real advisory-lock round trip, and on a loaded machine it + // expires and returns the same `Overloaded` this assertion reads as a drained + // pool: a correct run going red for a reason the test is not about. + let mut followup_cfg = (*state.config).clone(); + followup_cfg.git_acquire_timeout_secs = 120; + state.config = std::sync::Arc::new(followup_cfg); let followup = git_receive_pack( State(state.clone()), Path((owner.to_string(), name.to_string())), @@ -7184,6 +7192,39 @@ mod tests { assert!(b_ran.exists(), "push B ran its receive-pack once unblocked"); } + /// Backstop for the F1 wait loops below. It is a HANG detector, never the thing an + /// assertion rests on: every F1 conclusion is drawn from a state the loop actually + /// observed (a returned shed, a second reference on the lease entry), so a loaded + /// machine only spends more iterations getting there. It must stay comfortably under + /// `F1_HOLD_SECS` so push A is still holding the lease when the loop gives up. + #[cfg(unix)] + const F1_BACKSTOP: std::time::Duration = std::time::Duration::from_secs(30); + + /// How long `f1_hanging_first_git`'s first receive-pack holds the lease. Bounded so a + /// RED run leaks no permanent orphan, and well above the two sequential `F1_BACKSTOP` + /// windows the discriminator can spend, so the hold never expires mid-observation on + /// a loaded machine. The old bound was 10s, which is under the 19s the pair took on a + /// contended box: push A's git would exit and free the lease mid-test. + #[cfg(unix)] + const F1_HOLD_SECS: usize = 150; + + /// Poll `cond` until it holds, yielding between checks so spawned handlers make + /// progress. Returns false if `cap` elapses first. Callers assert on the state the + /// loop settled into, not on the elapsed time. + #[cfg(unix)] + async fn f1_wait_for(cap: std::time::Duration, mut cond: impl FnMut() -> bool) -> bool { + let deadline = std::time::Instant::now() + cap; + loop { + if cond() { + return true; + } + if std::time::Instant::now() >= deadline { + return false; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + } + /// Fake git for the U1 lease-park tests. The FIRST receive-pack invocation marks /// that it reached the pack and hangs in a BOUNDED loop (so a RED run leaks no /// permanent orphan); every later invocation marks that it got past the lease. @@ -7202,7 +7243,7 @@ mod tests { cat >/dev/null 2>/dev/null\n\ if mkdir \"{seq}\" 2>/dev/null; then\n\ echo 1 > \"{ainp}\"\n\ - i=0; while [ $i -lt 100 ]; do sleep 0.1; i=$((i+1)); done\n\ + i=0; while [ $i -lt {hold} ]; do sleep 1; i=$((i+1)); done\n\ else\n\ echo 1 > \"{later}\"\n\ fi ;;\n\ @@ -7213,6 +7254,7 @@ mod tests { seq = seq_a.display(), ainp = a_inpack.display(), later = later_ran.display(), + hold = F1_HOLD_SECS, ); (write_fake_git(tmp, &body), a_inpack, later_ran) } @@ -7228,10 +7270,13 @@ mod tests { /// them at `(max_concurrent_git_pushes / 8).max(1)` = 4 per source. /// /// Load-bearing: push A (source X) holds the lease and X's only per-caller slot. - /// Pre-fix push B (also X) parks on the lease and does not return inside the test - /// window (RED). Post-fix it sheds on the per-caller cap immediately (GREEN). The - /// elapsed assertion is the point: an error returned after `steal_after` would be - /// no fix at all. + /// Pre-fix push B (also X) reaches the lease and parks there (RED); post-fix it never + /// reaches the lease at all and sheds on the per-caller cap (GREEN). Shed-vs-park is + /// a question of WHICH PATH ran, so both outcomes are read as state, not as elapsed + /// time: the shed is B's returned `Overloaded`, and the park is the second reference + /// `RepoWriteLeases::acquire` takes on the lease entry (synchronously, before it + /// waits). The loop below waits for whichever of the two appears, so a machine that + /// is starved of CPU by a sibling test just iterates more times. #[cfg(unix)] #[sqlx::test] async fn f1_capped_source_sheds_instead_of_parking_on_a_contended_lease(pool: sqlx::PgPool) { @@ -7246,12 +7291,20 @@ mod tests { // Per-source write cap of 1 so push A alone exhausts source X's budget. The // global write pool stays generous (64), so nothing here sheds on it. state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + let repo_id = state + .db + .get_repo("z6f1cap", "c1") + .await + .unwrap() + .unwrap() + .id; let did = "did:key:z6MkF1CapPusherAAAAAAAAAAAAAAAAAAAAAAAAAA"; let src_x: SocketAddr = "203.0.113.61:5000".parse().unwrap(); - // Push A: drive its handler future in slices until it reaches receive-pack. It - // now holds the per-repo lease AND source X's single per-caller write slot. - let mut fut_a = Box::pin(git_receive_pack( + // Push A runs as its own task: the runtime schedules it, nothing here drives it in + // slices. It reaches receive-pack and then holds the per-repo lease AND source X's + // single per-caller write slot until it is aborted at the end. + let handle_a = tokio::spawn(git_receive_pack( State(state.clone()), Path(("z6f1cap".to_string(), "c1".to_string())), Extension(crate::auth::AuthenticatedDid(did.to_string())), @@ -7259,48 +7312,59 @@ mod tests { axum::http::HeaderMap::new(), axum::body::Bytes::from_static(b"0000"), )); - for _ in 0..1000 { - let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut_a).await; - if a_inpack.exists() { - break; - } - } assert!( - a_inpack.exists(), - "push A must reach receive-pack and hold the lease plus source X's write slot" + f1_wait_for(F1_BACKSTOP, || a_inpack.exists()).await, + "push A never reached receive-pack within {F1_BACKSTOP:?}" + ); + // Both preconditions read as state, so push B's shed below cannot be vacuous. + assert_eq!( + state.repo_write_leases.refs_for(&repo_id), + 1, + "push A must hold the repo write lease, and be the only handler on it" + ); + assert_eq!( + state.git_write_per_caller.tracked_keys(), + 1, + "push A must be holding a per-caller write permit, so the cap of 1 is spent" ); // Push B: SAME source, same repo. It must be turned away, not parked. - let started = std::time::Instant::now(); - let b = tokio::time::timeout( - std::time::Duration::from_secs(10), - git_receive_pack( - State(state.clone()), - Path(("z6f1cap".to_string(), "c1".to_string())), - Extension(crate::auth::AuthenticatedDid(did.to_string())), - crate::rate_limit::PeerAddr(Some(src_x)), - axum::http::HeaderMap::new(), - axum::body::Bytes::from_static(b"0000"), - ), - ) + let handle_b = tokio::spawn(git_receive_pack( + State(state.clone()), + Path(("z6f1cap".to_string(), "c1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(src_x)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + )); + // Wait for B to land in one of its two terminal states: returned (shed) or holding + // a reference on the lease entry (parked). They are mutually exclusive, and a + // parked waiter stays parked (steal_after is 1260s), so whichever the loop sees is + // stable rather than a sampled instant. + let settled = f1_wait_for(F1_BACKSTOP, || { + handle_b.is_finished() || state.repo_write_leases.refs_for(&repo_id) >= 2 + }) .await; - let elapsed = started.elapsed(); - let b = b.expect( + assert!( + settled, + "push B neither returned nor reached the repo write lease within {F1_BACKSTOP:?}; \ + it is stuck somewhere earlier in the handler" + ); + assert_eq!( + state.repo_write_leases.refs_for(&repo_id), + 1, "F1 RED: a source already at its per-source write cap PARKED on the contended \ repo lease holding a fully buffered pack body. The per-caller write permit \ must be taken BEFORE the lease so the request sheds instead of waiting out \ - steal_after (1260s at defaults)", + steal_after (1260s at defaults)" ); + let b = handle_b.await.expect("push B task must not panic"); assert!( matches!(b, Err(AppError::Overloaded(_))), "the capped source's second push must shed with Overloaded/503; got {b:?}" ); - assert!( - elapsed < std::time::Duration::from_secs(5), - "the shed must return promptly, not after the lease park; took {elapsed:?}" - ); - drop(fut_a); + handle_a.abort(); } /// #174 F1 (U1) THE DISCRIMINATOR: while source X sits at its per-source write cap @@ -7313,9 +7377,13 @@ mod tests { /// resolved source IP and Y sheds too, which is exactly the node-wide shed /// `f02c5e1` was written to prevent. /// - /// Load-bearing on both halves: X's second push sheds promptly (RED pre-fix, where - /// it parks) AND Y's push stays parked rather than shedding (RED under the - /// constant-key mutation). + /// Load-bearing on both halves: X's second push reaches the lease and parks (RED + /// pre-fix) AND Y's push parks rather than shedding (RED under the constant-key + /// mutation, where Y is denied at the per-caller gate and returns). Both halves are + /// read as state, never as elapsed time: parked is a reference on the lease entry, + /// shed is a returned `Overloaded`. The old `sleep(300ms)` + `!is_finished()` check + /// had the worse failure mode of the two, since a starved runtime that had not yet + /// polled push C would satisfy it vacuously and the mutation would go green. #[cfg(unix)] #[sqlx::test] async fn f1_capped_source_shed_leaves_a_different_source_parked_and_served(pool: sqlx::PgPool) { @@ -7328,12 +7396,19 @@ mod tests { let mut state = f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f1disc", "d1", false).await; state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + let repo_id = state + .db + .get_repo("z6f1disc", "d1") + .await + .unwrap() + .unwrap() + .id; let did = "did:key:z6MkF1DiscPusherAAAAAAAAAAAAAAAAAAAAAAAAA"; let src_x: SocketAddr = "203.0.113.63:5000".parse().unwrap(); let src_y: SocketAddr = "203.0.113.64:5000".parse().unwrap(); // Push A (source X) reaches receive-pack: holds the lease and X's only slot. - let mut fut_a = Box::pin(git_receive_pack( + let handle_a = tokio::spawn(git_receive_pack( State(state.clone()), Path(("z6f1disc".to_string(), "d1".to_string())), Extension(crate::auth::AuthenticatedDid(did.to_string())), @@ -7341,29 +7416,35 @@ mod tests { axum::http::HeaderMap::new(), axum::body::Bytes::from_static(b"0000"), )); - for _ in 0..1000 { - let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut_a).await; - if a_inpack.exists() { - break; - } - } - assert!(a_inpack.exists(), "push A must reach receive-pack"); + assert!( + f1_wait_for(F1_BACKSTOP, || a_inpack.exists()).await, + "push A never reached receive-pack within {F1_BACKSTOP:?}" + ); + assert_eq!( + state.repo_write_leases.refs_for(&repo_id), + 1, + "push A must hold the repo write lease, and be the only handler on it" + ); // Push C from source Y: it has its own budget, so it takes Y's permit and parks - // on the lease A holds. + // on the lease A holds. Wait for it to reach either terminal state. let state_c = state.clone(); - let handle_c = tokio::spawn(async move { - git_receive_pack( - State(state_c), - Path(("z6f1disc".to_string(), "d1".to_string())), - Extension(crate::auth::AuthenticatedDid(did.to_string())), - crate::rate_limit::PeerAddr(Some(src_y)), - axum::http::HeaderMap::new(), - axum::body::Bytes::from_static(b"0000"), - ) - .await - }); - tokio::time::sleep(std::time::Duration::from_millis(300)).await; + let handle_c = tokio::spawn(git_receive_pack( + State(state_c), + Path(("z6f1disc".to_string(), "d1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(src_y)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + )); + let c_settled = f1_wait_for(F1_BACKSTOP, || { + handle_c.is_finished() || state.repo_write_leases.refs_for(&repo_id) >= 2 + }) + .await; + assert!( + c_settled, + "push C neither returned nor reached the repo write lease within {F1_BACKSTOP:?}" + ); assert!( !handle_c.is_finished(), "F1 discriminator RED: a push from a DIFFERENT source was turned away while \ @@ -7371,28 +7452,34 @@ mod tests { source IP, so exhausting it denies only the exhausting source" ); - // Push B from source X: shed, promptly, while C stays parked. - let started = std::time::Instant::now(); - let b = tokio::time::timeout( - std::time::Duration::from_secs(10), - git_receive_pack( - State(state.clone()), - Path(("z6f1disc".to_string(), "d1".to_string())), - Extension(crate::auth::AuthenticatedDid(did.to_string())), - crate::rate_limit::PeerAddr(Some(src_x)), - axum::http::HeaderMap::new(), - axum::body::Bytes::from_static(b"0000"), - ), - ) - .await - .expect("the capped source must shed rather than park on the contended lease"); + // Push B from source X: shed, while C stays parked. A third reference on the + // lease entry would mean B parked too. + let handle_b = tokio::spawn(git_receive_pack( + State(state.clone()), + Path(("z6f1disc".to_string(), "d1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(src_x)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + )); + let b_settled = f1_wait_for(F1_BACKSTOP, || { + handle_b.is_finished() || state.repo_write_leases.refs_for(&repo_id) >= 3 + }) + .await; assert!( - matches!(b, Err(AppError::Overloaded(_))), - "source X's second push must shed with Overloaded/503; got {b:?}" + b_settled, + "push B neither returned nor reached the repo write lease within {F1_BACKSTOP:?}" + ); + assert_eq!( + state.repo_write_leases.refs_for(&repo_id), + 2, + "F1 RED: the capped source parked on the contended lease (A holding, C and B \ + waiting) instead of shedding on its per-source write cap" ); + let b = handle_b.await.expect("push B task must not panic"); assert!( - started.elapsed() < std::time::Duration::from_secs(5), - "source X's shed must be prompt" + matches!(b, Err(AppError::Overloaded(_))), + "source X's second push must shed with Overloaded/503; got {b:?}" ); assert!( !handle_c.is_finished(), @@ -7403,9 +7490,10 @@ mod tests { "no second receive-pack may run while A holds the lease" ); - // Free the lease: A disconnects. Source Y's parked push is served. - drop(fut_a); - let resp = tokio::time::timeout(std::time::Duration::from_secs(30), handle_c) + // Free the lease: A disconnects. Aborting drops its handler future at the child + // await, the same shape as a client hang-up. Source Y's parked push is served. + handle_a.abort(); + let resp = tokio::time::timeout(std::time::Duration::from_secs(60), handle_c) .await .expect("push C must complete once A's group is reaped and the lease frees") .expect("push C task must not panic") diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 16893e65..b7fb1492 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -589,6 +589,21 @@ impl RepoWriteLeases { pub fn is_empty(&self) -> bool { self.len() == 0 } + + /// How many handlers currently reference this repo's lease entry: the holder plus + /// every waiter parked in [`acquire`](Self::acquire). The ref is taken SYNCHRONOUSLY, + /// before the (cancellable) semaphore wait, so a waiter is visible here the moment it + /// reaches the lease. Tests use it to observe "this push parked" as a state rather + /// than inferring it from a deadline that never fired. + #[cfg(test)] + pub fn refs_for(&self, repo_id: &str) -> usize { + self.repos + .lock() + .expect("repo_write_leases mutex poisoned") + .get(repo_id) + .map(|slot| slot.refs) + .unwrap_or(0) + } } /// Shared-ownership handle to a held repo write lease (#174 U2/F3). `Clone` hands a From 8f0e9107a4044b6fa30a6ac8bf3aa3bf0ccb6342 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 28 Jul 2026 08:01:51 -0500 Subject: [PATCH 086/124] fix(node): bound lease waiters per repo, and stop spending cross-repo budget on a per-repo wait The previous attempt moved the per-source write permit above the lease park to bound buffered pack bodies. Two reviewers demonstrated that it denies far more than the source doing the parking. A source parked on one repo's lease was shed on repos with no contention at all, and because the trusted-proxy setting defaults to trusting nothing and keying on the socket peer, every pusher behind a proxy, NAT, or CI runner pool resolves to one key: four parked pushes shed every push from every pusher to every repo for up to twenty-one minutes. That is the node-wide shed the permit was moved below the lease to prevent in the first place, reintroduced at an eighth of the threshold. The permit goes back below the lease. What is actually unbounded is the waiter set, so that is what is bounded now, per repo, with a shed past the cap. The cap counts live waiters, never the entry refcount. The refcount includes the holder, so capping it would let a lease whose owner died without unwinding pin a slot forever, which is the permanent wedge the steal timeout exists to survive. The count is claimed in the same critical section as the refcount and held by a guard scoped to the cancellable wait, so a cancelled or shed waiter gives it back. An uncontended acquire takes the fast path and spends no waiter budget; that path does not barge a queued waiter, which was checked over two thousand rounds rather than assumed. Two tests now guard the defect that was just removed: a source parked on one repo is still served on another, and a parked push does not shed an unrelated pusher sharing its resolved key. Restoring the old ordering turns both red. Capping on the refcount instead of live waiters turns the steal-backstop test red. Holding the waiter slot past the wait turns the fast-path test red. What this bounds is the parked count, not the parked bytes. At a two gigabyte pack limit the byte figure stays large, and the total across repos, whose dominant term is the buffering the signature middleware does before any admission point at all, is tracked in #263. Past the cap, same-repo concurrency sheds instead of queueing; that is the trade the knob buys. --- crates/gitlawb-node/src/api/repos.rs | 489 ++++++++++++++---------- crates/gitlawb-node/src/auth/mod.rs | 2 +- crates/gitlawb-node/src/config.rs | 24 ++ crates/gitlawb-node/src/main.rs | 7 +- crates/gitlawb-node/src/state.rs | 372 ++++++++++++++++-- crates/gitlawb-node/src/test_support.rs | 2 +- 6 files changed, 643 insertions(+), 253 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 62516148..88138d0e 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1550,9 +1550,8 @@ pub async fn git_receive_pack( let name = smart_http_repo_name(&repo)?; // Fast-path shed BEFORE the DB lookup when the write pool is saturated, so a push // flood on a full pool does not hit Postgres per request. Best-effort (racy) and - // NON-holding: the authoritative, held GLOBAL write permit is taken after the per-repo - // lease below (so a lease-blocked waiter pins no global write slot, #174 F3 review; - // the per-SOURCE sub-cap is taken above the lease, see there). This peek only + // NON-holding: the authoritative, held permit is taken after the per-repo lease below + // (so a lease-blocked waiter pins no write slot — #174 F3 review). This peek only // restores the cheap pre-DB shed the permit reorder would otherwise have lost. if state.git_write_semaphore.available_permits() == 0 { return Err(AppError::Overloaded( @@ -1648,30 +1647,49 @@ pub async fn git_receive_pack( let lease_steal_after = std::time::Duration::from_secs(state.config.git_service_timeout_secs * 2 + 60); - // The two write admission permits are DELIBERATELY split around the lease park. - // - // Per-source sub-cap BEFORE the lease (#174 P1-d, and F1 review). One source IP - // cannot occupy the whole write pool via many slow pushes: owner enforcement defaults - // off, so any valid did:key is accepted (auth != authz) and the push rate limiter - // bounds arrival RATE, not in-flight concurrency. It is taken here, above the park, - // because `body: Bytes` means axum has ALREADY buffered the entire pack (up to - // max_pack_bytes) before this function runs, and `acquire` below is a block-and-wait - // serializer whose waiter set is unbounded (state.rs) and whose wait runs to - // steal_after (git_service_timeout_secs * 2 + 60 = 1260s at defaults). With both - // permits below the lease nothing accounted for parked waiters, so one source could - // stack unbounded buffered bodies on a contended repo. The sub-cap is - // (max_concurrent_git_pushes / 8).max(1) = 4 at defaults, so it bounds parked bodies - // at 4 per source. Keyed on the resolved source IP, NEVER the signed DID (a DID farm - // defeats a DID key); no resolvable key -> global write pool only. + // Parked waiters are bounded INSIDE the lease, not by an admission permit taken above + // it. `body: Bytes` means axum has already buffered the whole pack before this handler + // runs, and the park runs to steal_after (1260s at defaults), so an unbounded waiter + // set would let same-repo pushes stack buffered bodies. `acquire` therefore counts its + // LIVE WAITERS against GITLAWB_REPO_LEASE_MAX_WAITERS and returns None past the cap, + // which sheds here as a 503 + Retry-After like the other admission paths. The cap is + // per repo and counts only handlers actually parked, so it denies same-repo + // concurrency on the contended repo alone: a push to any other repo, from any source, + // is untouched. Taking a per-source or global admission permit above the park instead + // is what the F1 review rejected, since GITLAWB_TRUSTED_PROXY defaults to unset and + // every pusher behind a proxy/NAT then resolves to ONE key, turning one contended repo + // into a node-wide denial. + let lease = state + .repo_write_leases + .acquire(&record.id, lease_steal_after) + .await + .ok_or_else(|| { + tracing::warn!( + repo = %name, + "repo write-lease waiter cap reached; shedding with 503" + ); + AppError::Overloaded("repo is busy with another push, retry shortly".into()) + })?; + + // Admission permits are taken HERE, AFTER the per-repo lease and BEFORE acquire_write. + // Ordering is the fix (#174 P2 DoS): the lease is a block-and-wait serializer, so a + // second same-repo push can park on `acquire` above for up to steal_after. Taking the + // scarce write permits only once we own the lease means a lease-blocked waiter pins NO + // write-pool slot while it waits. Otherwise a few hostile sources could stack same-repo + // pushes, hold every global slot on zero-byte lease-waiters, and shed 503 on every push + // to every OTHER repo node-wide. The per-source sub-cap belongs below the park for the + // same reason: its key is the resolved source IP, which collapses to one key for every + // pusher when GITLAWB_TRUSTED_PROXY is unset (the default), so above the park it sheds + // cross-tenant too. Still before acquire_write, so the git op stays admission-gated + // (INV-10) and a saturated pool sheds 503 before spawning git. // - // The GLOBAL write permit deliberately stays BELOW the lease (#174 P2 DoS, f02c5e1). - // That is a scarce node-wide resource: a lease-blocked waiter holding one lets a few - // hostile sources stack same-repo pushes, pin every global slot on zero-byte - // lease-waiters, and shed 503 on every push to every OTHER repo node-wide. A - // per-source key is not global, so exhausting it denies only the exhausting source. - // Self-inflicted denial is the property that makes the sub-cap safe above the park and - // the global permit unsafe there. Both still land before acquire_write, so the git op - // stays admission-gated (INV-10) and a saturated pool sheds 503 before spawning git. + // Per-source sub-cap first (#174 P1-d): one source IP cannot occupy the whole write + // pool via many slow pushes. Owner enforcement defaults off, so any valid did:key is + // accepted (auth != authz) and the push rate limiter bounds arrival RATE, not in-flight + // concurrency. Keyed on the resolved source IP, NEVER the signed DID (a DID farm defeats + // a DID key); no resolvable key -> global write pool only. Then the global write permit: + // pushes draw from the dedicated WRITE pool, separate from reads, and it is held for the + // whole op (moved into the AdmissionGuard below). let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); let _caller_permit = acquire_read_caller_permit( &state.git_write_per_caller, @@ -1679,12 +1697,6 @@ pub async fn git_receive_pack( name, "receive-pack", )?; - - let lease = state - .repo_write_leases - .acquire(&record.id, lease_steal_after) - .await; - let _permit = git_permit(&state.git_write_semaphore)?; tracing::debug!(repo = %name, "acquiring write lock"); @@ -7259,134 +7271,148 @@ mod tests { (write_fake_git(tmp, &body), a_inpack, later_ran) } - /// #174 F1 (U1, RED-before/GREEN-after): a source already at its per-source WRITE - /// sub-cap must SHED on a contended repo rather than park on the per-repo write - /// lease. `git_receive_pack` takes `body: Bytes`, so axum has already buffered the - /// whole pack (up to `max_pack_bytes`, 2 GB by default) before the handler runs, - /// and the lease park runs to `lease_steal_after` = `git_service_timeout_secs * 2 - /// + 60` = 1260s at defaults. The waiter set in `RepoWriteLeases::acquire` is - /// unbounded, so with BOTH admission permits taken after the lease nothing accounts - /// for parked bodies. Taking the per-caller write permit BEFORE the lease bounds - /// them at `(max_concurrent_git_pushes / 8).max(1)` = 4 per source. + /// Add a second repo to a state built by `f4_state_with_repo`, returning its DB id. + /// The U1 tests need two repos in ONE state to show that shedding on a contended + /// lease is confined to that repo. + #[cfg(unix)] + async fn f1_add_repo(state: &AppState, owner: &str, name: &str) -> String { + state + .db + .upsert_mirror_repo(owner, name, &format!("/unused-{owner}-{name}"), None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + state + .repo_store + .init(&rec.owner_did, &rec.name) + .await + .unwrap(); + rec.id + } + + /// #174 U1 scenario 1 (RED-before/GREEN-after): parked pushes on one repo's write + /// lease are BOUNDED. `git_receive_pack` takes `body: Bytes`, so axum has already + /// buffered the whole pack (up to `max_pack_bytes`) before the handler runs, and the + /// park runs to `lease_steal_after` = `git_service_timeout_secs * 2 + 60` = 1260s at + /// defaults. An unbounded waiter set is therefore unbounded buffered memory held for + /// 21 minutes. With the cap at K, a holder plus K live waiters means the next push + /// sheds a 503 + Retry-After instead of joining the queue. /// - /// Load-bearing: push A (source X) holds the lease and X's only per-caller slot. - /// Pre-fix push B (also X) reaches the lease and parks there (RED); post-fix it never - /// reaches the lease at all and sheds on the per-caller cap (GREEN). Shed-vs-park is - /// a question of WHICH PATH ran, so both outcomes are read as state, not as elapsed - /// time: the shed is B's returned `Overloaded`, and the park is the second reference - /// `RepoWriteLeases::acquire` takes on the lease entry (synchronously, before it - /// waits). The loop below waits for whichever of the two appears, so a machine that - /// is starved of CPU by a sibling test just iterates more times. + /// Read as state, never as elapsed time: the shed is the returned `Overloaded`, and + /// "the queue did not grow" is the waiter count the loop polls to. The bound is on + /// LIVE WAITERS, so the holder is not counted; that is what keeps a leaked lease from + /// wedging the repo (see `steal_on_leaked_lease_still_works_under_the_waiter_cap`). #[cfg(unix)] #[sqlx::test] - async fn f1_capped_source_sheds_instead_of_parking_on_a_contended_lease(pool: sqlx::PgPool) { + async fn u1_push_past_the_lease_waiter_cap_sheds_with_503(pool: sqlx::PgPool) { use axum::extract::{Path, State}; + use axum::response::IntoResponse; use axum::Extension; use std::net::SocketAddr; let tmp = tempfile::TempDir::new().unwrap(); let (git_bin, a_inpack, _later_ran) = f1_hanging_first_git(tmp.path()); let mut state = - f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f1cap", "c1", false).await; - // Per-source write cap of 1 so push A alone exhausts source X's budget. The - // global write pool stays generous (64), so nothing here sheds on it. - state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6u1cap", "c1", false).await; + // Cap of ONE live waiter, so a holder plus one parked push fills the repo's queue. + state.repo_write_leases = crate::state::RepoWriteLeases::new(1); let repo_id = state .db - .get_repo("z6f1cap", "c1") + .get_repo("z6u1cap", "c1") .await .unwrap() .unwrap() .id; - let did = "did:key:z6MkF1CapPusherAAAAAAAAAAAAAAAAAAAAAAAAAA"; - let src_x: SocketAddr = "203.0.113.61:5000".parse().unwrap(); + let did = "did:key:z6MkU1CapPusherAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let push = |peer: SocketAddr| { + let st = state.clone(); + let did = did.to_string(); + async move { + git_receive_pack( + State(st), + Path(("z6u1cap".to_string(), "c1".to_string())), + Extension(crate::auth::AuthenticatedDid(did)), + crate::rate_limit::PeerAddr(Some(peer)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await + } + }; - // Push A runs as its own task: the runtime schedules it, nothing here drives it in - // slices. It reaches receive-pack and then holds the per-repo lease AND source X's - // single per-caller write slot until it is aborted at the end. - let handle_a = tokio::spawn(git_receive_pack( - State(state.clone()), - Path(("z6f1cap".to_string(), "c1".to_string())), - Extension(crate::auth::AuthenticatedDid(did.to_string())), - crate::rate_limit::PeerAddr(Some(src_x)), - axum::http::HeaderMap::new(), - axum::body::Bytes::from_static(b"0000"), - )); + // Push A holds the lease (its git hangs) until it is aborted below. + let handle_a = tokio::spawn(push("203.0.113.81:5000".parse().unwrap())); assert!( f1_wait_for(F1_BACKSTOP, || a_inpack.exists()).await, "push A never reached receive-pack within {F1_BACKSTOP:?}" ); - // Both preconditions read as state, so push B's shed below cannot be vacuous. assert_eq!( - state.repo_write_leases.refs_for(&repo_id), - 1, - "push A must hold the repo write lease, and be the only handler on it" + state.repo_write_leases.waiters_for(&repo_id), + 0, + "the uncontended holder must spend no waiter budget" ); - assert_eq!( - state.git_write_per_caller.tracked_keys(), - 1, - "push A must be holding a per-caller write permit, so the cap of 1 is spent" + + // Push B fills the single waiter slot. + let handle_b = tokio::spawn(push("203.0.113.82:5000".parse().unwrap())); + assert!( + f1_wait_for(F1_BACKSTOP, || state + .repo_write_leases + .waiters_for(&repo_id) + == 1) + .await, + "push B never parked on the contended lease within {F1_BACKSTOP:?}" ); - // Push B: SAME source, same repo. It must be turned away, not parked. - let handle_b = tokio::spawn(git_receive_pack( - State(state.clone()), - Path(("z6f1cap".to_string(), "c1".to_string())), - Extension(crate::auth::AuthenticatedDid(did.to_string())), - crate::rate_limit::PeerAddr(Some(src_x)), - axum::http::HeaderMap::new(), - axum::body::Bytes::from_static(b"0000"), - )); - // Wait for B to land in one of its two terminal states: returned (shed) or holding - // a reference on the lease entry (parked). They are mutually exclusive, and a - // parked waiter stays parked (steal_after is 1260s), so whichever the loop sees is - // stable rather than a sampled instant. - let settled = f1_wait_for(F1_BACKSTOP, || { - handle_b.is_finished() || state.repo_write_leases.refs_for(&repo_id) >= 2 - }) - .await; + // Push C is past the cap: it must be turned away, not queued. + let c = tokio::time::timeout(F1_BACKSTOP, push("203.0.113.83:5000".parse().unwrap())) + .await + .expect("a push past the waiter cap must return, not park"); assert!( - settled, - "push B neither returned nor reached the repo write lease within {F1_BACKSTOP:?}; \ - it is stuck somewhere earlier in the handler" + matches!(c, Err(AppError::Overloaded(_))), + "U1 RED: a push arriving past the repo's live-waiter cap joined the unbounded \ + park queue instead of shedding, holding its fully buffered pack for up to \ + steal_after (1260s at defaults); got {c:?}" ); + let resp = c.unwrap_err().into_response(); assert_eq!( - state.repo_write_leases.refs_for(&repo_id), + resp.status(), + 503, + "the shed must be a 503, consistent with the other admission paths" + ); + assert_eq!( + resp.headers().get("retry-after").unwrap().to_str().unwrap(), + "1", + "the shed must advertise Retry-After" + ); + assert_eq!( + state.repo_write_leases.waiters_for(&repo_id), 1, - "F1 RED: a source already at its per-source write cap PARKED on the contended \ - repo lease holding a fully buffered pack body. The per-caller write permit \ - must be taken BEFORE the lease so the request sheds instead of waiting out \ - steal_after (1260s at defaults)" + "the shed must not have joined the queue, and must leave no waiter residue" ); - let b = handle_b.await.expect("push B task must not panic"); - assert!( - matches!(b, Err(AppError::Overloaded(_))), - "the capped source's second push must shed with Overloaded/503; got {b:?}" + assert_eq!( + state.repo_write_leases.refs_for(&repo_id), + 2, + "only the holder and the one real waiter may reference the entry after a shed" ); handle_a.abort(); + let _ = tokio::time::timeout(std::time::Duration::from_secs(60), handle_b).await; } - /// #174 F1 (U1) THE DISCRIMINATOR: while source X sits at its per-source write cap - /// on a contended repo, a push from a DIFFERENT source Y must still park on the - /// lease and be served once the lease frees. This is what distinguishes moving the - /// PER-CALLER permit above the lease from the rejected alternative of moving the - /// GLOBAL write permit (or any node-wide waiter cap) above it: exhausting a - /// per-source key denies only the exhausting source, so the denial is - /// self-inflicted. Key the permit on a constant instead of `read_caller_key`'s - /// resolved source IP and Y sheds too, which is exactly the node-wide shed - /// `f02c5e1` was written to prevent. + /// #174 U1 scenario 2, THE REGRESSION GUARD for the rejected design. A source parked + /// on repo A's lease must still be served on an UNCONTENDED repo B. The rejected fix + /// bounded parked bodies by taking the per-source write permit ABOVE the lease, which + /// makes a parked push spend that source's node-wide budget: the same pusher is then + /// denied on every other repo, and (since `TrustedProxy` defaults to `None`, so every + /// pusher behind a proxy or NAT resolves to one key) so is everyone else. Moving the + /// per-caller permit back above the lease turns this red. /// - /// Load-bearing on both halves: X's second push reaches the lease and parks (RED - /// pre-fix) AND Y's push parks rather than shedding (RED under the constant-key - /// mutation, where Y is denied at the per-caller gate and returns). Both halves are - /// read as state, never as elapsed time: parked is a reference on the lease entry, - /// shed is a returned `Overloaded`. The old `sleep(300ms)` + `!is_finished()` check - /// had the worse failure mode of the two, since a starved runtime that had not yet - /// polled push C would satisfy it vacuously and the mutation would go green. + /// The per-source cap is 2 here: the holder spends one, so a parked push spending the + /// second is what denies repo B. Under the shipped ordering the parked push holds no + /// permit at all and repo B is served. #[cfg(unix)] #[sqlx::test] - async fn f1_capped_source_shed_leaves_a_different_source_parked_and_served(pool: sqlx::PgPool) { + async fn u1_a_source_parked_on_one_repo_is_still_served_on_another(pool: sqlx::PgPool) { use axum::extract::{Path, State}; use axum::Extension; use std::net::SocketAddr; @@ -7394,119 +7420,160 @@ mod tests { let tmp = tempfile::TempDir::new().unwrap(); let (git_bin, a_inpack, later_ran) = f1_hanging_first_git(tmp.path()); let mut state = - f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f1disc", "d1", false).await; - state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); - let repo_id = state + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6u1two", "r1", false).await; + state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(2, 100); + let repo1 = state .db - .get_repo("z6f1disc", "d1") + .get_repo("z6u1two", "r1") .await .unwrap() .unwrap() .id; - let did = "did:key:z6MkF1DiscPusherAAAAAAAAAAAAAAAAAAAAAAAAA"; - let src_x: SocketAddr = "203.0.113.63:5000".parse().unwrap(); - let src_y: SocketAddr = "203.0.113.64:5000".parse().unwrap(); + f1_add_repo(&state, "z6u1two", "r2").await; + let did = "did:key:z6MkU1TwoPusherAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let src: SocketAddr = "203.0.113.84:5000".parse().unwrap(); + let push = |repo: &'static str| { + let st = state.clone(); + let did = did.to_string(); + async move { + git_receive_pack( + State(st), + Path(("z6u1two".to_string(), repo.to_string())), + Extension(crate::auth::AuthenticatedDid(did)), + crate::rate_limit::PeerAddr(Some(src)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await + } + }; - // Push A (source X) reaches receive-pack: holds the lease and X's only slot. - let handle_a = tokio::spawn(git_receive_pack( - State(state.clone()), - Path(("z6f1disc".to_string(), "d1".to_string())), - Extension(crate::auth::AuthenticatedDid(did.to_string())), - crate::rate_limit::PeerAddr(Some(src_x)), - axum::http::HeaderMap::new(), - axum::body::Bytes::from_static(b"0000"), - )); + // Push A (this source) holds repo r1's lease; push B (same source) parks on it. + let handle_a = tokio::spawn(push("r1")); assert!( f1_wait_for(F1_BACKSTOP, || a_inpack.exists()).await, "push A never reached receive-pack within {F1_BACKSTOP:?}" ); - assert_eq!( - state.repo_write_leases.refs_for(&repo_id), - 1, - "push A must hold the repo write lease, and be the only handler on it" - ); - - // Push C from source Y: it has its own budget, so it takes Y's permit and parks - // on the lease A holds. Wait for it to reach either terminal state. - let state_c = state.clone(); - let handle_c = tokio::spawn(git_receive_pack( - State(state_c), - Path(("z6f1disc".to_string(), "d1".to_string())), - Extension(crate::auth::AuthenticatedDid(did.to_string())), - crate::rate_limit::PeerAddr(Some(src_y)), - axum::http::HeaderMap::new(), - axum::body::Bytes::from_static(b"0000"), - )); - let c_settled = f1_wait_for(F1_BACKSTOP, || { - handle_c.is_finished() || state.repo_write_leases.refs_for(&repo_id) >= 2 - }) - .await; + let handle_b = tokio::spawn(push("r1")); assert!( - c_settled, - "push C neither returned nor reached the repo write lease within {F1_BACKSTOP:?}" + f1_wait_for(F1_BACKSTOP, || state.repo_write_leases.waiters_for(&repo1) + == 1) + .await, + "push B never parked on r1's contended lease within {F1_BACKSTOP:?}" ); + + // Push C: same source, DIFFERENT repo, uncontended. It must be served. + let c = tokio::time::timeout(F1_BACKSTOP, push("r2")) + .await + .expect("a push to an uncontended repo must not park"); + let resp = c.unwrap_or_else(|e| { + panic!( + "U1 scenario 2 RED: a push to an UNCONTENDED repo was denied because the \ + same source had a push parked on a DIFFERENT repo's lease. A parked push \ + must hold no cross-repo admission budget; got {e:?}" + ) + }); + assert_eq!(resp.status(), 200, "the uncontended repo's push lands 200"); assert!( - !handle_c.is_finished(), - "F1 discriminator RED: a push from a DIFFERENT source was turned away while \ - source X was at its cap. The write sub-cap must be keyed on the resolved \ - source IP, so exhausting it denies only the exhausting source" + later_ran.exists(), + "the uncontended repo's push must have run its receive-pack" ); - // Push B from source X: shed, while C stays parked. A third reference on the - // lease entry would mean B parked too. - let handle_b = tokio::spawn(git_receive_pack( - State(state.clone()), - Path(("z6f1disc".to_string(), "d1".to_string())), - Extension(crate::auth::AuthenticatedDid(did.to_string())), - crate::rate_limit::PeerAddr(Some(src_x)), - axum::http::HeaderMap::new(), - axum::body::Bytes::from_static(b"0000"), + handle_a.abort(); + let _ = tokio::time::timeout(std::time::Duration::from_secs(60), handle_b).await; + } + + /// #174 U1 scenario 3, THE CROSS-TENANT GUARD. `GITLAWB_TRUSTED_PROXY` is unset by + /// default (`TrustedProxy::None`), so behind an edge proxy, a NAT, or a CI pool every + /// pusher resolves to the SAME source key. A push parked on one repo's lease must not + /// shed a DIFFERENT pusher's push to a DIFFERENT repo. This is the shape that made + /// the rejected design a cross-tenant denial rather than a self-inflicted one: with + /// the per-source permit above the park, four parked pushes deny every push on the + /// node for up to 1260s. + /// + /// Same one source IP for all three pushes (the collapsed-key shape), distinct pusher + /// DIDs, per-source cap 2 so the holder plus a parked push would exhaust it. + #[cfg(unix)] + #[sqlx::test] + async fn u1_parked_push_does_not_shed_another_pusher_behind_the_same_ip(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let tmp = tempfile::TempDir::new().unwrap(); + let (git_bin, a_inpack, later_ran) = f1_hanging_first_git(tmp.path()); + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6u1nat", "n1", false).await; + // The default proxy trust: the resolved key is the peer IP, which is the edge's. + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(2, 100); + let repo1 = state + .db + .get_repo("z6u1nat", "n1") + .await + .unwrap() + .unwrap() + .id; + f1_add_repo(&state, "z6u1nat", "n2").await; + // Every pusher arrives from the one edge IP, so they share a source key. + let edge: SocketAddr = "203.0.113.85:5000".parse().unwrap(); + let push = |pusher: &'static str, repo: &'static str| { + let st = state.clone(); + async move { + git_receive_pack( + State(st), + Path(("z6u1nat".to_string(), repo.to_string())), + Extension(crate::auth::AuthenticatedDid(pusher.to_string())), + crate::rate_limit::PeerAddr(Some(edge)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await + } + }; + + let handle_a = tokio::spawn(push( + "did:key:z6MkU1NatPusherOneAAAAAAAAAAAAAAAAAAAAAA", + "n1", )); - let b_settled = f1_wait_for(F1_BACKSTOP, || { - handle_b.is_finished() || state.repo_write_leases.refs_for(&repo_id) >= 3 - }) - .await; assert!( - b_settled, - "push B neither returned nor reached the repo write lease within {F1_BACKSTOP:?}" - ); - assert_eq!( - state.repo_write_leases.refs_for(&repo_id), - 2, - "F1 RED: the capped source parked on the contended lease (A holding, C and B \ - waiting) instead of shedding on its per-source write cap" - ); - let b = handle_b.await.expect("push B task must not panic"); - assert!( - matches!(b, Err(AppError::Overloaded(_))), - "source X's second push must shed with Overloaded/503; got {b:?}" - ); - assert!( - !handle_c.is_finished(), - "source Y must still be parked on the lease, unaffected by X hitting its cap" + f1_wait_for(F1_BACKSTOP, || a_inpack.exists()).await, + "pusher one never reached receive-pack within {F1_BACKSTOP:?}" ); + let handle_b = tokio::spawn(push( + "did:key:z6MkU1NatPusherTwoAAAAAAAAAAAAAAAAAAAAAA", + "n1", + )); assert!( - !later_ran.exists(), - "no second receive-pack may run while A holds the lease" + f1_wait_for(F1_BACKSTOP, || state.repo_write_leases.waiters_for(&repo1) + == 1) + .await, + "pusher two never parked on n1's contended lease within {F1_BACKSTOP:?}" ); - // Free the lease: A disconnects. Aborting drops its handler future at the child - // await, the same shape as a client hang-up. Source Y's parked push is served. - handle_a.abort(); - let resp = tokio::time::timeout(std::time::Duration::from_secs(60), handle_c) - .await - .expect("push C must complete once A's group is reaped and the lease frees") - .expect("push C task must not panic") - .expect("push C must succeed"); - assert_eq!( - resp.status(), - 200, - "the unaffected source's parked push lands 200 once the lease frees" - ); + // A third, unrelated pusher behind the same edge IP, on a different repo. + let c = tokio::time::timeout( + F1_BACKSTOP, + push("did:key:z6MkU1NatPusherThreeAAAAAAAAAAAAAAAAAA", "n2"), + ) + .await + .expect("an unrelated pusher's push to an uncontended repo must not park"); + let resp = c.unwrap_or_else(|e| { + panic!( + "U1 scenario 3 RED: one repo's parked push shed an UNRELATED pusher's push \ + to a DIFFERENT repo, because every pusher behind the proxy shares one \ + resolved source key. Contention on one repo must never deny another; \ + got {e:?}" + ) + }); + assert_eq!(resp.status(), 200, "the unrelated pusher lands 200"); assert!( later_ran.exists(), - "push C ran its receive-pack once the lease freed" + "the unrelated pusher must have run its receive-pack" ); + + handle_a.abort(); + let _ = tokio::time::timeout(std::time::Duration::from_secs(60), handle_b).await; } /// #174 F1 (U1) key shape: the write sub-cap is keyed on the SOURCE, not the repo. diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 431ac983..eee8fd8b 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -526,7 +526,7 @@ mod tests { git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), pin_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), encrypt_inflight: crate::state::EncryptInflight::new(), - repo_write_leases: crate::state::RepoWriteLeases::new(), + repo_write_leases: crate::state::RepoWriteLeases::new(8), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index d70be5ea..d1b98807 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -319,6 +319,30 @@ pub struct Config { )] pub max_concurrent_git_pushes: usize, + /// Maximum number of pushes that may be PARKED on one repo's in-process write lease + /// at once. Same-repo pushes are serialized (block-and-wait), and a parked push has + /// already had its entire pack buffered by axum, so an unbounded queue on a contended + /// repo is unbounded buffered memory held for up to `git_service_timeout_secs * 2 + + /// 60` (1260s at defaults). Past this cap the newest push sheds a clean 503 + + /// Retry-After instead of joining the queue. + /// + /// The trade: raising it lets more same-repo pushes wait their turn (fewer 503s for a + /// hot repo, more memory pinned by waiters); lowering it sheds sooner. Only pushes to + /// the SAME repo count, and only ones parked right now, so the cap can never deny a + /// push to a different repo. The holder is deliberately not counted: a holder whose + /// cleanup never ran would otherwise pin a slot forever and wedge the repo, which is + /// the failure the `steal_after` reclaim exists to survive. + /// + /// Default: 8, a quarter of the default `max_concurrent_git_pushes` (32). Raising the + /// push pool does not raise this; set it explicitly. Must be between 1 and 1_048_576. + #[arg( + long, + env = "GITLAWB_REPO_LEASE_MAX_WAITERS", + default_value_t = 8, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub repo_lease_max_waiters: usize, + /// Max concurrent post-push pin loops (`ipfs_pin` and `pinata` /// `pin_new_objects`) across all repos. `EncryptInflight` bounds the outstanding /// pin-task COUNT to one per repo, but each pin loop holds a full per-push diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index d6c0df46..a3e26753 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -412,9 +412,10 @@ async fn main() -> Result<()> { encrypt_inflight: crate::state::EncryptInflight::new(), // Per-repo in-process write-lease serializer (#174 U2/F3): supplements the pg // advisory lock so a disconnected push's still-reaping git group can't be raced - // by a second same-node push. Natural cap (one entry per contended repo, freed - // when unreferenced), no sized knob. - repo_write_leases: crate::state::RepoWriteLeases::new(), + // by a second same-node push. The map is naturally capped (one entry per contended + // repo, freed when unreferenced); the sized knob is how many pushes may PARK on + // one repo, since each parked push holds a fully buffered pack. + repo_write_leases: crate::state::RepoWriteLeases::new(config.repo_lease_max_waiters), git_read_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( config.max_concurrent_reads_per_caller, ), diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index b7fb1492..fa621aab 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -488,30 +488,54 @@ impl Drop for EncryptInflightGuard { /// block-and-wait, NOT coalesce. It mirrors [`EncryptInflight`]'s keyed-map + guard + /// Drop-frees-key STRUCTURE; the semantics differ (block-and-wait, so there is no /// lossy-coalesce degradation to fall back on). -#[derive(Clone, Default)] +#[derive(Clone)] pub struct RepoWriteLeases { // std::sync::Mutex: held only for O(1) map ops (get-or-create + refcount) in a sync // context, never across an await — the semaphore wait happens OUTSIDE this lock. repos: Arc>>, + /// Most handlers allowed to be PARKED on one repo's lease at once + /// (`GITLAWB_REPO_LEASE_MAX_WAITERS`). Past it `acquire` sheds instead of queueing. + max_waiters: usize, } -/// A per-repo lease entry: the one-permit semaphore plus a refcount of the handlers -/// currently referencing it (holding or waiting). While `refs > 0` every acquirer -/// shares the SAME semaphore, so mutual exclusion holds; the entry is removed only when -/// `refs` hits 0 (no one references it), so a fresh entry can never split serialization. +/// A per-repo lease entry: the one-permit semaphore, a refcount of the handlers +/// currently referencing it (holding or waiting), and a count of the ones actually +/// PARKED. While `refs > 0` every acquirer shares the SAME semaphore, so mutual exclusion +/// holds; the entry is removed only when `refs` hits 0 (no one references it), so a fresh +/// entry can never split serialization. +/// +/// `waiters` and `refs` are deliberately different counts, and the shed cap is on +/// `waiters`. `refs` includes the HOLDER, and a holder whose Drop never runs (task abort, +/// runtime teardown without unwind, `mem::forget`: precisely the leak `steal_after` exists +/// to survive) keeps its ref forever. Capping `refs` would let that leaked ref +/// permanently occupy a slot and wedge the repo, reintroducing the permanent wedge the +/// steal backstop was written to prevent. A waiter, by contrast, always leaves: it either +/// gets the permit, steals at `steal_after`, is shed, or is cancelled, and every one of +/// those paths drops the RAII waiter guard. struct LeaseSlot { sem: Arc, refs: usize, + waiters: usize, } impl RepoWriteLeases { - pub fn new() -> Self { - Self::default() + /// `max_waiters` is the per-repo live-waiter cap (see [`acquire`](Self::acquire)). + pub fn new(max_waiters: usize) -> Self { + Self { + repos: Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())), + max_waiters: max_waiters.max(1), + } } /// Acquire the per-repo write lease, blocking until it is free (a second same-repo /// writer waits). `steal_after` bounds that wait: past it the acquirer STEALS - /// (proceeds permit-less) rather than block forever. + /// (proceeds permit-less) rather than block forever. Returns `None` when this repo + /// already has `max_waiters` handlers parked, so the caller sheds (503) instead of + /// adding to an unbounded queue: `git_receive_pack` reaches here with the whole pack + /// already buffered by axum, and the park runs to `steal_after` (1260s at defaults), + /// so unbounded parking is unbounded buffered bytes. The cap is per repo and counts + /// only live waiters, so shedding is confined to the contended repo; a push to any + /// other repo, from any source, is unaffected. /// /// Why a bounded steal: block-and-wait has no degradation of its own (unlike the /// coalescing [`EncryptInflight`], whose lost key merely delays a best-effort copy), @@ -528,7 +552,11 @@ impl RepoWriteLeases { /// it. Nothing anywhere takes the pg lock before this lease, so the two serializers /// can never deadlock; taking the lease first also means a blocked second writer /// pins no pooled pg connection while it waits. - pub async fn acquire(&self, repo_id: &str, steal_after: std::time::Duration) -> RepoWriteLease { + pub async fn acquire( + &self, + repo_id: &str, + steal_after: std::time::Duration, + ) -> Option { // Take the entry refcount BEFORE the await, so the entry cannot be GC'd out from // under a waiter (a fresh entry for a new acquirer would split serialization). let sem = { @@ -536,6 +564,7 @@ impl RepoWriteLeases { let slot = map.entry(repo_id.to_string()).or_insert_with(|| LeaseSlot { sem: Arc::new(tokio::sync::Semaphore::new(1)), refs: 0, + waiters: 0, }); slot.refs += 1; Arc::clone(&slot.sem) @@ -550,30 +579,64 @@ impl RepoWriteLeases { repos: Arc::clone(&self.repos), repo_id: repo_id.to_string(), }; - let permit = match tokio::time::timeout(steal_after, Arc::clone(&sem).acquire_owned()).await - { - Ok(Ok(p)) => Some(p), - // The semaphore is never closed; treat the (unreachable) closed case as a - // steal so acquire always makes forward progress. - Ok(Err(_closed)) => None, - Err(_elapsed) => { - tracing::warn!( - repo = %repo_id, - steal_after_secs = steal_after.as_secs(), - "repo write-lease wait exceeded the steal bound; presuming a leaked \ - lease and proceeding permit-less (in-process serializer reclaim)" - ); - None + + // Uncontended fast path: take the free permit without spending waiter budget, so + // the cap bounds only handlers that actually park. `try_acquire_owned` on tokio's + // semaphore does NOT barge a queued FIFO waiter (it fails while anyone is queued), + // so the fast path is not a fairness hole; probed at 2000 rounds with a queued + // waiter present, 0 barges. + let permit = match Arc::clone(&sem).try_acquire_owned() { + Ok(p) => Some(p), + Err(_) => { + // Contended: park, holding a waiter slot for exactly the cancellable wait. + // Claimed in ONE critical section (check and increment together), so a + // burst of concurrent acquirers cannot all read an under-cap count and + // then all park. Past the cap, drop the reservation (freeing the ref, and + // the entry with it if nobody else holds one) and shed. + let waiter = match WaiterSlot::claim(&self.repos, repo_id, self.max_waiters) { + Some(w) => w, + None => { + tracing::warn!( + repo = %repo_id, + max_waiters = self.max_waiters, + "repo write-lease waiter cap reached; shedding this acquirer" + ); + drop(reservation); + return None; + } + }; + let parked = + tokio::time::timeout(steal_after, Arc::clone(&sem).acquire_owned()).await; + // Released HERE, at the end of the wait, not at the end of the push: past + // this point the handler is a holder, counted by `refs`, and a waiter slot + // it kept would be budget no parked request could ever use. + drop(waiter); + match parked { + Ok(Ok(p)) => Some(p), + // The semaphore is never closed; treat the (unreachable) closed case + // as a steal so acquire always makes forward progress. + Ok(Err(_closed)) => None, + Err(_elapsed) => { + tracing::warn!( + repo = %repo_id, + steal_after_secs = steal_after.as_secs(), + "repo write-lease wait exceeded the steal bound; presuming a \ + leaked lease and proceeding permit-less (in-process \ + serializer reclaim)" + ); + None + } + } } }; // Transfer the ref from the reservation to the guard: forget the reservation (so // it does NOT decrement) and let the guard own the single decrement on its Drop. std::mem::forget(reservation); - RepoWriteLease(Arc::new(LeaseGuardInner { + Some(RepoWriteLease(Arc::new(LeaseGuardInner { repos: Arc::clone(&self.repos), repo_id: repo_id.to_string(), _permit: permit, - })) + }))) } /// Number of repos with a live lease entry. Test/metrics observability. @@ -604,6 +667,19 @@ impl RepoWriteLeases { .map(|slot| slot.refs) .unwrap_or(0) } + + /// How many handlers are PARKED on this repo's lease right now (the count the shed + /// cap is enforced against, holder excluded). Tests use it to observe "this push + /// parked" and "the waiter slot was released" as state. + #[cfg(test)] + pub fn waiters_for(&self, repo_id: &str) -> usize { + self.repos + .lock() + .expect("repo_write_leases mutex poisoned") + .get(repo_id) + .map(|slot| slot.waiters) + .unwrap_or(0) + } } /// Shared-ownership handle to a held repo write lease (#174 U2/F3). `Clone` hands a @@ -654,6 +730,51 @@ impl Drop for RefReservation { } } +/// Holds one LIVE-WAITER slot on a lease entry for exactly the duration of the +/// cancellable park inside [`RepoWriteLeases::acquire`]. Every way out of that park +/// (permit acquired, steal on timeout, shed, or the acquire future being dropped on a +/// client disconnect) drops this guard, so the count can only reflect handlers that are +/// parked right now. That is why the shed cap is on this count and not on `refs`. +struct WaiterSlot { + repos: Arc>>, + repo_id: String, +} + +impl WaiterSlot { + /// Claim a waiter slot, or `None` when the repo already has `max_waiters` live + /// waiters. The entry always exists here: the caller took its `refs` reference in an + /// earlier critical section and still holds it. Check and increment share one + /// critical section, so concurrent acquirers cannot overshoot the cap. + fn claim( + repos: &Arc>>, + repo_id: &str, + max_waiters: usize, + ) -> Option { + if let Ok(mut map) = repos.lock() { + if let Some(slot) = map.get_mut(repo_id) { + if slot.waiters >= max_waiters { + return None; + } + slot.waiters += 1; + } + } + Some(Self { + repos: Arc::clone(repos), + repo_id: repo_id.to_string(), + }) + } +} + +impl Drop for WaiterSlot { + fn drop(&mut self) { + if let Ok(mut map) = self.repos.lock() { + if let Some(slot) = map.get_mut(&self.repo_id) { + slot.waiters = slot.waiters.saturating_sub(1); + } + } + } +} + /// Decrement a lease entry's refcount and remove it once no handler references it, so /// the map cannot grow without bound (Drop-frees-key, like `EncryptInflight`). Safe /// under block-and-wait: while `refs > 0` every acquirer shares the SAME semaphore, and @@ -713,11 +834,11 @@ mod repo_write_lease_tests { /// steal reclaim so a leaked (never-run Drop) holder cannot wedge the repo forever. #[tokio::test] async fn serializes_same_repo_frees_key_and_steals_on_leak() { - let leases = RepoWriteLeases::new(); + let leases = RepoWriteLeases::new(8); let big = Duration::from_secs(3600); // Block-and-wait: a second same-repo acquire waits while the first is held. - let a = leases.acquire("repo1", big).await; + let a = leases.acquire("repo1", big).await.expect("uncontended"); let blocked = tokio::time::timeout(Duration::from_millis(200), leases.acquire("repo1", big)).await; assert!( @@ -728,7 +849,8 @@ mod repo_write_lease_tests { drop(a); let b = tokio::time::timeout(Duration::from_millis(500), leases.acquire("repo1", big)) .await - .expect("the second acquire must proceed once the first lease frees"); + .expect("the second acquire must proceed once the first lease frees") + .expect("under the waiter cap, so it must not shed"); drop(b); // Drop-frees-key: with no holders the entry is removed (bounded map growth). @@ -738,35 +860,111 @@ mod repo_write_lease_tests { ); // Distinct repos never serialize against each other. - let x = leases.acquire("repoX", big).await; + let x = leases.acquire("repoX", big).await.expect("uncontended"); let _y = tokio::time::timeout(Duration::from_millis(200), leases.acquire("repoY", big)) .await - .expect("distinct repos must not serialize"); + .expect("distinct repos must not serialize") + .expect("a distinct repo has its own waiter budget"); drop(x); drop(_y); + } - // Steal-on-timeout reclaim: a leaked holder (never-run Drop, simulated by - // mem::forget) must NOT wedge the repo — the bounded wait proceeds permit-less. - let leaked = leases.acquire("repoZ", big).await; + /// Scenario 4: the steal backstop survives the waiter cap. A holder whose Drop never + /// runs (task abort, runtime teardown, `mem::forget`) keeps its entry `refs` forever; + /// the next acquirer must still proceed permit-less at `steal_after` and must not be + /// shed on the way in. This is why the cap counts LIVE WAITERS and never `refs`: + /// capping `refs` (which includes the holder) lets one leaked lease pin a slot + /// permanently and wedge the repo, the exact permanent wedge the steal exists to + /// prevent. `max_waiters` is 1 here, so a `refs`-based cap has no room at all. + #[tokio::test] + async fn steal_on_leaked_lease_still_works_under_the_waiter_cap() { + let leases = RepoWriteLeases::new(1); + let big = Duration::from_secs(3600); + + let leaked = leases.acquire("repoZ", big).await.expect("uncontended"); std::mem::forget(leaked); + assert_eq!( + leases.refs_for("repoZ"), + 1, + "the leaked holder keeps its entry reference forever (that is the leak)" + ); + let stolen = tokio::time::timeout( - Duration::from_secs(2), + Duration::from_secs(5), leases.acquire("repoZ", Duration::from_millis(150)), ) .await - .expect("a leaked lease must be reclaimed by the bounded-wait steal, not hang forever"); + .expect("a leaked lease must be reclaimed by the bounded-wait steal, not hang forever") + .expect( + "the waiter cap must not shed the stealer: it counts live waiters, and a leaked \ + HOLDER is not a waiter. Capping refs instead wedges the repo permanently", + ); + assert_eq!( + leases.waiters_for("repoZ"), + 0, + "the stealer must return its waiter slot when its wait ends" + ); drop(stolen); } + /// Scenario 5: a shed leaves no residue. Past the cap `acquire` returns `None` + /// without keeping either count, so the entry still GCs once the real holder and the + /// real waiter finish. A shed that stranded a ref would leak the map entry forever; + /// one that stranded a waiter slot would shrink the repo's budget permanently. + #[tokio::test] + async fn shed_waiter_leaves_no_ref_or_waiter_residue() { + let leases = RepoWriteLeases::new(1); + let big = Duration::from_secs(3600); + + let holder = leases.acquire("repoS", big).await.expect("uncontended"); + let waiting = leases.clone(); + let parked = tokio::spawn(async move { waiting.acquire("repoS", big).await }); + assert!( + wait_for(Duration::from_secs(5), || leases.waiters_for("repoS") == 1).await, + "the second acquire must park and be counted as a live waiter" + ); + + // At the cap (1 live waiter): the next acquire sheds instead of queueing. + let shed = tokio::time::timeout(Duration::from_secs(5), leases.acquire("repoS", big)) + .await + .expect("a shed must return immediately, not park"); + assert!( + shed.is_none(), + "past max_waiters the acquire must shed (None), not join the queue" + ); + assert_eq!( + leases.refs_for("repoS"), + 2, + "the shed must leave no refcount residue: only the holder and the real waiter" + ); + assert_eq!( + leases.waiters_for("repoS"), + 1, + "the shed must leave no waiter-count residue: only the real waiter" + ); + + // The entry still GCs once the real handlers finish. + drop(holder); + let promoted = parked + .await + .expect("the parked acquire task must not panic") + .expect("the parked acquire must be served once the holder frees"); + drop(promoted); + assert!( + wait_for(Duration::from_secs(5), || leases.is_empty()).await, + "the lease entry must still GC after a shed (no stranded ref)" + ); + } + /// Cancellation safety: dropping an acquire future while it is BLOCKED waiting for /// the lease (a client disconnect on a second same-repo push) must not strand the /// entry refcount — after the holder frees and the waiter is cancelled, the key GCs. #[tokio::test] async fn cancelled_waiter_does_not_strand_the_refcount() { - let leases = RepoWriteLeases::new(); + let leases = RepoWriteLeases::new(8); let big = Duration::from_secs(3600); - let a = leases.acquire("repoC", big).await; + let a = leases.acquire("repoC", big).await.expect("uncontended"); // A waiter blocks, then is cancelled (its acquire future dropped) mid-wait. let cancelled = tokio::time::timeout(Duration::from_millis(150), leases.acquire("repoC", big)).await; @@ -785,4 +983,104 @@ mod repo_write_lease_tests { "a cancelled waiter must not strand the entry refcount (key must GC)" ); } + + /// Scenario 6: a cancelled waiter returns its WAITER slot too, not just its ref. A + /// client that disconnects while parked is the common case, so a slot stranded here + /// would shrink the repo's waiter budget on every disconnect until the repo shed + /// every push. Observed as state: the count drops back to 0, and a later acquire on + /// the still-held lease parks rather than shedding, with the cap at 1. + #[tokio::test] + async fn cancelled_waiter_releases_its_waiter_slot() { + let leases = RepoWriteLeases::new(1); + let big = Duration::from_secs(3600); + + let holder = leases.acquire("repoX2", big).await.expect("uncontended"); + let cancelled = + tokio::time::timeout(Duration::from_millis(200), leases.acquire("repoX2", big)).await; + assert!( + cancelled.is_err(), + "the waiter must be parked, then cancelled mid-wait" + ); + assert_eq!( + leases.waiters_for("repoX2"), + 0, + "a cancelled waiter must release its waiter slot" + ); + + // The freed budget is usable: the next acquire parks (times out) instead of + // shedding (returning None immediately). + let next = + tokio::time::timeout(Duration::from_millis(300), leases.acquire("repoX2", big)).await; + assert!( + next.is_err(), + "the next acquire must be able to park on the freed waiter slot; it was shed \ + instead, so the cancelled waiter's slot was stranded" + ); + drop(holder); + } + + /// Scenario 7: an uncontended acquire spends no waiter budget, and a promoted waiter + /// hands its slot back at the END OF ITS WAIT rather than holding it for the whole + /// push. With `max_waiters` at 1: the uncontended holder leaves the count at 0, the + /// one waiter is counted while parked, and once that waiter is promoted to holder the + /// count returns to 0 so the NEXT push can still park. Holding the slot for the push + /// instead would let one waiter permanently occupy the only slot, shedding every + /// same-repo push behind it. + #[tokio::test] + async fn uncontended_acquire_spends_no_waiter_budget() { + let leases = RepoWriteLeases::new(1); + let big = Duration::from_secs(3600); + + // Fast path: the lease is free, so this takes it without parking. + let holder = leases.acquire("repoF", big).await.expect("uncontended"); + assert_eq!( + leases.waiters_for("repoF"), + 0, + "an uncontended acquire must take the free permit without spending waiter budget" + ); + + // One waiter parks behind it, spending the single slot. + let waiting = leases.clone(); + let parked = tokio::spawn(async move { waiting.acquire("repoF", big).await }); + assert!( + wait_for(Duration::from_secs(5), || leases.waiters_for("repoF") == 1).await, + "the second acquire must park and be counted" + ); + + // Promote it: the slot must come back at the end of its WAIT. + drop(holder); + let promoted = parked + .await + .expect("the parked acquire task must not panic") + .expect("the parked acquire must be served once the holder frees"); + assert!( + wait_for(Duration::from_secs(5), || leases.waiters_for("repoF") == 0).await, + "a promoted waiter must release its waiter slot when its wait ends, not when \ + its push finishes" + ); + + // ... so a third acquire can still park behind the new holder. + let third = + tokio::time::timeout(Duration::from_millis(300), leases.acquire("repoF", big)).await; + assert!( + third.is_err(), + "the freed waiter slot must be usable: this acquire was shed instead of parking" + ); + drop(promoted); + } + + /// Poll `cond` until it holds, yielding so spawned tasks progress. Returns false if + /// `cap` elapses first; callers assert on the state the loop settled into. + async fn wait_for(cap: Duration, mut cond: impl FnMut() -> bool) -> bool { + let deadline = std::time::Instant::now() + cap; + loop { + if cond() { + return true; + } + if std::time::Instant::now() >= deadline { + return false; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + } } diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 22bdf72d..831202ee 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -89,7 +89,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), pin_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), encrypt_inflight: crate::state::EncryptInflight::new(), - repo_write_leases: crate::state::RepoWriteLeases::new(), + repo_write_leases: crate::state::RepoWriteLeases::new(8), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( 8, From d753198755450598b4d78e4d83424dd24701127f Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:14:54 -0500 Subject: [PATCH 087/124] fix(node): stop a failed walk from taking a pin permit, and bound the close on an errored unlock Three findings from the review round. Moving the Pinata gate to the rules-only predicate was necessary, because a coalesced push has no walk of its own to consult. But it also meant a push whose OWN walk failed now acquired a global pin permit and ran a second full re-derivation that failed the same way. Measured on a repo whose rev-list always fails: twelve git children and three walk attempts, against four and one before. With eight pin slots that defer rather than shed, eight such pushes stall pins for every repo on the node. The gate now excludes a non-coalesced push whose own walk failed, leaving the coalesced path on the rules-only predicate it needs, and the pin helper returns before taking a permit for an empty object list. The connection close on the failing-unlock path had no deadline, and release is awaited inline while the write lease and both admission permits are still held. The condition that produces that branch is a connection whose last statement errored, and a blackholed socket to Postgres is one way to get there, so the await could pin three admission resources for a TCP timeout. It now runs under a five second bound; on expiry the future is dropped, which closes the socket and ends the session anyway, which is what frees the lock. What is tested is the deadline. That sqlx's close is the thing that stalls in production is reasoned from its source, not reproduced, and the test says so. The two tests for that path waited a fixed four hundred milliseconds before asserting. Replacing the sleep with polling was not enough on its own: the test pool sets a one second idle timeout, so a connection returned to the pool is reaped shortly after and the session ends without any help from the code under test. Both tests passed under mutation once the sleep was long enough to reach that reaper, which means a straightforward flake fix would have quietly gutted them. They now run against a pool with no idle reaper, so release is the only thing that can end the session, and both mutations go red again. Adds tokio's test-util as a dev-dependency for paused-time assertions. Production dependencies are unchanged and the build stays --locked clean. --- crates/gitlawb-node/Cargo.toml | 3 + crates/gitlawb-node/src/api/repos.rs | 177 +++++++++++++++++++- crates/gitlawb-node/src/git/repo_store.rs | 193 +++++++++++++++++++--- 3 files changed, 342 insertions(+), 31 deletions(-) diff --git a/crates/gitlawb-node/Cargo.toml b/crates/gitlawb-node/Cargo.toml index 72748fae..82854803 100644 --- a/crates/gitlawb-node/Cargo.toml +++ b/crates/gitlawb-node/Cargo.toml @@ -77,3 +77,6 @@ libp2p-dns = { version = "0.44.0", features = ["tokio"] } [dev-dependencies] mockito = "1" tempfile = "3" +# `test-util` for `#[tokio::test(start_paused = true)]`: lets the deadline tests assert +# on which timer fired instead of on wall clock. +tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 88138d0e..b89ebc36 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1101,6 +1101,14 @@ async fn pin_new_objects_gated( object_list: Vec, db: &Arc, ) -> Vec<(String, String)> { + // Nothing to pin: answer before taking a permit (#174 F2b). The permit bounds how + // many MB-scale object lists are held at once and an empty list holds none, so + // parking here would spend a global pin slot on no work. The pool DEFERS rather + // than sheds, so those calls stall pins for every other repo. Empty is the normal + // shape for a push whose walk failed or that may replicate nothing. + if object_list.is_empty() { + return Vec::new(); + } let _permit = pin_sem .clone() .acquire_owned() @@ -1960,6 +1968,13 @@ async fn post_receive_replication_tail( .1 }; + // #174 F2b: did THIS push run its own walk and have it fail? An admitted push that + // could not be vetted must not go on to take a pin permit and re-run the same + // failing walk in the Pinata worker below. Only the COALESCED path needs the + // rules-only predicate there: it has no walk of its own, so the worker's + // re-derivation is its only fail-closed source. + let own_walk_failed = announce_at_root && !coalesced && withheld.is_none(); + // Resolve the per-push pin candidate set once, off the async worker, then // filter to what may actually replicate. Delta path: the reachable-only // `withheld` set suffices (delta objects are reachable). Full-scan path: the @@ -2106,7 +2121,13 @@ async fn post_receive_replication_tail( // `replication_withheld_set` inside the pin permit: it returns an empty // list AND announce=false when the walk fails or the repo may not // replicate, so neither blobs nor announcements escape an unvetted push. - let do_pinata_replication = announce_at_root; + // + // #174 F2b: except when THIS push already ran that walk and it failed. The + // re-derivation would fail the same way, so it buys nothing, and it would buy + // it at the price of a global pin permit plus a second round of git children. + // The pin pool DEFERS rather than sheds, so enough such pushes stall pins + // node-wide. A coalesced push never walked, so it is unaffected. + let do_pinata_replication = announce_at_root && !own_walk_failed; // #174 F2 / KTD-3: capture only the small inputs the re-derivation needs; the // MB-scale object list is NOT moved in. `pinata_object_list_for_refs` recomputes // it from these once a pin slot frees. rules/owner/is_public drive the fresh @@ -5445,10 +5466,12 @@ mod tests { let held = pin_sem.clone().acquire_owned().await.unwrap(); // Empty ipfs_api makes the pin itself a no-op, but the loop must still DEFER on - // the exhausted pin pool rather than run. + // the exhausted pin pool rather than run. The object list is non-empty because + // an empty one takes no permit at all by design (#174 F2b). + let objects = vec!["0123456789abcdef0123456789abcdef01234567".to_string()]; let blocked = tokio::time::timeout( std::time::Duration::from_millis(500), - pin_new_objects_gated(&pin_sem, "", tmp.path(), vec![], &db), + pin_new_objects_gated(&pin_sem, "", tmp.path(), objects.clone(), &db), ) .await; assert!( @@ -5460,13 +5483,47 @@ mod tests { drop(held); let out = tokio::time::timeout( std::time::Duration::from_secs(5), - pin_new_objects_gated(&pin_sem, "", tmp.path(), vec![], &db), + pin_new_objects_gated(&pin_sem, "", tmp.path(), objects, &db), ) .await .expect("the pin loop completes once admission frees"); assert!(out.is_empty(), "an empty ipfs_api pins nothing"); } + /// #174 F2b: the pin permit bounds how many MB-scale object lists are held at once, + /// so a call with NOTHING to pin must not take one. It otherwise spends a global pin + /// slot on no work, and the pool DEFERS rather than sheds, so those calls stall pins + /// for every other repo. The empty case is the normal shape for a push whose walk + /// failed or that may replicate nothing. + /// + /// Load-bearing: without the guard this call parks on the exhausted pool exactly like + /// the non-empty one above, and the completion assertion fails. + #[sqlx::test] + async fn pin_new_objects_gated_takes_no_permit_for_an_empty_object_list(pool: sqlx::PgPool) { + use std::sync::Arc; + use tokio::sync::Semaphore; + + let state = crate::test_support::test_state(pool).await; + let db = state.db.clone(); + let tmp = tempfile::TempDir::new().unwrap(); + let pin_sem = Arc::new(Semaphore::new(1)); + // Hold the only pin permit for the whole call. + let _held = pin_sem.clone().acquire_owned().await.unwrap(); + + let out = tokio::time::timeout( + std::time::Duration::from_millis(500), + pin_new_objects_gated(&pin_sem, "", tmp.path(), vec![], &db), + ) + .await + .expect("an empty object list must not wait on pin admission (#174 F2b)"); + assert!(out.is_empty(), "and it pins nothing"); + assert_eq!( + pin_sem.available_permits(), + 0, + "the test still holds the only permit, so the call never took one" + ); + } + /// Shared fixture for the F4 handler-layer tests: a state whose repo_store and /// git_bin point at the given tempdir/fake-git, plus a seeded on-disk repo, /// optionally with a path-scoped rule (so the post-receive walks actually run). @@ -8312,4 +8369,116 @@ mod tests { .expect("the admitted push fires its announce"); assert_eq!(broadcast.new_sha, c2); } + + // ---- #174 F2b: a failed OWN walk does not buy a pin permit and a second walk ---- + + /// Withheld-walk commit-enumeration attempts so far. Each `replication_withheld_set` + /// runs exactly one `rev-list --all`, so this counts the walks that were attempted + /// (the `ls-tree` counter above cannot: a walk whose enumeration fails never gets + /// to `ls-tree`). + fn f2b_walk_attempts(log: &std::path::Path) -> usize { + f2a_log(log) + .lines() + .filter(|l| l.starts_with("rev-list --all")) + .count() + } + + /// A NON-coalesced push whose own withheld walk failed must not take a global pin + /// permit and must not re-run the same failing walk in the Pinata worker. + /// + /// The F2a change moved the Pinata gate from `withheld.is_some()` to the rules-only + /// `announce_at_root`, which a coalesced push genuinely needs (it has no walk of its + /// own). But it also let an ADMITTED push whose walk failed acquire `pin_semaphore` + /// and re-derive `replication_withheld_set`, which fails the same way. With + /// `max_concurrent_pin_tasks` defaulting to 8 and the pin pool DEFERRING rather than + /// shedding, eight such pushes stall pins node-wide. + /// + /// Asserted on observable work, twice over, with the pin pool's only permit held for + /// the whole first phase: + /// * the walk attempts while the permit is held. Two: the tail's own (which fails) + /// and the recovery task's recipients walk, which must NOT park on the pin pool + /// for its empty object list. The Pinata worker's is the third and must not exist. + /// * the walk attempts after the permit is released. Still two: nothing was left + /// waiting on the pin pool, which is the permit assertion. + /// + /// Load-bearing both ways. With the gate back on plain `announce_at_root` the Pinata + /// worker parks on the held permit and then walks once it is freed (phase 2 sees 3). + /// Without the empty-list guard on `pin_new_objects_gated` the recovery task parks + /// too, so phase 1 sees 1. + #[cfg(unix)] + #[sqlx::test] + async fn f2b_failed_own_walk_takes_no_pin_permit_and_runs_no_second_walk(pool: sqlx::PgPool) { + let repo = tempfile::TempDir::new().unwrap(); + let bin = tempfile::TempDir::new().unwrap(); + u5_init_repo(repo.path()); + let c1 = u5_commit_file(repo.path(), "a.txt", "one\n"); + let c2 = u5_commit_file(repo.path(), "secret/s.txt", "two\n"); + let log = bin.path().join("git.log"); + // Every withheld walk fails, so this push can never be vetted. + let git_bin = write_fake_git( + bin.path(), + &format!( + "#!/bin/sh\n\ + printf '%s\\n' \"$*\" >> \"{log}\"\n\ + case \"$*\" in 'rev-list --all'*) exit 1 ;; esac\n\ + exec git \"$@\"\n", + log = log.display(), + ), + ); + let (mut state, rec) = f2a_state(pool, &git_bin, "z6f2bfail", "w1", true).await; + let (_server, _cid) = f2a_pinata(&mut state).await; + // One pin permit, held: anything that reaches a pin-admission acquire parks + // instead of running, which is what makes "took no permit" observable. + state.pin_semaphore = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + let held = state.pin_semaphore.clone().acquire_owned().await.unwrap(); + + // Nothing pre-takes the coalescing key, so this push is ADMITTED and runs its + // own walk. + post_receive_replication_tail( + state.clone(), + rec.clone(), + vec![RefUpdate { + old_sha: c1.clone(), + new_sha: c2.clone(), + ref_name: "refs/heads/main".to_string(), + }], + repo.path().to_path_buf(), + F2A_PUSHER.to_string(), + ) + .await; + + f2a_wait_for( + || f2b_walk_attempts(&log) >= 2, + "the tail's own walk and the recovery task's recipients walk", + ) + .await; + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + assert_eq!( + f2b_walk_attempts(&log), + 2, + "a failed own walk must not buy a third walk in the Pinata worker; log:\n{}", + f2a_log(&log) + ); + + // Release pin admission. A task that had parked on it now wakes and walks; + // nothing should have been parked. + drop(held); + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + assert_eq!( + f2b_walk_attempts(&log), + 2, + "nothing may be left waiting on the pin permit for a push that pins \ + nothing; log:\n{}", + f2a_log(&log) + ); + assert!( + state + .db + .list_branch_cids(&f2a_slug(&rec)) + .await + .unwrap() + .is_empty(), + "and the unvetted push still maps no CID" + ); + } } diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 2bc15e0a..69d1bb33 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -403,6 +403,42 @@ pub struct RepoWriteGuard { test_pre_unlock_gate: Option>, } +/// Deadline for tearing down the connection that saw a failing `pg_advisory_unlock`. +/// Long enough that a healthy socket always finishes well inside it, short enough that +/// a blackholed one does not pin admission resources for a TCP timeout. +const UNLOCK_ERROR_CLOSE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); + +/// Await `close` under a deadline (#174 F3c). +/// +/// `release` awaits this INLINE while the global write permit, the per-source permit +/// and the write lease are all still held, and sqlx puts no deadline on `close()`: +/// it writes Terminate and then tears the socket down. The branch that reaches here is +/// by definition a connection whose last statement errored, and a blackholed TCP path +/// to Postgres (a cloud failover that drops packets without an RST) is a plausible +/// cause, so an unbounded await here parks every later push to the repo behind three +/// pinned admission resources until the steal bound. +/// +/// On elapsed the future is simply dropped, which drops the `PoolConnection` it owns. +/// Dropping it closes the socket, and closing the socket is what actually ends the +/// session and makes Postgres release the lock, so the deadline costs nothing the +/// graceful path was buying. +async fn close_conn_bounded( + repo_name: &str, + close: impl std::future::Future>, +) { + match tokio::time::timeout(UNLOCK_ERROR_CLOSE_TIMEOUT, close).await { + Ok(Ok(())) => {} + Ok(Err(e)) => { + warn!(repo = %repo_name, err = %e, + "closing the write-lock connection failed, the session teardown still frees the lock server-side"); + } + Err(_) => { + warn!(repo = %repo_name, timeout_secs = UNLOCK_ERROR_CLOSE_TIMEOUT.as_secs(), + "closing the write-lock connection timed out, dropping it instead; the socket goes down either way, which is what frees the lock server-side"); + } + } +} + impl RepoWriteGuard { /// Path to the bare repo on local disk. pub fn path(&self) -> &Path { @@ -476,11 +512,10 @@ impl RepoWriteGuard { // whenever the returned `PgConnection` is dropped and its // background close completes. If this future is cancelled during // `close()`, the connection is dropped mid-close, which still - // tears the session down. - if let Err(e) = conn.close().await { - warn!(repo = %self.repo_name, err = %e, - "closing the write-lock connection failed, the session teardown still frees the lock server-side"); - } + // tears the session down. That last point is also why the await is + // safe to bound: see `close_conn_bounded`, which gives it the + // deadline sqlx does not. + close_conn_bounded(&self.repo_name, conn.close()).await; } } } @@ -893,6 +928,108 @@ mod tests { ); } + /// How long the polls below may wait for an asynchronous server-side effect. + /// Postgres frees a session advisory lock when the backend exits, which happens + /// asynchronously to our socket close, so these are polled to a generous deadline + /// rather than slept for a fixed interval: a constant sleep is a flake under load, + /// and one that is long enough to be safe is dead time on every run. + const RELEASE_POLL_DEADLINE: std::time::Duration = std::time::Duration::from_secs(20); + + /// Poll `cond` until it holds, failing at the deadline so a regression fails the + /// test rather than hanging the suite. + async fn wait_until(mut cond: impl FnMut() -> bool, what: &str) { + let deadline = std::time::Instant::now() + RELEASE_POLL_DEADLINE; + while !cond() { + assert!( + std::time::Instant::now() < deadline, + "timed out waiting for {what}" + ); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + } + + /// Poll the advisory lock from `checker` until it is free. `pg_try_advisory_lock` + /// ACQUIRES on success, so a true result both answers the question and leaves the + /// checker session holding the lock; the caller unlocks it. + async fn wait_until_lock_free(checker: &mut sqlx::PgConnection, key: i64, what: &str) { + let deadline = std::time::Instant::now() + RELEASE_POLL_DEADLINE; + loop { + let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *checker) + .await + .unwrap(); + if free { + return; + } + assert!( + std::time::Instant::now() < deadline, + "timed out waiting for {what}" + ); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + } + + /// A second pool over the same test database with the idle reaper DISABLED. + /// + /// `#[sqlx::test]`'s own pool sets `idle_timeout(1s)`, so a connection returned to + /// it is closed by the reaper about a second later, which ends the session and + /// frees the advisory lock all on its own. The old fixed 400ms sleep landed inside + /// that window by luck; polling to a deadline long enough to be flake-proof would + /// land outside it and go green whether or not `release` disposed of the + /// connection, so the tests below would stop testing anything (measured: with the + /// reaper in play, the disposal shows up at ~2s even with the fix reverted). With + /// no reaper, `release` is the only thing that can end that session, so the poll + /// measures exactly the property these two tests exist for. + async fn pool_without_idle_reaper(pool: &sqlx::PgPool) -> sqlx::PgPool { + sqlx::postgres::PgPoolOptions::new() + .max_connections(5) + .idle_timeout(None) + .max_lifetime(None) + .connect_with(pool.connect_options().as_ref().clone()) + .await + .expect("a second pool over the test database") + } + + /// F3c (P2): the connection teardown on the failing-unlock path must be BOUNDED. + /// `release` awaits it inline while the global write permit, the per-source permit + /// and the write lease are all still held, and sqlx's `close()` carries no deadline + /// of its own, so a blackholed socket would park every later push to that repo + /// behind three pinned admission resources. + /// + /// What this covers: the deadline itself. A close that never resolves still lets + /// `close_conn_bounded` return, which is the property `release` depends on. What it + /// does NOT cover, and is reasoned rather than run: that sqlx's own `close()` is + /// what stalls in production. Making a real `PgConnection::close` hang needs a + /// blackholed TCP path to Postgres, and the flip has to land after the unlock + /// statement round-trips but before the Terminate write, which is not a seam this + /// module exposes. A never-resolving future is the faithful stand-in for that + /// close, and the F3b tests above already cover that `release` really routes its + /// close through here. + /// + /// Time is paused, so nothing here depends on wall clock: the runtime auto-advances + /// to the next timer, and the assertion is on which timer fired, not on elapsed + /// time. The outer bound is what turns a removed deadline into a failure rather + /// than a hung suite. + /// + /// Load-bearing: drop the `tokio::time::timeout` in `close_conn_bounded` and the + /// inner future never resolves, so the outer bound fires and this fails. + #[tokio::test(start_paused = true)] + async fn unlock_error_connection_close_is_bounded() { + let hanging = std::future::pending::>(); + let outcome = tokio::time::timeout( + UNLOCK_ERROR_CLOSE_TIMEOUT * 4, + close_conn_bounded("boundedclosetest", hanging), + ) + .await; + assert!( + outcome.is_ok(), + "a connection close that never completes must not hold the write lease and \ + both admission permits open-endedly: close_conn_bounded must give up and \ + drop the connection" + ); + } + /// F3b (P1): when `pg_advisory_unlock` ERRORS while the session is still alive /// (statement timeout, admin cancel, aborted transaction), the lock must not /// survive `release`. The old code discarded the error with `let _ =` and set @@ -910,7 +1047,8 @@ mod tests { #[sqlx::test] async fn write_guard_release_with_failing_unlock_frees_the_lock(pool: sqlx::PgPool) { let dir = tempfile::TempDir::new().unwrap(); - let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone()); + let store_pool = pool_without_idle_reaper(&pool).await; + let store = RepoStore::for_testing(dir.path().to_path_buf(), store_pool.clone()); let owner = "did:key:z6MkUnlockErrorProofFFFFFFFFFFFFFFFFFFFFFF"; let name = "unlockerrtest"; let slug = owner.replace([':', '/'], "_"); @@ -935,19 +1073,18 @@ mod tests { ); guard.release(false).await; - // Give a background close a moment to reach the server. - tokio::time::sleep(std::time::Duration::from_millis(400)).await; - let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") - .bind(key) - .fetch_one(&mut *checker) - .await - .unwrap(); - assert!( - free, - "an errored pg_advisory_unlock must not leave the lock held: release \ - must dispose of the connection so the session ends" - ); + // Postgres drops the lock when the disposed session's backend exits, which is + // asynchronous to our socket close: poll for it rather than sleeping a + // constant. The deadline is what keeps this load-bearing: a release that + // leaves the lock held never satisfies the probe and fails here. + wait_until_lock_free( + &mut checker, + key, + "an errored pg_advisory_unlock must not leave the lock held: release must \ + dispose of the connection so the session ends", + ) + .await; let _ = sqlx::query("SELECT pg_advisory_unlock($1)") .bind(key) .execute(&mut *checker) @@ -963,24 +1100,26 @@ mod tests { pool: sqlx::PgPool, ) { let dir = tempfile::TempDir::new().unwrap(); - let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone()); + let store_pool = pool_without_idle_reaper(&pool).await; + let store = RepoStore::for_testing(dir.path().to_path_buf(), store_pool.clone()); let owner = "did:key:z6MkUnlockErrorPoolProofGGGGGGGGGGGGGGGGGG"; let name = "unlockerrpooltest"; let mut guard = store.acquire_write(owner, name).await.expect("acquire"); poison_guard_connection(&mut guard).await; - let size_before = pool.size(); + let size_before = store_pool.size(); assert!(size_before > 0, "the pool owns the guard's connection"); guard.release(false).await; - tokio::time::sleep(std::time::Duration::from_millis(400)).await; - assert_eq!( - pool.size(), - size_before - 1, - "the connection that saw the unlock error must be closed, not returned \ - to the pool still holding the session lock" - ); + // The pool's size drops when the closed connection's slot is given up, which + // is not synchronous with `release` returning: poll rather than sleep. + wait_until( + || store_pool.size() == size_before - 1, + "the connection that saw the unlock error to be closed rather than returned \ + to the pool still holding the session lock", + ) + .await; } /// F3b regression guard on the success path: a normal unlock keeps the From b9566b77503a28a70186aab91461e7d62c73aafd Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 28 Jul 2026 16:01:30 -0500 Subject: [PATCH 088/124] fix(node): sweep the /ipfs rate limiter and document the lease waiter cap The 300s cleanup task swept the DID, create-IP, push, sync-trigger and peer-write limiters but not ipfs_rate_limiter, so its expired keys sat until the map hit max_keys and the inline capacity sweep reclaimed them. Drive the sweep off AppState in a named sweep_rate_limiters so a limiter added to the state and forgotten here fails a test instead of leaking keys, and cover all six with a regression that goes red when any one is dropped from the sweep. GITLAWB_REPO_LEASE_MAX_WAITERS was also missing from .env.example and the README settings table, leaving operators without a way to discover or tune the cap behind the "repo is busy with another push" 503. --- .env.example | 9 ++++ README.md | 1 + crates/gitlawb-node/src/main.rs | 71 +++++++++++++++++++++++---- crates/gitlawb-node/src/rate_limit.rs | 7 +++ 4 files changed, 78 insertions(+), 10 deletions(-) diff --git a/.env.example b/.env.example index 9913b41b..5ed087d9 100644 --- a/.env.example +++ b/.env.example @@ -144,6 +144,15 @@ GITLAWB_MAX_CONCURRENT_GIT_OPS=128 # Retry-After. Default 32. GITLAWB_MAX_CONCURRENT_GIT_PUSHES=32 +# Max pushes PARKED at once waiting for the per-repo write lease of the SAME repo. +# A parked push holds its already-buffered pack body in memory, so this bounds that +# memory for a hot repo. Past the cap the newest push sheds a clean 503 + +# Retry-After ("repo is busy with another push") instead of joining the queue; a +# push to a DIFFERENT repo is never affected. The lease holder is not counted. +# Raising GITLAWB_MAX_CONCURRENT_GIT_PUSHES does not raise this; set it explicitly. +# Default 8. +GITLAWB_REPO_LEASE_MAX_WAITERS=8 + # Max concurrent post-push pin loops (IPFS + Pinata pin_new_objects) across all # repos. Each loop holds a full per-push object-id list while pinning, so this # bounds that MB-scale memory even though the per-repo encrypt-task set already diff --git a/README.md b/README.md index cc32bcf7..6789cd80 100644 --- a/README.md +++ b/README.md @@ -345,6 +345,7 @@ Important node settings: | `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. | | `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. | | `GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS` | Max seconds the storage-acquisition phase (Tigris HEAD/GET, push advisory-lock) of a served git op may run before the request is shed with a 503, separate from the git-run timeout. The concurrency permit is released on expiry so a stalled backend cannot pin the pool. Default 30. | +| `GITLAWB_REPO_LEASE_MAX_WAITERS` | Max pushes parked at once waiting for the same repo's write lease. Each waiter pins its buffered pack body, so this bounds that memory for a hot repo; past the cap the newest push sheds a 503 + Retry-After instead of queueing. Pushes to other repos are unaffected, and the lease holder is not counted. Default 8. | | `GITLAWB_MAX_CONCURRENT_IPFS_WALKS` | Max concurrent `GET /ipfs/{cid}` visibility walks across all callers (own pool, disjoint from the served-git pools); over-cap sheds 503. Default 32. | | `GITLAWB_IPFS_WALK_PER_SOURCE` | Max concurrent `/ipfs` walks a single source IP may hold. Default 4. | | `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max expensive path-scope visibility walks per `/ipfs/{cid}` request; over-cap repos are skipped and the scan continues, shedding a retryable 503 (not a false 404) if the object is then found nowhere. Default 64. | diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index a3e26753..1fd26979 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -481,22 +481,14 @@ async fn main() -> Result<()> { // Periodic cleanup of expired rate limit entries + consumed-proof ledger { - let rl = state.rate_limiter.clone(); - let create_ip_rl = state.create_ip_rate_limiter.clone(); - let push_rl = state.push_rate_limiter.clone(); - let sync_trigger_rl = state.sync_trigger_rate_limiter.clone(); - let peer_write_rl = state.peer_write_rate_limiter.clone(); + let sweep_state = state.clone(); let db = state.db.clone(); let mut shutdown_rx = state.subscribe_shutdown(); tokio::spawn(async move { loop { tokio::select! { _ = tokio::time::sleep(std::time::Duration::from_secs(300)) => { - rl.cleanup().await; - create_ip_rl.cleanup().await; - push_rl.cleanup().await; - sync_trigger_rl.cleanup().await; - peer_write_rl.cleanup().await; + sweep_rate_limiters(&sweep_state).await; let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|d| d.as_secs() as i64) @@ -1032,6 +1024,65 @@ async fn gossip_task( } } +#[cfg(test)] +mod rate_limiter_sweep_tests { + use crate::rate_limit::RateLimiter; + use std::time::Duration; + + // Every per-key limiter the router mounts must be swept by the periodic + // task, the `/ipfs` one included: a limiter left out keeps expired keys + // until its map fills and the inline capacity sweep fires. Fails on the + // pre-fix sweeper, which skipped `ipfs_rate_limiter`. + #[tokio::test] + async fn sweep_evicts_expired_keys_from_every_limiter() { + let window = Duration::from_millis(30); + let mut state = crate::test_support::test_state_lazy(); + state.rate_limiter = RateLimiter::new(10, window); + state.create_ip_rate_limiter = RateLimiter::new(10, window); + state.push_rate_limiter = RateLimiter::new(10, window); + state.sync_trigger_rate_limiter = RateLimiter::new(10, window); + state.peer_write_rate_limiter = RateLimiter::new(10, window); + state.ipfs_rate_limiter = RateLimiter::new(10, window); + + let limiters = |s: &crate::state::AppState| { + [ + s.rate_limiter.clone(), + s.create_ip_rate_limiter.clone(), + s.push_rate_limiter.clone(), + s.sync_trigger_rate_limiter.clone(), + s.peer_write_rate_limiter.clone(), + s.ipfs_rate_limiter.clone(), + ] + }; + for l in limiters(&state) { + assert!(l.check("1.2.3.4").await); + assert_eq!(l.tracked_keys().await, 1); + } + + tokio::time::sleep(window * 3).await; + super::sweep_rate_limiters(&state).await; + + for (i, l) in limiters(&state).into_iter().enumerate() { + assert_eq!(l.tracked_keys().await, 0, "limiter {i} was not swept"); + } + } +} + +/// Evict expired entries from every per-key rate limiter on the state. +/// +/// Named and driven off `AppState` so the periodic sweeper stays in step with +/// the limiters the router actually mounts: adding a limiter field and +/// forgetting it here leaves its keys pinned until the map hits `max_keys` and +/// the inline capacity sweep runs (the `/ipfs` limiter was missed this way). +async fn sweep_rate_limiters(state: &AppState) { + state.rate_limiter.cleanup().await; + state.create_ip_rate_limiter.cleanup().await; + state.push_rate_limiter.cleanup().await; + state.sync_trigger_rate_limiter.cleanup().await; + state.peer_write_rate_limiter.cleanup().await; + state.ipfs_rate_limiter.cleanup().await; +} + /// Build the shared node HTTP client used for every outbound fan-out (sync /// trigger, profile/repo fetches, gossip announce + peer pings). /// diff --git a/crates/gitlawb-node/src/rate_limit.rs b/crates/gitlawb-node/src/rate_limit.rs index 22281691..a0dc6137 100644 --- a/crates/gitlawb-node/src/rate_limit.rs +++ b/crates/gitlawb-node/src/rate_limit.rs @@ -121,6 +121,13 @@ impl RateLimiter { true } + /// Number of keys currently tracked. Tests use it to observe what a sweep + /// reclaimed; there is no production reader. + #[cfg(test)] + pub async fn tracked_keys(&self) -> usize { + self.state.lock().await.len() + } + pub async fn cleanup(&self) { let now = Instant::now(); let mut state = self.state.lock().await; From 1564302dd96ed57ede0ab7adc0349bed426071f4 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 02:16:04 -0500 Subject: [PATCH 089/124] fix(node): write coalesced-drain pin metadata under the re-fetched repo id resolve_drain_object_list already re-fetches the repo row by owner/name and deliberately reads visibility rules under record.id, because a delete+recreate under the same slug gives the row a new id while the bare repo on disk is reused. The encrypt write a few lines later still took ctx.repo_id, frozen when the task was spawned, so recovery-pin metadata for a coalesced drain landed against the deleted row and was unreachable from the live one. Return the re-fetched id from the resolver and pass it explicitly to pin_and_encrypt_objects. The parameter is explicit rather than read from ctx so the two callers stay honest about which id they mean: the snapshot iteration passes its own spawn-time id, the drain passes the fresh one. The Arweave anchor is unaffected - EncryptedManifest carries no repo id and builds its slug from owner and name, both stable across the rotation. --- crates/gitlawb-node/src/api/repos.rs | 118 +++++++++++++++++++++++++-- 1 file changed, 109 insertions(+), 9 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index b89ebc36..d174a26b 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -869,17 +869,28 @@ async fn run_encrypt_pin_task( snapshot_rules: Option>, snapshot_is_public: bool, ) { - pin_and_encrypt_objects(&ctx, snapshot_objects, snapshot_rules, snapshot_is_public).await; + // The snapshot is this push's own work, resolved before spawn, so it belongs + // to the id captured at spawn. + pin_and_encrypt_objects( + &ctx, + &ctx.repo_id, + snapshot_objects, + snapshot_rules, + snapshot_is_public, + ) + .await; let mut guard = guard; loop { match guard.finish_or_take_pending() { crate::state::FinishOutcome::Finished(_) => break, crate::state::FinishOutcome::Pending(g, pending) => { guard = g; - if let Some((object_list, rules, is_public)) = + // `drain_repo_id`, not ctx.repo_id: see resolve_drain_object_list. + if let Some((drain_repo_id, object_list, rules, is_public)) = resolve_drain_object_list(&ctx, pending).await { - pin_and_encrypt_objects(&ctx, object_list, rules, is_public).await; + pin_and_encrypt_objects(&ctx, &drain_repo_id, object_list, rules, is_public) + .await; } } } @@ -893,10 +904,19 @@ async fn run_encrypt_pin_task( /// re-read) pins nothing at all (`None`). Returns the filtered object list plus /// the fresh rules/is_public snapshot for the encrypt stage — the same /// resolution → withheld-filter pipeline the receive-pack tail runs. +/// The returned `String` is the repo id the drain's encrypt/anchor writes must +/// use: the id from the FRESH re-fetch, never `ctx.repo_id` frozen at task spawn. +/// A delete+recreate under the same slug gives the row a new id, and metadata +/// written against the dead id is invisible to readers on the live row (#174 U3). async fn resolve_drain_object_list( ctx: &EncryptTaskCtx, pending: crate::state::PendingWork, -) -> Option<(Vec, Option>, bool)> { +) -> Option<( + String, + Vec, + Option>, + bool, +)> { let record = match ctx.db.get_repo(&ctx.owner_did, &ctx.repo_name).await { Ok(Some(r)) => r, Ok(None) => { @@ -985,7 +1005,10 @@ async fn resolve_drain_object_list( } else { crate::git::visibility_pack::replicable_objects(pin_set.candidates, &withheld_set) }; - Some((object_list, rules_opt, record.is_public)) + // record.id, never ctx.repo_id: the record above is re-fetched fresh by + // owner/name, and a delete+re-create between spawn and drain gives the row a + // NEW id. This is the same rule the rules read a few lines up already follows. + Some((record.id, object_list, rules_opt, record.is_public)) } /// Re-derive the Pinata replication object set for a push from its ref-update @@ -1117,8 +1140,13 @@ async fn pin_new_objects_gated( crate::ipfs_pin::pin_new_objects(ipfs_api, repo_path, object_list, db).await } +/// `repo_id` is passed explicitly rather than read from `ctx` so the two callers +/// stay honest about which id they mean: the snapshot iteration passes +/// `ctx.repo_id` (its own push's row), while a coalesced drain passes the id from +/// its fresh re-fetch, which differs after a delete+recreate (#174 U3). async fn pin_and_encrypt_objects( ctx: &EncryptTaskCtx, + repo_id: &str, object_list: Vec, rules: Option>, is_public: bool, @@ -1163,7 +1191,7 @@ async fn pin_and_encrypt_objects( &ctx.ipfs_api, &ctx.repo_path, &ctx.db, - &ctx.repo_id, + repo_id, &node_seed, &recipients, ) @@ -6316,6 +6344,78 @@ mod tests { } } + /// #174 U3: a coalesced drain that runs after the repo row was deleted and + /// recreated under the same owner/name must resolve the LIVE row's id, not the + /// id frozen into the task ctx at spawn. Encrypted-pin metadata written under + /// the dead id is invisible to authorized readers on the live row. + /// + /// `resolve_drain_object_list` already re-fetches by owner/name and uses + /// `record.id` for the visibility-rule read; this binds the same id to the + /// encrypt write, which was still taking `ctx.repo_id`. + #[sqlx::test] + async fn u3_drain_resolves_the_refetched_repo_id_after_an_id_rotation(pool: sqlx::PgPool) { + let raw = pool.clone(); + let state = crate::test_support::test_state(pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + u5_init_repo(tmp.path()); + let c1 = u5_commit_file(tmp.path(), "a.txt", "one\n"); + + state + .db + .upsert_mirror_repo("z6u3rot", "r", "/u3-rotation", None, false) + .await + .unwrap(); + let before = state.db.get_repo("z6u3rot", "r").await.unwrap().unwrap(); + let ctx = u5_ctx( + &state, + &before, + tmp.path().to_path_buf(), + "git", + std::sync::Arc::new(tokio::sync::Semaphore::new(4)), + ); + assert_eq!(ctx.repo_id, before.id, "ctx captures the spawn-time id"); + + // Delete + recreate under the SAME owner/name. This is the rotation: a new + // row id over the same on-disk bare repo. + sqlx::query("DELETE FROM repos WHERE id = $1") + .bind(&before.id) + .execute(&raw) + .await + .unwrap(); + let after = crate::db::RepoRecord { + id: Uuid::new_v4().to_string(), + name: before.name.clone(), + owner_did: before.owner_did.clone(), + description: None, + is_public: true, + default_branch: "main".to_string(), + created_at: Utc::now(), + updated_at: Utc::now(), + disk_path: before.disk_path.clone(), + forked_from: None, + machine_id: None, + }; + state.db.create_repo(&after).await.unwrap(); + assert_ne!(after.id, before.id, "the recreate really did rotate the id"); + + let (drain_id, _list, _rules, _pub) = resolve_drain_object_list( + &ctx, + crate::state::PendingWork::Tips(vec![(ZERO_SHA.to_string(), c1.clone())]), + ) + .await + .expect("a public repo drains to a pin list"); + + assert_eq!( + drain_id, after.id, + "the drain must write its encrypted-pin metadata under the LIVE row id" + ); + assert_ne!( + drain_id, ctx.repo_id, + "the spawn-time id is the deleted row; metadata written there is \ + unreachable from the live repo" + ); + } + /// The drain resolves a coalesced push's tip pair to exactly that push's /// introduced objects (delta semantics — the F5 observable: push B's pins are /// recorded by the drain, and pre-existing objects are not re-listed). @@ -6341,7 +6441,7 @@ mod tests { ); // Push B advanced main c1 -> c2 and lost try_begin; its pair was coalesced. - let (list, _rules, is_public) = resolve_drain_object_list( + let (_drain_id, list, _rules, is_public) = resolve_drain_object_list( &ctx, crate::state::PendingWork::Tips(vec![(c1.clone(), c2.clone())]), ) @@ -6385,7 +6485,7 @@ mod tests { std::sync::Arc::new(tokio::sync::Semaphore::new(4)), ); - let (list, _rules, _pub) = + let (_drain_id, list, _rules, _pub) = resolve_drain_object_list(&ctx, crate::state::PendingWork::FullScan) .await .expect("a public repo drains to a pin list"); @@ -6441,7 +6541,7 @@ mod tests { ) .await .unwrap(); - let (list, _rules, _pub) = resolve_drain_object_list(&ctx, pending()) + let (_drain_id, list, _rules, _pub) = resolve_drain_object_list(&ctx, pending()) .await .expect("still announceable at root"); let got: std::collections::HashSet = list.into_iter().collect(); From 19eaf9fccdb3708a594933650029c71d595b1f59 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 02:39:09 -0500 Subject: [PATCH 090/124] fix(node): key the write lease and encrypt coalescing on the stable disk identity repo_write_leases and encrypt_inflight keyed on record.id, but RepoStore::local_path and the pg advisory lock key on the sanitized owner slug plus repo name, and the bare repo on disk is reused when a repo is deleted and recreated under the same slug. The row id rotates across that recreate, so both in-process serializers stopped serializing exactly where the shared objects/ directory still needed them: a disconnected push's reaper could still be tearing down while a new push took a different key and proceeded. Add repo_identity_key(owner_did, repo_name) and use it at both sites. Three details are load-bearing and carry tests: the sanitization is repo_disk_path's replace([':','/'],"_") rather than normalize_owner_key, which folds a did:key: prefix and would disagree with the disk path; the separator is a NUL, without which owner 'a' + name 'b_c' and owner 'a_b' + name 'c' collide; and both call sites pass record.owner_did / record.name rather than the request path segments, since get_repo normalizes DID aliases and the raw segments would mint two keys for one directory. The sanitization is deliberately not injective. Two did:web spellings can fold to one slug, which would misroute encrypt coalescing if it were reachable, but repos.disk_path is NOT NULL UNIQUE and holds exactly this derivation, so two such rows cannot coexist. A test pins that fold in place: making the key injective is what would let two owners sharing one objects/ directory push concurrently. --- crates/gitlawb-node/src/api/repos.rs | 137 ++++++++++++++++++++------- crates/gitlawb-node/src/state.rs | 100 +++++++++++++++++++ 2 files changed, 205 insertions(+), 32 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index d174a26b..553d8234 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1695,9 +1695,13 @@ pub async fn git_receive_pack( // is what the F1 review rejected, since GITLAWB_TRUSTED_PROXY defaults to unset and // every pusher behind a proxy/NAT then resolves to ONE key, turning one contended repo // into a node-wide denial. + // The stable disk identity, never record.id: the row id rotates on a + // delete+recreate under the same slug while the bare repo on disk is reused, + // and an id-keyed lease stops serializing exactly across that rotation. + let repo_key = crate::state::repo_identity_key(&record.owner_did, &record.name); let lease = state .repo_write_leases - .acquire(&record.id, lease_steal_after) + .acquire(&repo_key, lease_steal_after) .await .ok_or_else(|| { tracing::warn!( @@ -1962,7 +1966,11 @@ async fn post_receive_replication_tail( .iter() .map(|u| (u.old_sha.clone(), u.new_sha.clone())) .collect(); - match state.encrypt_inflight.try_begin(&record.id, tip_pairs) { + // Same stable disk identity as the lease above (#174 U2): keyed on + // record.id, a post-recreate push would take a fresh key and run a second + // encrypt task against the same on-disk repo instead of coalescing. + let coalesce_key = crate::state::repo_identity_key(&record.owner_did, &record.name); + match state.encrypt_inflight.try_begin(&coalesce_key, tip_pairs) { crate::state::BeginOutcome::Coalesced => { coalesced = true; tracing::debug!( @@ -7447,6 +7455,69 @@ mod tests { rec.id } + /// #174 U2: the write lease must register under the STABLE DISK IDENTITY + /// (sanitized owner slug + repo name, what `RepoStore::local_path` and the pg + /// advisory lock key on), not `record.id`. + /// + /// The row id rotates on delete+recreate under the same slug while the bare + /// repo on disk is reused, so an id-keyed lease silently stops serializing + /// across that rotation and lets two writers onto one `objects/` directory. + /// Asserting on the key the holder actually registers under binds the + /// production call site — a helper unit test alone would not. + #[cfg(unix)] + #[sqlx::test] + async fn u2_lease_registers_under_the_disk_identity_not_the_row_id(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let tmp = tempfile::TempDir::new().unwrap(); + let (git_bin, a_inpack, _later_ran) = f1_hanging_first_git(tmp.path()); + let state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6u2key", "k1", false).await; + let rec = state.db.get_repo("z6u2key", "k1").await.unwrap().unwrap(); + let identity = crate::state::repo_identity_key(&rec.owner_did, &rec.name); + assert_ne!( + identity, rec.id, + "the identity key must differ from the row id, or this test proves nothing" + ); + + let did = "did:key:z6MkU2KeyPusherAAAAAAAAAAAAAAAAAAAAAAAA"; + let peer: SocketAddr = "203.0.113.91:5000".parse().unwrap(); + let handle = tokio::spawn({ + let st = state.clone(); + let did = did.to_string(); + async move { + git_receive_pack( + State(st), + Path(("z6u2key".to_string(), "k1".to_string())), + Extension(crate::auth::AuthenticatedDid(did)), + crate::rate_limit::PeerAddr(Some(peer)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await + } + }); + assert!( + f1_wait_for(F1_BACKSTOP, || a_inpack.exists()).await, + "the push never reached receive-pack within {F1_BACKSTOP:?}" + ); + + assert_eq!( + state.repo_write_leases.refs_for(&identity), + 1, + "the lease holder must be registered under the stable disk identity" + ); + assert_eq!( + state.repo_write_leases.refs_for(&rec.id), + 0, + "and never under the rotating row id" + ); + + handle.abort(); + } + /// #174 U1 scenario 1 (RED-before/GREEN-after): parked pushes on one repo's write /// lease are BOUNDED. `git_receive_pack` takes `body: Bytes`, so axum has already /// buffered the whole pack (up to `max_pack_bytes`) before the handler runs, and the @@ -7473,13 +7544,11 @@ mod tests { f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6u1cap", "c1", false).await; // Cap of ONE live waiter, so a holder plus one parked push fills the repo's queue. state.repo_write_leases = crate::state::RepoWriteLeases::new(1); - let repo_id = state - .db - .get_repo("z6u1cap", "c1") - .await - .unwrap() - .unwrap() - .id; + // The lease keys on the stable disk identity (#174 U2), not the row id. + let repo_id = { + let r = state.db.get_repo("z6u1cap", "c1").await.unwrap().unwrap(); + crate::state::repo_identity_key(&r.owner_did, &r.name) + }; let did = "did:key:z6MkU1CapPusherAAAAAAAAAAAAAAAAAAAAAAAAAA"; let push = |peer: SocketAddr| { let st = state.clone(); @@ -7579,13 +7648,10 @@ mod tests { let mut state = f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6u1two", "r1", false).await; state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(2, 100); - let repo1 = state - .db - .get_repo("z6u1two", "r1") - .await - .unwrap() - .unwrap() - .id; + let repo1 = { + let r = state.db.get_repo("z6u1two", "r1").await.unwrap().unwrap(); + crate::state::repo_identity_key(&r.owner_did, &r.name) + }; f1_add_repo(&state, "z6u1two", "r2").await; let did = "did:key:z6MkU1TwoPusherAAAAAAAAAAAAAAAAAAAAAAAAAA"; let src: SocketAddr = "203.0.113.84:5000".parse().unwrap(); @@ -7664,13 +7730,10 @@ mod tests { // The default proxy trust: the resolved key is the peer IP, which is the edge's. state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(2, 100); - let repo1 = state - .db - .get_repo("z6u1nat", "n1") - .await - .unwrap() - .unwrap() - .id; + let repo1 = { + let r = state.db.get_repo("z6u1nat", "n1").await.unwrap().unwrap(); + crate::state::repo_identity_key(&r.owner_did, &r.name) + }; f1_add_repo(&state, "z6u1nat", "n2").await; // Every pusher arrives from the one edge IP, so they share a source key. let edge: SocketAddr = "203.0.113.85:5000".parse().unwrap(); @@ -8030,7 +8093,9 @@ mod tests { f2a_log(&log) ); assert_eq!( - state.encrypt_inflight.pending_for(&rec.id), + state + .encrypt_inflight + .pending_for(&crate::state::repo_identity_key(&rec.owner_did, &rec.name)), Some(crate::state::PendingWork::Tips(vec![( ZERO_SHA.to_string(), c1.clone() @@ -8099,9 +8164,10 @@ mod tests { let c3 = u5_commit_file(repo.path(), "b.txt", "three\n"); assert!( matches!( - state - .encrypt_inflight - .try_begin(&rec.id, vec![(c2.clone(), c3.clone())]), + state.encrypt_inflight.try_begin( + &crate::state::repo_identity_key(&rec.owner_did, &rec.name), + vec![(c2.clone(), c3.clone())], + ), crate::state::BeginOutcome::Coalesced ), "a push arriving while a task is in flight must coalesce" @@ -8179,9 +8245,10 @@ mod tests { // ADMITTED here instead, and this assertion is what catches that.) assert!( matches!( - state - .encrypt_inflight - .try_begin(&rec.id, vec![(c2.clone(), c3.clone())]), + state.encrypt_inflight.try_begin( + &crate::state::repo_identity_key(&rec.owner_did, &rec.name), + vec![(c2.clone(), c3.clone())], + ), crate::state::BeginOutcome::Coalesced ), "a push arriving mid-walk must coalesce, not start a second task" @@ -8265,7 +8332,10 @@ mod tests { let mut updates = state.ref_update_tx.subscribe(); // A task for this repo is already in flight, so the push below coalesces. - let _inflight = match state.encrypt_inflight.try_begin(&rec.id, Vec::new()) { + let _inflight = match state.encrypt_inflight.try_begin( + &crate::state::repo_identity_key(&rec.owner_did, &rec.name), + Vec::new(), + ) { crate::state::BeginOutcome::Admitted(g) => g, crate::state::BeginOutcome::Coalesced => panic!("the first begin must admit"), }; @@ -8336,7 +8406,10 @@ mod tests { let (_server, _cid) = f2a_pinata(&mut state).await; let mut updates = state.ref_update_tx.subscribe(); - let _inflight = match state.encrypt_inflight.try_begin(&rec.id, Vec::new()) { + let _inflight = match state.encrypt_inflight.try_begin( + &crate::state::repo_identity_key(&rec.owner_did, &rec.name), + Vec::new(), + ) { crate::state::BeginOutcome::Admitted(g) => g, crate::state::BeginOutcome::Coalesced => panic!("the first begin must admit"), }; diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index fa621aab..278aa6ae 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -498,6 +498,42 @@ pub struct RepoWriteLeases { max_waiters: usize, } +/// The stable per-repo identity that [`RepoWriteLeases`] and [`EncryptInflight`] +/// key on (#174 U2). +/// +/// NOT `record.id`. A repo deleted and recreated under the same owner/name gets a +/// new row id while the bare repo on disk is reused, so an id-keyed serializer +/// stops serializing across that rotation and lets two writers onto one +/// `objects/` directory. `RepoStore::local_path` and the pg advisory lock both +/// key on the sanitized owner slug plus repo name, and this reproduces exactly +/// that identity so the in-process serializers agree with them. +/// +/// Three details are load-bearing: +/// +/// - The sanitization is [`crate::git::store::repo_disk_path`]'s +/// (`replace([':', '/'], "_")`), NOT `db::normalize_owner_key`, which strips a +/// `did:key:` prefix instead and would map the same input to a different +/// string. The disk path is authoritative because the `objects/` directory is +/// the resource being serialized. +/// - The separator is `\0`, which cannot occur in either component. A plain join +/// would collide (owner `a` + name `b_c` against owner `a_b` + name `c`), +/// letting one repo's push park another's. +/// - Callers pass `record.owner_did` / `record.name`, never the request's path +/// segments: `db::get_repo` normalizes DID aliases, so a caller could otherwise +/// mint two keys for one directory just by varying the DID spelling. +/// +/// The sanitization is not injective (`did:web:example.com:alice` and +/// `did:web:example.com/alice` fold to one slug), which would be a cross-tenant +/// hazard for the coalescing map if it were reachable. It is not: `repos.disk_path` +/// is `NOT NULL UNIQUE` and holds exactly this derivation, so two rows that fold +/// to one key cannot coexist. Do not "fix" this by making the key injective — +/// that is precisely what would let two owners sharing one `objects/` directory +/// push concurrently. +pub fn repo_identity_key(owner_did: &str, repo_name: &str) -> String { + let owner_slug = owner_did.replace([':', '/'], "_"); + format!("{owner_slug}\0{repo_name}") +} + /// A per-repo lease entry: the one-permit semaphore, a refcount of the handlers /// currently referencing it (holding or waiting), and a count of the ones actually /// PARKED. While `refs > 0` every acquirer shares the SAME semaphore, so mutual exclusion @@ -824,6 +860,70 @@ pub async fn acquire_scan_permit( permit } +#[cfg(test)] +mod repo_identity_key_tests { + use super::repo_identity_key; + + /// The key must reproduce `repo_disk_path`'s slug derivation exactly, because + /// the whole point is to agree with the on-disk identity the store and the pg + /// advisory lock already use. + #[test] + fn matches_repo_disk_paths_sanitization() { + let owner = "did:key:z6Mkfoo"; + let name = "r"; + let expected_slug = owner.replace([':', '/'], "_"); + assert_eq!(repo_identity_key(owner, name), format!("{expected_slug}\0{name}")); + + // The disk path for the same pair must carry the same slug component. + let disk = crate::git::store::repo_disk_path(std::path::Path::new("/srv"), owner, name); + assert!( + disk.to_string_lossy().contains(&expected_slug), + "the key's slug must be the one repo_disk_path puts on disk: {disk:?}" + ); + } + + /// Stability across the rotation is the entire reason this key exists: the row + /// id changes on delete+recreate, the identity does not. + #[test] + fn is_stable_across_a_row_id_rotation() { + assert_eq!( + repo_identity_key("did:key:z6Mkfoo", "r"), + repo_identity_key("did:key:z6Mkfoo", "r"), + ); + } + + /// The `\0` separator is what stops one repo's push parking another's. With a + /// plain join these two pairs would produce a single key. + #[test] + fn separator_prevents_the_owner_name_boundary_collision() { + assert_ne!( + repo_identity_key("a", "b_c"), + repo_identity_key("a_b", "c"), + "owner/name boundary must not be ambiguous" + ); + } + + /// Distinct repos and distinct owners never share a key. + #[test] + fn distinct_repos_and_owners_do_not_share_a_key() { + assert_ne!(repo_identity_key("did:key:z6A", "r"), repo_identity_key("did:key:z6A", "s")); + assert_ne!(repo_identity_key("did:key:z6A", "r"), repo_identity_key("did:key:z6B", "r")); + } + + /// Documents the ONE collision the sanitization admits, and why it is safe + /// rather than fixed: `repos.disk_path` is UNIQUE and holds this derivation, + /// so two rows folding to one key cannot coexist. If this assertion ever + /// flips to `assert_ne!`, the key became injective and two owners sharing one + /// `objects/` directory could push concurrently — the defect U2 closes. + #[test] + fn folds_did_web_alias_spellings_together_which_the_unique_disk_path_makes_unreachable() { + assert_eq!( + repo_identity_key("did:web:example.com:alice", "r"), + repo_identity_key("did:web:example.com/alice", "r"), + ); + } +} + #[cfg(test)] mod repo_write_lease_tests { use super::RepoWriteLeases; From 38bce707b8b0e4eeb89e59e3b30d0f6c2c9e20c7 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 02:46:51 -0500 Subject: [PATCH 091/124] fix(node): hold /ipfs walk admission through the blocking walk, not the handler future The global and per-source walk permits were handler locals, so a client disconnect released them the moment the future dropped while the uncancellable spawn_blocking probe/walk/read kept running against a slot the pool now considered free. A disconnect flood could admit replacements while prior blocking git children still occupied threads and PIDs, weakening GITLAWB_MAX_CONCURRENT_IPFS_WALKS and GITLAWB_IPFS_WALK_PER_SOURCE on a surface any unauthenticated caller can reach. Hold the pair in an Arc and clone it into each of the three blocking closures. The permits release when the last clone drops, which collapses both failure directions into one case: a disconnect leaves the abandoned closure holding the slot, and a panicking closure leaves the handler holding it. No loop arm changes and the JoinError arms keep today's taint-and-continue. A move-and-return shuttle was implemented and rejected. It is correct only if every arm that continues the loop re-binds the returned value, which the compiler cannot enforce - the absent-object arm Ok(Ok(None)) => continue, taken on nearly every iteration of a random-CID scan, drops it - and it forces a second decision about the panic arms whose least-bad answer would let one broken repo shed every other owner's /ipfs lookups. The regression strengthens the existing admission test: it now asserts the slot is still held after the future is dropped and before the child is killed, with a child-still-alive precondition so it cannot pass vacuously. Only the walk site runs under state.git_bin and can be pinned that way; the probe and read deliberately shell to real git, so a source-scan gate in inv22_gates.rs binds all three clone sites instead. Removing any one clone turns that gate red. --- crates/gitlawb-node/src/api/ipfs.rs | 80 +++++++++++++++++++++--- crates/gitlawb-node/tests/inv22_gates.rs | 57 +++++++++++++++++ 2 files changed, 129 insertions(+), 8 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 5e7ae05a..d9f4c01b 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -78,6 +78,20 @@ use crate::visibility::{visibility_check, Decision}; /// Scope: this closes the direct unauthenticated scan, including the dangling /// case. A stale-public mirror row still serves withheld content (tracked /// separately, #124). +/// +/// One `/ipfs` request's walk admission: the global pool permit plus the +/// optional per-source sub-permit, both RAII (#174 U1). +/// +/// Held behind an `Arc` whose clones go into every `spawn_blocking` walk this +/// request runs, so the permits release only when the last clone drops — the +/// handler's, or an abandoned/panicking closure's, whichever outlives the other. +/// Admission therefore tracks real blocking-thread occupancy rather than the +/// lifetime of the future that requested it. +struct WalkAdmission { + _global: tokio::sync::OwnedSemaphorePermit, + _per_source: Option, +} + pub async fn get_by_cid( Path(cid_str): Path, State(state): State, @@ -131,7 +145,7 @@ pub async fn get_by_cid( // (`client_key`), never the DID (`/ipfs` admits any `did:key` unthrottled, so a DID // key would be free to mint around); a `None` key (no trusted header, no peer) is // bounded by the global pool only, never the per-source sub-cap. - let _ipfs_walk_permit = state + let global_permit = state .git_ipfs_walk_semaphore .clone() .try_acquire_owned() @@ -140,13 +154,29 @@ pub async fn get_by_cid( AppError::Overloaded("ipfs service at capacity, retry shortly".into()) })?; let source_key = crate::rate_limit::client_key(&req_headers, peer, state.push_limiter_trust); - let _ipfs_caller_permit = match &source_key { + let caller_permit = match &source_key { Some(ip) => Some(state.git_ipfs_walk_per_caller.try_acquire(ip).ok_or_else(|| { tracing::warn!(key = %ip, "/ipfs per-source walk cap reached; shedding request with 503"); AppError::Overloaded("ipfs service at capacity for this source, retry shortly".into()) })?), None => None, }; + // Share the admission rather than holding it as a handler local (#174 U1). A + // clone goes into every `spawn_blocking` closure below, so the permits release + // only when the LAST holder drops. That makes the two failure directions one + // case: a client disconnect drops the handler's clone while the abandoned + // closure's clone keeps the slot taken for as long as its git child runs, and a + // panicking closure drops its clone while the handler's keeps the slot taken. + // + // Deliberately NOT a move-and-return shuttle. That shape is correct only if + // every arm continuing the loop re-binds the returned value, which the compiler + // cannot enforce — including the absent-object arm `Ok(Ok(None)) => continue` + // that a random-CID scan takes on nearly every iteration — and it forces a + // second decision about the arms where a panic destroys the moved-in permits. + let admission = std::sync::Arc::new(WalkAdmission { + _global: global_permit, + _per_source: caller_permit, + }); // 2. Search all repos for an object with this SHA-256. // @@ -378,7 +408,11 @@ pub async fn get_by_cid( // independent of `state.git_bin` (which tests point at a fake walk git). let probe_path = repo_path.clone(); let probe_sha = sha256_hex.clone(); + let probe_admission = std::sync::Arc::clone(&admission); let obj_type = match tokio::task::spawn_blocking(move || { + // Admission clone (#174 U1): the slot stays taken until this blocking + // work returns, even if the handler future was dropped or this closure panics. + let _admission = probe_admission; store::object_type_bounded("git", &probe_path, &probe_sha, probe_deadline) }) .await @@ -455,7 +489,11 @@ pub async fn get_by_cid( ); // Full-history walk shells out to git — keep it off the async runtime, // bounded and reaped like the served-git ops (#174). + let walk_admission = std::sync::Arc::clone(&admission); let walk = tokio::task::spawn_blocking(move || { + // Admission clone (#174 U1): the slot stays taken until this blocking + // work returns, even if the handler future was dropped or this closure panics. + let _admission = walk_admission; allowed_blob_set_for_caller_bounded( &rp, &git_bin, @@ -525,7 +563,11 @@ pub async fn get_by_cid( let read_path = repo_path.clone(); let read_sha = sha256_hex.clone(); let read_type = obj_type.clone(); + let read_admission = std::sync::Arc::clone(&admission); let content = match tokio::task::spawn_blocking(move || { + // Admission clone (#174 U1): the slot stays taken until this blocking + // work returns, even if the handler future was dropped or this closure panics. + let _admission = read_admission; store::read_object_content_bounded( "git", &read_path, @@ -2331,13 +2373,35 @@ mod tests { "a replacement must shed 503 while the prior request's blocking walk still runs" ); - // Drop the in-flight request; the detached blocking walk keeps running (a - // spawn_blocking cannot be cancelled), but on the fix the permit is a handler - // local, so dropping the future releases it once the blocking join is abandoned. - // Either way, kill the sleeping child so the slot frees promptly and poll for - // recovery — the point already proven above is that the slot stayed held for the - // duration of the blocking work. + // Drop the in-flight request — a client disconnect. The detached blocking walk + // keeps running (a spawn_blocking cannot be cancelled) and its git child is still + // occupying a blocking thread and a PID, so the slot it was admitted under must + // STAY TAKEN. Admission is released by the blocking work finishing, never by the + // handler future going away (#174 U1). + // + // MUTATION (RED): make the admission a handler local again (drop the Arc clone + // moved into the spawn_blocking closures) and this assertion fails immediately — + // the permit count returns to 1 the moment the future is dropped, while the + // sleeping child is still alive. drop(fut); + // Give the runtime a chance to actually run the drop and any woken tasks, so + // this is not merely observing a not-yet-processed release. + for _ in 0..10 { + tokio::task::yield_now().await; + } + assert!( + unsafe { libc::kill(pid, 0) } == 0, + "precondition: the blocking walk's git child must still be alive, or this \ + assertion proves nothing" + ); + assert_eq!( + sem.available_permits(), + 0, + "a client disconnect must NOT release the walk slot while the uncancellable \ + blocking walk it admitted is still running" + ); + + // Now end the blocking work; the slot frees when the closure returns. unsafe { libc::kill(pid, libc::SIGKILL); } diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index 0ce54088..6dd414cc 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -356,3 +356,60 @@ fn f3_second_writer_leased_until_reap() { with_lease, so the lease rides the guard into KillGroupOnDrop's reaper" ); } + +/// #174 U1 — every blocking walk in the `/ipfs` scan carries the request's walk +/// admission. +/// +/// The admission is an `Arc` cloned into each `spawn_blocking` +/// closure, so the global + per-source permits release only when the last holder +/// drops: a client disconnect leaves the abandoned closure holding the slot, and a +/// panicking closure leaves the handler holding it. +/// +/// Only the walk site runs under `state.git_bin`, so only it can be pinned by the +/// fake-git harness and mutation-verified dynamically (see +/// `get_by_cid_walk_permit_held_through_blocking_walk`). The probe and the content +/// read deliberately shell to the real `git`, which is why this structural check +/// exists: it binds all three sites, and any blocking site added to this loop +/// later, without reversing that deliberate independence. +/// +/// MUTATION (RED): delete any one `Arc::clone(&admission)` binding, or drop the +/// clone from inside its closure, and the count falls below three. +#[test] +fn inv22_ipfs_walk_admission_reaches_every_blocking_site() { + let ipfs = src("api/ipfs.rs"); + + // The shared owner must exist and be built once per request. + assert!( + ipfs.contains("struct WalkAdmission") && ipfs.contains("Arc::new(WalkAdmission {"), + "U1 gate missing: the /ipfs walk admission must be a shared WalkAdmission, \ + not a handler-local permit pair" + ); + + // Every blocking site in the scan takes its own clone... + let clones = ipfs.matches("Arc::clone(&admission)").count(); + assert!( + clones >= 3, + "U1 gate bypassed: expected an admission clone for each of the three \ + /ipfs spawn_blocking sites (probe, walk, read); found {clones}. A blocking \ + walk that does not hold the admission lets a disconnect or a panic free the \ + slot while its git child is still running." + ); + + // ...and each clone is actually moved INTO the blocking closure, not merely + // created in the async frame (which would hold nothing across the join). + let held = ipfs.matches("let _admission = ").count(); + assert!( + held >= 3, + "U1 gate bypassed: each admission clone must be bound inside its \ + spawn_blocking closure so the blocking work owns it; found {held} of 3." + ); + + // The count of blocking sites is itself the thing being covered: if a fourth + // appears, it needs an admission clone too and this gate must be revisited. + let sites = ipfs.matches("spawn_blocking(move ||").count(); + assert_eq!( + sites, 3, + "the /ipfs scan grew or lost a spawn_blocking site ({sites} found, expected 3); \ + give any new blocking walk its own Arc::clone(&admission) and update this gate" + ); +} From 43d35ebd70613e9d5695a7bd7fd92320ac650c80 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 02:55:20 -0500 Subject: [PATCH 092/124] fix(node): hold the encrypt-scan permit in its closure and run info/refs under git_bin Two single-site wiring fixes from the same review round. withheld_recipients_gated held its git_encrypt_semaphore permit in the async frame and then awaited spawn_blocking, so dropping the future released the permit while the uncancellable walk kept running. Every other post-receive scan helper moves the permit into the closure - replication_withheld_set says so in as many words - so this aligns the last site with the F4 contract. The regression drops the future mid-walk and asserts the pool stays at zero while the git child is still alive. git_info_refs passed a literal "git" to smart_http::info_refs while upload-pack and receive-pack both pass state.git_bin, so no fake-git harness could drive the advertisement path. The regression points git_bin at a fake emitting a distinctive advertisement and asserts it reaches the response body; before the fix the body came back from real system git (agent=git/2.43.0). --- crates/gitlawb-node/src/api/repos.rs | 162 ++++++++++++++++++++++++++- 1 file changed, 160 insertions(+), 2 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 553d8234..400e10e1 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -728,7 +728,13 @@ pub async fn git_info_refs( // are moved in, so admission tracks the real process lifetime. let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); - smart_http::info_refs("git", &service, &disk_path, git_timeout, Some(admission)) + smart_http::info_refs( + &state.git_bin, + &service, + &disk_path, + git_timeout, + Some(admission), + ) .await .map_err(|e| { let app = git_service_app_error(&e); @@ -820,11 +826,15 @@ async fn withheld_recipients_gated( anyhow::Result>>, tokio::task::JoinError, > { - let _permit = encrypt_sem + let permit = encrypt_sem .acquire_owned() .await .expect("git_encrypt_semaphore is never closed"); tokio::task::spawn_blocking(move || { + // The permit lives inside the blocking closure (#174 U4, the F4 contract): a + // started walk always completes holding it, so a dropped future cannot free + // the slot while this uncancellable walk still occupies a thread and a PID. + let _permit = permit; crate::git::visibility_pack::withheld_blob_recipients_bounded( &repo_path, &git_bin, timeout, &rules, is_public, &owner_did, ) @@ -3029,6 +3039,154 @@ mod tests { p.to_str().unwrap().to_string() } + /// #174 U6: the `info/refs` advertisement must run the CONFIGURED git binary, + /// like upload-pack and receive-pack already do. It passed a literal "git", so a + /// fake-git harness could drive the pack paths but never the advertisement. + /// + /// MUTATION (RED): restore the `"git"` literal at the `smart_http::info_refs` + /// call and the fake's marker never reaches the response body. + #[cfg(unix)] + #[sqlx::test] + async fn u6_info_refs_runs_the_configured_git_binary(pool: sqlx::PgPool) { + use axum::extract::{Path, Query, State}; + use http_body_util::BodyExt; + + let tmp = tempfile::TempDir::new().unwrap(); + // Distinctive advertisement so the assertion cannot pass on real git's output. + let body = "#!/bin/sh\n\ + case \"$1\" in\n\ + upload-pack) printf 'U6-FAKE-ADVERTISEMENT' ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + let state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6u6adv", "a1", false).await; + + let resp = git_info_refs( + State(state), + Path(("z6u6adv".to_string(), "a1".to_string())), + Query(InfoRefsQuery { + service: Some("git-upload-pack".to_string()), + }), + crate::rate_limit::PeerAddr(Some("203.0.113.95:5000".parse().unwrap())), + axum::http::HeaderMap::new(), + None, + ) + .await + .expect("the advertisement must succeed"); + + let bytes = resp.into_body().collect().await.unwrap().to_bytes(); + let text = String::from_utf8_lossy(&bytes); + assert!( + text.contains("U6-FAKE-ADVERTISEMENT"), + "info/refs must run state.git_bin, not a hardcoded \"git\"; body was: {text:?}" + ); + } + + /// #174 U4: `withheld_recipients_gated` must hold its encrypt-scan permit INSIDE + /// the blocking closure, matching every other post-receive scan helper (see + /// `replication_withheld_set`, where the permit is moved in with "a started walk + /// always completes holding it"). + /// + /// Holding it in the async frame instead means dropping the future releases the + /// permit while the uncancellable `spawn_blocking` walk keeps running, so the + /// encrypt pool admits a replacement scan against a slot still occupied by a live + /// git child. + /// + /// MUTATION (RED): hoist `_permit` back out of the closure and the + /// still-held-after-drop assertion fails — the count returns to 1 immediately. + #[cfg(unix)] + #[tokio::test] + async fn u4_encrypt_scan_permit_is_held_through_the_blocking_walk() { + use std::time::Duration; + let tmp = tempfile::TempDir::new().unwrap(); + let pidfile = tmp.path().join("u4_walk.pid"); + // Hang on the first real walk command, whatever it is; only rev-parse answers. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + rev-parse) echo deadbeef ;;\n\ + *) echo $$ > \"{pid}\"; while true; do sleep 1; done ;;\n\ + esac\n\ + exit 0\n", + pid = pidfile.display(), + ); + let git_bin = write_fake_git(tmp.path(), &body); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + + let mut fut = Box::pin(withheld_recipients_gated( + sem.clone(), + tmp.path().to_path_buf(), + git_bin, + Duration::from_secs(600), + vec![vis_rule("/secret/**", &[])], + true, + "did:key:z6MkU4OwnerAAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + )); + + // Drive until the blocking walk's git child records its pid. + let mut walk_pid: Option = None; + for _ in 0..500 { + let _ = tokio::time::timeout(Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&pidfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + walk_pid = Some(p); + break; + } + } + let pid = walk_pid.expect("the fake git walk command must have spawned"); + struct ReapOnDrop(i32); + impl Drop for ReapOnDrop { + fn drop(&mut self) { + unsafe { + libc::kill(self.0, libc::SIGKILL); + } + } + } + let _cleanup = ReapOnDrop(pid); + + assert_eq!( + sem.available_permits(), + 0, + "the scan permit must be held while the blocking walk runs" + ); + + drop(fut); + for _ in 0..10 { + tokio::task::yield_now().await; + } + assert!( + unsafe { libc::kill(pid, 0) } == 0, + "precondition: the walk's git child must still be alive, or this \ + assertion proves nothing" + ); + assert_eq!( + sem.available_permits(), + 0, + "dropping the future must NOT release the encrypt-scan permit while the \ + uncancellable blocking walk it admitted is still running" + ); + + unsafe { + libc::kill(pid, libc::SIGKILL); + } + let mut freed = false; + for _ in 0..400 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + assert!( + freed, + "once the blocking walk ends the scan permit must return to the pool" + ); + } + /// #174 (write-pool twin, vetted by execution not reasoning): the receive-pack /// post-push replication walk is bounded. Drive `replication_withheld_set` with an /// injected fake git that hangs on `rev-list` and a short budget: it must RETURN From 9fc68e2fc9d27ea77b8263156eb606b50a27db6e Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 03:04:30 -0500 Subject: [PATCH 093/124] fix(node): spawn the replication tail at the durability boundary The post-receive replication tail was spawned at the end of git_receive_pack, after touch_repo, the trust-score update, per-ref certificate issuance and webhook fan-out. A client or proxy disconnect anywhere in that window dropped the tail, and with it this push's pins, recovery copy, and announcements - even though the push was already durable on disk. Spawn it immediately after the receive_result ? instead. That is the first point where the push is known to have landed: guard.release() a few lines up runs on failure too, so anchoring there would spawn a tail for a push git rejected, pinning and announcing a half-applied repo - the state release() deliberately refuses to upload to Tigris, and one a hostile pusher can produce on demand by aborting a pack mid-transfer. RefUpdate gains Clone so the tail takes the parsed updates while the certificate and webhook loops keep iterating their own copy. This makes the tail run concurrently with certificate issuance rather than after it, so a ref can be announced before its signed certificate exists. Accepted, and stated here rather than left for a reviewer to find: cert issuance already fails open (errors are logged and skipped) and the gossip event carries cert_id: None regardless, so no announce consumer reads a certificate out of it. Evidence is an ordering gate in inv22_gates.rs rather than a cancellation-race test, matching the F3 gate's instrument: forcing a disconnect to land inside the cert/webhook window is timing-dependent and would be flaky, while the ordering is precisely what changed. The gate is mutation-verified in both directions - moving the spawn back below the webhooks fails the upper bound, hoisting it above the ? fails the lower one. --- crates/gitlawb-node/src/api/repos.rs | 50 +++++++++++++-------- crates/gitlawb-node/tests/inv22_gates.rs | 56 ++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 18 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 400e10e1..470badc7 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1823,6 +1823,34 @@ pub async fn git_receive_pack( app })?; + // #174 F2/U5: the post-receive replication tail runs in an independently owned + // task. It parks on `git_encrypt_semaphore` (withheld / candidate / full-scan + // resolution), so leaving it in the request future means a client/proxy disconnect + // while parked silently drops this push's pins, recovery copy, and announcements. + // + // Spawned HERE, at the durability boundary, rather than at the end of the handler. + // This is the first point where the push is known to have landed: `guard.release()` + // above runs on failure too, so anchoring on it would spawn a tail that pins and + // announces a half-applied repo — the state release() deliberately refuses to + // upload, and one a hostile pusher can produce on demand by aborting a pack + // mid-transfer. Everything below (touch_repo, metrics, trust score, certificates, + // webhooks) stays in the cancellable handler, so a disconnect during that window no + // longer takes the tail with it. + // + // The tail now runs CONCURRENTLY with certificate issuance rather than after it, so + // a ref can be announced before its signed certificate exists. That window is + // accepted: cert issuance already fails open (errors are logged and skipped) and the + // gossip event carries `cert_id: None` regardless, so no announce consumer reads a + // certificate out of it. Each push owns its own tail, including its own + // always-spawned announce, so per-push announcements are never coalesced away. + tokio::spawn(post_receive_replication_tail( + state.clone(), + record.clone(), + ref_updates.clone(), + disk_path.clone(), + auth.0.to_string(), + )); + // Update the repo's updated_at timestamp after a successful push let _ = state.db.touch_repo(&record.id).await; @@ -1912,24 +1940,6 @@ pub async fn git_receive_pack( } } - // #174 F2: move the whole post-receive replication tail into an independently owned - // task. It parks on `git_encrypt_semaphore` (withheld / candidate / full-scan - // resolution), so leaving it in the request future means a client/proxy disconnect - // while parked silently drops this push's pins, recovery copy, and announcements — - // the residual `state.rs` documented, because the durable `try_begin` gate sat after - // the park. The request future now returns the git response immediately and the tail - // runs detached; a disconnect can no longer drop it. Each push owns its own tail, - // including its own always-spawned announce, so per-push announcements are never - // coalesced away (the announce spawn stays out of the per-repo encrypt coalescing). - let did = did.to_string(); - tokio::spawn(post_receive_replication_tail( - state, - record, - ref_updates, - disk_path, - did, - )); - Ok(result) } @@ -2638,6 +2648,10 @@ pub async fn get_icaptcha_proof( // ── Pkt-line parsing ────────────────────────────────────────────────────── +/// `Clone` so `git_receive_pack` can hand the parsed updates to the detached +/// replication tail at the durability boundary while the certificate and webhook +/// loops below still iterate their own copy (#174 U5). +#[derive(Clone)] struct RefUpdate { old_sha: String, new_sha: String, diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index 6dd414cc..82712c23 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -413,3 +413,59 @@ fn inv22_ipfs_walk_admission_reaches_every_blocking_site() { give any new blocking walk its own Arc::clone(&admission) and update this gate" ); } + +/// #174 U5: the post-receive replication tail is spawned at the DURABILITY BOUNDARY, +/// not at the end of the handler. +/// +/// The tail owes this push its pins, recovery copy, and announcements. Spawned below +/// the certificate and webhook work, a client disconnect anywhere in that window took +/// the tail with it. Spawned above it, the tail is already detached and survives. +/// +/// The lower bound matters just as much as the upper one: `guard.release()` runs on +/// failure too, so a spawn anchored there would fire for a push git rejected, pinning +/// and announcing a half-applied repo. The `?` on `receive_result` is the first point +/// where the push is known to have landed, which is why the spawn must sit between +/// that `?` and `touch_repo`. +/// +/// This is an ordering check rather than a cancellation-race test on purpose: forcing +/// a disconnect to land inside the cert/webhook window is timing-dependent and would +/// be flaky, while the ordering is exactly what the fix changes. Same instrument the +/// F3 gate above uses. +/// +/// MUTATION (RED): move the `tokio::spawn(post_receive_replication_tail` call back +/// below the webhook block and the ordering assertion fails; move it above the `?` +/// and the failed-push assertion fails. +#[test] +fn inv22_replication_tail_spawns_at_the_durability_boundary() { + let repos = src("api/repos.rs"); + // Production half only — the tests below name these identifiers too. + let production = repos + .split("#[cfg(test)]") + .next() + .expect("split always yields a first chunk"); + + let receive_ok = production + .find("let result = receive_result.map_err(") + .expect("U5 gate stale: git_receive_pack no longer maps receive_result with ?"); + let spawn = production + .find("tokio::spawn(post_receive_replication_tail(") + .expect("U5 gate missing: the replication tail must be spawned by git_receive_pack"); + let touch = production + .find("state.db.touch_repo(") + .expect("U5 gate stale: git_receive_pack no longer calls touch_repo"); + let webhook = production + .find("webhooks::fire_event(") + .expect("U5 gate stale: git_receive_pack no longer fires push webhooks"); + + assert!( + receive_ok < spawn, + "U5 gate bypassed: the tail must be spawned AFTER the receive_result `?`, or a \ + rejected push spawns a tail that pins and announces a half-applied repo" + ); + assert!( + spawn < touch && spawn < webhook, + "U5 gate bypassed: the tail must be spawned BEFORE touch_repo and the webhook \ + fan-out, so a disconnect in that window cannot drop this push's pins, \ + recovery copy, and announcements" + ); +} From a4960020d58c9fc087daed6012b23e76e682dc54 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 03:04:58 -0500 Subject: [PATCH 094/124] style: rustfmt the repo_identity_key tests --- crates/gitlawb-node/src/state.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 278aa6ae..ef093661 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -872,7 +872,10 @@ mod repo_identity_key_tests { let owner = "did:key:z6Mkfoo"; let name = "r"; let expected_slug = owner.replace([':', '/'], "_"); - assert_eq!(repo_identity_key(owner, name), format!("{expected_slug}\0{name}")); + assert_eq!( + repo_identity_key(owner, name), + format!("{expected_slug}\0{name}") + ); // The disk path for the same pair must carry the same slug component. let disk = crate::git::store::repo_disk_path(std::path::Path::new("/srv"), owner, name); @@ -906,8 +909,14 @@ mod repo_identity_key_tests { /// Distinct repos and distinct owners never share a key. #[test] fn distinct_repos_and_owners_do_not_share_a_key() { - assert_ne!(repo_identity_key("did:key:z6A", "r"), repo_identity_key("did:key:z6A", "s")); - assert_ne!(repo_identity_key("did:key:z6A", "r"), repo_identity_key("did:key:z6B", "r")); + assert_ne!( + repo_identity_key("did:key:z6A", "r"), + repo_identity_key("did:key:z6A", "s") + ); + assert_ne!( + repo_identity_key("did:key:z6A", "r"), + repo_identity_key("did:key:z6B", "r") + ); } /// Documents the ONE collision the sanitization admits, and why it is safe From 62ec882bcb39bfa7a1afeea41f5af3fc8dbd8ede Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 05:31:13 -0500 Subject: [PATCH 095/124] fix(node): keep the repo identity key printable The key is logged as the `repo` field on the lease waiter-cap shed and the steal-bound warning (state.rs), and it was joined with a NUL. Those are the two messages an operator reads to find a contended repo, and a NUL-hostile log sink truncates the field at the separator, rendering two different repos' warnings identically. Before this PR that field was a UUID or a mirror id, both printable, so the NUL was a regression this change introduced. Join with '/' instead. It cannot occur in the owner slug by construction - the slug is owner_did.replace([':','/'], "_") - and it mirrors the shape of the disk path the key exists to reproduce. A test asserts the key carries no control characters so this cannot regress silently. Rotating the U2 lease test's row to a UUID id is not incidental: the mirror path that seeds it uses 'owner_short/name' as the row id, which now coincides with the identity key and would leave the test unable to tell the two keys apart. The rotated id is also the real shape of the bug. Re-verified RED against the reverted call site after the change. --- crates/gitlawb-node/src/api/repos.rs | 15 ++++++++++++++ crates/gitlawb-node/src/state.rs | 30 +++++++++++++++++++++++----- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 470badc7..5871601e 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -7647,7 +7647,22 @@ mod tests { let (git_bin, a_inpack, _later_ran) = f1_hanging_first_git(tmp.path()); let state = f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6u2key", "k1", false).await; + // Rotate the row to a UUID id first. f4_state_with_repo creates the repo + // through the mirror path, whose id is literally `owner_short/name` and so + // coincides with the identity key — which would leave this test unable to + // tell the two keys apart. A UUID id is also the real shape of the bug: an + // API-created repo, or any repo whose row was recreated under the same slug. + let seeded = state.db.get_repo("z6u2key", "k1").await.unwrap().unwrap(); + let rotated_id = Uuid::new_v4().to_string(); + sqlx::query("UPDATE repos SET id = $1 WHERE id = $2") + .bind(&rotated_id) + .bind(&seeded.id) + .execute(&pool) + .await + .unwrap(); + let rec = state.db.get_repo("z6u2key", "k1").await.unwrap().unwrap(); + assert_eq!(rec.id, rotated_id, "the row really did take the new id"); let identity = crate::state::repo_identity_key(&rec.owner_did, &rec.name); assert_ne!( identity, rec.id, diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index ef093661..1f0f8c81 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -515,9 +515,14 @@ pub struct RepoWriteLeases { /// `did:key:` prefix instead and would map the same input to a different /// string. The disk path is authoritative because the `objects/` directory is /// the resource being serialized. -/// - The separator is `\0`, which cannot occur in either component. A plain join -/// would collide (owner `a` + name `b_c` against owner `a_b` + name `c`), -/// letting one repo's push park another's. +/// - The separator is `/`, which cannot occur in the owner slug by construction +/// (`replace([':', '/'], "_")` removes it) and mirrors the shape of the disk +/// path this key exists to reproduce. A plain join would collide (owner `a` + +/// name `b_c` against owner `a_b` + name `c`), letting one repo's push park +/// another's. It must stay PRINTABLE: this key is logged as the `repo` field +/// on the lease shed and steal-bound warnings below, and an unprintable +/// separator (a NUL, a unit separator) truncates at a NUL-hostile log sink and +/// renders two different repos' warnings identically. /// - Callers pass `record.owner_did` / `record.name`, never the request's path /// segments: `db::get_repo` normalizes DID aliases, so a caller could otherwise /// mint two keys for one directory just by varying the DID spelling. @@ -531,7 +536,7 @@ pub struct RepoWriteLeases { /// push concurrently. pub fn repo_identity_key(owner_did: &str, repo_name: &str) -> String { let owner_slug = owner_did.replace([':', '/'], "_"); - format!("{owner_slug}\0{repo_name}") + format!("{owner_slug}/{repo_name}") } /// A per-repo lease entry: the one-permit semaphore, a refcount of the handlers @@ -874,7 +879,7 @@ mod repo_identity_key_tests { let expected_slug = owner.replace([':', '/'], "_"); assert_eq!( repo_identity_key(owner, name), - format!("{expected_slug}\0{name}") + format!("{expected_slug}/{name}") ); // The disk path for the same pair must carry the same slug component. @@ -906,6 +911,21 @@ mod repo_identity_key_tests { ); } + /// The key is logged as the `repo` field on the lease waiter-cap shed and the + /// steal-bound warning, so it must contain no control characters. A NUL (or a + /// unit separator) truncates at a NUL-hostile log sink, which would render two + /// different repos' shed warnings identically — an observability lie on + /// exactly the messages an operator reads to find a contended repo. + #[test] + fn key_is_printable_so_the_lease_warnings_stay_readable() { + let k = repo_identity_key("did:web:example.com:alice", "my-repo.git"); + assert!( + !k.chars().any(|c| c.is_control()), + "the identity key is logged; it must carry no control characters: {k:?}" + ); + assert_eq!(k, "did_web_example.com_alice/my-repo.git"); + } + /// Distinct repos and distinct owners never share a key. #[test] fn distinct_repos_and_owners_do_not_share_a_key() { From 929256579c7bf38983ba7a8f43bd0a71e8048bec Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 12:37:39 -0500 Subject: [PATCH 096/124] fix(node): spawn the replication tail above guard.release() (#174) The post-receive tail was spawned after `guard.release(...)`, which on a successful push awaits the Tigris upload and then the advisory unlock while the handler future is still tied to the client connection. The pack has already landed on disk by then, so a disconnect in that window left a durable push whose pins, recovery copy and announce were never spawned: the dropped tail F2 closed, one step earlier in the handler. Spawn it above `release`, gated on an explicit `receive_result.is_ok()`. The `?` below the spawn can no longer be what keeps a rejected push from pinning and announcing a half-applied repo, so the success check is now a named flag that `release` consumes too. The tail is read-only on the repo directory and holds neither the lease nor the advisory lock, so overlapping it with the upload waits on nothing the handler still owns. Tests: a handler-level disconnect driven through a parked `release` (new store-level test seam), plus the must-not direction, a failed receive-pack that spawns no tail. The U5 source gate now binds the new boundary: spawn inside `if push_succeeded`, above `guard.release(push_succeeded)`. --- crates/gitlawb-node/src/api/repos.rs | 259 +++++++++++++++++++--- crates/gitlawb-node/src/git/repo_store.rs | 20 +- crates/gitlawb-node/tests/inv22_gates.rs | 68 +++--- 3 files changed, 292 insertions(+), 55 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 5871601e..866df30b 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1800,10 +1800,50 @@ pub async fn git_receive_pack( ) .await; + // #174 F2/U5: the post-receive replication tail runs in an independently owned + // task. It parks on `git_encrypt_semaphore` (withheld / candidate / full-scan + // resolution), so leaving it in the request future means a client/proxy disconnect + // while parked silently drops this push's pins, recovery copy, and announcements. + // + // Spawned HERE, ABOVE `guard.release()`, because `release` is itself cancellable: + // on success it awaits the Tigris upload and then the advisory unlock, both while + // this future is still tied to the client connection, and the pack has ALREADY + // landed on disk by then. Spawning below `release` left exactly that window open, + // where a disconnect meant a durable push with no tail (the same class F2 closed, + // one step earlier). + // + // The success gate is explicit rather than the `?` below, because that `?` now + // sits under this spawn: `release` runs on failure too, so anchoring the tail on + // it would pin and announce a half-applied repo, the state `release(false)` + // deliberately refuses to upload and one a hostile pusher can produce on demand + // by aborting a pack mid-transfer. + // + // The tail is read-only on `disk_path` (walk plus plumbing) and takes neither the + // write lease nor the advisory lock, so running it concurrently with the upload + // below waits on nothing this handler still holds. Everything after (touch_repo, + // metrics, trust score, certificates, webhooks) stays in the cancellable handler. + // + // The tail also runs CONCURRENTLY with certificate issuance rather than after it, + // so a ref can be announced before its signed certificate exists. That window is + // accepted: cert issuance already fails open (errors are logged and skipped) and + // the gossip event carries `cert_id: None` regardless, so no announce consumer + // reads a certificate out of it. Each push owns its own tail, including its own + // always-spawned announce, so per-push announcements are never coalesced away. + let push_succeeded = receive_result.is_ok(); + if push_succeeded { + tokio::spawn(post_receive_replication_tail( + state.clone(), + record.clone(), + ref_updates.clone(), + disk_path.clone(), + auth.0.to_string(), + )); + } + // Always release the advisory lock — even on error — to prevent stale locks // from blocking subsequent pushes. Only upload to Tigris when the push // succeeded; uploading a half-applied repo would propagate corruption. - guard.release(receive_result.is_ok()).await; + guard.release(push_succeeded).await; // Clean path: clone (a) already dropped inside run_git_service when the receive-pack // group was reaped; clone (b) held here spanned the success-only Tigris upload that // ran inside release() above. Drop it now so a second same-repo push proceeds the @@ -1823,34 +1863,6 @@ pub async fn git_receive_pack( app })?; - // #174 F2/U5: the post-receive replication tail runs in an independently owned - // task. It parks on `git_encrypt_semaphore` (withheld / candidate / full-scan - // resolution), so leaving it in the request future means a client/proxy disconnect - // while parked silently drops this push's pins, recovery copy, and announcements. - // - // Spawned HERE, at the durability boundary, rather than at the end of the handler. - // This is the first point where the push is known to have landed: `guard.release()` - // above runs on failure too, so anchoring on it would spawn a tail that pins and - // announces a half-applied repo — the state release() deliberately refuses to - // upload, and one a hostile pusher can produce on demand by aborting a pack - // mid-transfer. Everything below (touch_repo, metrics, trust score, certificates, - // webhooks) stays in the cancellable handler, so a disconnect during that window no - // longer takes the tail with it. - // - // The tail now runs CONCURRENTLY with certificate issuance rather than after it, so - // a ref can be announced before its signed certificate exists. That window is - // accepted: cert issuance already fails open (errors are logged and skipped) and the - // gossip event carries `cert_id: None` regardless, so no announce consumer reads a - // certificate out of it. Each push owns its own tail, including its own - // always-spawned announce, so per-push announcements are never coalesced away. - tokio::spawn(post_receive_replication_tail( - state.clone(), - record.clone(), - ref_updates.clone(), - disk_path.clone(), - auth.0.to_string(), - )); - // Update the repo's updated_at timestamp after a successful push let _ = state.db.touch_repo(&record.id).await; @@ -8563,6 +8575,195 @@ mod tests { assert_eq!(broadcast.ref_name, "refs/heads/main"); } + /// A push handler whose `release` parks at its pre-unlock point, so a test can + /// drop the future from inside the cancellable post-receive window. Returns the + /// state, the seeded record and the git-invocation log. + /// + /// Path-scoped on purpose: the tail's withheld walk is the observable, and + /// without a path-scoped rule `replication_withheld_set` takes the no-walk + /// shortcut and spawns no git at all. + #[cfg(unix)] + async fn p2_parked_release_state( + pool: sqlx::PgPool, + tmp: &std::path::Path, + owner: &str, + name: &str, + git_body: Option<&str>, + ) -> (AppState, std::path::PathBuf) { + let log = tmp.join("git.log"); + let git_bin = match git_body { + Some(body) => write_fake_git(tmp, body), + None => f2a_logging_git(tmp, &log), + }; + let repos_dir = tmp.join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + + let mut state = crate::test_support::test_state(pool.clone()).await; + state.git_bin = git_bin; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Armed and never notified: every guard this store hands out parks in + // `release` right before the advisory unlock, which is inside the window a + // client disconnect can hit. + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool.clone()) + .with_pre_unlock_gate(std::sync::Arc::new(tokio::sync::Notify::new())); + state + .db + .upsert_mirror_repo(owner, name, &format!("/unused-{owner}-{name}"), None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkP2TailReaderAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + state + .repo_store + .init(&rec.owner_did, &rec.name) + .await + .unwrap(); + (state, log) + } + + const P2_PUSHER: &str = "did:key:z6MkP2TailPusherAAAAAAAAAAAAAAAAAAAAAA"; + + fn p2_push( + state: &AppState, + owner: &str, + name: &str, + ) -> impl std::future::Future> { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + git_receive_pack( + State(state.clone()), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid(P2_PUSHER.to_string())), + crate::rate_limit::PeerAddr(Some("203.0.113.90:5000".parse::().unwrap())), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + } + + fn p2_logged(log: &std::path::Path, prefix: &str) -> bool { + f2a_log(log).lines().any(|l| l.starts_with(prefix)) + } + + /// #174 (jatmn P2, RED-before/GREEN-after): a client disconnect DURING + /// `guard.release()` must not take the replication tail with it. On a + /// successful push `release` awaits the Tigris upload and then the advisory + /// unlock, both cancellation points, while the pack has already landed on disk. + /// Spawning the tail below `release` means a disconnect in that window drops + /// this push's pins, recovery copy and announce: the F2 dropped-tail class, one + /// step earlier in the handler. + /// + /// Load-bearing: with the spawn below `release` the walk's `for-each-ref` never + /// appears after the disconnect (RED). With it above, gated on + /// `receive_result.is_ok()`, it does (GREEN). + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_tail_survives_a_disconnect_during_release(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let (state, log) = p2_parked_release_state(pool, tmp.path(), "z6p2tail", "t1", None).await; + + let mut fut = Box::pin(p2_push(&state, "z6p2tail", "t1")); + + // Drive until receive-pack has run. Nothing between it and `release` awaits, + // so a future that stops completing after that point is parked on the gate. + let mut ran = false; + for _ in 0..1000 { + let step = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + assert!( + step.is_err(), + "the handler must park inside release, not return" + ); + if p2_logged(&log, "receive-pack") { + ran = true; + break; + } + } + assert!(ran, "the push must reach receive-pack"); + // Settle into the parked state. Whether the tail's walk has already started + // by now is immaterial: pre-fix no tail is ever spawned, because `release` + // never returns, so the marker below can only come from a spawn above it. + for _ in 0..5 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + } + + // The disconnect: drop the handler future while `release` is still awaiting. + drop(fut); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while !p2_logged(&log, "for-each-ref") { + assert!( + std::time::Instant::now() < deadline, + "RED: the pack landed but its replication tail never ran. A disconnect \ + during guard.release() took the tail with the handler future — spawn it \ + above release, gated on receive_result.is_ok()" + ); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + } + + /// The must-not direction of the same reorder. Moving the spawn above + /// `release` moves it above the `?` that used to gate it, so the success check + /// has to be explicit: a FAILED receive-pack must still spawn no tail, or a + /// pusher who aborts a pack mid-transfer gets a half-applied repo pinned and + /// announced on demand. + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_failure_spawns_no_tail_even_when_the_client_disconnects( + pool: sqlx::PgPool, + ) { + // receive-pack fails; everything else the handler or a tail might run is + // logged, so any walk child would show up. + let tmp = tempfile::TempDir::new().unwrap(); + let log = tmp.path().join("git.log"); + let body = format!( + "#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"{}\"\n\ + case \"$1\" in receive-pack) cat >/dev/null 2>/dev/null; exit 1 ;; esac\n\ + exec git \"$@\"\n", + log.display() + ); + let (state, log) = + p2_parked_release_state(pool, tmp.path(), "z6p2fail", "t1", Some(&body)).await; + + let mut fut = Box::pin(p2_push(&state, "z6p2fail", "t1")); + let mut ran = false; + for _ in 0..1000 { + let step = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if p2_logged(&log, "receive-pack") { + ran = true; + // A failed push still parks in `release` (the lock is freed on + // failure too); drop it there, the same disconnect as above. + assert!( + step.is_err(), + "the failed push must still park inside release" + ); + break; + } + } + assert!(ran, "the push must reach receive-pack"); + for _ in 0..5 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + } + drop(fut); + + tokio::time::sleep(std::time::Duration::from_millis(750)).await; + assert!( + !p2_logged(&log, "for-each-ref"), + "a failed receive-pack must spawn no replication tail: pinning and \ + announcing a half-applied repo is exactly what release(false) refuses \ + to upload" + ); + } + /// Scenario 5 (trap 3, fail-closed). On a repo whose withheld walk is failing, a /// coalesced push must not publish. Before the gate moved, every push on such a /// repo got `announce = false` from its own walk; a coalesced push has no walk, so diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 69d1bb33..2677cea2 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -31,6 +31,12 @@ pub struct RepoStore { /// Tracks repos already confirmed to exist in Tigris — avoids redundant /// HEAD checks and background uploads for repos we've already migrated. migrated: Arc>>, + /// Test-only seam: armed here, copied into every `RepoWriteGuard` this store + /// hands out, so a test that only holds the `AppState` (not the guard) can + /// still park `release` at its pre-unlock point. See + /// `RepoWriteGuard::test_pre_unlock_gate`. Never set outside tests. + #[cfg(test)] + pre_unlock_gate: Option>, } impl RepoStore { @@ -41,15 +47,27 @@ impl RepoStore { tigris: None, pool, migrated: Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new())), + pre_unlock_gate: None, } } + /// Test-only: every guard from this store parks in `release` right before the + /// `pg_advisory_unlock` await, until `gate` is notified. Dropping the future + /// while it is parked reproduces a client disconnect inside `release`. + #[cfg(test)] + pub fn with_pre_unlock_gate(mut self, gate: Arc) -> Self { + self.pre_unlock_gate = Some(gate); + self + } + pub fn new(repos_dir: PathBuf, tigris: Option, pool: PgPool) -> Self { Self { repos_dir, tigris, pool, migrated: Arc::new(Mutex::new(HashSet::new())), + #[cfg(test)] + pre_unlock_gate: None, } } @@ -181,7 +199,7 @@ impl RepoStore { released: false, tigris: self.tigris.clone(), #[cfg(test)] - test_pre_unlock_gate: None, + test_pre_unlock_gate: self.pre_unlock_gate.clone(), }; // Acquire the advisory lock with retry, through the guard's OWN connection, diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index 82712c23..32e23dd0 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -415,26 +415,30 @@ fn inv22_ipfs_walk_admission_reaches_every_blocking_site() { } /// #174 U5: the post-receive replication tail is spawned at the DURABILITY BOUNDARY, -/// not at the end of the handler. +/// which is the moment receive-pack returns success, not the end of the handler and +/// not after `guard.release()`. /// -/// The tail owes this push its pins, recovery copy, and announcements. Spawned below -/// the certificate and webhook work, a client disconnect anywhere in that window took -/// the tail with it. Spawned above it, the tail is already detached and survives. +/// The tail owes this push its pins, recovery copy, and announcements. Everything +/// below the spawn stays in the cancellable request future, so anything the tail is +/// spawned after is a window where a client disconnect drops that work while the pack +/// is already durable on disk. `guard.release()` is such a window: on success it +/// awaits the Tigris upload and then the advisory unlock. /// -/// The lower bound matters just as much as the upper one: `guard.release()` runs on -/// failure too, so a spawn anchored there would fire for a push git rejected, pinning -/// and announcing a half-applied repo. The `?` on `receive_result` is the first point -/// where the push is known to have landed, which is why the spawn must sit between -/// that `?` and `touch_repo`. +/// The lower bound matters just as much as the upper one: `release` runs on failure +/// too, so an ungated spawn would fire for a push git rejected, pinning and announcing +/// a half-applied repo. Above `release` the `?` on `receive_result` can no longer be +/// what gates it, so the success check is explicit and this gate binds it: the spawn +/// must sit inside `if push_succeeded`, and `release` must consume the same flag so +/// the two cannot drift apart. /// -/// This is an ordering check rather than a cancellation-race test on purpose: forcing -/// a disconnect to land inside the cert/webhook window is timing-dependent and would -/// be flaky, while the ordering is exactly what the fix changes. Same instrument the -/// F3 gate above uses. +/// This is an ordering check rather than a cancellation-race test on purpose: it is +/// the companion to `receive_pack_tail_survives_a_disconnect_during_release`, which +/// drives the actual disconnect through a parked `release`. Same instrument the F3 +/// gate above uses. /// -/// MUTATION (RED): move the `tokio::spawn(post_receive_replication_tail` call back -/// below the webhook block and the ordering assertion fails; move it above the `?` -/// and the failed-push assertion fails. +/// MUTATION (RED): move the `tokio::spawn(post_receive_replication_tail` call below +/// `guard.release(` and the ordering assertion fails; take it out of the +/// `if push_succeeded` block and the failed-push assertion fails. #[test] fn inv22_replication_tail_spawns_at_the_durability_boundary() { let repos = src("api/repos.rs"); @@ -444,12 +448,20 @@ fn inv22_replication_tail_spawns_at_the_durability_boundary() { .next() .expect("split always yields a first chunk"); - let receive_ok = production - .find("let result = receive_result.map_err(") - .expect("U5 gate stale: git_receive_pack no longer maps receive_result with ?"); + let success_flag = production + .find("let push_succeeded = receive_result.is_ok();") + .expect( + "U5 gate missing: the tail's success gate must be bound from receive_result.is_ok()", + ); + let gate_open = production + .find("if push_succeeded {") + .expect("U5 gate missing: the tail spawn must be gated on the push having succeeded"); let spawn = production .find("tokio::spawn(post_receive_replication_tail(") .expect("U5 gate missing: the replication tail must be spawned by git_receive_pack"); + let release = production + .find("guard.release(push_succeeded)") + .expect("U5 gate stale: release must consume the same success flag as the tail gate"); let touch = production .find("state.db.touch_repo(") .expect("U5 gate stale: git_receive_pack no longer calls touch_repo"); @@ -458,14 +470,20 @@ fn inv22_replication_tail_spawns_at_the_durability_boundary() { .expect("U5 gate stale: git_receive_pack no longer fires push webhooks"); assert!( - receive_ok < spawn, - "U5 gate bypassed: the tail must be spawned AFTER the receive_result `?`, or a \ + success_flag < gate_open && gate_open < spawn, + "U5 gate bypassed: the tail must be spawned inside `if push_succeeded`, or a \ rejected push spawns a tail that pins and announces a half-applied repo" ); + // Still inside that block: no `}` may close it between the gate and the spawn. + assert!( + !production[gate_open + "if push_succeeded {".len()..spawn].contains('}'), + "U5 gate bypassed: the tail spawn left the `if push_succeeded` block, so a \ + rejected push now spawns a tail" + ); assert!( - spawn < touch && spawn < webhook, - "U5 gate bypassed: the tail must be spawned BEFORE touch_repo and the webhook \ - fan-out, so a disconnect in that window cannot drop this push's pins, \ - recovery copy, and announcements" + spawn < release && spawn < touch && spawn < webhook, + "U5 gate bypassed: the tail must be spawned BEFORE guard.release, touch_repo \ + and the webhook fan-out, so a disconnect in any of those windows cannot drop \ + this push's pins, recovery copy, and announcements" ); } From 13af73fc3e55288fc0b411cd42c29431d48b7eb5 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 13:46:33 -0500 Subject: [PATCH 097/124] test(node): record why the handler path cannot probe the stranded-lock case A handler-level "the next acquire_write succeeds" check does not discriminate: it still passes with the RepoWriteGuard::Drop backstop disabled. Measured with the backstop off, the lock is held at the drop and reads free about two seconds later from a session outside the pool. #[sqlx::test] builds its pool with idle_timeout(1s), so the guard's connection is reaped once the handler future drops it, the session ends, and postgres frees the lock with no help from the code under test. acquire_write retries for far longer than that window, so it waits the release out and reports success either way. write_guard_release_cancelled_mid_unlock_frees_the_lock is the real proof: it probes from a connection held out of the pool, 400ms after the drop, inside that window rather than past it. --- crates/gitlawb-node/src/api/repos.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 866df30b..3236fa99 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -8709,6 +8709,19 @@ mod tests { ); tokio::time::sleep(std::time::Duration::from_millis(25)).await; } + + // Not asserted here: that the same disconnect leaves the session advisory + // lock free. A handler-level "the next acquire_write succeeds" probe does not + // discriminate. It still passes with the `Drop` backstop disabled, because the + // guard's `PoolConnection` goes back to the pool when this future is dropped + // and `#[sqlx::test]` builds the pool with `idle_timeout(1s)`, so the session + // ends on its own a beat later and postgres frees the lock with no help from + // the code under test. Measured with the backstop disabled: held at the drop, + // free ~2s later, observed from a session outside the pool. `acquire_write` + // retries for far longer than that, so it waits the release out and reports + // success either way. `write_guard_release_cancelled_mid_unlock_frees_the_lock` + // is the real proof: it probes from a connection held OUT of the pool, 400ms + // after the drop, which is inside that window rather than past it. } /// The must-not direction of the same reorder. Moving the spawn above From a9d8906fa03415a749aa2927aedb948eb3cf829d Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 14:31:51 -0500 Subject: [PATCH 098/124] docs(node): record the Tigris ordering residual the tail reorder introduces Spawning the replication tail above guard.release() also moves it above the Tigris upload, so a ref can be announced while the shared durable copy is still the old one, and a disconnect there cancels the upload while the detached tail still pins and announces. Accepted rather than fixed. Upload-then-announce was never guaranteed: release tolerates a failed upload by design, warning and continuing to the unlock, so an announce over a stale Tigris copy was already reachable before this reorder, and it self-heals through acquire_fresh's local fallback plus the next push. The alternative, detaching release and the tail together, would return 200 before the durable copy lands, which changes the client contract more than the window it closes. --- crates/gitlawb-node/src/api/repos.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 3236fa99..c60f8c19 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1829,6 +1829,19 @@ pub async fn git_receive_pack( // the gossip event carries `cert_id: None` regardless, so no announce consumer // reads a certificate out of it. Each push owns its own tail, including its own // always-spawned announce, so per-push announcements are never coalesced away. + // + // ACCEPTED RESIDUAL, and it is the cost of this ordering: the tail also runs + // concurrently with the Tigris upload inside `release` below, where before it ran + // after. So a ref can be announced while the shared durable copy is still the old + // one, and on a disconnect here the upload is cancelled outright while the + // detached tail still pins and announces. What makes that acceptable is that + // upload-then-announce was never actually guaranteed: `release` tolerates a failed + // upload by design (it warns and continues to the unlock), so an announce over a + // stale Tigris copy was already reachable before this reorder, and it self-heals, + // since `acquire_fresh` falls back to the local copy and the next push re-uploads. + // The alternative, detaching `release` and the tail together to keep the ordering, + // would return 200 to the pusher before the durable copy lands, which is a larger + // change to the client contract than the window it closes. let push_succeeded = receive_result.is_ok(); if push_succeeded { tokio::spawn(post_receive_replication_tail( From 4f3278f28c69777471bb12c95ce97c0834c39422 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 21:44:28 -0500 Subject: [PATCH 099/124] fix(node): share one deadline across the withheld walk and the pack serve (#62) The path-scoped git-upload-pack branch spent git_service_timeout_secs twice: once on the withheld-blob classification walk, then a second full budget on the serve. One clone could hold a read-pool permit for ~2x the configured budget, which is the occupancy hole this PR otherwise exists to close. Thread one Instant deadline through both phases, mirroring fail_closed_full_scan_objects and build_filtered_pack: the walk runs against the remainder and the serve gets what the walk left, computed after the blocking task's await so it cannot pick up a fresh budget. A walk that consumes the whole budget saturates the serve to zero, which surfaces as GitServiceTimeout -> 504 rather than running unbounded. Under-serving one slow clone is the safe direction versus over-holding the pool. Both serve arms are covered because a fix that reached only the plain arm would leave the ~2x hold reachable by any clone of a repo that actually withholds a blob. Each test was observed RED before the change (status 200, the serve completing on a fresh budget) and GREEN after, and reverting the filtered arm alone flips only its own test, so neither guard is a proxy for the other. The knob's meaning changed for this path, so .env.example now says the value bounds the two phases combined rather than each separately. --- .env.example | 5 + crates/gitlawb-node/src/api/repos.rs | 192 ++++++++++++++++++++++++++- 2 files changed, 194 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index 5ed087d9..a4d66a7a 100644 --- a/.env.example +++ b/.env.example @@ -115,6 +115,11 @@ GITLAWB_MAX_PACK_BYTES=2147483648 # worker and, on push, the repo write lock. Also bounds both info/refs # advertisements, the withheld-blob pack build, and the push-side candidate # discovery (rev-list / cat-file), all reaped via process-group teardown (#174). +# On the path-scoped upload-pack path the withheld-blob classification walk and +# the pack serve share ONE deadline, so this value bounds their COMBINED duration +# rather than giving each stage a full budget: a walk that consumes it leaves the +# serve nothing and the clone gets a 504. Serving large path-scoped repos may +# need a higher value here than when each stage was budgeted separately. # Default 600. GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600 diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index c60f8c19..aedf03e6 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1361,6 +1361,16 @@ pub async fn git_upload_pack( // task hands the permits back so the serve phase below keeps them; on a // dropped future the returned tuple (with the permits) is discarded only when // the blocking task completes, so admission tracks the real git work. + // ONE deadline spans the walk AND the serve below (#174 U1 follow-up). A fresh + // `git_timeout` for the serve let a slow-but-successful walk plus a full serve + // hold this read permit ~2x the configured budget. Sharing the deadline caps + // that at ~1x: the walk runs against the remaining budget, and if it consumes + // the budget the serve gets what is left and is reaped rather than over-holding + // — the safe direction, same tradeoff as `fail_closed_full_scan_objects` and + // `build_filtered_pack`. The cost is honest: a genuinely slow walk on a large + // repo 504s this clone instead of silently holding the pool for ~2x, so size + // `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` so both phases normally fit. + let deadline = std::time::Instant::now() + git_timeout; let (withheld, _permit, _caller_permit) = { let path = disk_path.clone(); let rules = rules.clone(); @@ -1368,11 +1378,12 @@ pub async fn git_upload_pack( let caller_owned = caller.map(str::to_string); let is_public = record.is_public; let git_bin = state.git_bin.clone(); + let walk_budget = deadline.saturating_duration_since(std::time::Instant::now()); tokio::task::spawn_blocking(move || { let withheld = visibility_pack::withheld_blob_oids_bounded( &path, &git_bin, - git_timeout, + walk_budget, &rules, is_public, &owner_did, @@ -1393,10 +1404,22 @@ pub async fn git_upload_pack( // The handler keeps no copy (F1: handler-local permits would drop the // instant a disconnect drops this future, mid-reap). let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + // Computed AFTER the walk's await, so the serve gets what the walk left, not a + // second full budget. A walk that consumed the whole budget saturates this to + // zero, which the serve surfaces as GitServiceTimeout -> 504 rather than + // running unbounded. + let serve_budget = deadline.saturating_duration_since(std::time::Instant::now()); if withheld.is_empty() { // No blobs to withhold: serve the plain pack (the walk already held the // permits per be0cdd6; the guard hands them to the serve). - smart_http::upload_pack(&state.git_bin, &disk_path, body, git_timeout, Some(admission)).await + smart_http::upload_pack( + &state.git_bin, + &disk_path, + body, + serve_budget, + Some(admission), + ) + .await } else { tracing::info!(repo = %name, caller = ?caller, withheld = withheld.len(), "serving filtered pack"); // The guard threads through both filtered-pack stages (rev-list, then @@ -1407,7 +1430,7 @@ pub async fn git_upload_pack( &disk_path, body, &withheld, - git_timeout, + serve_budget, Some(admission), ) .await @@ -3332,6 +3355,169 @@ mod tests { ); } + /// #174 U1 follow-up (RED-before/GREEN-after): the path-scoped upload-pack branch + /// shares ONE deadline across the withheld-blob walk and the pack serve, so one + /// clone cannot hold a read permit for ~2x `git_service_timeout_secs`. A walk that + /// consumes most of the budget must leave the serve only the REMAINDER, so the + /// serve is reaped and the request is a 504. + /// + /// Load-bearing: give the serve a fresh `git_timeout` instead of the remainder and + /// the fake `upload-pack` (1.2s) fits inside a fresh 2s budget, completes, and the + /// status is no longer 504 (RED). This is the ~2x-budget hold the unit removes. + /// + /// Plain-serve arm: the fake git lists no refs and fails `rev-parse`, so the walk + /// yields an empty withheld set and the branch takes `upload_pack`. `rev-list` + /// carries the walk's cost. + #[cfg(unix)] + #[sqlx::test] + async fn upload_pack_shares_one_deadline_across_walk_and_plain_serve(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + // Walk: no refs (for-each-ref empty), HEAD does not resolve (rev-parse exit 1), + // rev-list burns 1.2s of the 2s budget and lists no commits -> empty withheld. + // Serve: upload-pack needs 1.2s, which does NOT fit the ~0.8s remainder but + // WOULD fit a fresh 2s budget. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-parse) exit 1 ;;\n rev-list) sleep 1.2 ;;\n upload-pack) sleep 1.2 ;;\n *) : ;;\nesac\nexit 0\n"; + let fake = write_fake_git(tmp.path(), body); + + let mut state = crate::test_support::test_state(pool).await; + state.git_bin = fake; + let mut cfg = (*state.config).clone(); + cfg.git_service_timeout_secs = 2; + state.config = std::sync::Arc::new(cfg); + state + .db + .upsert_mirror_repo("z6shared1", "sv", "/tmp/z6shared1-sv", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6shared1", "sv").await.unwrap().unwrap(); + // Path-scoped rule so has_path_scoped_rule() is true and the walk runs. + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &[], + OWNER_DID, + ) + .await + .unwrap(); + let disk = std::path::Path::new("/tmp/z6shared1/sv.git"); + std::fs::create_dir_all(disk).unwrap(); + + let peer: SocketAddr = "203.0.113.92:7000".parse().unwrap(); + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::POST) + .uri("/z6shared1/sv/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let status = router.oneshot(req).await.unwrap().status(); + let _ = std::fs::remove_dir_all("/tmp/z6shared1"); + assert_eq!( + status, + StatusCode::GATEWAY_TIMEOUT, + "the walk and the serve must share ONE deadline, so a walk that burns most \ + of the budget leaves the serve only the remainder and the serve is reaped; \ + a non-504 here means the serve got a fresh full budget (the ~2x hold)" + ); + } + + /// #174 U1 follow-up, FILTERED arm (RED-before/GREEN-after): the shared deadline + /// must reach `upload_pack_excluding` too, not just the plain `upload_pack`. Same + /// property as the plain-arm test, different serve function, because the branch + /// threads the remainder into both arms and a fix that missed one would leave the + /// ~2x hold reachable by any clone of a repo that actually withholds something. + /// + /// The fake git yields a NON-EMPTY withheld set: one ref peeling to a commit, a + /// resolvable HEAD, one commit, and an `ls-tree -rz` record placing a blob under + /// `/secret/`, which the path-scoped rule denies to an anonymous caller. `ls-tree` + /// carries the walk's cost (walk-only), and `pack-objects` carries the serve's, so + /// the two phases are independently attributable. + /// + /// Load-bearing: hand the serve a fresh `git_timeout` and `pack-objects` (1.2s) + /// fits a fresh 2s budget, completes, and the status is no longer 504 (RED). + #[cfg(unix)] + #[sqlx::test] + async fn upload_pack_shares_one_deadline_across_walk_and_filtered_serve(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + let commit = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + let blob = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + // for-each-ref lists one ref; cat-file peels it to a commit (the fail-closed + // ref check); rev-parse resolves HEAD; rev-list lists the one commit; ls-tree + // emits " blob \t" (NUL-delimited) under secret/ and burns + // 1.2s of the 2s budget; pack-objects is the serve's 1.2s cost. + let body = format!( + "#!/bin/sh\ncase \"$1\" in\n \ + for-each-ref) echo refs/heads/main ;;\n \ + cat-file) echo commit ;;\n \ + rev-parse) echo {commit} ;;\n \ + rev-list) echo {commit} ;;\n \ + ls-tree) printf '100644 blob {blob}\\tsecret/f.txt' ; sleep 1.2 ;;\n \ + pack-objects) sleep 1.2 ;;\n \ + *) : ;;\nesac\nexit 0\n" + ); + let fake = write_fake_git(tmp.path(), &body); + + let mut state = crate::test_support::test_state(pool).await; + state.git_bin = fake; + let mut cfg = (*state.config).clone(); + cfg.git_service_timeout_secs = 2; + state.config = std::sync::Arc::new(cfg); + state + .db + .upsert_mirror_repo("z6shared2", "sv", "/tmp/z6shared2-sv", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6shared2", "sv").await.unwrap().unwrap(); + // Denies /secret/** to an anonymous caller, so the blob above is withheld and + // the branch takes upload_pack_excluding rather than the plain serve. + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &[], + OWNER_DID, + ) + .await + .unwrap(); + let disk = std::path::Path::new("/tmp/z6shared2/sv.git"); + std::fs::create_dir_all(disk).unwrap(); + + let peer: SocketAddr = "203.0.113.93:7000".parse().unwrap(); + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::POST) + .uri("/z6shared2/sv/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let status = router.oneshot(req).await.unwrap().status(); + let _ = std::fs::remove_dir_all("/tmp/z6shared2"); + assert_eq!( + status, + StatusCode::GATEWAY_TIMEOUT, + "the FILTERED serve must also take the shared deadline's remainder; a \ + non-504 here means upload_pack_excluding got a fresh full budget and the \ + ~2x hold is still reachable whenever a repo withholds a blob" + ); + } + /// #174 (F2 sizing edge, vetted by execution): the receive-pack advertisement /// per-source cap is derived in main.rs as `(max_concurrent_git_pushes / 8).max(1)`, /// so it is never 0 even at the minimum write-pool size (1). A 0 cap would make From 6b76600755e184cedb19db0a443876060e82d039 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 21:44:53 -0500 Subject: [PATCH 100/124] docs(node): correct stale admission-bound comments and document derived caps (#62) Four prose corrections on this PR's surfaces, no behavior change. The receive-pack advertisement no longer draws from the write pool, it draws from a dedicated advert pool, but five comments still described the old shape: the info/refs per-source block, the cap construction in main.rs, the State field doc, and a test doc plus its assertion message. The state.rs and main.rs ones mattered most because they contradicted the operator text being written in this same commit. The /ipfs probe and content-read subprocesses each run under their own deadline now (the lesser of git_service_timeout_secs and the remaining budget, reaped by process-group teardown), but four surfaces still claimed they had no duration bound past the pre-start budget check. The correction stays scoped rather than declaring the route safe: the probe's object_store_readable check is a synchronous filesystem sweep with nothing to reap, so a wedged filesystem can still hold the walk slot past the deadline, and that residual is now stated where the old warning used to be. A grep for the full phrase misses two of these surfaces, since one wraps across doc-comment lines and one says "duration clamp" instead, so the sweep uses the fragment. Both push-side per-source concurrency caps are derived as max_concurrent_git_pushes / 8 with a floor of 1 and have no environment variable, so neither was documented anywhere an operator reads. The write-side one is load-bearing for push availability: it is acquired before the global write permit, and owner enforcement defaults off, so without it one host minting disposable did:key identities could monopolize the write pool. They are described in .env.example beside the read cap rather than in README's table, which is keyed by variable name. The hung-walk assertion in visibility_pack printed {err}, showing only the top anyhow context and dropping the underlying io error. A beta-lane failure surfaced as "failed to spawn git for-each-ref" with no errno, which made it undiagnosable. {err:#} prints the chain. The underlying flake is still undiagnosed: it did not reproduce in 123 local runs and the stable lane passed on the same commit, so this only improves the next occurrence. --- .env.example | 26 ++++++++++++++++--- crates/gitlawb-node/src/api/ipfs.rs | 21 ++++++++++----- crates/gitlawb-node/src/api/repos.rs | 21 ++++++++------- crates/gitlawb-node/src/config.rs | 9 ++++--- .../gitlawb-node/src/git/visibility_pack.rs | 5 +++- crates/gitlawb-node/src/main.rs | 8 +++--- crates/gitlawb-node/src/state.rs | 13 ++++++---- 7 files changed, 71 insertions(+), 32 deletions(-) diff --git a/.env.example b/.env.example index a4d66a7a..aea17f61 100644 --- a/.env.example +++ b/.env.example @@ -175,6 +175,23 @@ GITLAWB_MAX_CONCURRENT_PIN_TASKS=8 # Default 16. GITLAWB_MAX_CONCURRENT_READS_PER_CALLER=16 +# Two further per-source concurrency caps exist on the PUSH side but have NO +# environment variable of their own: both are DERIVED as +# GITLAWB_MAX_CONCURRENT_GIT_PUSHES / 8, with a floor of 1. They are not settable +# independently, so raising the push pool raises both proportionally, and the +# default push pool of 32 gives each a cap of 4. +# * The anon receive-pack info/refs advertisement cap: one source IP may hold at +# most this many slots in the dedicated advert pool, so saturating that pool +# takes ~8 distinct source IPs. +# * The authenticated receive-pack POST cap: one source IP may hold at most this +# many WRITE-pool slots. This one is load-bearing for push availability, since +# it is acquired before the global write permit: without it, one host minting +# disposable did:key identities could open enough slow pushes to monopolize the +# write pool and 503 every other source (owner enforcement defaults off, and +# the push rate limiter caps arrival rate, not in-flight concurrency). +# Keyed on the resolved source IP, never the DID, so a DID farm does not defeat +# them; keying granularity follows GITLAWB_TRUSTED_PROXY like the read cap above. + # ── Push rate limiting (git-receive-pack flood brake) ───────────────────── # Max receive-pack requests (info/refs advertisement + push POST) per client # IP per hour. 0 disables. Default 600. @@ -207,9 +224,12 @@ GITLAWB_IPFS_MAX_REPO_VISITS=1024 # lifetime (all stages of the whole scan). Per-stage clamps bound the acquire # and walk stages to the remaining budget, and no stage starts once it is # exhausted (the scan then sheds a retryable 503). The object-type probe and -# content-read cat-file subprocesses are budget-checked before starting but -# have NO duration bound of their own: a hung git probe (corrupt pack, stuck -# filesystem) holds the request's walk slot for the full duration of the hang. +# content-read cat-file subprocesses are budget-checked before starting AND each +# run under their own deadline (the lesser of GITLAWB_GIT_SERVICE_TIMEOUT_SECS and +# the remaining budget), reaped via process-group teardown, so a hung cat-file +# cannot hold the request's walk slot past it. One hang path is still unbounded: +# the probe's object-store readability check is a plain filesystem sweep with +# nothing to reap, so a wedged filesystem can hold the slot past the deadline. # Default 600. GITLAWB_IPFS_REQUEST_BUDGET_SECS=600 # Max /ipfs/{cid} requests per client IP per hour (route flood brake, distinct diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index d9f4c01b..f5420324 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -71,9 +71,16 @@ use crate::visibility::{visibility_check, Decision}; /// Request budget (F3): one absolute clock (`ipfs_request_budget_secs`) spans /// the whole admitted request. No stage (acquire, probe, walk, content read) /// starts once it is exhausted, and the acquire wait and walk deadline are -/// clamped to the remainder. The probe and content-read subprocesses have no -/// duration bound of their own past their pre-start budget check, so a hung -/// git probe holds the request's walk slot for the full duration of the hang. +/// clamped to the remainder. The probe and content-read subprocesses each ALSO +/// run under their own deadline, the lesser of `git_service_timeout_secs` and the +/// remaining budget, reaped by process-group teardown at that deadline, so a hung +/// `cat-file` cannot hold the request's walk slot past it. +/// +/// Residual, still true: the probe's object-store readability check +/// (`store::object_store_readable`, reached on the `missing` branch that a +/// random-CID spray drives) is a synchronous `read_dir` + `File::open` sweep with +/// no deadline and nothing to reap, so a wedged filesystem can still hold the +/// walk slot past the deadline. Same class as the D-state git survivor residual. /// /// Scope: this closes the direct unauthenticated scan, including the dangling /// case. A stale-public mirror row still serves withheld content (tracked @@ -274,10 +281,10 @@ pub async fn get_by_cid( // knob and tainting "budget"; the call site only breaks (the scan STOPS, // leaving this and every later candidate unproven, never a false 404). A // stage is never started with zero remaining; the probe and read - // subprocesses carry no internal duration clamp, so this pre-start check is - // their entire bound (a hung one holds the request's walk slot for the - // duration of the hang). The acquire and walk stages clamp their deadlines - // to the returned remainder. + // subprocesses additionally carry their own deadline (the lesser of + // git_service_timeout_secs and the remainder, reaped by process-group + // teardown), so this pre-start check is a gate, not their entire bound. The + // acquire and walk stages clamp their deadlines to the returned remainder. fn budget_gate( truncated_by: &mut Vec<&'static str>, deadline: std::time::Instant, diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index aedf03e6..3244bc29 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -647,11 +647,12 @@ pub async fn git_info_refs( // acquired AFTER the visibility + push-rate gates (KTD7) so a denied or // rate-limited request never consumes a slot; held for the whole op. The // upload-pack advertisement is bounded on the read pool (git_read_per_caller). - // The receive-pack advertisement draws from the write pool, so it is bounded per - // source (git_push_advert_per_caller) instead: without this, an anonymous - // multi-source flood of push-handshake advertisements could hold the write pool's - // slots across acquire_fresh and shed authenticated pushes, since the per-IP push - // rate limiter caps rate, not concurrency (#174 review fix). + // The receive-pack advertisement draws from its own dedicated advert pool + // (git_push_advert_semaphore, see the _permit block below), so it is bounded per + // source by git_push_advert_per_caller instead: without this, an anonymous + // multi-source flood of push-handshake advertisements could hold every advert-pool + // slot across acquire_fresh and shed other sources' advertisements, since the + // per-IP push rate limiter caps rate, not concurrency (#174 review fix). let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); let _caller_permit = if service == "git-receive-pack" { acquire_read_caller_permit( @@ -4316,10 +4317,10 @@ mod tests { ); } - /// #174 (review fix): the anon-reachable receive-pack advertisement draws from - /// the write pool, so it is bounded per source by `git_push_advert_per_caller` to - /// stop one source from monopolizing the write pool and shedding authenticated - /// pushes. Fill one source IP's advert slot; its next receive-pack advertisement + /// #174 (review fix): the anon-reachable receive-pack advertisement draws from its + /// own dedicated advert pool, so it is bounded per source by + /// `git_push_advert_per_caller` to stop one source from monopolizing that pool and + /// shedding other sources' advertisements. Fill one source IP's advert slot; its next receive-pack advertisement /// sheds 503, while a different source and the upload-pack advertisement are /// unaffected. Remove the advert-cap acquisition and the same-source assertion /// goes green-not-503. @@ -4358,7 +4359,7 @@ mod tests { assert_eq!( router.oneshot(req).await.unwrap().status(), StatusCode::SERVICE_UNAVAILABLE, - "a source at its receive-pack advertisement cap must shed 503, so it cannot monopolize the write pool" + "a source at its receive-pack advertisement cap must shed 503, so it cannot monopolize the advert pool" ); // A DIFFERENT source keeps its own advert budget -> not shed. diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index d1b98807..5ba10584 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -470,9 +470,12 @@ pub struct Config { /// only the acquire and walk stages (overshoot there is the walk watchdog's /// SIGTERM grace + SIGKILL settle); the `object_type` / /// `read_object_content` probe subprocesses are budget-checked before they - /// start but have NO duration bound of their own, so a hung git probe - /// (corrupt pack, stuck filesystem) holds the request's walk slot for the - /// full duration of the hang. + /// start AND each run under their own deadline (the lesser of + /// `git_service_timeout_secs` and the remaining budget), reaped by + /// process-group teardown, so a hung `cat-file` cannot hold the request's walk + /// slot past it. Still unbounded: the probe's `object_store_readable` check is a + /// synchronous filesystem sweep with nothing to reap, so a wedged filesystem can + /// hold the slot past the deadline. /// Must be positive. Default: 600s (10 min), matching /// `git_service_timeout_secs` so a single full-length walk still fits. #[arg( diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index e69c4ceb..2d94b84d 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -801,7 +801,10 @@ mod tests { assert!( err.downcast_ref::() .is_some(), - "a hung walk must abort with GitServiceTimeout (mapped to 504), got: {err}" + // `{err:#}` prints the whole anyhow chain. Plain `{err}` shows only the top + // context ("failed to spawn git for-each-ref") and drops the underlying io + // error, which left a real beta-lane CI failure undiagnosable. + "a hung walk must abort with GitServiceTimeout (mapped to 504), got: {err:#}" ); // The recorded process-group leader must be gone: the watchdog reaps the diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 1fd26979..5843d0ab 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -420,9 +420,11 @@ async fn main() -> Result<()> { config.max_concurrent_reads_per_caller, ), // Per-source cap on the receive-pack advertisement, sized to an eighth of the - // write pool (min 1): a single source can hold at most this many write-pool - // slots via the anon advertisement, so saturating the pool takes ~8 distinct - // source IPs, each also rate-limited (#174). + // write pool (min 1): a single source can hold at most this many slots in the + // DEDICATED advert pool (git_push_advert_semaphore, disjoint from the write + // pool), so saturating that pool takes ~8 distinct source IPs, each also + // rate-limited (#174). Sized off the write pool only because the advert pool + // is created at the same size; an advert flood cannot touch a write permit. git_push_advert_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( (config.max_concurrent_git_pushes / 8).max(1), ), diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 1f0f8c81..77755932 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -169,11 +169,14 @@ pub struct AppState { /// advertisement. pub git_read_per_caller: crate::rate_limit::PerCallerConcurrency, /// Per-source concurrency sub-cap on the anon-reachable receive-pack `info/refs` - /// advertisement: each source IP may hold at most a small share of the write - /// pool, so a multi-source flood of push-handshake advertisements cannot - /// saturate `git_write_semaphore` and shed authenticated pushes (#174). Sized as - /// a fraction of `max_concurrent_git_pushes`, so filling the write pool takes many - /// distinct source IPs (each also braked by the per-IP push rate limiter). + /// advertisement: each source IP may hold at most a small share of the DEDICATED + /// advert pool (`git_push_advert_semaphore`), so a multi-source flood of + /// push-handshake advertisements cannot saturate that pool and shed other sources' + /// advertisements (#174). An advert flood cannot reach `git_write_semaphore` at + /// all, since the two pools are disjoint. Sized as a fraction of + /// `max_concurrent_git_pushes` because the advert pool is created at the same size, + /// so filling it takes many distinct source IPs (each also braked by the per-IP + /// push rate limiter). pub git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency, /// Per-source concurrency sub-cap on the authenticated `git-receive-pack` POST: /// each source IP may hold at most a small share of `git_write_semaphore`, so one From 701a5e3428e1c1e758147d8bb63bbe41c272da74 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 21:48:49 -0500 Subject: [PATCH 101/124] docs: normalize README punctuation and record the two admission doc changes (#62) Two table rows needed correcting for the changes in this PR: the GITLAWB_GIT_SERVICE_TIMEOUT_SECS row now says the classification walk and the pack serve share one deadline on the path-scoped upload-pack path, so the value bounds them combined; and the GITLAWB_IPFS_REQUEST_BUDGET_SECS row no longer claims the probe and content-read subprocesses are unbounded, while keeping the object-store-readability sweep named as a live residual. The other thirteen changed lines are punctuation only, no wording changes: em dashes replaced with colons, commas, or sentence breaks, and one pair of curly quotes straightened. The repo's prose-style hook validates a whole file rather than a diff, so these pre-existing characters were rejecting every edit to README.md, including the two rows above. --- README.md | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 6789cd80..f4ba0f9e 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Gitlawb Node is the open-source node software behind the Gitlawb network. It lets anyone run a self-hosted node, publish repositories under a DID, sign writes with Ed25519 HTTP signatures, replicate git activity across peers, and move toward a resilient app-delivery network where code and build assets can be served closer to users. -Gitlawb is not trying to be only “another git host.” The long-term direction is: +Gitlawb is not trying to be only "another git host." The long-term direction is: ```txt Decentralized GitHub @@ -34,12 +34,12 @@ This is a Rust workspace with four crates: Most git hosting today depends on a small number of centralized platforms. Gitlawb Nodes are designed for a different model: -- **Own your identity** — every user, agent, and node is an Ed25519 keypair represented as `did:key:z6Mk...`. -- **Signed writes by default** — write requests use RFC 9421 HTTP Signatures instead of passwords. -- **Git-native transport** — repositories are still real git repositories served over smart HTTP. -- **Agent-native workflows** — the `gl` CLI and MCP server expose repo, issue, task, PR, and UCAN flows to AI agents. -- **Peer-aware delivery** — nodes can announce, discover, gossip, and sync with each other. -- **App CDN direction** — the network can evolve from decentralized code storage into code + asset + app delivery. +- **Own your identity**: every user, agent, and node is an Ed25519 keypair represented as `did:key:z6Mk...`. +- **Signed writes by default**: write requests use RFC 9421 HTTP Signatures instead of passwords. +- **Git-native transport**: repositories are still real git repositories served over smart HTTP. +- **Agent-native workflows**: the `gl` CLI and MCP server expose repo, issue, task, PR, and UCAN flows to AI agents. +- **Peer-aware delivery**: nodes can announce, discover, gossip, and sync with each other. +- **App CDN direction**: the network can evolve from decentralized code storage into code + asset + app delivery. --- @@ -189,14 +189,14 @@ For public-network use, make sure `GITLAWB_NODE` points to the node you want. Th Public nodes (e.g. `node.gitlawb.com`) require two things on writes: -1. **RFC 9421 HTTP Signatures** — every write is signed by your identity key. `gl` +1. **RFC 9421 HTTP Signatures**: every write is signed by your identity key. `gl` and the `git-remote-gitlawb` helper do this automatically. An old/unsigned CLI fails with `401 not_an_agent`; `gl` will tell you to upgrade and register. 2. **An iCaptcha proof** on the spam-gated writes (**repo create, fork, register**). `gl` solves this for you: on the node's `403 icaptcha_proof_required` it reads the `x-icaptcha-url` / `x-icaptcha-level` hints, requests a challenge, solves it locally (arithmetic / algebra / sequence), and **retries the same signed request** - with the `x-icaptcha-proof` header — no manual steps, no env vars. + with the `x-icaptcha-proof` header. No manual steps, no env vars. ```bash gl identity new # create did:key identity @@ -215,14 +215,14 @@ Notes: - **Proofs are short-lived (~5 min TTL) and single-use.** If one expires between solving and use, the client transparently solves a fresh one and retries. - **What needs what:** create / fork / register are signed **and** iCaptcha-gated; - `git push` is **signed-only** (owner signature is the gate — no per-push challenge); + `git push` is **signed-only** (owner signature is the gate, no per-push challenge); reads (clone / fetch / `repo info`) need no proof. A non-existent repo returns a clear `404`, never a placeholder. - **API-key iCaptcha deployments:** set `GITLAWB_ICAPTCHA_URL` to your iCaptcha origin and `GITLAWB_ICAPTCHA_API_KEY` to its key. The client only talks to an `https` origin whose host is allowlisted (that URL or the public default), and - sends the bearer token **only** to your configured origin — never to a URL a - node advertises — so a hostile node can't capture the key or redirect the solve. + sends the bearer token **only** to your configured origin, never to a URL a + node advertises, so a hostile node can't capture the key or redirect the solve. --- @@ -343,14 +343,14 @@ Important node settings: | `GITLAWB_REQUIRE_SIGNED_PEER_WRITES` | Require signed peer announce/sync writes. | | `GITLAWB_AUTO_SYNC` | Enable automatic sync from known peers. | | `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. | -| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. | +| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. On the path-scoped upload-pack path the classification walk and the pack serve share ONE deadline, so this value bounds their combined duration rather than granting each stage a full budget: a walk that consumes it leaves the serve nothing and the clone gets a 504. Serving large path-scoped repos may therefore need a higher value than they did when each stage was budgeted separately. | | `GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS` | Max seconds the storage-acquisition phase (Tigris HEAD/GET, push advisory-lock) of a served git op may run before the request is shed with a 503, separate from the git-run timeout. The concurrency permit is released on expiry so a stalled backend cannot pin the pool. Default 30. | | `GITLAWB_REPO_LEASE_MAX_WAITERS` | Max pushes parked at once waiting for the same repo's write lease. Each waiter pins its buffered pack body, so this bounds that memory for a hot repo; past the cap the newest push sheds a 503 + Retry-After instead of queueing. Pushes to other repos are unaffected, and the lease holder is not counted. Default 8. | | `GITLAWB_MAX_CONCURRENT_IPFS_WALKS` | Max concurrent `GET /ipfs/{cid}` visibility walks across all callers (own pool, disjoint from the served-git pools); over-cap sheds 503. Default 32. | | `GITLAWB_IPFS_WALK_PER_SOURCE` | Max concurrent `/ipfs` walks a single source IP may hold. Default 4. | | `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max expensive path-scope visibility walks per `/ipfs/{cid}` request; over-cap repos are skipped and the scan continues, shedding a retryable 503 (not a false 404) if the object is then found nowhere. Default 64. | -| `GITLAWB_IPFS_MAX_REPO_VISITS` | Ceiling on repos one `/ipfs/{cid}` request may visit (acquire + probe) past the visibility gate — also the worst-case per-request Tigris fetch count. On exhaustion the scan stops with a retryable 503. Default 1024. | -| `GITLAWB_IPFS_REQUEST_BUDGET_SECS` | Absolute wall-clock budget for one admitted `/ipfs/{cid}` request's acquire+walk lifetime. Per-stage clamps bound the acquire and walk stages to the remaining budget, and no stage starts once it is exhausted; the scan then stops with a retryable 503. The object-type probe and content-read `cat-file` subprocesses are budget-checked before starting but have no duration bound of their own, so a hung git probe (corrupt pack, stuck filesystem) holds the request's walk slot for the full duration of the hang. Default 600. | +| `GITLAWB_IPFS_MAX_REPO_VISITS` | Ceiling on repos one `/ipfs/{cid}` request may visit (acquire + probe) past the visibility gate. Also the worst-case per-request Tigris fetch count. On exhaustion the scan stops with a retryable 503. Default 1024. | +| `GITLAWB_IPFS_REQUEST_BUDGET_SECS` | Absolute wall-clock budget for one admitted `/ipfs/{cid}` request's acquire+walk lifetime. Per-stage clamps bound the acquire and walk stages to the remaining budget, and no stage starts once it is exhausted; the scan then stops with a retryable 503. The object-type probe and content-read `cat-file` subprocesses are budget-checked before starting and each also run under their own deadline (the lesser of `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` and the remaining budget), reaped via process-group teardown, so a hung `cat-file` cannot hold the request's walk slot past it. One hang path is still unbounded: the probe's object-store readability check is a plain filesystem sweep with nothing to reap, so a wedged filesystem can hold the slot past the deadline. Default 600. | | `GITLAWB_IPFS_RATE_LIMIT` | Max `/ipfs/{cid}` requests per client IP per hour (route flood brake). 0 disables. Default 600. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | From 28a6ca4082f2ccff8e0ddac11cb8caf9b2703dff Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 22:11:43 -0500 Subject: [PATCH 102/124] fix(node): charge blocking-queue delay against the shared upload-pack deadline (#62) The shared deadline computed the walk's budget on the async side, before the task was queued on the blocking pool. withheld_blob_oids_bounded starts its own clock when it actually runs, so a budget measured at queue time handed the walk a full budget from whenever the pool got to it: the read permit was held for queue delay PLUS the budget, and only the serve phase saw the outer deadline already spent. Computing the remainder inside the closure charges that delay against the shared deadline, which is what makes the ~1x bound in the docs true rather than approximate. Not a regression from the previous commit (the old code passed a constant git_timeout, so the delay went uncharged there too, and the combined hold was queue delay plus two full budgets), but the claim this PR now prints in README and .env.example should be exact. The two shared-deadline tests still pass and still fail on revert; they cover the budget arithmetic, not the queue-saturation path. Reproducing that needs a runtime with the blocking pool pinned and parked, which the sqlx::test harness does not give a seam for, so the queue-delay path is reasoned, not executed. --- crates/gitlawb-node/src/api/repos.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 3244bc29..de357059 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1379,8 +1379,16 @@ pub async fn git_upload_pack( let caller_owned = caller.map(str::to_string); let is_public = record.is_public; let git_bin = state.git_bin.clone(); - let walk_budget = deadline.saturating_duration_since(std::time::Instant::now()); tokio::task::spawn_blocking(move || { + // Derive the walk's budget from the shared deadline HERE, inside the + // closure, not on the async side before the task is queued. The walk + // starts its own clock when it runs, so a budget computed at queue time + // would hand it a full budget measured from whenever the blocking pool + // got to it, leaving the permit held for queue-delay PLUS the budget. + // Computing it at task start charges that queue delay against the + // shared deadline, which is what makes the ~1x bound true rather than + // approximate. + let walk_budget = deadline.saturating_duration_since(std::time::Instant::now()); let withheld = visibility_pack::withheld_blob_oids_bounded( &path, &git_bin, From 4b69c450070c20d30244f25871a0c1c6632e6947 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 22:56:47 -0500 Subject: [PATCH 103/124] fix(node): skip an /ipfs absence re-probe the budget cannot pay for (#62) The absent-CID path disambiguates a clean `missing` by re-running the probe, but the re-probe took the same deadline with no check that any budget was left. A first probe that nearly exhausted the budget still spawned a second child that could only be reaped, and the watchdog's SIGTERM grace plus SIGKILL settle carried the whole call to roughly twice the budget. Measured on object_type_bounded with a 1s budget and a probe that burns 0.9s of it: two spawns and 2021ms elapsed. That is the same ~2x occupancy shape this PR exists to remove, on a different handler, and it is worse here than on the served-git path: /ipfs/{cid} carries only optional_signature, and an absent CID drives this branch once per repo, so an unauthenticated caller spraying well-formed absent CIDs pays the overshoot per repo. It also degraded an honest absence into a retryable 503. Re-probe only when the remaining budget covers what the first probe actually took, since the re-probe runs the same command. Below that, taint to Transient rather than spawn: an inconclusive disambiguation is not an absence verdict, so it must not become a false 404 either. The same 1s case is now one spawn and finishes inside the budget. Both introduced by this PR (d39fc8d, the F5 disambiguation); neither object_type_bounded nor the re-probe exists at the merge base, so this is not a pre-existing condition. The companion test pins that an ample budget still re-probes, so the check cannot silently disable the disambiguation it guards. --- crates/gitlawb-node/src/git/store.rs | 127 +++++++++++++++++++++++++-- 1 file changed, 122 insertions(+), 5 deletions(-) diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index a18ab472..084fa20e 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -438,6 +438,7 @@ pub fn object_type_bounded( sha256_hex: &str, deadline: std::time::Instant, ) -> std::result::Result, ProbeError> { + let probe_started = std::time::Instant::now(); match batch_check_probe(git_bin, repo_path, sha256_hex, deadline)? { BatchProbe::Present(ty) => Ok(Some(ty)), BatchProbe::Fault(detail) => Err(classify_store_fault(repo_path, detail)), @@ -453,11 +454,27 @@ pub fn object_type_bounded( repo_path.display() ))); } - // Store readable: re-probe once. Still `missing` on a confirmed-readable - // store is very likely truly absent (Ok(None)); a mid-repack race that - // resolved returns the type. This narrows, but cannot fully close, the - // concurrent-repack window (the readability check samples a different - // instant than the failing probe). + // Only re-probe if the budget can actually pay for it. The re-probe runs the + // SAME command, so it needs roughly what the first probe took; with less than + // that left, the child is spawned only to be reaped, and the watchdog's + // SIGTERM grace plus SIGKILL settle carries this call well past `deadline` + // (measured ~2x a 1s budget before this check existed). `/ipfs/{cid}` is + // anon-reachable and an absent CID drives this branch once per repo, so an + // unaffordable re-probe is pure overshoot on a permissionless path. An + // inconclusive disambiguation is NOT an absence verdict, so taint to a + // retryable Transient rather than spawn or return a false Ok(None). + let first_probe_took = probe_started.elapsed(); + if deadline.saturating_duration_since(std::time::Instant::now()) < first_probe_took { + return Err(ProbeError::Transient(anyhow::anyhow!( + "git cat-file inconclusive: no budget left for the confirming re-probe at {} (not an absence verdict)", + repo_path.display() + ))); + } + // Store readable and budget available: re-probe once. Still `missing` on a + // confirmed-readable store is very likely truly absent (Ok(None)); a + // mid-repack race that resolved returns the type. This narrows, but cannot + // fully close, the concurrent-repack window (the readability check samples a + // different instant than the failing probe). match batch_check_probe(git_bin, repo_path, sha256_hex, deadline)? { BatchProbe::Present(ty) => Ok(Some(ty)), BatchProbe::Fault(detail) => Err(classify_store_fault(repo_path, detail)), @@ -1021,4 +1038,104 @@ mod tests { never a false Ok(None) 404; got {res:?}" ); } + + /// #174 U1 follow-up (RED-before/GREEN-after): the absent-CID path must not spawn a + /// confirming re-probe it cannot afford. `object_type_bounded` disambiguates a clean + /// `missing` by re-running the probe, but the re-probe took the SAME deadline with no + /// check that any budget was left, so a first probe that nearly exhausted the budget + /// still spawned a second child that could only be reaped. The watchdog's SIGTERM + /// grace plus SIGKILL settle then carried the whole call to ~2x the budget, on a + /// route an unauthenticated caller drives for every repo by spraying absent CIDs. + /// + /// Load-bearing: remove the affordability check and this goes RED on both assertions + /// (a second spawn appears, and elapsed crosses 2x the budget). Measured before the + /// fix: spawns=2, elapsed 2021ms against a 1000ms budget. + #[cfg(unix)] + #[test] + fn absent_probe_skips_a_reprobe_it_cannot_afford() { + use std::os::unix::fs::PermissionsExt; + let td = tempfile::TempDir::new().unwrap(); + let bare = td.path().join("bare.git"); + std::fs::create_dir_all(bare.join("objects/pack")).unwrap(); + let log = td.path().join("spawns.log"); + let fake = td.path().join("fakegit"); + // Burns 0.9s of a 1s budget, then reports the structured absence token cleanly. + std::fs::write( + &fake, + format!( + "#!/bin/sh\necho call >> {}\nsleep 0.9\necho 'deadbeef missing'\nexit 0\n", + log.display() + ), + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + + let budget = std::time::Duration::from_millis(1000); + let deadline = std::time::Instant::now() + budget; + let started = std::time::Instant::now(); + let res = super::object_type_bounded(fake.to_str().unwrap(), &bare, "deadbeef", deadline); + let elapsed = started.elapsed(); + let spawns = std::fs::read_to_string(&log) + .map(|s| s.lines().count()) + .unwrap_or(0); + + assert_eq!( + spawns, 1, + "a re-probe with no remaining budget must not be spawned at all; a second \ + spawn here is a child created only to be reaped, and its teardown grace is \ + what pushes this call past the deadline" + ); + assert!( + elapsed < budget + std::time::Duration::from_millis(400), + "the call must not overshoot its deadline by the reap grace; got {elapsed:?} \ + against a {budget:?} budget (pre-fix this was ~2x the budget)" + ); + assert!( + matches!(res, Err(super::ProbeError::Transient(_))), + "an unaffordable disambiguation is NOT an absence verdict: it must taint to \ + a retryable Transient, never a false Ok(None) 404; got {res:?}" + ); + } + + /// Companion must-not-regress case for the affordability check above: with an ample + /// budget the confirming re-probe MUST still run, so the #174 F5 disambiguation is + /// intact and the check did not simply disable it. Two spawns, and a clean absence. + #[cfg(unix)] + #[test] + fn absent_probe_still_reprobes_when_the_budget_allows() { + use std::os::unix::fs::PermissionsExt; + let td = tempfile::TempDir::new().unwrap(); + let bare = td.path().join("bare.git"); + std::fs::create_dir_all(bare.join("objects/pack")).unwrap(); + let log = td.path().join("spawns.log"); + let fake = td.path().join("fakegit"); + std::fs::write( + &fake, + format!( + "#!/bin/sh\necho call >> {}\necho 'deadbeef missing'\nexit 0\n", + log.display() + ), + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + let res = super::object_type_bounded(fake.to_str().unwrap(), &bare, "deadbeef", deadline); + let spawns = std::fs::read_to_string(&log) + .map(|s| s.lines().count()) + .unwrap_or(0); + assert_eq!( + spawns, 2, + "with budget to spare the confirming re-probe must still run, or the \ + absence-vs-unreadable-pack disambiguation is gone" + ); + assert!( + matches!(res, Ok(None)), + "a clean `missing` twice on a readable store is a genuine absence; got {res:?}" + ); + } } From 28cbb50ec0de8f83675ca4b62d791772e6ff5d3d Mon Sep 17 00:00:00 2001 From: t Date: Thu, 30 Jul 2026 10:49:00 -0500 Subject: [PATCH 104/124] fix(node): bound the IPFS pin batch and let a slow healthy upload finish (#174) pin_new_objects runs under a pin_semaphore permit and that pool defers rather than sheds. Both sinks in ipfs_pin.rs built a bare reqwest::Client::new(), which carries no timeout, so a silent endpoint parked the pool indefinitely. Both now use the shared no-redirect client from build_http_client() through a module-local OnceLock, rather than the hand-rolled twin its docstring forbids. The client alone does not bound the hold: without a ceiling the loop is O(N) with N chosen by the pusher. A 120s batch budget is taken once at loop start and gated at the top of every iteration, so no object's work begins with zero budget left. The remainder is also passed to each add as a per-request timeout, which is what lets one large healthy upload run past the client's 10s default without letting the batch run forever. RequestBuilder::timeout replaces the client value for that request only and leaks nothing to other callers, verified by execution against the pinned reqwest. An earlier version of this commit classified a dead endpoint by reqwest::Error::is_timeout() || is_connect() and broke the batch. That was wrong in both directions and is gone. It missed the ordinary dead-peer shapes, since a peer closing mid-request yields IncompleteMessage and a peer RST mid-upload yields ConnectionReset, both observed with is_timeout and is_connect false, so a load-balancer idle timeout evaded it entirely. And it fired on a slow but alive endpoint, because .timeout() is a total deadline covering body upload, so a healthy multi-megabyte add was read as a dead endpoint and abandoned the rest of the push. A wall-clock bound needs neither judgement. The docstring now states what is bounded and what is not, rather than implying the hold is closed. Not bounded: the git read, since store::read_object shells two bare Command::output() calls with no timeout or process-group reaping and this loop does not run it under spawn_blocking; the DB round-trips; and the pool itself, since the Pinata replication task holds the same semaphore across a full git re-derivation with no deadline. Truncation is stated plainly: a batch stopped at the deadline leaves the rest unpinned and nothing sweeps them up, so recovery is opportunistic through a later full-scan push. Tests written first, both RED by assertion rather than by a harness timeout: pin_new_objects_stops_the_batch_at_its_deadline ... FAILED the batch must stop partway, not pin all five and not stall on the first: pinned 5 pin_new_objects_does_not_abandon_the_batch_on_a_slow_but_alive_endpoint ... FAILED a slow but progressing endpoint must pin both objects: an upload past the client's 10s default is not a dead endpoint left: 1, right: 2 Both guards mutation-proved load-bearing against present-but-wrong mutants, each red on its own message: the loop-top break widened to continue, and the per-request timeout replaced with None. Also bounds encrypted_pin.rs's pin_git_object call, where a hang wedged a repo's EncryptInflight key. Its Drop backstop covers unwind, not hang. Full suite 730 passed, 0 failed. fmt and clippy clean. --- crates/gitlawb-node/src/api/ipfs.rs | 24 +- crates/gitlawb-node/src/api/repos.rs | 9 +- crates/gitlawb-node/src/encrypted_pin.rs | 2 +- crates/gitlawb-node/src/ipfs_pin.rs | 522 ++++++++++++++++++++++- crates/gitlawb-node/src/sync.rs | 2 +- crates/gitlawb-node/src/test_support.rs | 19 + 6 files changed, 542 insertions(+), 36 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index f5420324..a539ff34 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -804,24 +804,6 @@ mod tests { .to_string() } - /// A local endpoint whose TCP accept succeeds instantly but that never writes - /// an HTTP response, so a Tigris HEAD against it stalls deterministically - /// until the caller's timeout. (A non-routable address hangs only if the - /// network blackholes the SYN — a fast RST would end the stall early.) The - /// accepted sockets are parked in the spawned task, which dies with the - /// test's runtime, so the peer never sees a close mid-test. - async fn silent_tigris_endpoint() -> String { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let endpoint = format!("http://{}", listener.local_addr().unwrap()); - tokio::spawn(async move { - let mut held = Vec::new(); - while let Ok((sock, _)) = listener.accept().await { - held.push(sock); - } - }); - endpoint - } - /// Fake git for the WALK only (`state.git_bin`): empty refs, `rev-parse` /// resolves, and each `rev-list` appends one line to `log` and prints nothing — /// every walked repo yields an EMPTY allowed-set (path-gate deny verdict) and @@ -1090,7 +1072,7 @@ mod tests { // Endpoint-pinned test client (no AWS_* env reads — env is racy under a // parallel test run); the silent local endpoint stalls the HEAD // deterministically. - let endpoint = silent_tigris_endpoint().await; + let endpoint = crate::test_support::silent_http_endpoint().await; let tigris = crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) .await; @@ -1154,7 +1136,7 @@ mod tests { // repo stays a fast local hit) and add a NEWER ghost row with no local // copy: its acquire consults the silent local endpoint and stalls to the // 1s timeout (endpoint-pinned test client, no AWS_* env reads). - let endpoint = silent_tigris_endpoint().await; + let endpoint = crate::test_support::silent_http_endpoint().await; let tigris = crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) .await; @@ -1679,7 +1661,7 @@ mod tests { // repo stays a fast local hit) and add a NEWER ghost row with no local // copy: its acquire consults the silent local endpoint and stalls past // the budget (endpoint-pinned test client, no AWS_* env reads). - let endpoint = silent_tigris_endpoint().await; + let endpoint = crate::test_support::silent_http_endpoint().await; let tigris = crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) .await; diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index de357059..67c2ddd0 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1148,7 +1148,14 @@ async fn pin_new_objects_gated( .acquire_owned() .await .expect("pin_semaphore is never closed"); - crate::ipfs_pin::pin_new_objects(ipfs_api, repo_path, object_list, db).await + crate::ipfs_pin::pin_new_objects( + ipfs_api, + repo_path, + object_list, + db, + crate::ipfs_pin::PIN_BATCH_BUDGET, + ) + .await } /// `repo_id` is passed explicitly rather than read from `ctx` so the two callers diff --git a/crates/gitlawb-node/src/encrypted_pin.rs b/crates/gitlawb-node/src/encrypted_pin.rs index 40e444e5..19b80651 100644 --- a/crates/gitlawb-node/src/encrypted_pin.rs +++ b/crates/gitlawb-node/src/encrypted_pin.rs @@ -170,7 +170,7 @@ pub async fn encrypt_and_pin( continue; } }; - let cid = match crate::ipfs_pin::pin_git_object(ipfs_api, oid, &envelope).await { + let cid = match crate::ipfs_pin::pin_git_object(ipfs_api, oid, &envelope, None).await { Ok(c) if !c.is_empty() => c, Ok(_) => { tracing::warn!(oid = %oid, "pin_git_object returned no cid; skipping"); diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 3b346190..765bd6b0 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -9,6 +9,36 @@ use anyhow::Result; use gitlawb_core::cid::Cid; +use std::time::{Duration, Instant}; + +/// Wall-clock ceiling on one [`pin_new_objects`] batch. +/// +/// The loop runs under a `pin_semaphore` permit and that pool defers rather than +/// sheds, so without a ceiling the hold is O(N) with N (the push's object count) +/// chosen by the pusher. This bounds the drain of a saturated pool instead. +/// +/// 120s is 12x the shared client's 10s whole-request ceiling, so a single large +/// healthy upload that needs more than the client default still has room to +/// finish (the per-request timeout is set to the remainder, not the default), +/// while a batch of them still cannot hold the permit indefinitely. Deliberately +/// a constant and not a config knob: the value only has to be large enough to be +/// uninteresting on a healthy node, and a knob is operator surface that would +/// have to be documented, validated, and kept meaningful. +pub const PIN_BATCH_BUDGET: Duration = Duration::from_secs(120); + +/// The shared outbound client for both IPFS sinks. +/// +/// `pin_new_objects` runs while holding a `pin_semaphore` permit and that pool +/// defers rather than sheds, so an unbounded await here parks the pool. A bare +/// `reqwest::Client::new()` has no timeout, which is exactly that. Built from +/// `crate::build_http_client` rather than a local builder: its docstring forbids +/// hand-rolling an equivalent, so that the redirect and timeout guarantees the +/// node's tests bind stay bound to the client every outbound path actually uses. +fn http_client() -> &'static reqwest::Client { + static CLIENT: std::sync::OnceLock = std::sync::OnceLock::new(); + CLIENT + .get_or_init(|| crate::build_http_client().expect("failed to build production http client")) +} /// Pin a single git object to the local IPFS/Kubo node. /// @@ -16,9 +46,20 @@ use gitlawb_core::cid::Cid; /// If empty the function returns `Ok("")` immediately. /// - `sha256_hex`: the git SHA-256 hex object ID (used only for logging). /// - `data`: raw git object content bytes (same bytes used for CID computation). +/// - `request_timeout`: overrides the shared client's whole-request timeout for +/// THIS request only. `RequestBuilder::timeout` replaces the client-level value +/// per request and leaks nothing to other calls on the same client, so the +/// batch loop can hand each add whatever is left of its budget without +/// loosening or tightening any other outbound path. `None` keeps the client's +/// own ceiling. /// /// Returns the CID string on success, or `""` when IPFS is not configured. -pub async fn pin_git_object(ipfs_api: &str, sha256_hex: &str, data: &[u8]) -> Result { +pub async fn pin_git_object( + ipfs_api: &str, + sha256_hex: &str, + data: &[u8], + request_timeout: Option, +) -> Result { if ipfs_api.is_empty() { return Ok(String::new()); } @@ -37,13 +78,26 @@ pub async fn pin_git_object(ipfs_api: &str, sha256_hex: &str, data: &[u8]) -> Re .mime_str("application/octet-stream")?; let form = reqwest::multipart::Form::new().part("file", part); - let client = reqwest::Client::new(); - let resp = client - .post(&url) - .multipart(form) + let mut req = http_client().post(&url).multipart(form); + if let Some(t) = request_timeout { + req = req.timeout(t); + } + + let resp = req .send() .await - .map_err(|e| anyhow::anyhow!("IPFS add request failed: {e}"))?; + // Keep the `reqwest::Error` as this error's source rather than + // formatting it away. Operators reading a pin failure want the concrete + // transport cause in the logged chain, not a single flattened line, and + // this module's tests downcast to it to prove a silent endpoint really + // surfaces as a timeout rather than as some other failure that happens + // to arrive in time. + // The context keeps the old message verbatim so the callers that log + // this at `%e` (here, `sync.rs`, `encrypted_pin.rs`) read the same. + .map_err(|e| { + let msg = format!("IPFS add request failed: {e}"); + anyhow::Error::new(e).context(msg) + })?; if !resp.status().is_success() { let status = resp.status(); @@ -55,7 +109,10 @@ pub async fn pin_git_object(ipfs_api: &str, sha256_hex: &str, data: &[u8]) -> Re // Kubo returns newline-delimited JSON; we only care about the last object // (there's typically just one for a single-file add). - let body = resp.text().await?; + let body = resp + .text() + .await + .map_err(|e| anyhow::anyhow!("IPFS add response body read failed: {e}"))?; let cid = body .lines() .filter(|l| !l.trim().is_empty()) @@ -76,13 +133,33 @@ pub async fn cat(ipfs_api: &str, cid: &str) -> Result> { return Err(anyhow::anyhow!("IPFS not configured")); } let url = format!("{}/api/v0/cat?arg={}", ipfs_api.trim_end_matches('/'), cid); - let resp = reqwest::Client::new().post(&url).send().await?; + let resp = http_client().post(&url).send().await?; if !resp.status().is_success() { return Err(anyhow::anyhow!("ipfs cat {cid}: {}", resp.status())); } Ok(resp.bytes().await?.to_vec()) } +/// The batch's remaining wall-clock, or `None` once it is spent, after logging +/// the truncation exactly once. +/// +/// Shaped like `api::ipfs`'s `budget_gate` on purpose: the nonzero-ness rides in +/// the returned value, so every call site must consume it as +/// `let Some(x) = ... else { break }` and a zero `Duration` can never reach a +/// request as its timeout. +fn batch_budget_gate(deadline: Instant, pinned: usize, unattempted: usize) -> Option { + let left = deadline.saturating_duration_since(Instant::now()); + if left.is_zero() { + tracing::warn!( + pinned, + unattempted, + "IPFS pin batch deadline reached; the remaining objects are left unpinned" + ); + return None; + } + Some(left) +} + /// Pin any of the given candidate git objects that are not yet recorded in /// `pinned_cids`. /// @@ -90,8 +167,50 @@ pub async fn cat(ipfs_api: &str, cid: &str) -> Result> { /// applies `visibility_pack::replicable_objects` on the delta path or the /// `..._fail_closed` filter on the full-scan path before calling, so this /// function never sees a withheld blob. `repo_path` is still needed to read each -/// object's bytes. The twin in `pinata.rs` mirrors this shape — change both in -/// lockstep. +/// object's bytes. +/// +/// # What `batch_budget` does and does not bound +/// +/// The loop holds a `pin_semaphore` permit and that pool defers rather than +/// sheds, so the hold has to be bounded by something other than the pusher's +/// object count. Two things here are: +/// +/// - this loop's own wall-clock: the deadline is taken once at loop start and +/// checked at the top of every iteration, so no object's work begins with zero +/// budget left. It is a gate, not a hard ceiling, since a started iteration +/// still runs to completion; +/// - each HTTP add: `pin_git_object` is handed the remainder measured at the top +/// of the iteration as its per-request timeout, which is what lets one large +/// healthy upload run past the shared client's 10s default without letting the +/// batch run forever. The DB check and the git read spend a little of that +/// remainder before the request starts, so the add can finish marginally after +/// the deadline. +/// +/// Three things are NOT bounded, and the gate cannot fix any of them: +/// +/// - the git read. `store::read_object` composes two bare +/// `std::process::Command::output()` calls with no timeout and no +/// process-group reaping, and this loop does not run it under +/// `spawn_blocking`, so one hung `git cat-file` overruns the deadline and +/// blocks a runtime worker thread while it does. +/// - the DB round-trips (`is_pinned`, `record_pinned_cid`). +/// - the pool. `api::repos` acquires the same `pin_semaphore` for the Pinata +/// replication task and holds it across a full git re-derivation plus +/// `pinata::pin_new_objects`, neither of which has a deadline. This change +/// bounds this loop's hold, not the semaphore's worst-case queue. +/// +/// # Truncation semantics +/// +/// A batch stopped at the deadline leaves its remaining objects unpinned, and +/// nothing sweeps them up afterwards. There is no reconciliation pass over +/// `pinned_cids`; recovery is opportunistic, happening only if some later push +/// on the repo takes the full-scan fallback (`push_delta::list_all_objects`) and +/// re-derives the whole object set, which then re-offers the skipped OIDs. +/// +/// The twin in `pinata.rs` mirrors this loop's shape but NOT its bound: it has +/// no batch deadline and no per-request override, so the two are no longer at +/// parity here. Everything else about the shape (the skip-if-pinned check, the +/// warn-and-continue arm, the returned pairs) still changes in lockstep. /// /// Returns a list of `(sha256_hex, cid)` pairs for objects pinned this call. pub async fn pin_new_objects( @@ -99,14 +218,23 @@ pub async fn pin_new_objects( repo_path: &std::path::Path, object_list: Vec, db: &crate::db::Db, + batch_budget: Duration, ) -> Vec<(String, String)> { if ipfs_api.is_empty() { return vec![]; } + let deadline = Instant::now() + batch_budget; + let total = object_list.len(); let mut pinned = Vec::new(); - for sha in object_list { + for (attempted, sha) in object_list.into_iter().enumerate() { + // Top of the iteration, before any of this object's work: an object is + // never started with zero budget left. The remainder becomes this add's + // request timeout below. + let Some(budget_left) = batch_budget_gate(deadline, pinned.len(), total - attempted) else { + break; + }; // Skip if already pinned match db.is_pinned(&sha).await { Ok(true) => continue, @@ -128,7 +256,7 @@ pub async fn pin_new_objects( }; // Pin to IPFS - match pin_git_object(ipfs_api, &sha, &data).await { + match pin_git_object(ipfs_api, &sha, &data, Some(budget_left)).await { Ok(cid) if !cid.is_empty() => { if let Err(e) = db.record_pinned_cid(&sha, &cid).await { tracing::warn!(sha = %sha, err = %e, "failed to record pinned CID in DB"); @@ -144,3 +272,373 @@ pub async fn pin_new_objects( pinned } + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + /// Write `n` loose blobs into a fresh bare repo and return their oids. + /// `read_object` shells to `git cat-file`, so the objects must genuinely + /// exist on disk — a fabricated oid would `continue` past the pin call and + /// the loop scenario below would prove nothing. + fn seed_loose_blobs(repo_path: &std::path::Path, n: usize) -> Vec { + crate::git::store::init_bare(repo_path).expect("init bare repo"); + (0..n) + .map(|i| { + let mut cmd = std::process::Command::new("git"); + cmd.args(["hash-object", "-w", "--stdin"]) + .current_dir(repo_path) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()); + let mut child = cmd.spawn().expect("spawn git hash-object"); + { + use std::io::Write; + child + .stdin + .as_mut() + .expect("stdin") + .write_all(format!("pin loop object {i}\n").as_bytes()) + .expect("write stdin"); + } + let out = child.wait_with_output().expect("hash-object output"); + assert!( + out.status.success(), + "git hash-object: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }) + .collect() + } + + /// A live endpoint that answers every add with `500`. Counts the requests it + /// received so a test can tell "the loop kept going" from "the loop stopped", + /// which the returned pin list cannot (it is empty either way). Reads the + /// full request, headers plus the `Content-Length` body, before answering: + /// responding early and closing would surface as a write failure on the + /// client and turn a rejection into something else. + async fn rejecting_endpoint( + requests: std::sync::Arc, + ) -> String { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { + while let Ok((mut sock, _)) = listener.accept().await { + requests.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + tokio::spawn(async move { + let mut acc = Vec::new(); + let mut buf = [0u8; 4096]; + loop { + let n = match sock.read(&mut buf).await { + Ok(0) | Err(_) => break, + Ok(n) => n, + }; + acc.extend_from_slice(&buf[..n]); + // Once the headers are complete, keep reading until the + // declared body has arrived. + if let Some(hdr_end) = + acc.windows(4).position(|w| w == b"\r\n\r\n").map(|p| p + 4) + { + let headers = String::from_utf8_lossy(&acc[..hdr_end]).to_lowercase(); + let len: usize = headers + .lines() + .find_map(|l| l.strip_prefix("content-length:")) + .and_then(|v| v.trim().parse().ok()) + .unwrap_or(0); + if acc.len() >= hdr_end + len { + break; + } + } + } + let _ = sock + .write_all( + b"HTTP/1.1 500 Internal Server Error\r\nContent-Length: 0\r\n\r\n", + ) + .await; + let _ = sock.flush().await; + }); + } + }); + endpoint + } + + /// A sleeping-but-live endpoint. Answers `200` with an empty body after + /// `delays[i]` for the i-th request it accepts (the last entry repeats), so + /// a test can make one add slow and the next fast. Drains the full request, + /// headers plus the declared `Content-Length` body, before sleeping: exactly + /// as in `rejecting_endpoint`, answering early and closing would surface as + /// a write failure on the client and turn a slow-but-healthy add into a + /// different failure shape. + /// + /// An empty body is a successful pin: `pin_git_object` falls back to the CID + /// it computed from the bytes when the response carries no `Hash`. + async fn delaying_endpoint(delays: Vec) -> String { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { + let mut seen = 0usize; + while let Ok((mut sock, _)) = listener.accept().await { + let delay = *delays + .get(seen) + .or_else(|| delays.last()) + .unwrap_or(&Duration::ZERO); + seen += 1; + tokio::spawn(async move { + let mut acc = Vec::new(); + let mut buf = [0u8; 4096]; + loop { + let n = match sock.read(&mut buf).await { + Ok(0) | Err(_) => break, + Ok(n) => n, + }; + acc.extend_from_slice(&buf[..n]); + if let Some(hdr_end) = + acc.windows(4).position(|w| w == b"\r\n\r\n").map(|p| p + 4) + { + let headers = String::from_utf8_lossy(&acc[..hdr_end]).to_lowercase(); + let len: usize = headers + .lines() + .find_map(|l| l.strip_prefix("content-length:")) + .and_then(|v| v.trim().parse().ok()) + .unwrap_or(0); + if acc.len() >= hdr_end + len { + break; + } + } + } + tokio::time::sleep(delay).await; + let _ = sock + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n") + .await; + let _ = sock.flush().await; + }); + } + }); + endpoint + } + + /// A `tracing` sink a test can read back, so the deadline warn can be + /// asserted on rather than assumed. Installed with `set_default`, which is + /// thread-local and scoped to the guard, so it cannot bleed into any other + /// test in the binary. + #[derive(Clone, Default)] + struct CapturedLogs(std::sync::Arc>>); + + impl CapturedLogs { + fn text(&self) -> String { + String::from_utf8_lossy(&self.0.lock().unwrap()).to_string() + } + } + + impl std::io::Write for CapturedLogs { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for CapturedLogs { + type Writer = CapturedLogs; + fn make_writer(&'a self) -> Self::Writer { + self.clone() + } + } + + fn capture_logs() -> (CapturedLogs, tracing::subscriber::DefaultGuard) { + let logs = CapturedLogs::default(); + let subscriber = tracing_subscriber::fmt() + .with_writer(logs.clone()) + .with_max_level(tracing::Level::WARN) + .with_ansi(false) + .finish(); + let guard = tracing::subscriber::set_default(subscriber); + (logs, guard) + } + + /// The add sink must be built from the shared no-redirect client, and the + /// per-request override must actually reach the request. Against a silent + /// endpoint (accept succeeds, no response ever written) a bare + /// `reqwest::Client::new()` blocks forever. With `Some(2s)` the call must + /// come back well inside that, and as a reqwest timeout: the elapsed + /// assertion is the real RED signal, and the outer `tokio::time::timeout` + /// is only a wedge guard so a regression fails the suite instead of hanging + /// it (`cargo test` has no per-test timeout). The old "no elapsed assertion + /// because the timeout is a process-global `OnceLock`" caveat no longer + /// holds now that `request_timeout` overrides it per call. + #[tokio::test] + async fn pin_git_object_against_silent_endpoint_errors_within_its_own_timeout() { + let endpoint = crate::test_support::silent_http_endpoint().await; + let started = std::time::Instant::now(); + let inner = tokio::time::timeout( + Duration::from_secs(30), + pin_git_object( + &endpoint, + "deadbeef", + b"some object bytes\n", + Some(Duration::from_secs(2)), + ), + ) + .await + .expect("wedge guard: pin_git_object must return long before 30s"); + let elapsed = started.elapsed(); + let err = inner.expect_err("a silent endpoint must not surface as a successful pin"); + assert!( + elapsed < Duration::from_secs(5), + "the 2s per-request override must bound this call, not the client's own ceiling (took {elapsed:?})" + ); + assert!( + err.downcast_ref::() + .is_some_and(|e| e.is_timeout()), + "a silent endpoint must surface as a reqwest timeout, preserved as the error's source: {err:#}" + ); + } + + /// The second unhardened sink, reached from `sync.rs`. Same shape as above. + #[tokio::test] + async fn cat_against_silent_endpoint_errors_within_its_own_timeout() { + let endpoint = crate::test_support::silent_http_endpoint().await; + let inner = tokio::time::timeout(Duration::from_secs(30), cat(&endpoint, "bafkqaaa")) + .await + .expect( + "cat must return before the outer 30s timeout — an unbounded client hangs here", + ); + assert!( + inner.is_err(), + "a silent endpoint must surface as a transport error, not successful bytes" + ); + } + + /// The permit-hold bound. `pin_new_objects` runs under a deferring + /// `pin_semaphore`, so without a batch deadline the hold is O(N) with N + /// chosen by the pusher. Five objects against an endpoint that takes 2s + /// each, under a 5.5s budget, must stop partway: only the first two can + /// finish inside the budget, so the batch is truncated and the remainder is + /// left unattempted with one warn naming how many. + /// + /// The windows are deliberately loose. Three pins would need every add to + /// answer in under 1.83s, which the endpoint's own 2s sleep forbids, and one + /// pin needs only the first add to land inside 5.5s, so both bounds hold + /// with more than a second of slack on a loaded box. + #[sqlx::test] + async fn pin_new_objects_stops_the_batch_at_its_deadline(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("slow.git"); + let oids = seed_loose_blobs(&repo_path, 5); + let endpoint = delaying_endpoint(vec![Duration::from_secs(2)]).await; + + let (logs, _guard) = capture_logs(); + let pinned = tokio::time::timeout( + Duration::from_secs(30), + pin_new_objects( + &endpoint, + &repo_path, + oids, + &db, + Duration::from_millis(5500), + ), + ) + .await + .expect("wedge guard: a 5.5s budget cannot take 30s"); + + assert!( + (1..=3).contains(&pinned.len()), + "the batch must stop partway, not pin all five and not stall on the first: pinned {}", + pinned.len() + ); + let text = logs.text(); + let warns: Vec<&str> = text + .lines() + .filter(|l| l.contains("pin batch deadline reached")) + .collect(); + assert_eq!( + warns.len(), + 1, + "the deadline must be reported exactly once for the batch, not per object: {text}" + ); + let unattempted: usize = warns[0] + .split("unattempted=") + .nth(1) + .and_then(|s| { + s.split(|c: char| !c.is_ascii_digit()) + .next() + .and_then(|d| d.parse().ok()) + }) + .unwrap_or_else(|| panic!("the deadline warn must name the unattempted count: {text}")); + assert!( + unattempted >= 1 && unattempted + pinned.len() <= 5, + "unattempted={unattempted} with {} pinned is not a partial batch of five", + pinned.len() + ); + } + + /// The must-not case, and the regression that killed the old transport + /// classifier: an endpoint that is slow but genuinely alive must NOT cost + /// the rest of the batch. The first add takes 13s, past the shared client's + /// 10s ceiling, which is exactly what the classifier used to read as a dead + /// endpoint; the second is immediate. Under a 90s budget both must pin, so + /// this fails if the per-request timeout is left at the client default and + /// fails if any error arm breaks the loop. Two objects, not one, because + /// with one object "did not abandon the rest" would be vacuous. + #[sqlx::test] + async fn pin_new_objects_does_not_abandon_the_batch_on_a_slow_but_alive_endpoint( + pool: sqlx::PgPool, + ) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("slow_alive.git"); + let oids = seed_loose_blobs(&repo_path, 2); + let endpoint = + delaying_endpoint(vec![Duration::from_secs(13), Duration::from_millis(0)]).await; + + let pinned = tokio::time::timeout( + Duration::from_secs(60), + pin_new_objects(&endpoint, &repo_path, oids, &db, Duration::from_secs(90)), + ) + .await + .expect("wedge guard: a 13s add plus an immediate one cannot take 60s"); + assert_eq!( + pinned.len(), + 2, + "a slow but progressing endpoint must pin both objects: an upload past the client's \ + 10s default is not a dead endpoint" + ); + } + + /// The must-not case for the warn-and-continue arm: a live endpoint + /// rejecting each object with `500` is a per-object failure, so the loop + /// must still warn and continue and every object must be attempted. Without + /// this, a `break` arm could be reintroduced and the deadline test above + /// would not notice. + #[sqlx::test] + async fn pin_new_objects_continues_past_a_per_object_rejection(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("rejecting.git"); + let oids = seed_loose_blobs(&repo_path, 4); + let requests = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let endpoint = rejecting_endpoint(std::sync::Arc::clone(&requests)).await; + + let pinned = tokio::time::timeout( + Duration::from_secs(30), + pin_new_objects(&endpoint, &repo_path, oids, &db, Duration::from_secs(60)), + ) + .await + .expect("a rejecting endpoint answers immediately, so this cannot take 30s"); + assert!(pinned.is_empty(), "every add was rejected"); + assert_eq!( + requests.load(std::sync::atomic::Ordering::SeqCst), + 4, + "a non-2xx rejection is per-object: all four objects must still be attempted" + ); + } +} diff --git a/crates/gitlawb-node/src/sync.rs b/crates/gitlawb-node/src/sync.rs index 5c6864fd..ab97747f 100644 --- a/crates/gitlawb-node/src/sync.rs +++ b/crates/gitlawb-node/src/sync.rs @@ -605,7 +605,7 @@ async fn replicate_encrypted_blobs( continue; } }; - match crate::ipfs_pin::pin_git_object(ipfs_api, &blob.oid, &envelope).await { + match crate::ipfs_pin::pin_git_object(ipfs_api, &blob.oid, &envelope, None).await { Ok(cid) if !cid.is_empty() => { if cid != blob.cid { warn!(oid = %blob.oid, expected = %blob.cid, got = %cid, "replicated envelope CID mismatch; skipping record"); diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 831202ee..0eb11481 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -129,6 +129,25 @@ pub(crate) fn signed_request_as(did: &str, method: Method, uri: &str, body: Body .expect("request builder") } +/// A local endpoint whose TCP accept succeeds instantly but that never writes an +/// HTTP response, so any request against it stalls deterministically until the +/// caller's own timeout. (A non-routable address hangs only if the network +/// blackholes the SYN — a fast RST would end the stall early and make a timeout +/// test pass for the wrong reason.) The accepted sockets are parked in the +/// spawned task, which dies with the test's runtime, so the peer never sees a +/// close mid-test. +pub(crate) async fn silent_http_endpoint() -> String { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { + let mut held = Vec::new(); + while let Ok((sock, _)) = listener.accept().await { + held.push(sock); + } + }); + endpoint +} + #[cfg(test)] mod tests { use super::*; From 17cbdf7ff318a7fbbb4a7dc76c043338ce9028a1 Mon Sep 17 00:00:00 2001 From: t Date: Thu, 30 Jul 2026 10:59:27 -0500 Subject: [PATCH 105/124] fix(node): charge the /ipfs walk budget for blocking-queue delay (#174) The walk's deadline clamp was computed on the async side, before spawn_blocking was called, then used inside the closure. Time spent queued for a blocking thread went uncharged, so the walk ran with a budget larger than the request's true remainder. The clamp is now derived inside the closure at task start. This is the sibling of the upload-pack site fixed in 28a6ca4, and unlike that one it sits on the anonymously reachable route. A queue delay that eats the whole remainder saturates the clamp to zero and the bounded walk fails closed through the existing taint arm, no verdict, which is the safe direction. Testing gap, recorded at the site rather than papered over: the queue-delay path is reasoned, not executed. Observing it needs a runtime with the blocking pool pinned and parked, and the sqlx::test harness gives no seam for that. 28a6ca4 shipped with the same gap for the same reason, verified against its diff and message rather than assumed. The tests in this file saturate the admission semaphores, never the blocking pool, so there was nothing to reuse. An assertion on the arithmetic would only restate the implementation. cargo test ipfs::: 24 passed, 0 failed. --- crates/gitlawb-node/src/api/ipfs.rs | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index a539ff34..f9e501ff 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -460,7 +460,7 @@ pub async fn get_by_cid( // started walk runs its git children under a deadline clamped // to the remainder (the min below), so a walk can never // complete past the budget. - let Some(budget_left) = budget_gate( + let Some(_budget_left) = budget_gate( &mut truncated_by, request_deadline, state.config.ipfs_request_budget_secs, @@ -490,10 +490,9 @@ pub async fn get_by_cid( let owner = repo.owner_did.clone(); let caller_for_walk = caller_owned.clone(); let git_bin = state.git_bin.clone(); - let walk_timeout = std::cmp::min( - std::time::Duration::from_secs(state.config.git_service_timeout_secs), - budget_left, - ); + let git_service_timeout = + std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let walk_deadline = request_deadline; // Full-history walk shells out to git — keep it off the async runtime, // bounded and reaped like the served-git ops (#174). let walk_admission = std::sync::Arc::clone(&admission); @@ -501,6 +500,24 @@ pub async fn get_by_cid( // Admission clone (#174 U1): the slot stays taken until this blocking // work returns, even if the handler future was dropped or this closure panics. let _admission = walk_admission; + // Derive the walk's budget from the request deadline HERE, inside the + // closure, not on the async side before the task is queued. The walk + // starts its own clock when it runs, so a budget computed at queue time + // would hand it the full remainder measured from whenever the blocking + // pool got to it — the queue delay would go uncharged and the walk could + // finish past the request budget. Computing it at task start charges the + // delay against the deadline; a queue delay that eats the whole remainder + // saturates this to zero and the walk fails closed (no verdict, taint), + // which is the safe direction. Same fix as the upload-pack walk in + // `api/repos.rs`, and this route is anonymously reachable. + // TESTING GAP: the queue-delay path is reasoned, not executed. Observing + // it needs a runtime with the blocking pool pinned and parked, and the + // `#[sqlx::test]` harness gives no seam for that (the sibling fix, + // 28a6ca4, shipped with the same gap for the same reason). + let walk_timeout = std::cmp::min( + git_service_timeout, + walk_deadline.saturating_duration_since(std::time::Instant::now()), + ); allowed_blob_set_for_caller_bounded( &rp, &git_bin, From 2413a71aa3b4c29b2148826a539185bec8cc1176 Mon Sep 17 00:00:00 2001 From: t Date: Thu, 30 Jul 2026 11:01:14 -0500 Subject: [PATCH 106/124] test(node): bind the repo-key boundary test to the separator it names (#174) separator_prevents_the_owner_name_boundary_collision asserted that ("a","b_c") and ("a_b","c") produce different keys. Those differ under any join at all, bare included (ab_c against a_bc), so deleting the separator from repo_identity_key left the test green. It could not fail for the reason it named. The inputs are now ("a","bc") against ("ab","c"), both abc under a bare join. The test docstring said the separator was \0; the code joins with /. Corrected. The docstring on repo_identity_key itself carried the same wrong collision example, so it is corrected too. Proven by mutation rather than argued, joining with no separator: old inputs, injected, re-running ... still green VACUOUS the guard's own defect was injected and nothing failed 0/1 load-bearing new inputs, injected, re-running ... red, on the expected message LOAD-BEARING matched 'owner/name boundary must not be ambiguous' 1/1 load-bearing Separator restored, no mutation residue. All 6 repo_identity_key_tests pass. --- crates/gitlawb-node/src/state.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 77755932..a189548b 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -521,9 +521,9 @@ pub struct RepoWriteLeases { /// - The separator is `/`, which cannot occur in the owner slug by construction /// (`replace([':', '/'], "_")` removes it) and mirrors the shape of the disk /// path this key exists to reproduce. A plain join would collide (owner `a` + -/// name `b_c` against owner `a_b` + name `c`), letting one repo's push park -/// another's. It must stay PRINTABLE: this key is logged as the `repo` field -/// on the lease shed and steal-bound warnings below, and an unprintable +/// name `bc` against owner `ab` + name `c`, both `abc`), letting one repo's +/// push park another's. It must stay PRINTABLE: this key is logged as the +/// `repo` field on the lease shed and steal-bound warnings below, and an unprintable /// separator (a NUL, a unit separator) truncates at a NUL-hostile log sink and /// renders two different repos' warnings identically. /// - Callers pass `record.owner_did` / `record.name`, never the request's path @@ -903,13 +903,14 @@ mod repo_identity_key_tests { ); } - /// The `\0` separator is what stops one repo's push parking another's. With a - /// plain join these two pairs would produce a single key. + /// The `/` separator is what stops one repo's push parking another's. These + /// two pairs both concatenate to `abc`, so with a plain join they would + /// produce a single key. #[test] fn separator_prevents_the_owner_name_boundary_collision() { assert_ne!( - repo_identity_key("a", "b_c"), - repo_identity_key("a_b", "c"), + repo_identity_key("a", "bc"), + repo_identity_key("ab", "c"), "owner/name boundary must not be ambiguous" ); } From a56f094affad2901f68b701f60b750070a99b5b8 Mon Sep 17 00:00:00 2001 From: t Date: Thu, 30 Jul 2026 11:06:21 -0500 Subject: [PATCH 107/124] test(node): bind the per-source cap test to the production derivation (#174) advert_per_caller_cap_sizing_is_never_zero opened with a local closure re-implementing (pushes / 8).max(1), so it bound nothing. Production derives that expression twice, and the second site was added after the test was written. per_source_push_cap now lives in rate_limit.rs next to PerCallerConcurrency, the type the value feeds, and both main.rs sites and the test call it. The finding was confirmed by execution before the fix. Deleting .max(1) from both production sites with the closure still in place: injected (2 sites), re-running ... still green VACUOUS the guard's own defect was injected and nothing failed 0/1 load-bearing And after, deleting .max(1) from the helper: injected (1 site), re-running ... red, on the expected message LOAD-BEARING matched 'advert cap must be >= 1 for pushes=1' 1/1 load-bearing The first attempt at that second proof came back RED-WRONG-REASON against 'minimum write pool must derive cap 1, not 0': the loop's assertion for pushes=1 fires first and aborts before that line is reached. Same property, earlier assertion. No mutation residue. Test green, fmt and clippy clean. --- crates/gitlawb-node/src/api/repos.rs | 9 +++++---- crates/gitlawb-node/src/main.rs | 4 ++-- crates/gitlawb-node/src/rate_limit.rs | 10 ++++++++++ 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 67c2ddd0..b3051db3 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -3535,12 +3535,13 @@ mod tests { } /// #174 (F2 sizing edge, vetted by execution): the receive-pack advertisement - /// per-source cap is derived in main.rs as `(max_concurrent_git_pushes / 8).max(1)`, - /// so it is never 0 even at the minimum write-pool size (1). A 0 cap would make - /// PerCallerConcurrency shed EVERY receive-pack advertisement and break all pushes. + /// per-source cap comes from `rate_limit::per_source_push_cap`, the same helper + /// both main.rs derivation sites call, so it is never 0 even at the minimum + /// write-pool size (1). A 0 cap would make PerCallerConcurrency shed EVERY + /// receive-pack advertisement and break all pushes. #[test] fn advert_per_caller_cap_sizing_is_never_zero() { - let cap = |pushes: usize| (pushes / 8).max(1); + let cap = crate::rate_limit::per_source_push_cap; for pushes in [1usize, 4, 8, 32, 256] { assert!( cap(pushes) >= 1, diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 5843d0ab..30c2ba37 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -426,14 +426,14 @@ async fn main() -> Result<()> { // rate-limited (#174). Sized off the write pool only because the advert pool // is created at the same size; an advert flood cannot touch a write permit. git_push_advert_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( - (config.max_concurrent_git_pushes / 8).max(1), + rate_limit::per_source_push_cap(config.max_concurrent_git_pushes), ), // Per-source cap on the authenticated receive-pack POST, sized like the advert // cap: one source IP can hold at most this many write-pool slots, so // monopolizing the pool takes ~8 distinct source IPs, each also rate-limited // (#174 P1-d). git_write_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( - (config.max_concurrent_git_pushes / 8).max(1), + rate_limit::per_source_push_cap(config.max_concurrent_git_pushes), ), // Bounds concurrent /ipfs visibility walks — a distinct public cost center, so // its own pool + per-source sub-cap + per-IP rate limiter, never a git pool diff --git a/crates/gitlawb-node/src/rate_limit.rs b/crates/gitlawb-node/src/rate_limit.rs index a0dc6137..6e2a7f3c 100644 --- a/crates/gitlawb-node/src/rate_limit.rs +++ b/crates/gitlawb-node/src/rate_limit.rs @@ -139,6 +139,16 @@ impl RateLimiter { } } +/// Per-source concurrency cap derived from the write-pool size: one source may hold +/// at most an eighth of the pool, so saturating it takes ~8 distinct source IPs. +/// +/// Floored at 1 because the value feeds [`PerCallerConcurrency`], where a cap of 0 +/// would shed EVERY receive-pack advertisement and break all pushes. The floor is +/// load-bearing at the minimum write-pool size (1), which integer-divides to 0. +pub(crate) fn per_source_push_cap(max_concurrent_git_pushes: usize) -> usize { + (max_concurrent_git_pushes / 8).max(1) +} + /// A bounded per-caller CONCURRENCY limiter — distinct from [`RateLimiter`], which /// caps request RATE. Each caller key may hold at most `per_caller` in-flight /// permits at once; beyond that [`try_acquire`](Self::try_acquire) returns `None` From 15aceb462d884fc6b69821adaa9162203418af2d Mon Sep 17 00:00:00 2001 From: t Date: Thu, 30 Jul 2026 11:08:43 -0500 Subject: [PATCH 108/124] docs(node): drop a backstop claim with no backstop behind it (#174) The pinata_object_list_for_refs docstring said a visibility tightening that lands after tail-start is covered by the reconciliation sweep. No such sweep exists; state.rs says so directly, that a dropped job would be lost forever precisely because there is none. The claim is dropped and nothing replaces it. An earlier draft wanted to substitute a next-push claim, which no site establishes either, and swapping one unbacked claim for another is the same defect. Four pre-existing sites make the same claim (repos.rs:30, push_delta.rs at 8, 189 and 281 with warns at 349 and 358). They predate this PR and are left for a follow-up rather than widening a nine-round diff. --- crates/gitlawb-node/src/api/repos.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index b3051db3..72650086 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1047,10 +1047,10 @@ async fn resolve_drain_object_list( /// and the filter always fails closed (a newly-withheld blob is not pinned; a /// no-longer-announceable repo pins nothing). A tightening AFTER tail-start — before /// a slow re-derivation runs — is not reflected, matching the old retained-list -/// behavior; the reconciliation sweep is the backstop. Every git child runs -/// through the same INV-22 bounded, process-group-reaped helpers the sibling -/// post-receive scans use (`replication_withheld_set`, -/// `resolve_candidates_for_push`, `fail_closed_full_scan_objects`). +/// behavior. Every git child runs through the same INV-22 bounded, +/// process-group-reaped helpers the sibling post-receive scans use +/// (`replication_withheld_set`, `resolve_candidates_for_push`, +/// `fail_closed_full_scan_objects`). #[allow(clippy::too_many_arguments)] async fn pinata_object_list_for_refs( encrypt_sem: Arc, From 43d2dd786044f589a92acc9eee1bd6f66989bee5 Mon Sep 17 00:00:00 2001 From: t Date: Thu, 30 Jul 2026 11:09:08 -0500 Subject: [PATCH 109/124] docs(node): stop the per-source cap comments claiming a bound the keying lacks (#174) Both new comments said saturating the pool takes ~8 distinct source IPs, each also rate-limited. Neither half holds for an IPv6 caller with a routed /64: the sub-cap and the rate limiter both key on rate_limit::client_key, which with TrustedProxy::None falls through to the full peer address with no prefix folding, so one caller has 2^64 keys. max_keys does not compensate, since keys are freed on permit drop and live keys never approach the 100,000 default. The comments now say what the cap actually bounds, concurrent slots per resolved client key, and name the residual. The keying fix stays deferred as a design call, and that deferral is defensible: full-IP keying predates this PR, so these caps help IPv4 and single-address callers without regressing anything. Landing a comment that overstates the guarantee is not defensible, in a round that corrects a weaker overclaim one screen away. The per_source_push_cap docstring added earlier in this round repeated the same claim, so it is corrected here rather than shipping fresh. Comments only, no behaviour change. check, fmt clean. --- crates/gitlawb-node/src/main.rs | 20 ++++++++++++-------- crates/gitlawb-node/src/rate_limit.rs | 8 ++++++-- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 30c2ba37..486bfa0e 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -420,18 +420,22 @@ async fn main() -> Result<()> { config.max_concurrent_reads_per_caller, ), // Per-source cap on the receive-pack advertisement, sized to an eighth of the - // write pool (min 1): a single source can hold at most this many slots in the - // DEDICATED advert pool (git_push_advert_semaphore, disjoint from the write - // pool), so saturating that pool takes ~8 distinct source IPs, each also - // rate-limited (#174). Sized off the write pool only because the advert pool - // is created at the same size; an advert flood cannot touch a write permit. + // write pool (min 1): one resolved client key (rate_limit::client_key) can hold + // at most this many slots in the DEDICATED advert pool (git_push_advert_semaphore, + // disjoint from the write pool), so saturating that pool takes ~8 distinct keys + // (#174). That bounds an IPv4 or single-address caller; a caller controlling many + // addresses (an IPv6 /64 is 2^64 keys) still gets one cap per address, since + // client_key uses the full IP with no prefix folding. Narrowing the keying is a + // deferred design call, not something these caps claim to solve. Sized off the + // write pool only because the advert pool is created at the same size; an advert + // flood cannot touch a write permit. git_push_advert_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( rate_limit::per_source_push_cap(config.max_concurrent_git_pushes), ), // Per-source cap on the authenticated receive-pack POST, sized like the advert - // cap: one source IP can hold at most this many write-pool slots, so - // monopolizing the pool takes ~8 distinct source IPs, each also rate-limited - // (#174 P1-d). + // cap: one resolved client key can hold at most this many write-pool slots, so + // monopolizing the pool takes ~8 distinct keys (#174 P1-d). Same residual as + // above: keys are full IPs, so a caller with many addresses has many caps. git_write_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( rate_limit::per_source_push_cap(config.max_concurrent_git_pushes), ), diff --git a/crates/gitlawb-node/src/rate_limit.rs b/crates/gitlawb-node/src/rate_limit.rs index 6e2a7f3c..d203f69f 100644 --- a/crates/gitlawb-node/src/rate_limit.rs +++ b/crates/gitlawb-node/src/rate_limit.rs @@ -139,8 +139,12 @@ impl RateLimiter { } } -/// Per-source concurrency cap derived from the write-pool size: one source may hold -/// at most an eighth of the pool, so saturating it takes ~8 distinct source IPs. +/// Per-source concurrency cap derived from the write-pool size: one resolved client +/// key (see [`client_key`]) may hold at most an eighth of the pool, so saturating it +/// takes ~8 distinct keys. Real for an IPv4 or single-address caller; a caller with a +/// routed IPv6 /64 has 2^64 keys, because `client_key` returns the full address with +/// no prefix folding. Narrowing that keying is a deferred decision, so this cap is a +/// bound per key, not a bound per operator. /// /// Floored at 1 because the value feeds [`PerCallerConcurrency`], where a cap of 0 /// would shed EVERY receive-pack advertisement and break all pushes. The floor is From b9d901f978fe2f29be68e2b56cdef0d9a0192501 Mon Sep 17 00:00:00 2001 From: t Date: Thu, 30 Jul 2026 11:15:17 -0500 Subject: [PATCH 110/124] fix(node): dispose the write-guard connection when the Drop unlock errors (#174) Review finding from jatmn, round nine. RepoWriteGuard::release closes the connection when pg_advisory_unlock returns Err (F3b): the await resolved, so the session is alive and still holds the lock, and returning that connection to the pool hands the next caller a lock nobody tracks. The Drop backstop carried the same hazard and only logged. Both arms are fixed. The spawned detached unlock now closes the connection through close_conn_bounded on the error path, instead of letting the async block end and return it to the pool. The off-runtime arm was worse than the finding described. It dropped an already-taken connection with no unlock attempted at all, and this turned out to panic rather than leak: PoolConnection::drop calls rt::spawn for its return-to-pool task, whose no-runtime fallback panics with "this functionality requires a Tokio context". So any off-runtime guard drop panicked in a destructor today. It now detaches, which gives up the pool slot and yields a plain PgConnection whose drop closes the socket, ending the session that holds the lock. Drop cannot await, so detach is the whole disposal available here. Confirmed against the vendored sqlx 0.8.6 source rather than assumed, and min_connections is 0 everywhere in this repo. Three tests, modeled on the release F3b cases, on a pool with the idle reaper disabled so a reaped session cannot pass the assertion for us. Observed RED first: write_guard_dropped_off_runtime_disposes_the_connection ... FAILED panicked at sqlx-core-0.8.6/src/pool/connection.rs:208: this functionality requires a Tokio context then: dropping a write guard off a Tokio runtime must not panic write_guard_drop_with_failing_unlock_does_not_return_the_connection ... FAILED timed out waiting for the connection whose detached unlock errored to be closed rather than returned to the pool still holding the session lock write_guard_drop_with_successful_unlock_keeps_the_connection ... ok The success case is green on both sides on purpose: it is the must-not case, so it only ever catches a widening of disposal onto the healthy path. GREEN after, and all 12 guard tests pass, so the release coverage did not regress. Both new guards mutation-proved load-bearing, each red on its own message, 2/2. Full suite 729 passed, 0 failed. fmt and clippy clean. --- crates/gitlawb-node/src/git/repo_store.rs | 188 +++++++++++++++++++++- 1 file changed, 180 insertions(+), 8 deletions(-) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index afc0d6e8..6fcc5a92 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -674,8 +674,8 @@ impl Drop for RepoWriteGuard { /// error: that case is closed inside `release` by disposing of the connection, /// because `Drop` early-returns on `conn == None` and the two mechanisms cannot /// both apply (#174 F3b). `Drop` cannot await, so spawn a detached unlock — it runs on the - /// same session (connection-affine). An off-runtime drop falls back to a log; - /// the ~60s stale-lock retry loop in `acquire_write` reclaims it. On runtime + /// same session (connection-affine). An off-runtime drop has nothing to spawn onto, + /// so it disposes of the connection instead. On runtime /// SHUTDOWN the spawned unlock task may be dropped before it polls, so the unlock /// may not run — but shutdown tears down the pool, and closing the connection /// releases the session-level advisory lock server-side, so this too is bounded. @@ -691,20 +691,38 @@ impl Drop for RepoWriteGuard { match tokio::runtime::Handle::try_current() { Ok(handle) => { handle.spawn(async move { - if let Err(e) = sqlx::query("SELECT pg_advisory_unlock($1)") + let unlock = sqlx::query("SELECT pg_advisory_unlock($1)") .bind(lock_key) .execute(&mut *conn) - .await - { - warn!(repo = %repo_name, err = %e, "detached advisory-unlock on write-guard drop failed"); + .await; + // Same failure as `release`'s (#174 F3b), one level down: the await + // RESOLVED with an error, so the session is alive and still holds + // the lock. Letting this async block end here would drop `conn` and + // RETURN it to the pool, handing the next caller a connection + // holding a lock nobody tracks. Close it instead, which both keeps + // it out of the pool and ends the session that holds the lock. + if let Err(e) = unlock { + warn!(repo = %repo_name, err = %e, "detached advisory-unlock on write-guard drop failed, closing the connection so the session ends and postgres drops the lock"); + close_conn_bounded(&repo_name, conn.close()).await; } }); } Err(_) => { + // No runtime to spawn the unlock onto, and the connection is already + // out of the guard, so there is no path that unlocks on this session. + // Returning it to the pool would hand the next caller a connection + // still holding the lock. `PoolConnection`'s own drop also spawns its + // return-to-pool task, which panics with no runtime. `detach` gives up + // the pool slot and yields a plain `PgConnection`; dropping that closes + // the socket, which ends the session and is what frees the lock + // server-side. `Drop` cannot await, so this is the whole disposal: + // `close_conn_bounded` is not available here. + drop(conn.detach()); warn!( repo = %repo_name, - "RepoWriteGuard dropped off a Tokio runtime; advisory lock not released \ - synchronously — the stale-lock retry loop will reclaim it" + "RepoWriteGuard dropped off a Tokio runtime; no detached unlock is \ + possible, so the pinned connection is disposed of instead: ending \ + the session is what releases the advisory lock" ); } } @@ -1544,6 +1562,160 @@ mod tests { .await; } + // ── the Drop backstop disposes its connection too (#174 U8) ───────────── + + /// U8 (P2): the `Drop` backstop carries the same hazard F3b closed in `release`. + /// When the detached `pg_advisory_unlock` ERRORS on a live session, the async + /// block ends and drops the moved `PoolConnection`, which RETURNS it to the pool + /// while that session may still hold the lock, handing the next caller a + /// connection holding a lock nobody tracks. The errored connection must be closed + /// instead: that keeps it out of the pool and ends the session, which is what + /// frees the lock server-side. + /// + /// Run against a pool with the idle reaper disabled, for the reason spelled out on + /// `pool_without_idle_reaper`: with the reaper in play the session dies on its own + /// about a second later and the assertion stops measuring the disposal. + /// + /// Load-bearing: RED before the fix (the connection goes back to the pool, so the + /// size never drops and this times out), GREEN after. + #[sqlx::test] + async fn write_guard_drop_with_failing_unlock_does_not_return_the_connection( + pool: sqlx::PgPool, + ) { + let dir = tempfile::TempDir::new().unwrap(); + let store_pool = pool_without_idle_reaper(&pool).await; + let store = RepoStore::for_testing(dir.path().to_path_buf(), store_pool.clone()); + let owner = "did:key:z6MkDropUnlockErrProofIIIIIIIIIIIIIIIIIIII"; + let name = "dropunlockerrtest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + // Distinct session for the probe, held out of the store's pool entirely. + let mut checker = pool.acquire().await.expect("checker connection"); + + let mut guard = store.acquire_write(owner, name).await.expect("acquire"); + poison_guard_connection(&mut guard).await; + let size_before = store_pool.size(); + assert!(size_before > 0, "the pool owns the guard's connection"); + + // The backstop shape: dropped without release(), with an unlock that errors. + drop(guard); + + wait_until( + || store_pool.size() == size_before - 1, + "the connection whose detached unlock errored to be closed rather than \ + returned to the pool still holding the session lock", + ) + .await; + wait_until_lock_free( + &mut checker, + key, + "an errored detached unlock must not leave the lock held: Drop must dispose \ + of the connection so the session ends", + ) + .await; + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } + + /// U8 regression guard on the success path: a detached unlock that SUCCEEDS must + /// still return the connection to the pool. Without this, "close the connection on + /// Drop" could be widened to "always close" and the test above would not notice. + #[sqlx::test] + async fn write_guard_drop_with_successful_unlock_keeps_the_connection(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let store_pool = pool_without_idle_reaper(&pool).await; + let store = RepoStore::for_testing(dir.path().to_path_buf(), store_pool.clone()); + let owner = "did:key:z6MkDropUnlockOkProofJJJJJJJJJJJJJJJJJJJJ"; + let name = "dropunlockoktest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + let mut checker = pool.acquire().await.expect("checker connection"); + let guard = store.acquire_write(owner, name).await.expect("acquire"); + let size_before = store_pool.size(); + assert!(size_before > 0, "the pool owns the guard's connection"); + + drop(guard); + + // The connection goes back only once the detached unlock task has finished. + wait_until( + || store_pool.num_idle() > 0, + "the detached unlock to finish and hand the connection back", + ) + .await; + assert_eq!( + store_pool.size(), + size_before, + "a successful detached unlock must leave the connection in the pool" + ); + wait_until_lock_free( + &mut checker, + key, + "the Drop backstop's successful unlock to free the lock", + ) + .await; + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } + + /// U8, the off-runtime arm: with no Tokio runtime there is nothing to spawn the + /// unlock onto, and the connection has already been taken out of the guard, so the + /// old code dropped it with no unlock attempted at all, back to the pool, session + /// lock still held. Dropping a `PoolConnection` off a runtime is worse than that: + /// sqlx's return-to-pool path spawns, and its no-runtime fallback panics, so the + /// old arm also panicked in a destructor. + /// + /// Reached by dropping the guard on a plain `std::thread`, where + /// `Handle::try_current()` fails. + /// + /// Load-bearing: RED before the fix (the join sees the "requires a Tokio context" + /// panic from sqlx's return-to-pool spawn), GREEN after (`detach` gives up the + /// pool slot, so nothing is spawned and dropping the detached connection closes + /// the socket, which ends the session and frees the lock). + #[sqlx::test] + async fn write_guard_dropped_off_runtime_disposes_the_connection(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let store_pool = pool_without_idle_reaper(&pool).await; + let store = RepoStore::for_testing(dir.path().to_path_buf(), store_pool.clone()); + let owner = "did:key:z6MkDropOffRuntimeProofKKKKKKKKKKKKKKKKKK"; + let name = "dropoffruntimetest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + let mut checker = pool.acquire().await.expect("checker connection"); + let guard = store.acquire_write(owner, name).await.expect("acquire"); + let size_before = store_pool.size(); + assert!(size_before > 0, "the pool owns the guard's connection"); + + let dropped = std::thread::spawn(move || drop(guard)).join(); + assert!( + dropped.is_ok(), + "dropping a write guard off a Tokio runtime must not panic" + ); + + wait_until( + || store_pool.size() == size_before - 1, + "the connection of a guard dropped off a runtime to be disposed of rather \ + than returned to the pool with no unlock attempted", + ) + .await; + wait_until_lock_free( + &mut checker, + key, + "a guard dropped off a runtime to end its session so postgres drops the lock", + ) + .await; + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } + /// F4: releasing a guard that never took the lock (`locked == false`, the state /// acquire_write leaves after a failed acquisition) must not unlock or panic. #[sqlx::test] From fa9bf08739f0ced4de8d2f36b8ebfbba28370030 Mon Sep 17 00:00:00 2001 From: t Date: Thu, 30 Jul 2026 17:28:25 -0500 Subject: [PATCH 111/124] fix(node): saturate the derived lease steal bound instead of overflowing it (#174) GITLAWB_GIT_SERVICE_TIMEOUT_SECS accepts every positive u64, so deriving the per-repo write-lease steal bound as `* 2 + 60` panics every push in a debug build across the top of the accepted range, and in release wraps to a short Duration that would let a waiter steal a live push's lease. Saturating keeps the knob's documented meaning: a timeout that large means the service bound is effectively off, so the steal bound should be effectively infinite rather than tiny. The regression drives git_receive_pack rather than the arithmetic, so the call-site wiring is what is under test. RED before at repos.rs:1733 with "attempt to multiply with overflow", GREEN after; re-confirmed load-bearing by reinstating the unchecked expression. --- crates/gitlawb-node/src/api/repos.rs | 71 +++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 2 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 72650086..61cc42cf 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1729,8 +1729,19 @@ pub async fn git_receive_pack( // bound — on the non-disconnect path the retained pg advisory lock still serializes // the stealer at acquire_write (a spurious 503, not a race); the only corruption-capable // overlap is the ~4s disconnect/reap window, which the reaper-carried clone (a) covers. - let lease_steal_after = - std::time::Duration::from_secs(state.config.git_service_timeout_secs * 2 + 60); + // Saturating, not unchecked: `git_service_timeout_secs` accepts every positive u64 and + // its help text explicitly permits setting it very large to disable the service bound, + // so `* 2 + 60` overflows across the top of the accepted range — a debug panic on every + // push, and in release a WRAPPED bound short enough for a waiter to steal a live push's + // lease. Saturating keeps the documented semantic intact: a timeout that large means no + // steal, which is what an effectively-disabled service bound implies. + let lease_steal_after = std::time::Duration::from_secs( + state + .config + .git_service_timeout_secs + .saturating_mul(2) + .saturating_add(60), + ); // Parked waiters are bounded INSIDE the lease, not by an admission permit taken above // it. `body: Bytes` means axum has already buffered the whole pack before this handler @@ -7933,6 +7944,62 @@ mod tests { handle.abort(); } + /// #174 (RED-before/GREEN-after): a large-but-valid `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` + /// must not overflow the lease steal bound derived from it. The config help explicitly + /// permits setting the timeout very large to disable the bound and clap accepts every + /// positive `u64`, so an unchecked `* 2 + 60` panics the push in a debug build and + /// wraps to a short `Duration` in release — a wrapped bound would let a waiter steal + /// the lease out from under a live push. Drives the handler rather than the + /// arithmetic, so the call-site wiring is what is under test. + #[cfg(unix)] + #[sqlx::test] + async fn push_survives_a_git_service_timeout_that_overflows_the_lease_bound( + pool: sqlx::PgPool, + ) { + use axum::extract::{Path, State}; + use axum::response::IntoResponse; + use axum::Extension; + use std::net::SocketAddr; + + let tmp = tempfile::TempDir::new().unwrap(); + let git_bin = write_fake_git( + tmp.path(), + "#!/bin/sh\n\ + case \"$1\" in\n\ + receive-pack) cat > /dev/null 2>/dev/null ;;\n\ + rev-parse) echo deadbeef ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + ); + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6ovflow", "o1", false).await; + // The largest value clap accepts. Any value above (u64::MAX - 60) / 2 overflows the + // derived bound, so this is the whole tail of the accepted range, not a corner. + let mut cfg = (*state.config).clone(); + cfg.git_service_timeout_secs = u64::MAX; + state.config = std::sync::Arc::new(cfg); + + let resp = git_receive_pack( + State(state), + Path(("z6ovflow".to_string(), "o1".to_string())), + Extension(crate::auth::AuthenticatedDid( + "did:key:z6MkOverflowPusherAAAAAAAAAAAAAAAAAAAAAA".to_string(), + )), + crate::rate_limit::PeerAddr(Some("203.0.113.90:5000".parse::().unwrap())), + axum::http::HeaderMap::new(), + ref_update_body("1111111111111111111111111111111111111111"), + ) + .await + .expect("push must succeed under a maximal git_service_timeout_secs") + .into_response(); + assert_eq!( + resp.status(), + 200, + "a maximal service timeout must disable the steal bound, not break the push" + ); + } + /// #174 U1 scenario 1 (RED-before/GREEN-after): parked pushes on one repo's write /// lease are BOUNDED. `git_receive_pack` takes `body: Bytes`, so axum has already /// buffered the whole pack (up to `max_pack_bytes`) before the handler runs, and the From 88f5e938fdaeea91f5ab1745586a8ed319c22b8f Mon Sep 17 00:00:00 2001 From: t Date: Thu, 30 Jul 2026 17:28:36 -0500 Subject: [PATCH 112/124] fix(node): bound git_service_timeout_secs to a range its deadlines can represent (#174) Tracing the lease-bound overflow to its other consumers turned up a worse instance of the same defect. This PR routed git_service_timeout_secs into build_filtered_pack and blob_paths, and both build a deadline as `Instant::now() + Duration::from_secs(..)`. That addition panics on overflow in RELEASE as well as debug, so a large-but-accepted value crashes the serve path rather than merely disabling a timeout. Hardening each arithmetic site as it is found would leave the next one open, so the bound goes where the value enters: clap now rejects anything above 365 days at parse time, which keeps every derived duration in range at once and fails fast at boot instead of at the first path-scoped clone. A year is many orders of magnitude above any real clone or push, so it remains the practical way to disable the bound. The test pins both derivations at the maximum and rejects the value one past it and u64::MAX. RED when the upper bound is dropped from the value_parser. --- .env.example | 3 +- README.md | 2 +- crates/gitlawb-node/src/config.rs | 72 ++++++++++++++++++++++++++++--- 3 files changed, 69 insertions(+), 8 deletions(-) diff --git a/.env.example b/.env.example index aea17f61..c719fa1f 100644 --- a/.env.example +++ b/.env.example @@ -120,7 +120,8 @@ GITLAWB_MAX_PACK_BYTES=2147483648 # rather than giving each stage a full budget: a walk that consumes it leaves the # serve nothing and the clone gets a 504. Serving large path-scoped repos may # need a higher value here than when each stage was budgeted separately. -# Default 600. +# Must be 1..=31536000 (365 days): the node derives deadlines from this value, +# and a larger one cannot be represented. Default 600. GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600 # Max seconds the storage-ACQUISITION phase of a served git op may run before the diff --git a/README.md b/README.md index f4ba0f9e..73c8127d 100644 --- a/README.md +++ b/README.md @@ -343,7 +343,7 @@ Important node settings: | `GITLAWB_REQUIRE_SIGNED_PEER_WRITES` | Require signed peer announce/sync writes. | | `GITLAWB_AUTO_SYNC` | Enable automatic sync from known peers. | | `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. | -| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. On the path-scoped upload-pack path the classification walk and the pack serve share ONE deadline, so this value bounds their combined duration rather than granting each stage a full budget: a walk that consumes it leaves the serve nothing and the clone gets a 504. Serving large path-scoped repos may therefore need a higher value than they did when each stage was budgeted separately. | +| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. On the path-scoped upload-pack path the classification walk and the pack serve share ONE deadline, so this value bounds their combined duration rather than granting each stage a full budget: a walk that consumes it leaves the serve nothing and the clone gets a 504. Serving large path-scoped repos may therefore need a higher value than they did when each stage was budgeted separately. Accepted range is 1 to 31536000 (365 days), since the node derives deadlines from this value and a larger one cannot be represented. | | `GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS` | Max seconds the storage-acquisition phase (Tigris HEAD/GET, push advisory-lock) of a served git op may run before the request is shed with a 503, separate from the git-run timeout. The concurrency permit is released on expiry so a stalled backend cannot pin the pool. Default 30. | | `GITLAWB_REPO_LEASE_MAX_WAITERS` | Max pushes parked at once waiting for the same repo's write lease. Each waiter pins its buffered pack body, so this bounds that memory for a hot repo; past the cap the newest push sheds a 503 + Retry-After instead of queueing. Pushes to other repos are unaffected, and the lease holder is not counted. Default 8. | | `GITLAWB_MAX_CONCURRENT_IPFS_WALKS` | Max concurrent `GET /ipfs/{cid}` visibility walks across all callers (own pool, disjoint from the served-git pools); over-cap sheds 503. Default 32. | diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 5ba10584..fbe894db 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -1,6 +1,20 @@ use clap::Parser; use std::path::PathBuf; +/// Upper bound on `git_service_timeout_secs`, in seconds (365 days). +/// +/// The knob is not just stored, it is arithmetic input: the write path derives the +/// per-repo lease steal bound from it (`* 2 + 60`), and #174 routed it into +/// `build_filtered_pack` and `blob_paths`, which each build a deadline as +/// `Instant::now() + Duration::from_secs(this)`. That addition panics on overflow in +/// RELEASE as well as debug, so an unbounded `u64` here turns an operator typo into a +/// serve-path crash rather than a very long timeout. Bounding at parse time keeps every +/// derived duration in range at once, instead of hardening each site as it is found. +/// +/// A year is many orders of magnitude above any real clone or push, so this remains the +/// practical way to disable the bound. +pub const GIT_SERVICE_TIMEOUT_SECS_MAX: u64 = 365 * 24 * 60 * 60; + #[derive(Parser, Debug, Clone)] #[command(name = "gitlawb-node", about = "gitlawb node daemon", version)] pub struct Config { @@ -170,16 +184,18 @@ pub struct Config { /// Maximum wall-clock time a single served git operation (upload-pack / /// receive-pack through `run_git_service`) may run before it is aborted and /// its process group torn down, in seconds. Bounds a git that neither - /// finishes nor disconnects. Must be positive; set it very large to - /// effectively disable the bound. Default: 600s (10 min), generous for large - /// clones. Also bounds the ref advertisement (`info/refs`) and the withheld-blob - /// pack build (`upload_pack_excluding`'s pack-objects stage), which now share the - /// same timeout + process-group teardown (#174). + /// finishes nor disconnects. Must be positive and at most + /// [`GIT_SERVICE_TIMEOUT_SECS_MAX`] (365 days), which is far above any real git + /// operation and is the practical way to disable the bound. Default: 600s + /// (10 min), generous for large clones. Also bounds the ref advertisement + /// (`info/refs`) and the withheld-blob pack build (`upload_pack_excluding`'s + /// pack-objects stage), which now share the same timeout + process-group + /// teardown (#174). #[arg( long, env = "GITLAWB_GIT_SERVICE_TIMEOUT_SECS", default_value_t = 600, - value_parser = clap::value_parser!(u64).range(1..) + value_parser = clap::value_parser!(u64).range(1..=GIT_SERVICE_TIMEOUT_SECS_MAX) )] pub git_service_timeout_secs: u64, @@ -563,6 +579,50 @@ mod tests { ); } + /// #174 (RED-before/GREEN-after): the upper bound is what keeps every duration + /// derived from this knob in range — the lease steal bound's `* 2 + 60` on the write + /// path, and the `Instant::now() + Duration::from_secs(..)` deadlines in + /// `build_filtered_pack` / `blob_paths` on the serve path, which panic on overflow in + /// release builds too. Checked at parse time so no reachable configuration can carry a + /// value those sites cannot represent. + #[test] + fn git_service_timeout_rejects_values_no_derived_duration_can_represent() { + // At the bound: accepted, and every derived duration still fits. + let at_max = Config::try_parse_from([ + "gitlawb-node", + "--git-service-timeout-secs", + &GIT_SERVICE_TIMEOUT_SECS_MAX.to_string(), + ]) + .expect("the documented maximum must parse"); + assert_eq!( + at_max.git_service_timeout_secs, + GIT_SERVICE_TIMEOUT_SECS_MAX + ); + assert!(at_max + .git_service_timeout_secs + .checked_mul(2) + .and_then(|v| v.checked_add(60)) + .is_some()); + assert!(std::time::Instant::now() + .checked_add(std::time::Duration::from_secs( + at_max.git_service_timeout_secs + )) + .is_some()); + + // One past the bound, and the top of the u64 range clap used to accept. + for over in [GIT_SERVICE_TIMEOUT_SECS_MAX + 1, u64::MAX] { + assert!( + Config::try_parse_from([ + "gitlawb-node", + "--git-service-timeout-secs", + &over.to_string(), + ]) + .is_err(), + "{over} exceeds the derivable range and must be rejected at parse time" + ); + } + } + #[test] fn max_concurrent_pin_tasks_defaults_and_rejects_out_of_range() { assert_eq!( From 578ae5b99d4275898d546bc15dd21823353cf0e2 Mon Sep 17 00:00:00 2001 From: t Date: Thu, 30 Jul 2026 18:07:16 -0500 Subject: [PATCH 113/124] fix(node): set the git_service_timeout ceiling by representability, not tidiness (#174) A second-model pass on the previous two commits caught an upgrade footgun in the first and stale prose in the second. The 365-day ceiling was tighter than the defect required. The help text has always told operators to set this very large to disable the bound, and clap accepted every positive u64, so values like 999999999 (~31 years) are working production "off" settings that neither overflow `* 2 + 60` nor exceed an Instant deadline. A node running one of those would have failed to start after this upgrade, over a value that was never the defect. The ceiling moves to 100 years, which clears every such setting and stays an order of magnitude under the ~584-year limit of a u64-nanosecond Instant, the tightest representation on any platform we build for. Every value that worked before still parses; only the ones that would have panicked are rejected. The test now pins the pre-existing disable values alongside the rejections so the distinction cannot quietly regress. Clamp-and-warn was the suggested remedy and is not what landed: it would keep accepting a value the node cannot honor and silently substitute a different one, where the range says what is representable and fails fast, matching how every other bound in this file behaves. The lease-bound comment and its test still described the pre-ceiling world. Both now say what is true, and the test deliberately keeps u64::MAX rather than moving to the ceiling as suggested: Config is reachable by direct construction, and at the ceiling the arithmetic does not overflow, so a test pinned there would pass with the saturation removed and prove nothing. Re-confirmed load-bearing after the rewording. --- .env.example | 2 +- README.md | 2 +- crates/gitlawb-node/src/api/repos.rs | 37 +++++++++------ crates/gitlawb-node/src/config.rs | 70 +++++++++++++++++++--------- 4 files changed, 72 insertions(+), 39 deletions(-) diff --git a/.env.example b/.env.example index c719fa1f..5a3a5acc 100644 --- a/.env.example +++ b/.env.example @@ -120,7 +120,7 @@ GITLAWB_MAX_PACK_BYTES=2147483648 # rather than giving each stage a full budget: a walk that consumes it leaves the # serve nothing and the clone gets a 504. Serving large path-scoped repos may # need a higher value here than when each stage was budgeted separately. -# Must be 1..=31536000 (365 days): the node derives deadlines from this value, +# Must be 1..=3153600000 (100 years): the node derives deadlines from this value, # and a larger one cannot be represented. Default 600. GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600 diff --git a/README.md b/README.md index 73c8127d..d5b9c6b9 100644 --- a/README.md +++ b/README.md @@ -343,7 +343,7 @@ Important node settings: | `GITLAWB_REQUIRE_SIGNED_PEER_WRITES` | Require signed peer announce/sync writes. | | `GITLAWB_AUTO_SYNC` | Enable automatic sync from known peers. | | `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. | -| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. On the path-scoped upload-pack path the classification walk and the pack serve share ONE deadline, so this value bounds their combined duration rather than granting each stage a full budget: a walk that consumes it leaves the serve nothing and the clone gets a 504. Serving large path-scoped repos may therefore need a higher value than they did when each stage was budgeted separately. Accepted range is 1 to 31536000 (365 days), since the node derives deadlines from this value and a larger one cannot be represented. | +| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. On the path-scoped upload-pack path the classification walk and the pack serve share ONE deadline, so this value bounds their combined duration rather than granting each stage a full budget: a walk that consumes it leaves the serve nothing and the clone gets a 504. Serving large path-scoped repos may therefore need a higher value than they did when each stage was budgeted separately. Accepted range is 1 to 3153600000 (100 years), since the node derives deadlines from this value and a larger one cannot be represented. | | `GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS` | Max seconds the storage-acquisition phase (Tigris HEAD/GET, push advisory-lock) of a served git op may run before the request is shed with a 503, separate from the git-run timeout. The concurrency permit is released on expiry so a stalled backend cannot pin the pool. Default 30. | | `GITLAWB_REPO_LEASE_MAX_WAITERS` | Max pushes parked at once waiting for the same repo's write lease. Each waiter pins its buffered pack body, so this bounds that memory for a hot repo; past the cap the newest push sheds a 503 + Retry-After instead of queueing. Pushes to other repos are unaffected, and the lease holder is not counted. Default 8. | | `GITLAWB_MAX_CONCURRENT_IPFS_WALKS` | Max concurrent `GET /ipfs/{cid}` visibility walks across all callers (own pool, disjoint from the served-git pools); over-cap sheds 503. Default 32. | diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 61cc42cf..22a216fe 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1729,12 +1729,14 @@ pub async fn git_receive_pack( // bound — on the non-disconnect path the retained pg advisory lock still serializes // the stealer at acquire_write (a spurious 503, not a race); the only corruption-capable // overlap is the ~4s disconnect/reap window, which the reaper-carried clone (a) covers. - // Saturating, not unchecked: `git_service_timeout_secs` accepts every positive u64 and - // its help text explicitly permits setting it very large to disable the service bound, - // so `* 2 + 60` overflows across the top of the accepted range — a debug panic on every - // push, and in release a WRAPPED bound short enough for a waiter to steal a live push's - // lease. Saturating keeps the documented semantic intact: a timeout that large means no - // steal, which is what an effectively-disabled service bound implies. + // Saturating, not unchecked. `GIT_SERVICE_TIMEOUT_SECS_MAX` now keeps every parsed + // value inside this arithmetic, so on the configured path this cannot overflow; it is + // deliberate defense in depth for the construction paths clap does not cover (tests + // build `Config` by mutation, and nothing stops a future caller doing the same). The + // failure it holds off is not cosmetic: unchecked, `* 2 + 60` panics the push in debug + // and in release WRAPS to a bound short enough for a waiter to steal a live push's + // lease. Saturating also states the intent — a timeout that large means no steal, which + // is what an effectively-disabled service bound implies. let lease_steal_after = std::time::Duration::from_secs( state .config @@ -7944,13 +7946,18 @@ mod tests { handle.abort(); } - /// #174 (RED-before/GREEN-after): a large-but-valid `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` - /// must not overflow the lease steal bound derived from it. The config help explicitly - /// permits setting the timeout very large to disable the bound and clap accepts every - /// positive `u64`, so an unchecked `* 2 + 60` panics the push in a debug build and - /// wraps to a short `Duration` in release — a wrapped bound would let a waiter steal - /// the lease out from under a live push. Drives the handler rather than the - /// arithmetic, so the call-site wiring is what is under test. + /// #174 (RED-before/GREEN-after): the lease steal bound derived from + /// `git_service_timeout_secs` must not overflow. Unchecked, `* 2 + 60` panics the push + /// in a debug build and wraps to a short `Duration` in release, and a wrapped bound + /// would let a waiter steal the lease out from under a live push. Drives the handler + /// rather than the arithmetic, so the call-site wiring is what is under test. + /// + /// `GIT_SERVICE_TIMEOUT_SECS_MAX` means clap no longer admits a value this large, and + /// the test keeps `u64::MAX` anyway rather than moving to the ceiling. Two reasons, and + /// the second is the load-bearing one: `Config` is reachable by direct construction, + /// which is how this test and every other one build it; and at the ceiling the + /// arithmetic does not overflow, so a test pinned there would pass with the saturation + /// removed and prove nothing about this line. #[cfg(unix)] #[sqlx::test] async fn push_survives_a_git_service_timeout_that_overflows_the_lease_bound( @@ -7974,8 +7981,8 @@ mod tests { ); let mut state = f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6ovflow", "o1", false).await; - // The largest value clap accepts. Any value above (u64::MAX - 60) / 2 overflows the - // derived bound, so this is the whole tail of the accepted range, not a corner. + // Every value above (u64::MAX - 60) / 2 overflows the derived bound; u64::MAX is + // the top of that tail, and the value clap accepted before the ceiling landed. let mut cfg = (*state.config).clone(); cfg.git_service_timeout_secs = u64::MAX; state.config = std::sync::Arc::new(cfg); diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index fbe894db..c038a525 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -1,7 +1,7 @@ use clap::Parser; use std::path::PathBuf; -/// Upper bound on `git_service_timeout_secs`, in seconds (365 days). +/// Upper bound on `git_service_timeout_secs`, in seconds (100 years). /// /// The knob is not just stored, it is arithmetic input: the write path derives the /// per-repo lease steal bound from it (`* 2 + 60`), and #174 routed it into @@ -11,9 +11,16 @@ use std::path::PathBuf; /// serve-path crash rather than a very long timeout. Bounding at parse time keeps every /// derived duration in range at once, instead of hardening each site as it is found. /// -/// A year is many orders of magnitude above any real clone or push, so this remains the -/// practical way to disable the bound. -pub const GIT_SERVICE_TIMEOUT_SECS_MAX: u64 = 365 * 24 * 60 * 60; +/// The ceiling is representability, NOT a policy view of a sane timeout, and that +/// distinction is what sets the number. The help text has always told operators to set +/// this very large to disable the bound, so values like `999999999` (~31 years) are +/// working production "off" settings; a tighter, tidier cap would fail those nodes at +/// boot on upgrade over a value that was never the defect. 100 years clears every such +/// setting while staying an order of magnitude under the ~584-year ceiling of a +/// `u64`-nanosecond `Instant`, which is the tightest representation on any platform we +/// build for. Every value that worked before still parses; only the ones that would have +/// panicked are rejected. +pub const GIT_SERVICE_TIMEOUT_SECS_MAX: u64 = 100 * 365 * 24 * 60 * 60; #[derive(Parser, Debug, Clone)] #[command(name = "gitlawb-node", about = "gitlawb node daemon", version)] @@ -185,9 +192,10 @@ pub struct Config { /// receive-pack through `run_git_service`) may run before it is aborted and /// its process group torn down, in seconds. Bounds a git that neither /// finishes nor disconnects. Must be positive and at most - /// [`GIT_SERVICE_TIMEOUT_SECS_MAX`] (365 days), which is far above any real git - /// operation and is the practical way to disable the bound. Default: 600s - /// (10 min), generous for large clones. Also bounds the ref advertisement + /// [`GIT_SERVICE_TIMEOUT_SECS_MAX`] (100 years, the largest value every derived + /// deadline can represent); setting it very large is still the way to disable the + /// bound. Default: 600s (10 min), generous for large clones. Also bounds the ref + /// advertisement /// (`info/refs`) and the withheld-blob pack build (`upload_pack_excluding`'s /// pack-objects stage), which now share the same timeout + process-group /// teardown (#174). @@ -585,15 +593,34 @@ mod tests { /// `build_filtered_pack` / `blob_paths` on the serve path, which panic on overflow in /// release builds too. Checked at parse time so no reachable configuration can carry a /// value those sites cannot represent. + /// + /// The pre-existing "set it very large to disable the bound" settings are asserted + /// alongside the rejections on purpose. The bound exists to exclude unrepresentable + /// values, not to impose a view of a reasonable timeout, so a node that has been + /// running on ~31 years must not start failing at boot on upgrade. #[test] fn git_service_timeout_rejects_values_no_derived_duration_can_represent() { - // At the bound: accepted, and every derived duration still fits. - let at_max = Config::try_parse_from([ - "gitlawb-node", - "--git-service-timeout-secs", - &GIT_SERVICE_TIMEOUT_SECS_MAX.to_string(), - ]) - .expect("the documented maximum must parse"); + let parse = |secs: u64| { + Config::try_parse_from([ + "gitlawb-node", + "--git-service-timeout-secs", + &secs.to_string(), + ]) + }; + + // Large "disable the bound" values that predate the ceiling still parse. + for disable in [1_000_000_000, 999_999_999] { + assert_eq!( + parse(disable) + .unwrap_or_else(|e| panic!("{disable} was a working setting: {e}")) + .git_service_timeout_secs, + disable + ); + } + + // At the ceiling: accepted, and every derived duration still fits. + let at_max = + parse(GIT_SERVICE_TIMEOUT_SECS_MAX).expect("the documented maximum must parse"); assert_eq!( at_max.git_service_timeout_secs, GIT_SERVICE_TIMEOUT_SECS_MAX @@ -609,18 +636,17 @@ mod tests { )) .is_some()); - // One past the bound, and the top of the u64 range clap used to accept. + // Past the ceiling, and the top of the u64 range clap used to accept — the value + // that panics `Instant::now() + Duration::from_secs(..)` outright. for over in [GIT_SERVICE_TIMEOUT_SECS_MAX + 1, u64::MAX] { assert!( - Config::try_parse_from([ - "gitlawb-node", - "--git-service-timeout-secs", - &over.to_string(), - ]) - .is_err(), - "{over} exceeds the derivable range and must be rejected at parse time" + parse(over).is_err(), + "{over} is past the representable ceiling and must be rejected at parse time" ); } + assert!(std::time::Instant::now() + .checked_add(std::time::Duration::from_secs(u64::MAX)) + .is_none()); } #[test] From 44efb996674aba19f8a23d9b734387a398f6c9e8 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 31 Jul 2026 00:56:04 -0500 Subject: [PATCH 114/124] fix(node): bound GITLAWB_IPFS_REQUEST_BUDGET_SECS by representability ipfs_request_budget_secs parsed range(1..), so every positive u64 was accepted, but get_by_cid derives the request deadline as Instant::now() + Duration::from_secs(that). That addition is an explicit overflow check rather than a debug-only one, so an oversized value aborts every /ipfs/{cid} request in a release build instead of setting a very long budget. The route is anon-reachable, so an operator typo is felt publicly. Reuse GIT_SERVICE_TIMEOUT_SECS_MAX, the ceiling the sibling knob already carries: the bound is representability, not a policy view of a sane budget, so the large "set it very large to disable" values operators were told to use still parse. Reject rather than clamp, matching the other bounds in this config. Execution evidence: - RED before the change: the new test failed with "3153600001 is past the representable ceiling and must be rejected at parse time" while its disable-value assertions passed, so the test is not vacuous in either direction. - GREEN after: config::tests 13 passed, including the sibling git_service_timeout regression test. Sweep for the same defect class, re-run with a widened shape because the original grep missed a site. Passes: Instant::now() sites; from_secs() fed by config fields; config fields reaching a Duration through an arithmetic expression; config fields reaching a tokio timer constructor. The arithmetic and timer passes each return exactly one hit, heartbeat_interval_hours (config.rs 150, main.rs 916, operator.rs 161), which has no range at all and multiplies by 3600 before the Duration. That knob is a separate defect on a path this branch does not touch, so it is recorded for a follow-up rather than widened into here. Every other Instant addition derives from the already-bounded git_service_timeout_secs or from a constant. --- crates/gitlawb-node/src/config.rs | 76 +++++++++++++++++++++++++++++-- 1 file changed, 73 insertions(+), 3 deletions(-) diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index c038a525..2d4299c2 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -500,13 +500,21 @@ pub struct Config { /// slot past it. Still unbounded: the probe's `object_store_readable` check is a /// synchronous filesystem sweep with nothing to reap, so a wedged filesystem can /// hold the slot past the deadline. - /// Must be positive. Default: 600s (10 min), matching - /// `git_service_timeout_secs` so a single full-length walk still fits. + /// Must be positive, and no larger than `GIT_SERVICE_TIMEOUT_SECS_MAX`. The ceiling is + /// representability, NOT a policy view of a sane budget: `get_by_cid` derives the + /// request deadline as `Instant::now() + Duration::from_secs(this)`, and that addition + /// is an explicit overflow check rather than a debug-only one, so a value past the + /// ceiling aborts every `/ipfs/{cid}` request in a release build instead of setting a + /// very long budget. Rejecting at parse time keeps that value out of every reachable + /// configuration. Setting it very large is still the way to effectively disable the + /// budget; the ceiling clears every such setting (see the constant's own note). + /// Default: 600s (10 min), matching `git_service_timeout_secs` so a single full-length + /// walk still fits. #[arg( long, env = "GITLAWB_IPFS_REQUEST_BUDGET_SECS", default_value_t = 600, - value_parser = clap::value_parser!(u64).range(1..) + value_parser = clap::value_parser!(u64).range(1..=GIT_SERVICE_TIMEOUT_SECS_MAX) )] pub ipfs_request_budget_secs: u64, @@ -819,6 +827,68 @@ mod tests { ); } + /// #174 (RED-before/GREEN-after): the upper bound is what keeps the deadline derived + /// from this knob in range. `get_by_cid` builds the request budget as + /// `Instant::now() + Duration::from_secs(this)` (api/ipfs.rs), and that addition is an + /// explicit overflow check rather than a debug-only one, so an oversized value aborts + /// every `/ipfs/{cid}` request in a release build instead of setting a very long budget. + /// The route is anon-reachable, so the failure is operator-triggered but publicly felt. + /// Checked at parse time so no reachable configuration can carry a value the deadline + /// cannot represent. + /// + /// The large "disable the bound" settings are asserted alongside the rejections on + /// purpose, the same way the `git_service_timeout_secs` sibling does it. The ceiling + /// exists to exclude unrepresentable values, not to impose a view of a reasonable + /// budget, so a node already running on such a value must not start failing at boot on + /// upgrade. A test that only checked the boundary would pass with a far tighter cap. + #[test] + fn ipfs_request_budget_rejects_values_no_derived_duration_can_represent() { + let parse = |secs: u64| { + Config::try_parse_from([ + "gitlawb-node", + "--ipfs-request-budget-secs", + &secs.to_string(), + ]) + }; + + // Large "disable the bound" values that predate the ceiling still parse. + for disable in [1_000_000_000, 999_999_999] { + assert_eq!( + parse(disable) + .unwrap_or_else(|e| panic!("{disable} was a working setting: {e}")) + .ipfs_request_budget_secs, + disable + ); + } + + // At the ceiling: accepted, and the derived deadline still fits. This knob feeds + // only the `Instant` addition (no multiply derivation like the lease steal bound), + // so there is no `checked_mul` clause to carry over from the sibling test. + let at_max = + parse(GIT_SERVICE_TIMEOUT_SECS_MAX).expect("the documented maximum must parse"); + assert_eq!( + at_max.ipfs_request_budget_secs, + GIT_SERVICE_TIMEOUT_SECS_MAX + ); + assert!(std::time::Instant::now() + .checked_add(std::time::Duration::from_secs( + at_max.ipfs_request_budget_secs + )) + .is_some()); + + // Past the ceiling, and the top of the u64 range clap used to accept — the value + // that panics `Instant::now() + Duration::from_secs(..)` outright. + for over in [GIT_SERVICE_TIMEOUT_SECS_MAX + 1, u64::MAX] { + assert!( + parse(over).is_err(), + "{over} is past the representable ceiling and must be rejected at parse time" + ); + } + assert!(std::time::Instant::now() + .checked_add(std::time::Duration::from_secs(u64::MAX)) + .is_none()); + } + #[test] fn max_concurrent_reads_per_caller_defaults_and_rejects_out_of_range() { assert_eq!( From 87bdc0cebab6ad603a8625e53efadd3b7fbfce76 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 31 Jul 2026 00:57:53 -0500 Subject: [PATCH 115/124] docs(node): state what the pin-admission pool actually bounds The pin pool's docs claimed it bounded "how many MB-scale lists are held at once across all repos". It does not. On the local IPFS path the caller materializes the object list before pin_new_objects_gated acquires, so a task parked waiting for a permit is still holding its full list, and how many tasks park is capped only per repo by EncryptInflight. Across distinct repos that retained memory is unbounded, which is the gap a reviewer raised against this claim. Correct the claim at each site rather than implying the pool closes an exposure it cannot see. The pool does bound concurrent pin loops, and therefore lists held while being pinned; that part was always true and is kept. The residual is now named where each claim lives, including on the config knob, so an operator does not read the knob as a memory cap it cannot deliver. Bounding the retention is a real change to the capture shape (the Pinata twin already avoids it by acquiring before it derives) and is not attempted here. Sites corrected: pin_new_objects_gated's docstring and its F2b empty-list comment, the empty-list test's docstring, AppState::pin_semaphore and AppState::encrypt_inflight field docs, the max_concurrent_pin_tasks knob doc, and the construction comment in main. Sweep over pin_semaphore, F6, and "held at once". Left alone with reason: ipfs_pin.rs 16/31/174-200/518 describe the loop that genuinely holds the permit, so their claims are about hold duration and are true; repos.rs 1029-1036 and 2276-2294 describe the Pinata path, which acquires before materializing, so its "at most pin_semaphore permits' worth exist at once" holds; repos.rs 5898 claims concurrent loops are bounded, which is the true concurrency claim; the F6 hits in api/ipfs.rs are that file's own review numbering for walk-permit ordering, unrelated to this pool; constructions and test fixtures carry no claim. No behavior change: the diff is comments and docstrings only, checked by filtering the diff for non-comment lines. --- crates/gitlawb-node/src/api/repos.rs | 27 ++++++++++++++++++--------- crates/gitlawb-node/src/config.rs | 10 +++++++--- crates/gitlawb-node/src/main.rs | 6 ++++-- crates/gitlawb-node/src/state.rs | 13 +++++++++++-- 4 files changed, 40 insertions(+), 16 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 22a216fe..5f4ab9b2 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1126,8 +1126,17 @@ async fn pinata_object_list_for_refs( /// Pin `object_list` to the local IPFS node under the global pin-admission permit /// (#174 F6). `EncryptInflight` bounds the pin-task COUNT to one per repo, but each /// pin loop holds a full per-push object-id list while walking it; this permit bounds -/// how many such MB-scale lists are held at once across all repos. DEFERS (waits) when -/// the pool is full — never drops, since a dropped pin loses the replication copy. +/// how many pin loops RUN CONCURRENTLY, and therefore how many such MB-scale lists are +/// held WHILE BEING PINNED. DEFERS (waits) when the pool is full — never drops, since a +/// dropped pin loses the replication copy. +/// +/// What it does NOT bound, stated plainly rather than implied closed: on this path the +/// caller materializes `object_list` BEFORE this function acquires, so a task that is +/// parked here waiting for a permit is still holding its full list. The parked-task +/// count is capped only per repo by `EncryptInflight`, so across distinct repos the +/// retained list memory is not bounded by this pool. Bounding that is a real change to +/// the capture shape and is deliberately not attempted here; the Pinata twin below +/// avoids it by acquiring BEFORE it derives its list. async fn pin_new_objects_gated( pin_sem: &Arc, ipfs_api: &str, @@ -1136,8 +1145,8 @@ async fn pin_new_objects_gated( db: &Arc, ) -> Vec<(String, String)> { // Nothing to pin: answer before taking a permit (#174 F2b). The permit bounds how - // many MB-scale object lists are held at once and an empty list holds none, so - // parking here would spend a global pin slot on no work. The pool DEFERS rather + // many pin loops run concurrently, and an empty list does no pinning, so parking + // here would spend a global pin slot on no work. The pool DEFERS rather // than sheds, so those calls stall pins for every other repo. Empty is the normal // shape for a push whose walk failed or that may replicate nothing. if object_list.is_empty() { @@ -5939,11 +5948,11 @@ mod tests { assert!(out.is_empty(), "an empty ipfs_api pins nothing"); } - /// #174 F2b: the pin permit bounds how many MB-scale object lists are held at once, - /// so a call with NOTHING to pin must not take one. It otherwise spends a global pin - /// slot on no work, and the pool DEFERS rather than sheds, so those calls stall pins - /// for every other repo. The empty case is the normal shape for a push whose walk - /// failed or that may replicate nothing. + /// #174 F2b: the pin permit bounds how many pin loops run concurrently, so a call + /// with NOTHING to pin must not take one. It otherwise spends a global pin slot on no + /// work, and the pool DEFERS rather than sheds, so those calls stall pins for every + /// other repo. The empty case is the normal shape for a push whose walk failed or + /// that may replicate nothing. /// /// Load-bearing: without the guard this call parks on the exhausted pool exactly like /// the non-empty one above, and the completion assertion fails. diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 2d4299c2..d01fdaa6 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -372,9 +372,13 @@ pub struct Config { /// pin-task COUNT to one per repo, but each pin loop holds a full per-push /// object-id list (up to `git_max_pack_bytes` worth of OIDs) while it walks it, /// so N distinct repos could hold N such lists at once. This caps how many run - /// concurrently, bounding that MB-scale memory regardless of how many repos an - /// authenticated actor pushes to (#174 F6). Beyond it a pin loop DEFERS (waits), - /// never drops — a dropped pin would lose the object's replication copy. + /// concurrently (#174 F6). Beyond it a pin loop DEFERS (waits), never drops — a + /// dropped pin would lose the object's replication copy. + /// + /// It does not cap the memory itself: the local IPFS path builds its list before + /// taking a permit, so tasks parked on this pool still hold theirs, and how many + /// park is capped only per repo. Lowering this knob bounds concurrent pinning, not + /// how much an actor pushing to many repos can retain. /// /// Default: 8. Must be between 1 and 1_048_576. #[arg( diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 486bfa0e..4c6cf905 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -402,8 +402,10 @@ async fn main() -> Result<()> { git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new( config.max_concurrent_git_pushes, )), - // Bounds concurrent post-push pin loops (their MB-scale object-id lists) across - // all repos (#174 F6), independent of the per-repo encrypt-task coalescing below. + // Bounds how many post-push pin loops run concurrently across all repos (#174 F6), + // independent of the per-repo encrypt-task coalescing below. Not a bound on the + // MB-scale object-id lists themselves: parked tasks still hold theirs (see the + // field doc on AppState::pin_semaphore). pin_semaphore: Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_pin_tasks)), // Coalesces the DETACHED post-push encryption tasks per repo so a rapid pusher // cannot grow the outstanding parked-waiter set past one task per repo (#174 diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index a189548b..df78668e 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -133,13 +133,22 @@ pub struct AppState { /// per repo, but each pin loop holds a full per-push object-id list while walking /// it, so N distinct repos could hold N such MB-scale lists at once. This caps how /// many run concurrently; a loop DEFERS (waits) when the pool is full, never drops. + /// + /// It does NOT bound the lists held by tasks PARKED on it. The local IPFS path + /// materializes its list before acquiring, so a parked task still holds one, and the + /// parked-task count is capped only per repo by `encrypt_inflight`. Cross-repo + /// retained memory is therefore not bounded by this pool. The Pinata twin acquires + /// before it derives and does not carry that residual. pub pin_semaphore: Arc, /// Bounds the outstanding post-push encryption-task set to at most one PER REPO by /// coalescing (#174 P2-2). This is NOT a global cap: N distinct repos still admit N /// tasks; the cross-repo residual (an authenticated actor pushing to many repos /// leaves many parked tasks) is throttled by auth plus the per-IP/per-DID rate - /// limits, and its real cost — the MB-scale per-push object-id list each pin loop - /// holds — is bounded by `pin_semaphore`, not this. `git_encrypt_semaphore` caps + /// limits. Its real cost, the MB-scale per-push object-id list each parked task + /// holds, is NOT bounded by `pin_semaphore` either: on the local IPFS path the list + /// is materialized before that permit is acquired, so a task parked on the pin pool + /// still holds one. That pool bounds concurrent pin loops, not parked retention, so + /// nothing currently bounds this memory across repos. `git_encrypt_semaphore` caps /// *active* walks; this caps duplicate SPAWNS per repo. Before spawning a per-push /// encryption task, the receive-pack handler consults this set: if the repo already /// has a task in flight it coalesces (skips the duplicate spawn) rather than parking From 41325c34c4106f57fd877496d0245f02fcf57036 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 31 Jul 2026 00:59:51 -0500 Subject: [PATCH 116/124] docs(node): state what the permit-less load-shed peek actually delivers The three handler peeks said they shed "before any DB/disk work", which reads as an admission bound on the DB window. It is not one. The peek holds no permit, and permits are only held from the authoritative acquire below, after the visibility and rate gates, so a burst arriving while permits are free all proceeds into the DB and none of it sheds. What the peek actually gives is a cheap 503 for one request once the pool has ALREADY filled, sparing that request's DB work. The gate ordering that makes it a peek is deliberate and stays: a denied or rate-limited request must consume no slot, and one source must not hold global slots through the DB/visibility window. Reordering to bound the window would undo that. So this corrects the claims and changes no behavior; bounding the DB window would need an admission mechanism the peek is not. test_support.rs's receive-pack docstring additionally called the peek "the permit" and "the first statement". Both were wrong at this head: it is a non-holding peek, and the authoritative write-pool acquire sits after the per-repo lease. Corrected along with the shed claim. The "remove the block and this goes red" sentences stay, since those tests drive an already-saturated pool, which is the case the peek does handle. Sweep over "before any DB", "before DB", "pre-DB". Left alone with reason: api/ipfs.rs:142 describes a HELD walk-admission acquire, so its claim is true; graphql/mutation.rs:201 and test_support.rs:528 are about auth ordering, unrelated; the repos.rs test docstrings at 4194/4213/4556/ 4618/4682 describe test mechanics against a saturated pool, which is the case the peek delivers. No behavior change: the diff is comments and docstrings only, checked by filtering the diff for non-comment lines. --- crates/gitlawb-node/src/api/repos.rs | 35 +++++++++++++++---------- crates/gitlawb-node/src/test_support.rs | 34 +++++++++++++++--------- 2 files changed, 42 insertions(+), 27 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 5f4ab9b2..7cec9914 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -567,12 +567,16 @@ pub async fn git_info_refs( "unsupported git service: {service}" ))); } - // #62 cheap pre-DB load shed: if the pool this service draws from is already - // saturated, shed with 503 before any DB/disk work. Best-effort (holds no - // permit); the authoritative hold is `git_permit` below, after the per-source - // cap. Restores the shed-before-DB property the reordered held acquire alone - // would drop, while the reorder still prevents one source from occupying global - // slots during the DB/visibility window. + // #62 cheap load shed: if the pool this service draws from is ALREADY saturated, + // shed this request with a 503 before it does any DB/disk work. Best-effort and + // permit-less, so it is a snapshot, not admission: it spares THIS request's DB + // work once the pool has filled, and nothing more. It is NOT a bound on the DB + // window. Permits are only held from `git_permit` below, after the visibility and + // rate gates, so a burst arriving while permits are free all proceeds into the DB + // and none of it sheds here. That ordering is deliberate (a denied or rate-limited + // request must consume no slot, and one source must not hold global slots through + // the DB/visibility window); bounding the DB window itself would need an admission + // mechanism this peek is not. { // The receive-pack advertisement peeks its DEDICATED advert pool, not the // write pool the authenticated POST uses (#174) — matching the held acquire @@ -1291,9 +1295,10 @@ pub async fn git_upload_pack( headers: axum::http::HeaderMap, body: Bytes, ) -> Result { - // #62 cheap pre-DB load shed (see git_info_refs): shed before DB when the read - // pool is saturated; the authoritative hold is `git_permit` below, after the - // per-source cap. + // #62 cheap load shed (see git_info_refs for the full contract): spares THIS + // request's DB work when the read pool is ALREADY saturated. Permit-less snapshot, + // not admission, so it does not bound the DB window; the authoritative hold is + // `git_permit` below, after the per-source cap. if state.git_read_semaphore.available_permits() == 0 { tracing::warn!("served-git concurrency cap reached; shedding request with 503 (pre-DB)"); return Err(AppError::Overloaded( @@ -1642,11 +1647,13 @@ pub async fn git_receive_pack( body: Bytes, ) -> Result { let name = smart_http_repo_name(&repo)?; - // Fast-path shed BEFORE the DB lookup when the write pool is saturated, so a push - // flood on a full pool does not hit Postgres per request. Best-effort (racy) and - // NON-holding: the authoritative, held permit is taken after the per-repo lease below - // (so a lease-blocked waiter pins no write slot — #174 F3 review). This peek only - // restores the cheap pre-DB shed the permit reorder would otherwise have lost. + // Fast-path shed before the DB lookup when the write pool is ALREADY saturated, so a + // push flood against a full pool does not hit Postgres per request. Best-effort + // (racy) and NON-holding: a snapshot, not admission. It spares this request's DB + // work once the pool has filled; pushes arriving while permits are free all proceed + // into the DB, so it does not bound that window. The authoritative, held permit is + // taken after the per-repo lease below (so a lease-blocked waiter pins no write slot + // — #174 F3 review). if state.git_write_semaphore.available_permits() == 0 { return Err(AppError::Overloaded( "git service at capacity, retry shortly".into(), diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 0eb11481..a93c882d 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -228,10 +228,12 @@ mod tests { /// PR3 (#62): the served-git concurrency cap sheds at the HTTP layer before the /// DB. The held `git_permit` acquire now sits after the per-source cap, so the - /// shed-before-DB property is carried by an explicit `available_permits() == 0` - /// early check at the top of the handler (the held permit remains the - /// authoritative bound further down). DB-free: an exhausted semaphore sheds - /// before any DB/disk access, so a lazy state works. Remove the early-shed block + /// cheap early shed is carried by an explicit `available_permits() == 0` check at + /// the top of the handler (the held permit remains the authoritative bound further + /// down). That check is a permit-less snapshot: it spares a request's DB work once + /// the pool is ALREADY saturated, which is the case this test drives, and it does + /// not bound the DB window in general. DB-free here because an exhausted semaphore + /// sheds before any DB/disk access, so a lazy state works. Remove the early-shed block /// from git_info_refs and this goes red (the request falls through to the DB and /// returns something other than 503). #[tokio::test] @@ -267,10 +269,11 @@ mod tests { } /// PR3 (#62) sibling of the info/refs shed test: git-upload-pack carries the same - /// explicit `available_permits() == 0` early-shed check at the top, so an - /// exhausted semaphore must shed it with a 503 before any DB/disk work. - /// Anonymous-reachable, so no auth injection is needed. Remove the early-shed - /// block from git_upload_pack and this goes red. + /// explicit `available_permits() == 0` early check at the top, so an ALREADY + /// exhausted semaphore must shed the request with a 503 before its DB/disk work. + /// That is the case the permit-less snapshot does deliver; it is not an admission + /// bound on the DB window. Anonymous-reachable, so no auth injection is needed. + /// Remove the early-shed block from git_upload_pack and this goes red. #[tokio::test] async fn git_upload_pack_sheds_with_503_when_semaphore_exhausted() { let mut state = test_state_lazy(); @@ -305,8 +308,10 @@ mod tests { /// PR3 (#62) receive-pack sibling of the info/refs shed test: the early shed /// selects the dedicated ADVERT pool for a git-receive-pack advertisement (#174), - /// so an exhausted advert pool sheds the advert with 503 before any DB/disk work - /// — while the write pool (reserved for authenticated POSTs) is left free here. + /// so an ALREADY exhausted advert pool sheds the advert with 503 before its DB/disk + /// work (the case the permit-less snapshot delivers, not an admission bound on the + /// DB window) — while the write pool (reserved for authenticated POSTs) is left + /// free here. /// Flip the pool selection back to the write pool, or remove the early-shed /// block, and this goes red. #[tokio::test] @@ -344,9 +349,12 @@ mod tests { /// PR3 (#62) sibling for the push path: git-receive-pack requires an /// AuthenticatedDid extension (production: require_signature injects it), so the /// request carries one via signed_request_as — without it the Extension - /// extractor 500s before the handler body reaches git_permit. The permit is the - /// first statement, so an exhausted semaphore still sheds 503 before any DB - /// work. Remove the permit line from git_receive_pack and this goes red. + /// extractor 500s before the handler body reaches the shed. What sits at the top of + /// the handler is a permit-less `available_permits() == 0` peek, NOT the permit + /// itself: the authoritative held acquire is taken after the per-repo lease, so a + /// lease-blocked waiter pins no write slot. An ALREADY exhausted pool is the case + /// the peek delivers, so the request sheds 503 before its DB work here. Remove the + /// early-shed block from git_receive_pack and this goes red. #[tokio::test] async fn git_receive_pack_sheds_with_503_when_semaphore_exhausted() { let mut state = test_state_lazy(); From 807bc935b362bab129a3a8a78bcbb4682e0df6a7 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 31 Jul 2026 01:24:46 -0500 Subject: [PATCH 117/124] fix(review): correct the budget-knob claims and mirror the range in operator docs Review round on the three preceding commits. The knob's doc said a value past the ceiling "aborts every /ipfs/{cid} request in a release build". That is wrong, and three reviewers plus an outside model landed on it independently: the ceiling is 100 years, the Instant addition does not actually overflow until roughly 584 years on a u64-nanosecond Instant, so values in between are representable and are rejected as a deliberate safety margin, not because they panic. Reworded to say that, since a confidently wrong comment is worse than the vague one it replaced. Also on the knob: - GIT_SERVICE_TIMEOUT_SECS_MAX has two consumers now, so its own doc says so, and notes this knob derives only the Instant addition and not the lease-steal multiply. - The "clears every disable setting" phrasing overclaimed: u32::MAX sits above the ceiling. Now names the documented sentinels instead. Operator-facing docs: the sibling knob's ceiling commit updated both README.md and .env.example in the same change, and this one had not. Both now carry the accepted range, matching that precedent. House style: five comment lines added by the preceding commits used em dashes, which this project does not allow. Four were carried over from the text being edited and one was copied from the sibling test. All replaced. Checked mechanically over the added lines rather than by eye. Two comments restated an argument in full immediately after pointing at the canonical site for it. Trimmed to the pointer plus the local delta. Scoping correction worth recording: the two preceding commits claimed "no behavior change". That holds for repos.rs, state.rs, main.rs and test_support.rs, but a config.rs field doc IS clap --help output, so the pin-knob doc edit changes operator-visible help text. No runtime behavior changes either way. Verification: full suite 733 passed, fmt clean, clippy clean under the CI form. The range guard was separately proven load-bearing in both directions by mutation: reverting to range(1..) goes RED on the rejection assertions, and tightening the ceiling to 500_000_000 goes RED on the disable-value assertions. Known residual, not fixed here: the bound is parse-side only, so a Config built by direct construction rather than through clap can still hold an unrepresentable value. --- .env.example | 3 ++- README.md | 2 +- crates/gitlawb-node/src/api/repos.rs | 15 ++++++------- crates/gitlawb-node/src/config.rs | 30 ++++++++++++++++--------- crates/gitlawb-node/src/state.rs | 6 ++--- crates/gitlawb-node/src/test_support.rs | 2 +- 6 files changed, 33 insertions(+), 25 deletions(-) diff --git a/.env.example b/.env.example index 5a3a5acc..1d728f88 100644 --- a/.env.example +++ b/.env.example @@ -231,7 +231,8 @@ GITLAWB_IPFS_MAX_REPO_VISITS=1024 # cannot hold the request's walk slot past it. One hang path is still unbounded: # the probe's object-store readability check is a plain filesystem sweep with # nothing to reap, so a wedged filesystem can hold the slot past the deadline. -# Default 600. +# Must be 1..=3153600000 (100 years): the node derives an Instant deadline from +# this value, and a larger one cannot be represented. Default 600. GITLAWB_IPFS_REQUEST_BUDGET_SECS=600 # Max /ipfs/{cid} requests per client IP per hour (route flood brake, distinct # from the concurrency caps above). 0 disables. Default 600. diff --git a/README.md b/README.md index d5b9c6b9..2c382d7c 100644 --- a/README.md +++ b/README.md @@ -350,7 +350,7 @@ Important node settings: | `GITLAWB_IPFS_WALK_PER_SOURCE` | Max concurrent `/ipfs` walks a single source IP may hold. Default 4. | | `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max expensive path-scope visibility walks per `/ipfs/{cid}` request; over-cap repos are skipped and the scan continues, shedding a retryable 503 (not a false 404) if the object is then found nowhere. Default 64. | | `GITLAWB_IPFS_MAX_REPO_VISITS` | Ceiling on repos one `/ipfs/{cid}` request may visit (acquire + probe) past the visibility gate. Also the worst-case per-request Tigris fetch count. On exhaustion the scan stops with a retryable 503. Default 1024. | -| `GITLAWB_IPFS_REQUEST_BUDGET_SECS` | Absolute wall-clock budget for one admitted `/ipfs/{cid}` request's acquire+walk lifetime. Per-stage clamps bound the acquire and walk stages to the remaining budget, and no stage starts once it is exhausted; the scan then stops with a retryable 503. The object-type probe and content-read `cat-file` subprocesses are budget-checked before starting and each also run under their own deadline (the lesser of `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` and the remaining budget), reaped via process-group teardown, so a hung `cat-file` cannot hold the request's walk slot past it. One hang path is still unbounded: the probe's object-store readability check is a plain filesystem sweep with nothing to reap, so a wedged filesystem can hold the slot past the deadline. Default 600. | +| `GITLAWB_IPFS_REQUEST_BUDGET_SECS` | Absolute wall-clock budget for one admitted `/ipfs/{cid}` request's acquire+walk lifetime. Per-stage clamps bound the acquire and walk stages to the remaining budget, and no stage starts once it is exhausted; the scan then stops with a retryable 503. The object-type probe and content-read `cat-file` subprocesses are budget-checked before starting and each also run under their own deadline (the lesser of `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` and the remaining budget), reaped via process-group teardown, so a hung `cat-file` cannot hold the request's walk slot past it. One hang path is still unbounded: the probe's object-store readability check is a plain filesystem sweep with nothing to reap, so a wedged filesystem can hold the slot past the deadline. Default 600. Accepted range is 1 to 3153600000 (100 years), since the node derives a deadline from this value and a larger one cannot be represented. | | `GITLAWB_IPFS_RATE_LIMIT` | Max `/ipfs/{cid}` requests per client IP per hour (route flood brake). 0 disables. Default 600. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 7cec9914..8ce60e7e 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1131,8 +1131,8 @@ async fn pinata_object_list_for_refs( /// (#174 F6). `EncryptInflight` bounds the pin-task COUNT to one per repo, but each /// pin loop holds a full per-push object-id list while walking it; this permit bounds /// how many pin loops RUN CONCURRENTLY, and therefore how many such MB-scale lists are -/// held WHILE BEING PINNED. DEFERS (waits) when the pool is full — never drops, since a -/// dropped pin loses the replication copy. +/// held WHILE BEING PINNED. DEFERS (waits) when the pool is full and never drops, since +/// a dropped pin loses the replication copy. /// /// What it does NOT bound, stated plainly rather than implied closed: on this path the /// caller materializes `object_list` BEFORE this function acquires, so a task that is @@ -1295,10 +1295,9 @@ pub async fn git_upload_pack( headers: axum::http::HeaderMap, body: Bytes, ) -> Result { - // #62 cheap load shed (see git_info_refs for the full contract): spares THIS - // request's DB work when the read pool is ALREADY saturated. Permit-less snapshot, - // not admission, so it does not bound the DB window; the authoritative hold is - // `git_permit` below, after the per-source cap. + // #62 cheap load shed. Permit-less snapshot, not admission; see git_info_refs for + // what it does and does not bound. The authoritative hold is `git_permit` below, + // after the per-source cap. if state.git_read_semaphore.available_permits() == 0 { tracing::warn!("served-git concurrency cap reached; shedding request with 503 (pre-DB)"); return Err(AppError::Overloaded( @@ -1652,8 +1651,8 @@ pub async fn git_receive_pack( // (racy) and NON-holding: a snapshot, not admission. It spares this request's DB // work once the pool has filled; pushes arriving while permits are free all proceed // into the DB, so it does not bound that window. The authoritative, held permit is - // taken after the per-repo lease below (so a lease-blocked waiter pins no write slot - // — #174 F3 review). + // taken after the per-repo lease below, so a lease-blocked waiter pins no write slot + // (#174 F3 review). if state.git_write_semaphore.available_permits() == 0 { return Err(AppError::Overloaded( "git service at capacity, retry shortly".into(), diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index d01fdaa6..aab8e758 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -1,7 +1,13 @@ use clap::Parser; use std::path::PathBuf; -/// Upper bound on `git_service_timeout_secs`, in seconds (100 years). +/// Upper bound on `git_service_timeout_secs` and `ipfs_request_budget_secs`, in seconds +/// (100 years). +/// +/// Two consumers now, so a future tightening moves both. `ipfs_request_budget_secs` +/// derives only the `Instant` addition in `get_by_cid`, not the lease-steal multiply +/// below, but it shares this ceiling because the defect class and the "set it very large +/// to disable" contract are the same. /// /// The knob is not just stored, it is arithmetic input: the write path derives the /// per-repo lease steal bound from it (`* 2 + 60`), and #174 routed it into @@ -372,8 +378,8 @@ pub struct Config { /// pin-task COUNT to one per repo, but each pin loop holds a full per-push /// object-id list (up to `git_max_pack_bytes` worth of OIDs) while it walks it, /// so N distinct repos could hold N such lists at once. This caps how many run - /// concurrently (#174 F6). Beyond it a pin loop DEFERS (waits), never drops — a - /// dropped pin would lose the object's replication copy. + /// concurrently (#174 F6). Beyond it a pin loop DEFERS (waits) and never drops, + /// since a dropped pin would lose the object's replication copy. /// /// It does not cap the memory itself: the local IPFS path builds its list before /// taking a permit, so tasks parked on this pool still hold theirs, and how many @@ -507,11 +513,14 @@ pub struct Config { /// Must be positive, and no larger than `GIT_SERVICE_TIMEOUT_SECS_MAX`. The ceiling is /// representability, NOT a policy view of a sane budget: `get_by_cid` derives the /// request deadline as `Instant::now() + Duration::from_secs(this)`, and that addition - /// is an explicit overflow check rather than a debug-only one, so a value past the - /// ceiling aborts every `/ipfs/{cid}` request in a release build instead of setting a - /// very long budget. Rejecting at parse time keeps that value out of every reachable - /// configuration. Setting it very large is still the way to effectively disable the - /// budget; the ceiling clears every such setting (see the constant's own note). + /// is an explicit overflow check rather than a debug-only one, so a value near the top + /// of the `u64` range aborts every `/ipfs/{cid}` request in a release build instead of + /// setting a very long budget. The ceiling sits an order of magnitude below where that + /// starts (see the constant's own note), so it is a conservative margin rather than the + /// exact overflow point; rejecting at parse time keeps the unrepresentable values out + /// of every reachable configuration. Setting it very large is still the way to + /// effectively disable the budget, and the documented sentinels (`999999999`, + /// `1000000000`) are well inside the range. /// Default: 600s (10 min), matching `git_service_timeout_secs` so a single full-length /// walk still fits. #[arg( @@ -880,8 +889,9 @@ mod tests { )) .is_some()); - // Past the ceiling, and the top of the u64 range clap used to accept — the value - // that panics `Instant::now() + Duration::from_secs(..)` outright. + // Past the ceiling, and the top of the u64 range clap used to accept. Only the + // latter actually panics `Instant::now() + Duration::from_secs(..)`; the ceiling + // sits well below that, which is the conservative margin the constant documents. for over in [GIT_SERVICE_TIMEOUT_SECS_MAX + 1, u64::MAX] { assert!( parse(over).is_err(), diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index df78668e..c1086841 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -145,10 +145,8 @@ pub struct AppState { /// tasks; the cross-repo residual (an authenticated actor pushing to many repos /// leaves many parked tasks) is throttled by auth plus the per-IP/per-DID rate /// limits. Its real cost, the MB-scale per-push object-id list each parked task - /// holds, is NOT bounded by `pin_semaphore` either: on the local IPFS path the list - /// is materialized before that permit is acquired, so a task parked on the pin pool - /// still holds one. That pool bounds concurrent pin loops, not parked retention, so - /// nothing currently bounds this memory across repos. `git_encrypt_semaphore` caps + /// holds, is NOT bounded by `pin_semaphore` either; see that field's doc above for + /// why. Nothing currently bounds this memory across repos. `git_encrypt_semaphore` caps /// *active* walks; this caps duplicate SPAWNS per repo. Before spawning a per-push /// encryption task, the receive-pack handler consults this set: if the repo already /// has a task in flight it coalesces (skips the duplicate spawn) rather than parking diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index a93c882d..2b5aef95 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -310,7 +310,7 @@ mod tests { /// selects the dedicated ADVERT pool for a git-receive-pack advertisement (#174), /// so an ALREADY exhausted advert pool sheds the advert with 503 before its DB/disk /// work (the case the permit-less snapshot delivers, not an admission bound on the - /// DB window) — while the write pool (reserved for authenticated POSTs) is left + /// DB window), while the write pool (reserved for authenticated POSTs) is left /// free here. /// Flip the pool selection back to the write pool, or remove the early-shed /// block, and this goes red. From b0482480a873d7436c032b3de5fccbb41fe46a9c Mon Sep 17 00:00:00 2001 From: t Date: Fri, 31 Jul 2026 02:09:32 -0500 Subject: [PATCH 118/124] docs(node): correct the ceiling margin, it is 5.85x not an order of magnitude Both doc comments describing GIT_SERVICE_TIMEOUT_SECS_MAX said the 100-year ceiling sits "an order of magnitude" under the ~584-year limit of a u64-nanosecond Instant. The real margin is 3,153,600,000 seconds against 18,446,744,074, a factor of about 5.85. An order of magnitude means ten. Worth noting how it spread, since the fix is trivial and the pattern is not. The phrase was written once on the constant, then copied into the ipfs_request_budget_secs doc by the commit that was at that moment correcting a different false claim in the same file. Nobody re-ran the division at either site, because quoting an existing sentence feels like citation rather than assertion. The direction of the error is the part that matters: a vague "well under" would have been true and would have invited a reader to check. The precise-sounding "order of magnitude" was false and discouraged checking. Comments only; the diff carries no executable line, checked by filtering it for non-comment changes. fmt and clippy clean under the CI form. --- crates/gitlawb-node/src/config.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index aab8e758..fefa063e 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -22,7 +22,7 @@ use std::path::PathBuf; /// this very large to disable the bound, so values like `999999999` (~31 years) are /// working production "off" settings; a tighter, tidier cap would fail those nodes at /// boot on upgrade over a value that was never the defect. 100 years clears every such -/// setting while staying an order of magnitude under the ~584-year ceiling of a +/// setting while staying a factor of about 5.85 under the ~584-year ceiling of a /// `u64`-nanosecond `Instant`, which is the tightest representation on any platform we /// build for. Every value that worked before still parses; only the ones that would have /// panicked are rejected. @@ -515,8 +515,8 @@ pub struct Config { /// request deadline as `Instant::now() + Duration::from_secs(this)`, and that addition /// is an explicit overflow check rather than a debug-only one, so a value near the top /// of the `u64` range aborts every `/ipfs/{cid}` request in a release build instead of - /// setting a very long budget. The ceiling sits an order of magnitude below where that - /// starts (see the constant's own note), so it is a conservative margin rather than the + /// setting a very long budget. The ceiling sits well below where that starts, about a + /// factor of 5.85 (see the constant's own note), so it is a conservative margin rather than the /// exact overflow point; rejecting at parse time keeps the unrepresentable values out /// of every reachable configuration. Setting it very large is still the way to /// effectively disable the budget, and the documented sentinels (`999999999`, From b4deb21b673796a888dad32ca14931b1e8239768 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 2 Aug 2026 18:39:33 -0500 Subject: [PATCH 119/124] test(node): restore three guard tests the #174 merge dropped The merge deleted coverage that no longer compiled against the reconciled designs, rather than re-seaming it. Three of those tests have counterparts in the merged tree after all: unlock_error_connection_close_is_bounded and write_guard_dropped_off_runtime_disposes_the_connection came back with close_conn_bounded and the Drop backstop, which the lock-pool design needed after all (the after_release hook cannot free a lock when the unlock errors on a live session). Both now observe the store's derived lock pool rather than the pool handed to for_testing, which is where the guard's connection actually lives. get_by_cid_caps_repos_walked_knob_bounds_the_walks was orphaned when its body was spliced into a sibling test. It drives GITLAWB_IPFS_MAX_REPOS_WALKED, which is the binding cap only now that the gate takes the tighter of the two walk ceilings. The one test not restored is write_guard_release_when_not_locked: the guard has a single construction site and it sits below the lock-or-bail, so a guard that never took the lock cannot exist. --- crates/gitlawb-node/src/api/ipfs.rs | 172 ++++++++++++++++++++++ crates/gitlawb-node/src/git/repo_store.rs | 95 ++++++++++++ 2 files changed, 267 insertions(+) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 02e9add1..04cf75c6 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -3263,6 +3263,178 @@ mod tests { ); } + /// Loop bound (cap N) + F2 truncation verdict: one `/ipfs/{cid}` request against a + /// CID present in many path-scoped repos must not serialize an unbounded number of + /// full-history walks — and cutting a candidate WITHOUT a verdict must not report + /// the object absent. With `ipfs_max_repos_walked = 1` and TWO public, path-scoped + /// repos both carrying the blob, the first candidate is walked (empty allowed-set → + /// a deny VERDICT) and the second is cut by the cap (no verdict), so the fake git's + /// `rev-list` runs exactly once and the request sheds a retryable 503 + Retry-After + /// — never the old false 404 (the blob genuinely sits in the second repo). + /// This drives the GITLAWB_IPFS_MAX_REPOS_WALKED knob specifically. The merge left + /// two walk caps in play, this one and the branch's own history-walk ceiling, and + /// the gate takes the tighter of the two; setting this knob to 1 is what makes it + /// the binding one here. A sibling case covers the ceiling. + /// + /// MUTATION (RED): drop `config.ipfs_max_repos_walked` from the `min()` in the walk + /// gate and both repos are walked (count 2); drop the truncation taint on the skip + /// and the 503 decays to a 404. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_caps_repos_walked_knob_bounds_the_walks(pool: sqlx::PgPool) { + use std::process::Command; + + let tmp = tempfile::TempDir::new().unwrap(); + let walk_log = tmp.path().join("walks.log"); + // Fake git for the WALK: empty refs, `rev-parse` resolves, and each `rev-list` + // appends one line to a log (so the number of walks == the line count) and exits + // with EMPTY output (the allowed-set is empty, so every repo path-gates to a + // `continue` and the request 404s after walking). object_type uses the REAL git, + // so the seeded blob below must genuinely exist. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + for-each-ref) : ;;\n\ + rev-parse) echo deadbeef ;;\n\ + rev-list) echo walk >> \"{}\" ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + walk_log.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // The bound under test: walk at most one candidate repo per request. + let mut cfg = (*state.config).clone(); + cfg.ipfs_max_repos_walked = 1; + state.config = Arc::new(cfg); + + // Seed TWO public repos, each with the SAME blob (same content -> same sha256 OID + // -> same CID) under a path-scoped rule, so both are walk candidates for one CID. + let run = |args: &[&str], cwd: &std::path::Path| { + let out = Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + }; + let mut oid = String::new(); + for (i, name) in ["ipa", "ipb"].iter().enumerate() { + let owner = "z6ipfsN"; + state + .db + .upsert_mirror_repo(owner, name, &format!("/unused-{name}"), None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + let _ = std::fs::remove_dir_all(&bare); + std::fs::create_dir_all(&bare).unwrap(); + let work = tmp.path().join(format!("work{i}")); + std::fs::create_dir_all(work.join("src")).unwrap(); + // Identical content in both repos -> identical sha256 blob OID -> one CID. + std::fs::write(work.join("src/secret.txt"), b"loop bound proof\n").unwrap(); + run( + &["init", "-q", "--object-format=sha256", "-b", "main"], + &work, + ); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "src/secret.txt"], &work); + run(&["commit", "-q", "-m", "seed"], &work); + run( + &[ + "clone", + "--bare", + "-q", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + tmp.path(), + ); + if oid.is_empty() { + let out = Command::new("git") + .args(["rev-parse", "HEAD:src/secret.txt"]) + .current_dir(&work) + .output() + .expect("git rev-parse runs"); + oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); + } + state + .db + .set_visibility_rule( + &rec.id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderBBBBBBBBBBBBBBBBBBBBBBBB".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + } + // The resolver maps a requested CID back to an oid through the CID index, so a + // bare digest-as-oid CID resolves to nothing and 404s before any repo is + // visited. Register a legacy NULL-provenance row, which is also what routes the + // request to the bounded legacy scan this cap governs. Neither repo serves, so + // the key need not be the content CID. + let cid = seed_legacy_pin(&state, &oid).await; + + let peer: SocketAddr = "203.0.113.90:5000".parse().unwrap(); + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let resp = ipfs_router(state).oneshot(req).await.unwrap(); + // The first repo's walk yields the empty allowed-set (deny verdict); the second + // repo NEEDS a walk the cap forbids, so the scan is truncated without a verdict + // on it: retryable 503, never a false 404 for the blob it genuinely carries. + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a walk-cap truncation must shed a retryable 503, not report the object absent" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + + let walks = std::fs::read_to_string(&walk_log) + .map(|s| s.lines().count()) + .unwrap_or(0); + assert_eq!( + walks, 1, + "with the per-request repo-walk cap at 1, only the first candidate repo is \ + walked (the second is cut by the cap), so exactly one walk runs; got {walks}" + ); + } + /// Route rate limit is WIRED (not a silent no-op): the production `build_router` /// attaches an `IpRateLimiter` extension to the `/ipfs/{cid}` route, so a per-IP /// flood is braked with 429. A bare `rate_limit_by_ip` layer with no extension does diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 0b28963d..fb32a0b3 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -2015,6 +2015,101 @@ mod tests { } } + /// F3c (P2): the connection teardown on the failing-unlock path must be BOUNDED. + /// `release` awaits it inline while the global write permit, the per-source permit + /// and the write lease are all still held, and sqlx's `close()` carries no deadline + /// of its own, so a blackholed socket would park every later push to that repo + /// behind three pinned admission resources. + /// + /// What this covers: the deadline itself. A close that never resolves still lets + /// `close_conn_bounded` return, which is the property `release` depends on. What it + /// does NOT cover, and is reasoned rather than run: that sqlx's own `close()` is + /// what stalls in production. Making a real `PgConnection::close` hang needs a + /// blackholed TCP path to Postgres, and the flip has to land after the unlock + /// statement round-trips but before the Terminate write, which is not a seam this + /// module exposes. A never-resolving future is the faithful stand-in for that + /// close, and the F3b tests above already cover that `release` really routes its + /// close through here. + /// + /// Time is paused, so nothing here depends on wall clock: the runtime auto-advances + /// to the next timer, and the assertion is on which timer fired, not on elapsed + /// time. The outer bound is what turns a removed deadline into a failure rather + /// than a hung suite. + /// + /// Load-bearing: drop the `tokio::time::timeout` in `close_conn_bounded` and the + /// inner future never resolves, so the outer bound fires and this fails. + #[tokio::test(start_paused = true)] + async fn unlock_error_connection_close_is_bounded() { + let hanging = std::future::pending::>(); + let outcome = tokio::time::timeout( + UNLOCK_ERROR_CLOSE_TIMEOUT * 4, + close_conn_bounded("boundedclosetest", hanging), + ) + .await; + assert!( + outcome.is_ok(), + "a connection close that never completes must not hold the write lease and \ + both admission permits open-endedly: close_conn_bounded must give up and \ + drop the connection" + ); + } + + /// U8, the off-runtime arm: with no Tokio runtime there is nothing to spawn the + /// unlock onto, and the connection has already been taken out of the guard, so + /// dropping it with no unlock attempted returns it to the pool with the session + /// lock still held. Dropping a `PoolConnection` off a runtime is worse than that: + /// sqlx's return-to-pool path spawns, and its no-runtime fallback panics, so that + /// arm also panics in a destructor. + /// + /// Reached by dropping the guard on a plain `std::thread`, where + /// `Handle::try_current()` fails. + /// + /// Load-bearing: replace the `detach` arm with a plain `drop(conn)` and the join + /// sees sqlx's "requires a Tokio context" panic; `detach` gives up the pool slot, + /// so nothing is spawned and dropping the detached connection closes the socket, + /// which ends the session and frees the lock. + #[sqlx::test] + async fn write_guard_dropped_off_runtime_disposes_the_connection(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let store_pool = pool_without_idle_reaper(&pool).await; + let store = RepoStore::for_testing(dir.path().to_path_buf(), store_pool.clone()); + let owner = "did:key:z6MkDropOffRuntimeProofKKKKKKKKKKKKKKKKKK"; + let name = "dropoffruntimetest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + let mut checker = pool.acquire().await.expect("checker connection"); + let guard = store.acquire_write(owner, name).await.expect("acquire"); + // The guard's connection lives in the store's DERIVED lock pool, not the pool + // handed to `for_testing`; see `RepoStore::lock_pool`. + let lock_pool = store.lock_pool().clone(); + let size_before = lock_pool.size(); + assert!(size_before > 0, "the lock pool owns the guard's connection"); + + let dropped = std::thread::spawn(move || drop(guard)).join(); + assert!( + dropped.is_ok(), + "dropping a write guard off a Tokio runtime must not panic" + ); + + wait_until( + || lock_pool.size() == size_before - 1, + "the connection of a guard dropped off a runtime to be disposed of rather \ + than returned to the pool with no unlock attempted", + ) + .await; + wait_until_lock_free( + &mut checker, + key, + "a guard dropped off a runtime to end its session so postgres drops the lock", + ) + .await; + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } + /// A second pool over the same test database with the idle reaper DISABLED. /// /// `#[sqlx::test]`'s own pool sets `idle_timeout(1s)`, so a connection returned to From 3354c08b512b2d792be900f06c2d8c573da90674 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 2 Aug 2026 18:39:33 -0500 Subject: [PATCH 120/124] test(node): restore the four u3 drain tests onto the coalescer These covered post-push work being requeued rather than dropped, and were removed in the merge because they drove the replaced requeue loop. The behaviour they check is still there, so they re-seam onto the coalescer without any production change. The seam moved in three ways. The inflight key is the repo identity key, not the row id. A coalescing push now merges its (old, new) tip pairs into the pending slot, so a test that passes an empty vec merges nothing and the drain never laps: every coalesce carries real commit oids. And the full-scan arm is reached through the pending-slot overflow, so the leak test coalesces 1025 pairs and asserts the slot degraded to FullScan before the run. run_encrypt_pin_task_for_test builds the task context the way the production spawn site does and takes a real owner and name, because the drain re-fetches by owner and name; a blank name would resolve Gone on every lap. All four are mutation-proven: drain-does-nothing, stale-empty-rules, full-scan-skips-the-fail-closed-filter, and a leaked inflight key each turn their own test red. --- crates/gitlawb-node/src/api/repos.rs | 48 ++++ crates/gitlawb-node/src/test_support.rs | 362 ++++++++++++++++++++++++ 2 files changed, 410 insertions(+) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index beaf3a01..ae8b752d 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -912,6 +912,54 @@ async fn run_encrypt_pin_task( } } +/// Test-only entry point: build an [`EncryptTaskCtx`] from a test `AppState` (with +/// an overridable `ipfs_api` for a mock Kubo server and an explicit `disk_path` for +/// the fixture repo) and run the real drain task. Keeps `EncryptTaskCtx` and +/// `run_encrypt_pin_task` private to this module. +/// +/// `owner_did` and `repo_name` must name the real seeded row: the drain re-fetches +/// the record by owner/name every lap, so a blank name resolves `Gone` and every +/// lap would pin nothing. +#[cfg(test)] +#[allow(clippy::too_many_arguments)] +pub(crate) async fn run_encrypt_pin_task_for_test( + state: &AppState, + guard: crate::state::EncryptInflightGuard, + disk_path: std::path::PathBuf, + repo_id: String, + owner_did: String, + repo_name: String, + ipfs_api: String, + snapshot_objects: Vec, + snapshot_rules: Option>, + snapshot_is_public: bool, +) { + let ctx = EncryptTaskCtx { + ipfs_api, + repo_path: disk_path, + db: state.db.clone(), + repo_id, + owner_did, + repo_name, + irys_url: String::new(), + http_client: std::sync::Arc::clone(&state.http_client), + node_did: state.node_did.to_string(), + node_keypair: std::sync::Arc::clone(&state.node_keypair), + git_bin: state.git_bin.clone(), + git_timeout: std::time::Duration::from_secs(state.config.git_service_timeout_secs), + encrypt_sem: state.git_encrypt_semaphore.clone(), + pin_sem: state.pin_semaphore.clone(), + }; + run_encrypt_pin_task( + ctx, + guard, + snapshot_objects, + snapshot_rules, + snapshot_is_public, + ) + .await; +} + /// Resolve a coalesced-drain iteration's replicable object list. Re-fetches the /// repo record and visibility rules FRESH — rules tightened between the coalesced /// push and its drain must be honored, fail closed: a newly-withheld blob is not diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index ae99a988..3a1edea7 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -10914,4 +10914,366 @@ mod tests { "result includes the deep cert matching the prefix" ); } + + /// Coalesced-drain behavior of the detached post-push encrypt/pin task. + /// + /// A push arriving while a task is in flight does not spawn a second task; its + /// (old_sha, new_sha) tip pairs are merged into the in-flight key's pending slot + /// and the task loop-drains them before releasing the key. These tests drive the + /// real task through `run_encrypt_pin_task_for_test` and assert on the WORK + /// PERFORMED (what is pinned, what is sealed, whether the key is released), not + /// on control flow. The drain re-reads repo state FRESH, so a rule tightened + /// between the coalesced push and its drain must be honored, fail closed. + mod u3_requeue { + use super::*; + use crate::db::VisibilityMode; + use crate::state::{BeginOutcome, PendingWork}; + use std::path::{Path, PathBuf}; + use std::process::Command; + + fn git(args: &[&str], dir: &Path) { + let ok = Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(); + assert!(ok, "git {args:?} failed"); + } + fn oid(rev: &str, dir: &Path) -> String { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(dir) + .output() + .unwrap(); + assert!(out.status.success(), "rev-parse {rev}: {out:?}"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + struct Repo { + _td: tempfile::TempDir, + path: PathBuf, + } + fn init_repo() -> Repo { + let td = tempfile::TempDir::new().unwrap(); + let path = td.path().to_path_buf(); + git(&["init", "-q"], &path); + git(&["config", "user.email", "t@t"], &path); + git(&["config", "user.name", "t"], &path); + Repo { _td: td, path } + } + /// Commit `content` at `rel`, return the blob oid. + fn commit(repo: &Path, rel: &str, content: &str) -> String { + let full = repo.join(rel); + std::fs::create_dir_all(full.parent().unwrap()).unwrap(); + std::fs::write(&full, content).unwrap(); + git(&["add", "."], repo); + git(&["commit", "-qm", rel], repo); + oid(&format!("HEAD:{rel}"), repo) + } + /// Write a loose, UNREACHABLE blob (dangling object). + fn write_dangling_blob(repo: &Path, content: &str) -> String { + let out = Command::new("git") + .args(["hash-object", "-w", "--stdin"]) + .current_dir(repo) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap(); + use std::io::Write; + out.stdin + .as_ref() + .unwrap() + .write_all(content.as_bytes()) + .unwrap(); + let o = out.wait_with_output().unwrap(); + assert!(o.status.success()); + String::from_utf8_lossy(&o.stdout).trim().to_string() + } + fn new_did() -> String { + Keypair::generate().did().to_string() + } + /// Admit push A on the in-flight key, or fail the test. + fn admit(state: &AppState, key: &str) -> crate::state::EncryptInflightGuard { + match state.encrypt_inflight.try_begin(key, Vec::new()) { + BeginOutcome::Admitted(g) => g, + BeginOutcome::Coalesced => panic!("push A must be admitted, nothing is in flight"), + } + } + /// Coalesce push B's tip pairs into the in-flight key, or fail the test. + fn coalesce(state: &AppState, key: &str, pairs: Vec<(String, String)>) { + match state.encrypt_inflight.try_begin(key, pairs) { + BeginOutcome::Coalesced => {} + BeginOutcome::Admitted(_) => { + panic!("push B must coalesce, a task is already in flight") + } + } + } + + /// SCENARIO 2 + 5 (pin half, TAIL-PLACEMENT guard). A coalesced push on a PUBLIC + /// repo with NO path-scoped rule must still drain its pin half: the second + /// push's new object is pinned after the task. RED without the drain (the stale + /// spawn object_list never lists obj2), and RED if the drain sits inside the + /// `has_path_scoped_rule` block (a rules-free repo would never reach it). + #[sqlx::test] + async fn u3_rules_free_public_repo_requeues_pin_half(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u3-pin"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let obj1 = commit(&git_repo.path, "a.txt", "one\n"); + let tip_a = oid("HEAD", &git_repo.path); + // The coalesced push B adds obj2 (present at drain time, NOT in the stale + // push-A spawn object_list). + let obj2 = commit(&git_repo.path, "b.txt", "two\n"); + let tip_b = oid("HEAD", &git_repo.path); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // Push A admits (guard); push B coalesces its tip pair into the slot. + let guard = admit(&state, &key); + coalesce(&state, &key, vec![(tip_a, tip_b)]); + + // Spawn-time (push A) captures are STALE: object_list lists only obj1, no rule. + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![obj1.clone()], + Some(vec![]), + true, + ) + .await; + + assert!( + state.db.is_pinned(&obj1).await.unwrap(), + "push A's object is pinned on the first pass" + ); + assert!( + state.db.is_pinned(&obj2).await.unwrap(), + "the coalesced push's new object is pinned by the DRAIN lap (RED without \ + the drain, or if the drain sits inside the encrypt gate)" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is released once the task exits clean" + ); + } + + /// SCENARIO 1 + 3 (encrypt half, FRESH re-read). A coalesced push adds a + /// path-scoped rule withholding a blob. The task must re-read rules FRESH on + /// the drain lap and seal the newly-withheld blob's recovery copy. RED without + /// the fresh read (pass one's stale empty rule set seals nothing). + #[sqlx::test] + async fn u3_requeue_seals_blob_withheld_by_coalesced_rule_change(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let reader = new_did(); + let repo = seed_repo(&owner, "u3-enc"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let _pub_oid = commit(&git_repo.path, "public/a.txt", "public\n"); + let tip_a = oid("HEAD", &git_repo.path); + let secret_oid = commit(&git_repo.path, "secret/b.txt", "TOP SECRET\n"); + let tip_b = oid("HEAD", &git_repo.path); + + // Coalesced push B changes .gitlawb: withhold /secret/** from anon, grant reader. + state + .db + .set_visibility_rule(&repo.id, "/secret/**", VisibilityMode::B, &[reader], &owner) + .await + .expect("set rule"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + let guard = admit(&state, &key); + coalesce(&state, &key, vec![(tip_a, tip_b)]); + + // Push A captures are STALE: no rule, public repo. + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![], + Some(vec![]), + true, + ) + .await; + + assert!( + state + .db + .encrypted_blob_recipients_tag(&repo.id, &secret_oid) + .await + .unwrap() + .is_some(), + "the coalesced push's newly-withheld blob is sealed after the DRAIN re-read \ + (RED without the fresh read: pass one's stale empty rules seal nothing)" + ); + assert!(state.encrypt_inflight.is_empty(), "guard key released"); + } + + /// SCENARIO 4 (visibility-leak negative). The drain's full scan must feed + /// `list_all_objects` through the fail-closed filter, never pin it bare: a + /// withheld secret blob and a dangling blob must NOT land in the public pin set. + /// + /// The full scan is forced through the public API: one coalescing push carrying + /// more than the pending tip-pair cap degrades the slot to `PendingWork::FullScan`, + /// which is also the overflow path itself. + #[sqlx::test] + async fn u3_requeue_full_scan_does_not_publicly_pin_withheld_or_dangling(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let reader = new_did(); + let repo = seed_repo(&owner, "u3-leak"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let pub_oid = commit(&git_repo.path, "public/a.txt", "public\n"); + let secret_oid = commit(&git_repo.path, "secret/b.txt", "TOP SECRET\n"); + state + .db + .set_visibility_rule(&repo.id, "/secret/**", VisibilityMode::B, &[reader], &owner) + .await + .expect("set rule"); + // Coalesced push adds a new public object and a dangling blob. + let new_pub_oid = commit(&git_repo.path, "public/c.txt", "more public\n"); + let tip = oid("HEAD", &git_repo.path); + let dangling_oid = write_dangling_blob(&git_repo.path, "orphan bytes\n"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + let rules = state.db.list_visibility_rules(&repo.id).await.unwrap(); + + let guard = admit(&state, &key); + // 1025 pairs is one past the pending cap, so the slot degrades to FullScan. + coalesce(&state, &key, vec![(tip.clone(), tip.clone()); 1025]); + assert_eq!( + state.encrypt_inflight.pending_for(&key), + Some(PendingWork::FullScan), + "an overflowing coalesce degrades the pending slot to a forced full scan" + ); + + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![pub_oid.clone()], + Some(rules), + true, + ) + .await; + + assert!( + state.db.is_pinned(&new_pub_oid).await.unwrap(), + "the coalesced push's new PUBLIC object is pinned by the drain full scan" + ); + assert!( + !state.db.is_pinned(&secret_oid).await.unwrap(), + "a WITHHELD blob is never publicly pinned by the drain enumeration (leak guard)" + ); + assert!( + !state.db.is_pinned(&dangling_oid).await.unwrap(), + "a DANGLING blob is never publicly pinned by the drain enumeration (leak guard)" + ); + // The withheld blob still gets its ENCRYPTED recovery copy (not a public pin). + assert!( + state + .db + .encrypted_blob_recipients_tag(&repo.id, &secret_oid) + .await + .unwrap() + .is_some(), + "withheld blob is sealed as an encrypted recovery copy, not pinned in the clear" + ); + } + + /// SCENARIO 8 (no-coalesce happy path). A single push with no coalesced follower + /// runs exactly one pass, pins its object, and releases the key. No drain lap. + #[sqlx::test] + async fn u3_no_coalesce_single_pass_pins_and_releases(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u3-happy"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let obj1 = commit(&git_repo.path, "a.txt", "one\n"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // No second try_begin: nothing is ever merged into the pending slot. + let guard = admit(&state, &key); + assert_eq!( + state.encrypt_inflight.pending_for(&key), + Some(PendingWork::Tips(vec![])), + "clean, no coalesce" + ); + + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![obj1.clone()], + Some(vec![]), + true, + ) + .await; + + assert!( + state.db.is_pinned(&obj1).await.unwrap(), + "the single push's object is pinned" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the key is released after one pass" + ); + } + } } From 8c1b80027d1220c39215a0608cb42010ea5e4370 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 2 Aug 2026 18:50:03 -0500 Subject: [PATCH 121/124] test(node): add a test-only fault seam for the drain's two re-reads A real Postgres pool will not fail on demand, so the drain's error arms have no way to be driven from a test. This ports the injection table the removed requeue loop used, keyed on the task's repo id, which is a fresh uuid per test so parallel cases cannot cross-inject. The two reads now go through drain_get_repo and drain_list_rules, whose bodies consult the seam under cfg(test) and then make the same call as before. Error handling is untouched: the repo read keeps its arms, the rules read keeps its ok() collapse, and no retry exists yet. The suite is unchanged at 856, which is the point. inject and counters carry a dead_code allow until the tests that call them land. --- crates/gitlawb-node/src/api/repos.rs | 102 ++++++++++++++++++++++++++- 1 file changed, 100 insertions(+), 2 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index ae8b752d..e3303855 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -960,6 +960,104 @@ pub(crate) async fn run_encrypt_pin_task_for_test( .await; } +/// Test-only fault-injection seam for the drain re-reads in +/// `resolve_drain_object_list`. The behavior worth testing lives on the `Err` arm of +/// the two re-reads, which a real Postgres pool will not produce on demand, so the two +/// reads go through the wrappers below and consult this table first. Keyed by `repo_id` +/// (a fresh uuid per test) so tests running in parallel in one process cannot see each +/// other's injections, and it also records the ATTEMPT counts the retry-bound +/// assertions key on. +#[cfg(test)] +pub(crate) mod drain_faults { + use std::collections::HashMap; + use std::sync::{Mutex, OnceLock}; + + #[derive(Default, Clone, Copy, Debug)] + pub(crate) struct Counters { + pub(crate) repo_read_failures_left: usize, + pub(crate) rules_read_failures_left: usize, + pub(crate) repo_read_attempts: usize, + pub(crate) rules_read_attempts: usize, + } + + fn table() -> &'static Mutex> { + static TABLE: OnceLock>> = OnceLock::new(); + TABLE.get_or_init(|| Mutex::new(HashMap::new())) + } + + /// Make the next `repo_read_failures` repo re-reads and the next + /// `rules_read_failures` rule re-reads for `repo_id` return `Err`, then succeed. + /// Only the tests that drive the retry arms call it, so it is dead until they land. + #[allow(dead_code)] + pub(crate) fn inject(repo_id: &str, repo_read_failures: usize, rules_read_failures: usize) { + table().lock().unwrap().insert( + repo_id.to_string(), + Counters { + repo_read_failures_left: repo_read_failures, + rules_read_failures_left: rules_read_failures, + ..Default::default() + }, + ); + } + + /// Observed attempt counts (and remaining injections) for `repo_id`. + /// Only the tests that assert the retry bound call it, so it is dead until they land. + #[allow(dead_code)] + pub(crate) fn counters(repo_id: &str) -> Counters { + table() + .lock() + .unwrap() + .get(repo_id) + .copied() + .unwrap_or_default() + } + + /// Production-path hook: count one repo re-read attempt, return whether it must fail. + pub(crate) fn take_repo_read(repo_id: &str) -> bool { + let mut map = table().lock().unwrap(); + let c = map.entry(repo_id.to_string()).or_default(); + c.repo_read_attempts += 1; + if c.repo_read_failures_left > 0 { + c.repo_read_failures_left -= 1; + return true; + } + false + } + + /// Production-path hook: count one rules re-read attempt, return whether it must fail. + pub(crate) fn take_rules_read(repo_id: &str) -> bool { + let mut map = table().lock().unwrap(); + let c = map.entry(repo_id.to_string()).or_default(); + c.rules_read_attempts += 1; + if c.rules_read_failures_left > 0 { + c.rules_read_failures_left -= 1; + return true; + } + false + } +} + +/// The drain's repo re-read, behind the test-only fault seam above. +async fn drain_get_repo(ctx: &EncryptTaskCtx) -> anyhow::Result> { + #[cfg(test)] + if drain_faults::take_repo_read(&ctx.repo_id) { + return Err(anyhow::anyhow!("injected repo re-read failure")); + } + ctx.db.get_repo(&ctx.owner_did, &ctx.repo_name).await +} + +/// The drain's visibility-rule re-read, behind the test-only fault seam above. +async fn drain_list_rules( + ctx: &EncryptTaskCtx, + record_id: &str, +) -> anyhow::Result> { + #[cfg(test)] + if drain_faults::take_rules_read(&ctx.repo_id) { + return Err(anyhow::anyhow!("injected visibility-rule re-read failure")); + } + ctx.db.list_visibility_rules(record_id).await +} + /// Resolve a coalesced-drain iteration's replicable object list. Re-fetches the /// repo record and visibility rules FRESH — rules tightened between the coalesced /// push and its drain must be honored, fail closed: a newly-withheld blob is not @@ -980,7 +1078,7 @@ async fn resolve_drain_object_list( Option>, bool, )> { - let record = match ctx.db.get_repo(&ctx.owner_did, &ctx.repo_name).await { + let record = match drain_get_repo(ctx).await { Ok(Some(r)) => r, Ok(None) => { tracing::warn!( @@ -1002,7 +1100,7 @@ async fn resolve_drain_object_list( // fresh by owner/name, and a delete+re-create between spawn and drain gives // the row a NEW id — rules read against the stale id come back empty and // would fail open for the new row. - let rules_opt = ctx.db.list_visibility_rules(&record.id).await.ok(); + let rules_opt = drain_list_rules(ctx, &record.id).await.ok(); let (_announce, withheld) = replication_withheld_set( ctx.encrypt_sem.clone(), rules_opt.clone(), From 5b15817df366e03f805832d0483ecc772f469b3f Mon Sep 17 00:00:00 2001 From: t Date: Sun, 2 Aug 2026 20:16:58 -0500 Subject: [PATCH 122/124] fix(node): retry the coalesced drain's re-read instead of losing the push The drain read the repo row and its visibility rules once each. Either read failing dropped that coalesced push's pins and recovery copy for good: the repo read returned None on error, and the rules read collapsed through ok(), which makes the repo read as not announceable. Both fail closed, so nothing leaked, but nothing sweeps the lost work up either. A single database blip was enough. resolve_drain_object_list now refreshes through a bounded retry: three attempts, 50ms doubling, and either read failing is transient. A missing repo row is the one terminal answer, so it releases at once and spends no retry budget. Exhaustion logs at ERROR with the repo id and attempt count, so giving up is observable rather than silent. The coalescing loop is untouched. It already has the shape the retry needs: a refresh that gives up skips the pin call, and the next finish_or_take_pending picks up anything that coalesced while the retry was running. Eight restored tests cover it, written and observed red against the seam-only tree first. The five the plan predicted red came up red for the reasons it predicted, and the three controls stayed green. All eight are mutation-proven: no-retry, an off-by-one bound, gone burning budget, a rules error read as empty, an unfiltered candidate set, a duplicated refresh, break-on-giveup, and a pending slot that never drains each turn their own test red. --- crates/gitlawb-node/src/api/repos.rs | 100 +++- crates/gitlawb-node/src/test_support.rs | 673 ++++++++++++++++++++++++ 2 files changed, 759 insertions(+), 14 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index e3303855..fa812069 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -987,8 +987,6 @@ pub(crate) mod drain_faults { /// Make the next `repo_read_failures` repo re-reads and the next /// `rules_read_failures` rule re-reads for `repo_id` return `Err`, then succeed. - /// Only the tests that drive the retry arms call it, so it is dead until they land. - #[allow(dead_code)] pub(crate) fn inject(repo_id: &str, repo_read_failures: usize, rules_read_failures: usize) { table().lock().unwrap().insert( repo_id.to_string(), @@ -1001,8 +999,6 @@ pub(crate) mod drain_faults { } /// Observed attempt counts (and remaining injections) for `repo_id`. - /// Only the tests that assert the retry bound call it, so it is dead until they land. - #[allow(dead_code)] pub(crate) fn counters(repo_id: &str) -> Counters { table() .lock() @@ -1058,6 +1054,84 @@ async fn drain_list_rules( ctx.db.list_visibility_rules(record_id).await } +/// Attempts allowed for the drain re-read before the lap gives up. The coalesced +/// push's work is already out of the pending slot (`finish_or_take_pending` took it +/// in the same critical section that kept the key), and there is no reconciliation +/// sweep to re-derive it: a transient read error must be RETRIED here or that push's +/// pin/encrypt pass is gone. The bound keeps a sustained outage from spinning +/// forever; on exhaustion the work is still lost (the pre-existing residual), but +/// the give-up is logged at ERROR so it is observable instead of silent. +const DRAIN_REREAD_MAX_ATTEMPTS: usize = 3; + +/// Backoff before the next re-read attempt. Doubles per attempt. +const DRAIN_REREAD_BACKOFF: std::time::Duration = std::time::Duration::from_millis(50); + +/// The outcome of the drain's fresh state re-read, keeping the three cases distinct +/// that a single `Err => None` collapses into one: a usable refresh, a repo that +/// genuinely no longer exists (terminal, and NOT a retry), and a transient read +/// failure (retryable). +enum DrainRefresh { + State { + /// Boxed only to keep the enum small: `RepoRecord` dwarfs the other two + /// variants, which carry nothing (clippy::large_enum_variant). + record: Box, + rules: Vec, + }, + Gone, + Failed, +} + +/// Re-read repo state for a drain lap, retrying transient read errors. +/// +/// Both reads are retryable and neither may be read as an absence: an `Err` from the +/// repo row is not "the repo is gone", and an `Err` from the rule list is not "this +/// repo has no rules" (`.ok()` made those indistinguishable, and a `None` rule set +/// makes `replication_withheld_set` return `None`, which skips the entire lap). Only +/// `Ok(None)` on the repo row is a terminal absence, and it consumes no retry budget. +/// +/// The whole `RepoRecord` comes back, not just its rules and flags: the caller writes +/// against `record.id` from this FRESH re-fetch, never `ctx.repo_id` frozen at spawn. +async fn drain_refresh_state(ctx: &EncryptTaskCtx) -> DrainRefresh { + let mut backoff = DRAIN_REREAD_BACKOFF; + for attempt in 1..=DRAIN_REREAD_MAX_ATTEMPTS { + let record = match drain_get_repo(ctx).await { + Ok(Some(rec)) => Box::new(rec), + Ok(None) => return DrainRefresh::Gone, + Err(e) => { + tracing::warn!( + repo = %ctx.repo_id, err = %e, attempt, + "coalesced drain: repo re-read failed; retrying" + ); + tokio::time::sleep(backoff).await; + backoff *= 2; + continue; + } + }; + // record.id, never the spawn-time ctx.repo_id: the record above is re-fetched + // fresh by owner/name, and a delete+re-create between spawn and drain gives + // the row a NEW id - rules read against the stale id come back empty and + // would fail open for the new row. + match drain_list_rules(ctx, &record.id).await { + Ok(rules) => return DrainRefresh::State { record, rules }, + Err(e) => { + tracing::warn!( + repo = %ctx.repo_id, err = %e, attempt, + "coalesced drain: visibility-rule re-read failed; retrying" + ); + tokio::time::sleep(backoff).await; + backoff *= 2; + } + } + } + tracing::error!( + repo = %ctx.repo_id, + attempts = DRAIN_REREAD_MAX_ATTEMPTS, + "coalesced drain: re-read failed on every attempt; the coalesced push's \ + pin/encrypt pass is dropped (no reconciliation sweep re-derives it)" + ); + DrainRefresh::Failed +} + /// Resolve a coalesced-drain iteration's replicable object list. Re-fetches the /// repo record and visibility rules FRESH — rules tightened between the coalesced /// push and its drain must be honored, fail closed: a newly-withheld blob is not @@ -1078,29 +1152,27 @@ async fn resolve_drain_object_list( Option>, bool, )> { - let record = match drain_get_repo(ctx).await { - Ok(Some(r)) => r, - Ok(None) => { + // Both re-reads are bounded-retried: a transient blip must not discard the + // coalesced push's work (`finish_or_take_pending` already took it out of the + // pending slot and no sweep re-derives it). The rules come back from the same + // refresh, read against the FRESH record.id, never the spawn-time ctx.repo_id. + let (record, rules_opt) = match drain_refresh_state(ctx).await { + DrainRefresh::State { record, rules } => (*record, Some(rules)), + DrainRefresh::Gone => { tracing::warn!( repo = %ctx.repo_id, "coalesced drain: repo record is gone; dropping the pending work" ); return None; } - Err(e) => { + DrainRefresh::Failed => { tracing::warn!( repo = %ctx.repo_id, - err = %e, "coalesced drain: repo re-fetch failed; pinning nothing (fail closed)" ); return None; } }; - // record.id, never the spawn-time ctx.repo_id: the record above is re-fetched - // fresh by owner/name, and a delete+re-create between spawn and drain gives - // the row a NEW id — rules read against the stale id come back empty and - // would fail open for the new row. - let rules_opt = drain_list_rules(ctx, &record.id).await.ok(); let (_announce, withheld) = replication_withheld_set( ctx.encrypt_sem.clone(), rules_opt.clone(), diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 3a1edea7..7da4e269 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -11275,5 +11275,678 @@ mod tests { "the key is released after one pass" ); } + + mod u2_reread_retry { + use super::*; + use crate::api::repos::drain_faults; + + /// Process-wide tracing capture so a test can assert the give-up is logged at + /// ERROR. A global default subscriber can only be installed once per process, + /// so it is shared by every test here and assertions filter on the repo id, + /// which is a fresh uuid per test. + mod logcap { + use std::sync::{Arc, Mutex, OnceLock}; + use tracing::{Event, Level, Subscriber}; + use tracing_subscriber::layer::{Context, Layer}; + use tracing_subscriber::prelude::*; + + type Lines = Arc>>; + + fn lines() -> &'static Lines { + static LINES: OnceLock = OnceLock::new(); + LINES.get_or_init(|| Arc::new(Mutex::new(Vec::new()))) + } + + struct Capture; + impl Layer for Capture { + fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) { + struct V(String); + impl tracing::field::Visit for V { + fn record_debug( + &mut self, + field: &tracing::field::Field, + value: &dyn std::fmt::Debug, + ) { + self.0.push_str(&format!(" {}={:?}", field.name(), value)); + } + } + let mut v = V(String::new()); + event.record(&mut v); + lines() + .lock() + .unwrap() + .push((*event.metadata().level(), v.0)); + } + } + + pub(super) fn install() { + static ONCE: OnceLock<()> = OnceLock::new(); + ONCE.get_or_init(|| { + let _ = tracing::subscriber::set_global_default( + tracing_subscriber::registry().with(Capture), + ); + }); + } + + pub(super) fn errors_containing(needle: &str) -> Vec { + lines() + .lock() + .unwrap() + .iter() + .filter(|(lvl, msg)| *lvl == Level::ERROR && msg.contains(needle)) + .map(|(_, msg)| msg.clone()) + .collect() + } + } + + /// SCENARIO 1. The repo re-read fails once, then succeeds: the drain lap + /// must still RUN, under the refreshed state, and pin the coalesced push's + /// object. RED before the fix (the single `Err` returned `None`, the lap + /// pinned nothing, and `finish_or_take_pending` had already taken the + /// pending work out of the slot, so it was gone for good). + #[sqlx::test] + async fn u2_transient_repo_reread_failure_is_retried_and_work_lands(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u2-retry"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let obj1 = commit(&git_repo.path, "a.txt", "one\n"); + let tip_a = oid("HEAD", &git_repo.path); + // The coalesced push B adds obj2, absent from push A's spawn captures. + let obj2 = commit(&git_repo.path, "b.txt", "two\n"); + let tip_b = oid("HEAD", &git_repo.path); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // One transient repo re-read failure, then the real DB answers. + drain_faults::inject(&repo.id, 1, 0); + + let guard = admit(&state, &key); + coalesce(&state, &key, vec![(tip_a, tip_b)]); + + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![obj1.clone()], + Some(vec![]), + true, + ) + .await; + + assert!( + state.db.is_pinned(&obj2).await.unwrap(), + "the coalesced push's object is pinned after the retried re-read (RED \ + before this unit: the Err arm dropped the lap and the work with it)" + ); + let c = drain_faults::counters(&repo.id); + assert_eq!( + c.repo_read_attempts, 2, + "the failed re-read is retried exactly once before it succeeds" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is released once the task exits" + ); + } + + /// SCENARIO 2. Every re-read attempt fails: the loop must give up on a BOUND + /// (asserted as a literal, so raising or removing the bound goes RED) and log + /// the give-up at ERROR so the residual loss is observable rather than silent. + #[sqlx::test] + async fn u2_sustained_repo_reread_failure_is_bounded_and_logged(pool: PgPool) { + logcap::install(); + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u2-bounded"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let obj1 = commit(&git_repo.path, "a.txt", "one\n"); + let tip_a = oid("HEAD", &git_repo.path); + let obj2 = commit(&git_repo.path, "b.txt", "two\n"); + let tip_b = oid("HEAD", &git_repo.path); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // Far more failures than the bound allows: the outage never clears. + drain_faults::inject(&repo.id, 10_000, 0); + + let guard = admit(&state, &key); + coalesce(&state, &key, vec![(tip_a, tip_b)]); + + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![obj1.clone()], + Some(vec![]), + true, + ) + .await; + + let c = drain_faults::counters(&repo.id); + assert_eq!( + c.repo_read_attempts, 3, + "the re-read is bounded at 3 attempts; unbounded retry or a raised \ + bound must fail here" + ); + assert!( + !state.db.is_pinned(&obj2).await.unwrap(), + "with the read never succeeding there is nothing fresh to act on" + ); + let errs = logcap::errors_containing(&repo.id); + assert!( + !errs.is_empty(), + "the exhausted drain re-read is logged at ERROR with the repo id, so \ + the residual work loss is observable; captured: {errs:?}" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is still released on the give-up path" + ); + } + + /// SCENARIO 3. `Ok(None)` (the repo was deleted during the in-flight window) + /// is NOT a transient failure: it must release immediately without burning the + /// retry budget. The repo row is never created, so the re-read legitimately + /// returns `Ok(None)`. + #[sqlx::test] + async fn u2_repo_gone_releases_without_consuming_retries(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let missing_id = uuid::Uuid::new_v4().to_string(); + let missing_name = "u2-gone".to_string(); + let key = crate::state::repo_identity_key(&owner, &missing_name); + let git_repo = init_repo(); + let _obj1 = commit(&git_repo.path, "a.txt", "one\n"); + let tip_a = oid("HEAD", &git_repo.path); + let _obj2 = commit(&git_repo.path, "b.txt", "two\n"); + let tip_b = oid("HEAD", &git_repo.path); + + let server = mockito::Server::new_async().await; + + drain_faults::inject(&missing_id, 0, 0); + + let guard = admit(&state, &key); + // A real pair, so a drain lap actually runs and reaches the re-read. + coalesce(&state, &key, vec![(tip_a, tip_b)]); + + // Empty object list: pass one touches no pin rows for a repo that is gone. + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + missing_id.clone(), + owner.clone(), + missing_name.clone(), + server.url(), + vec![], + Some(vec![]), + true, + ) + .await; + + let c = drain_faults::counters(&missing_id); + assert_eq!( + c.repo_read_attempts, 1, + "a deleted repo is a terminal answer, never retried" + ); + assert_eq!( + c.rules_read_attempts, 0, + "no rules read is attempted once the repo row is gone" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is released cleanly" + ); + } + + /// SCENARIO 4. A failed visibility-rule read is transient, never an empty + /// policy. RED before the fix, where `.ok()` made "the rules read failed" and + /// "this repo has no rules" the same value: the withheld blob was then neither + /// sealed nor covered, because a `None` rule set skips the entire lap. + #[sqlx::test] + async fn u2_transient_rules_read_failure_is_retried_not_read_as_empty(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let reader = new_did(); + let repo = seed_repo(&owner, "u2-rules"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let pub_oid = commit(&git_repo.path, "public/a.txt", "public\n"); + let tip_a = oid("HEAD", &git_repo.path); + let secret_oid = commit(&git_repo.path, "secret/b.txt", "TOP SECRET\n"); + let tip_b = oid("HEAD", &git_repo.path); + + // The coalesced push B is what added the path-scoped rule. + state + .db + .set_visibility_rule( + &repo.id, + "/secret/**", + VisibilityMode::B, + &[reader], + &owner, + ) + .await + .expect("set rule"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // The repo row reads fine; the RULES read is the one that blips. + drain_faults::inject(&repo.id, 0, 1); + + let guard = admit(&state, &key); + coalesce(&state, &key, vec![(tip_a, tip_b)]); + + // Push A's captures are stale: no rule, nothing withheld. + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![pub_oid.clone()], + Some(vec![]), + true, + ) + .await; + + assert!( + state + .db + .encrypted_blob_recipients_tag(&repo.id, &secret_oid) + .await + .unwrap() + .is_some(), + "the withheld blob is sealed under the RETRIED rule set (RED with \ + list_visibility_rules(..).ok(): an empty policy seals nothing)" + ); + let c = drain_faults::counters(&repo.id); + assert_eq!( + c.rules_read_attempts, 2, + "the failed rules read is retried, not collapsed into an empty rule set" + ); + assert!( + !state.db.is_pinned(&secret_oid).await.unwrap(), + "the withheld blob is never pinned in the clear by the drain" + ); + } + + /// SCENARIO 5. The fault-free control for scenario 4: the rules applied by the + /// drain are the COALESCED push's fresh ones, never the spawn-time capture, + /// and the retry path does not perturb that (exactly one read of each). + #[sqlx::test] + async fn u2_requeue_applies_fresh_rules_not_spawn_captures(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let reader = new_did(); + let repo = seed_repo(&owner, "u2-fresh"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let pub_oid = commit(&git_repo.path, "public/a.txt", "public\n"); + let tip_a = oid("HEAD", &git_repo.path); + let secret_oid = commit(&git_repo.path, "secret/b.txt", "TOP SECRET\n"); + let tip_b = oid("HEAD", &git_repo.path); + state + .db + .set_visibility_rule( + &repo.id, + "/secret/**", + VisibilityMode::B, + &[reader], + &owner, + ) + .await + .expect("set rule"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + drain_faults::inject(&repo.id, 0, 0); + + let guard = admit(&state, &key); + coalesce(&state, &key, vec![(tip_a, tip_b)]); + + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![pub_oid.clone()], + Some(vec![]), + true, + ) + .await; + + let c = drain_faults::counters(&repo.id); + assert_eq!( + (c.repo_read_attempts, c.rules_read_attempts), + (1, 1), + "a healthy DB is read exactly once per drain lap" + ); + assert!( + state.db.is_pinned(&pub_oid).await.unwrap(), + "the visible object is pinned under the fresh rules" + ); + assert!( + !state.db.is_pinned(&secret_oid).await.unwrap(), + "the freshly-read rule withholds the secret blob (the spawn-time \ + capture had no rules at all)" + ); + } + + /// SCENARIO 6. Regression guard on the property the fix must not disturb: the + /// finish-or-take critical section is atomic, so a push coalescing during it is + /// still covered by exactly one more lap, and the key is released after. + #[sqlx::test] + async fn u2_coalesced_push_still_covered_by_exactly_one_requeue_pass(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u2-coalesce"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let obj1 = commit(&git_repo.path, "a.txt", "one\n"); + let tip_a = oid("HEAD", &git_repo.path); + let obj2 = commit(&git_repo.path, "b.txt", "two\n"); + let tip_b = oid("HEAD", &git_repo.path); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + drain_faults::inject(&repo.id, 0, 0); + + let guard = admit(&state, &key); + // Push B lands during the in-flight window: its tip pair is merged. + coalesce(&state, &key, vec![(tip_a.clone(), tip_b.clone())]); + assert_eq!( + state.encrypt_inflight.pending_for(&key), + Some(PendingWork::Tips(vec![(tip_a, tip_b)])), + "the coalesced push recorded its work in the pending slot" + ); + + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![obj1.clone()], + Some(vec![]), + true, + ) + .await; + + assert_eq!( + drain_faults::counters(&repo.id).repo_read_attempts, + 1, + "one coalesced push means exactly one drain lap, no re-spin" + ); + assert!( + state.db.is_pinned(&obj1).await.unwrap(), + "push A's object is pinned" + ); + assert!( + state.db.is_pinned(&obj2).await.unwrap(), + "the coalesced push's object is covered by the drain lap" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the key is released once the task is clean" + ); + } + + /// Wait for `finish_or_take_pending` to take the pending work out of the slot + /// (`Tips(nonempty)` -> `Tips(empty)`), which is the exact instant the task + /// enters `drain_refresh_state`'s retry window. Deterministic, so the + /// coalescing push below lands INSIDE that window rather than on a sleep + /// guess. `None` means the key is already gone (the task exited), which the + /// caller reports as its own failure. + async fn wait_for_drain_window( + inflight: &crate::state::EncryptInflight, + key: &str, + ) -> bool { + for _ in 0..5_000 { + match inflight.pending_for(key) { + Some(PendingWork::Tips(acc)) if acc.is_empty() => return true, + None => return false, + Some(_) => {} + } + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + } + false + } + + /// SCENARIO 7 (RED-before/GREEN-after). A push that coalesces WHILE the + /// re-read is retrying must not be thrown away when that re-read finally + /// gives up. Breaking the drain loop on the give-up would let + /// `EncryptInflightGuard::drop` remove the key with push C's work still + /// recorded, and push C's lap would never run: a silent drop with no + /// reconciliation sweep behind it. + /// + /// Exactly `DRAIN_REREAD_MAX_ATTEMPTS` injected repo-read faults, so the + /// first refresh exhausts its budget and the DB is healthy for the next one. + /// Push C coalesces inside that window. + #[sqlx::test] + async fn u2_failed_reread_keeps_a_push_that_coalesced_during_the_window(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u2-window"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let obj_a = commit(&git_repo.path, "a.txt", "one\n"); + let tip_a = oid("HEAD", &git_repo.path); + let _obj_b = commit(&git_repo.path, "b.txt", "two\n"); + let tip_b = oid("HEAD", &git_repo.path); + let obj_c = commit(&git_repo.path, "c.txt", "three\n"); + let tip_c = oid("HEAD", &git_repo.path); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // Exactly the bound: the FIRST refresh burns all three attempts and gives + // up; every later refresh sees a healthy DB. + drain_faults::inject(&repo.id, 3, 0); + + let guard = admit(&state, &key); + coalesce(&state, &key, vec![(tip_a.clone(), tip_b.clone())]); + + // Push C lands during the retry window, after the loop already took push + // B's pending work out of the slot. It MUST carry a real tip pair: an + // empty merge leaves the slot empty and no extra lap runs at all. + let inflight = state.encrypt_inflight.clone(); + let watch_key = key.clone(); + let coalesced = tokio::spawn(async move { + if !wait_for_drain_window(&inflight, &watch_key).await { + return false; + } + matches!( + inflight.try_begin(&watch_key, vec![(tip_b, tip_c)]), + BeginOutcome::Coalesced + ) + }); + + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![obj_a.clone()], + Some(vec![]), + true, + ) + .await; + + assert!( + coalesced.await.expect("coalescing task"), + "push C must have coalesced inside the retry window for this test to \ + mean anything" + ); + assert!( + state.db.is_pinned(&obj_c).await.unwrap(), + "the push that coalesced during the retry window must still get a lap \ + once the DB recovers (RED if the give-up breaks the loop: the pending \ + work was already taken, so the lap was dropped with nothing to \ + re-derive it)" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is released once the task exits" + ); + } + + /// SCENARIO 8 (the sustained-outage guard on the fall-through). Continuing the + /// loop on a give-up means `finish_or_take_pending` runs again, so a DB that + /// never recovers must still TERMINATE rather than spin. It does: an extra lap + /// only happens when a push actually coalesced, and each lap pays a full + /// bounded re-read (3 attempts with backoff). One coalescing push during the + /// window buys exactly one extra lap: 6 repo-read attempts, then exit. + #[sqlx::test] + async fn u2_sustained_failure_with_a_coalesce_terminates_after_one_more_lap( + pool: PgPool, + ) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u2-sustained-window"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let obj_a = commit(&git_repo.path, "a.txt", "one\n"); + let tip_a = oid("HEAD", &git_repo.path); + let _obj_b = commit(&git_repo.path, "b.txt", "two\n"); + let tip_b = oid("HEAD", &git_repo.path); + let _obj_c = commit(&git_repo.path, "c.txt", "three\n"); + let tip_c = oid("HEAD", &git_repo.path); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // The outage never clears. + drain_faults::inject(&repo.id, 10_000, 0); + + let guard = admit(&state, &key); + coalesce(&state, &key, vec![(tip_a.clone(), tip_b.clone())]); + + let inflight = state.encrypt_inflight.clone(); + let watch_key = key.clone(); + let coalesced = tokio::spawn(async move { + if !wait_for_drain_window(&inflight, &watch_key).await { + return false; + } + matches!( + inflight.try_begin(&watch_key, vec![(tip_b, tip_c)]), + BeginOutcome::Coalesced + ) + }); + + // The watchdog is the real assertion: a loop that re-spins without the + // pending gate would never return here. + tokio::time::timeout( + std::time::Duration::from_secs(60), + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![obj_a.clone()], + Some(vec![]), + true, + ), + ) + .await + .expect( + "the task must terminate under a sustained outage; a fall-through that \ + does not gate on the pending slot spins forever", + ); + + assert!( + coalesced.await.expect("coalescing task"), + "push C must have coalesced inside the retry window" + ); + assert_eq!( + drain_faults::counters(&repo.id).repo_read_attempts, + 6, + "one coalescing push buys exactly one more bounded re-read lap \ + (3 + 3 attempts), never an unbounded retry" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is released on the give-up path" + ); + } + } } } From 8f9ed070cd843bdd8b6c6540b2bd4325f41aa9b8 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 2 Aug 2026 20:16:58 -0500 Subject: [PATCH 123/124] test(node): stop the repos.rs structural gates splitting at the wrong marker Three gates scanned the "production half" of api/repos.rs by splitting at the first cfg(test) attribute. Adding test-only items above the code they check moved that split point above every line being scanned, so all three failed looking for production code that was still right there. They now split at the test module, which is what they meant. This is the same defect already fixed in the ipfs gate during the merge, and no split-on-the- attribute site remains. Both behavior-bearing gates were re-proven load-bearing: F2 goes red when the Pinata object set is re-derived before the pin permit is taken, and the tail gate goes red when release stops consuming the same success flag. F3 is a presence check whose marker cannot be removed by a single compiling edit, so it is not mutation-proven here. --- crates/gitlawb-node/tests/inv22_gates.rs | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index 7fb25512..33815022 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -147,8 +147,12 @@ fn inv22_concurrency_gates_present_and_not_bypassed() { // recovery copies are absent until an unrelated later push. Scan only the // production half of the file — the u5 tests in its `mod tests` also name the // drain call, and matching them would make this check vacuous. + // Split at the TEST MODULE, not at the first `#[cfg(test)]`. api/repos.rs carries + // test-only items (the drain fault seam, the task entry point) ABOVE the production + // code these gates scan for, so splitting on the attribute truncated the production + // half above every line being checked and the gate went vacuously blind. let repos_production = repos - .split("#[cfg(test)]") + .split("\nmod tests {") .next() .expect("split always yields a first chunk"); assert!( @@ -326,8 +330,12 @@ fn f6_ipfs_metadata_queries_are_deadline_wrapped() { #[test] fn f2_pinata_enqueues_refs_not_retained_object_lists() { let repos = src("api/repos.rs"); + // Split at the TEST MODULE, not at the first `#[cfg(test)]`. api/repos.rs carries + // test-only items (the drain fault seam, the task entry point) ABOVE the production + // code these gates scan for, so splitting on the attribute truncated the production + // half above every line being checked and the gate went vacuously blind. let production = repos - .split("#[cfg(test)]") + .split("\nmod tests {") .next() .expect("split always yields a first chunk"); @@ -386,8 +394,12 @@ fn f2_pinata_enqueues_refs_not_retained_object_lists() { fn f3_second_writer_leased_until_reap() { let repos = src("api/repos.rs"); let smart_http = src("git/smart_http.rs"); + // Split at the TEST MODULE, not at the first `#[cfg(test)]`. api/repos.rs carries + // test-only items (the drain fault seam, the task entry point) ABOVE the production + // code these gates scan for, so splitting on the attribute truncated the production + // half above every line being checked and the gate went vacuously blind. let repos_production = repos - .split("#[cfg(test)]") + .split("\nmod tests {") .next() .expect("split always yields a first chunk"); @@ -511,8 +523,12 @@ fn inv22_ipfs_walk_admission_reaches_every_blocking_site() { fn inv22_replication_tail_spawns_at_the_durability_boundary() { let repos = src("api/repos.rs"); // Production half only — the tests below name these identifiers too. + // Split at the TEST MODULE, not at the first `#[cfg(test)]`. api/repos.rs carries + // test-only items (the drain fault seam, the task entry point) ABOVE the production + // code these gates scan for, so splitting on the attribute truncated the production + // half above every line being checked and the gate went vacuously blind. let production = repos - .split("#[cfg(test)]") + .split("\nmod tests {") .next() .expect("split always yields a first chunk"); From f4a8b8b94f0ceb05c13c2567632e9930eb350e34 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 3 Aug 2026 01:16:52 -0500 Subject: [PATCH 124/124] test(node): restore the ipfs-max-repos-walked range test the #174 merge dropped The merge kept the knob and its clap range but carried over only the default-value assertion, so nothing asserted that 0 and 1048577 are rejected. Restored verbatim from the #174 side, beside its ipfs_max_repo_visits sibling. Load-bearing: widening the range to 0..=2_097_152 turns it red. --- crates/gitlawb-node/src/config.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 38c92197..e88611d6 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -1025,6 +1025,24 @@ mod tests { }); } + #[test] + fn ipfs_max_repos_walked_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).ipfs_max_repos_walked, + 64 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "8"]) + .ipfs_max_repos_walked, + 8 + ); + // 0 would walk no repos (serve nothing); clap must reject it. + assert!(Config::try_parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "0"]).is_err()); + assert!( + Config::try_parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "1048577"]).is_err() + ); + } + #[test] fn ipfs_max_repo_visits_defaults_and_rejects_out_of_range() { assert_eq!(