From 65ba1374af2526f8fe9d950d1a946056fc7bdb9d Mon Sep 17 00:00:00 2001 From: t Date: Thu, 9 Jul 2026 21:43:38 -0500 Subject: [PATCH 01/95] feat(node,git): cap concurrent served git ops with a 503 load-shed (#62) PR3 of the #62 served-git hardening stack (timeout #165 and teardown wiring #150 are merged). A bounded semaphore caps how many upload-pack / receive-pack / info-refs operations run at once; past the cap a request is shed with a clean 503 + Retry-After before spawning another git subprocess, instead of exhausting the PID/thread table. A permit is acquired at the top of each of the three handlers and held for the whole op, releasing on return. The cap is a portable backstop: the compose pids_limit is absent on Fly, whose 500-connection cap is a different axis. Size --max-concurrent-git-ops (GITLAWB_MAX_CONCURRENT_GIT_OPS, default 128) below the process budget. Range 1..=1_048_576 so 0 (shed everything) and an oversized value that would panic tokio's Semaphore at boot are clean CLI errors. Known gap, tracked separately: info/refs and the withheld-blob (upload_pack_excluding) path are not duration-bounded and do not reap their git child on client disconnect, so a hung git on those two paths holds its slot until it exits and live git can briefly exceed the cap. The main pack path (run_git_service) tears its group down on drop. Tests: Overloaded maps to 503 + Retry-After; the config knob defaults and rejects out-of-range; git_permit sheds at capacity and releases; and each of the three endpoints sheds with 503 when the semaphore is exhausted (load-bearing: drop the permit line and the endpoint test goes red). --- crates/gitlawb-node/src/api/repos.rs | 37 ++++++++ crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/config.rs | 61 ++++++++++++ crates/gitlawb-node/src/error.rs | 31 +++++- crates/gitlawb-node/src/main.rs | 1 + crates/gitlawb-node/src/state.rs | 4 + crates/gitlawb-node/src/test_support.rs | 119 ++++++++++++++++++++++++ 7 files changed, 253 insertions(+), 1 deletion(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index b9cbc352..4bc1f737 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -507,6 +507,9 @@ pub async fn git_info_refs( headers: axum::http::HeaderMap, auth: Option>, ) -> Result { + // Shed with a 503 before spawning git when the concurrency cap is saturated; + // held for the whole op (incl. the smart_http call), released on return. + let _permit = git_permit(&state.git_semaphore)?; let name = smart_http_repo_name(&repo)?; tracing::info!(owner = %owner, repo = %name, "info/refs request"); let record = state @@ -589,6 +592,22 @@ pub async fn git_info_refs( }) } +/// Acquire a permit from the served-git concurrency semaphore, or shed the +/// request with a 503 + Retry-After when every slot is in use. Bind the returned +/// permit to a named local so it is held for the whole git op (it releases on +/// drop); a bare `_` would release it immediately. +fn git_permit( + sem: &std::sync::Arc, +) -> Result { + sem.clone().try_acquire_owned().map_err(|_| { + // Surface the shed so operators can see the cap engaging, mirroring the + // receive-pack rate-limit warn above. A silent 503 makes a saturated or + // misconfigured cap look like a client problem instead of a capacity one. + tracing::warn!("served-git concurrency cap reached; shedding request with 503"); + AppError::Overloaded("git service at capacity, retry shortly".into()) + }) +} + /// Map an error from a `smart_http` git service call to the right `AppError`: /// [`smart_http::GitServiceTimeout`] to 504, a malformed client request to 400, /// anything else to a 500 git error. Pure (no logging) so it is unit-testable; @@ -616,6 +635,9 @@ pub async fn git_upload_pack( auth: Option>, body: Bytes, ) -> Result { + // Shed with a 503 before spawning git when the concurrency cap is saturated; + // held for the whole op (incl. the smart_http call), released on return. + let _permit = git_permit(&state.git_semaphore)?; let name = smart_http_repo_name(&repo)?; let record = state .db @@ -853,6 +875,10 @@ pub async fn git_receive_pack( Extension(auth): Extension, body: Bytes, ) -> Result { + // Shed with a 503 before spawning git when the concurrency cap is saturated. + // Acquired at the very top so it wraps the write-guard below; held for the + // whole op (incl. the smart_http call), released on return. + let _permit = git_permit(&state.git_semaphore)?; let name = smart_http_repo_name(&repo)?; tracing::info!(owner = %owner, repo = %name, "receive-pack request"); let record = state @@ -1862,6 +1888,17 @@ mod tests { assert!(matches!(git_service_app_error(&other), AppError::Git(_))); } + #[test] + fn git_permit_sheds_at_capacity_and_releases() { + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + let p1 = git_permit(&sem).expect("first acquire succeeds"); + // At capacity the next request is shed with Overloaded (-> 503), not queued. + assert!(matches!(git_permit(&sem), Err(AppError::Overloaded(_)))); + // Releasing the permit frees the slot for the next request. + drop(p1); + assert!(git_permit(&sem).is_ok()); + } + fn repo_owned_by(owner_did: &str) -> crate::db::RepoRecord { let now = chrono::Utc::now(); crate::db::RepoRecord { diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 720fb3ae..b541b57b 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -520,6 +520,7 @@ mod tests { sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), shutdown_tx: tokio::sync::watch::channel(false).0, + git_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), } } diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index fc2247d9..688e99f4 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -234,6 +234,40 @@ pub struct Config { value_parser = clap::value_parser!(u64).range(1..) )] pub db_retry_max_secs: u64, + + /// Maximum number of served git operations (upload-pack / receive-pack / + /// info-refs) allowed to run concurrently. Beyond this the node sheds the + /// request with a clean 503 + Retry-After instead of spawning another git + /// subprocess and risking PID/thread exhaustion. Portable backstop: the + /// compose `pids_limit` is not present on Fly, whose connection-concurrency + /// cap is a different axis (500 connections each fan out to git + + /// pack-objects + threads). Size below the process budget with headroom. + /// + /// A permit is held for the whole op. upload-pack/receive-pack are + /// duration-bounded by `git_service_timeout_secs`, but the `info/refs` + /// advertisement and the withheld-blob (`upload_pack_excluding`) path are + /// not, so a hung git on those two paths holds its slot until it exits, so a + /// stuck advertisement permanently costs one unit of capacity. Bounding + /// those paths' duration is tracked as separate follow-up. + /// + /// The same two paths have the mirror gap on cancellation: the permit frees + /// when the handler future drops (client disconnect), but neither reaps its + /// git child — `info/refs` runs a bare `Command`, and `upload_pack_excluding` + /// a blocking `spawn_blocking` a dropped future cannot cancel — so the child + /// runs to completion after its slot is freed and live git can briefly exceed + /// the cap. The main pack path (`run_git_service`) tears its process group + /// down on drop, so this is confined to the same two follow-up paths. + /// + /// Default: 128. Must be between 1 and 1_048_576; the ceiling keeps the value + /// well under tokio's `Semaphore` permit limit so an oversized value is a + /// clean CLI error rather than a boot-time panic. + #[arg( + long, + env = "GITLAWB_MAX_CONCURRENT_GIT_OPS", + default_value_t = 128, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub max_concurrent_git_ops: usize, } impl Config { @@ -272,4 +306,31 @@ mod tests { Config::try_parse_from(["gitlawb-node", "--git-service-timeout-secs", "0"]).is_err() ); } + + #[test] + fn max_concurrent_git_ops_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).max_concurrent_git_ops, + 128 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-git-ops", "8"]) + .max_concurrent_git_ops, + 8 + ); + // 0 permits would shed every served-git request with a 503; clap must reject it. + assert!(Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-ops", "0"]).is_err()); + // Above the ceiling would panic tokio's Semaphore::new at boot (permits > + // usize::MAX >> 3); clap must reject it as a clean CLI error instead. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-ops", "1048577"]) + .is_err() + ); + // The ceiling itself is accepted. + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-git-ops", "1048576"]) + .max_concurrent_git_ops, + 1_048_576 + ); + } } diff --git a/crates/gitlawb-node/src/error.rs b/crates/gitlawb-node/src/error.rs index bdd5aec9..a07235fa 100644 --- a/crates/gitlawb-node/src/error.rs +++ b/crates/gitlawb-node/src/error.rs @@ -47,6 +47,9 @@ pub enum AppError { #[error("git service timed out: {0}")] Timeout(String), + #[error("server overloaded: {0}")] + Overloaded(String), + #[error("database error: {0}")] Db(#[from] sqlx::Error), @@ -147,6 +150,12 @@ impl IntoResponse for AppError { DB_UNAVAILABLE_CODE, DB_UNAVAILABLE_MESSAGE.into(), ), + // 503 with a Retry-After (attached after this match — the shared tail + // can't carry per-variant headers). This is the single place Overloaded + // becomes a response, so it can never ship a 503 without the retry hint. + AppError::Overloaded(msg) => { + (StatusCode::SERVICE_UNAVAILABLE, "overloaded", msg.clone()) + } AppError::Db(e) => (StatusCode::INTERNAL_SERVER_ERROR, "db_error", e.to_string()), AppError::Internal(e) => ( StatusCode::INTERNAL_SERVER_ERROR, @@ -160,7 +169,17 @@ impl IntoResponse for AppError { "message": message, })); - (status, body).into_response() + let mut resp = (status, body).into_response(); + // Overloaded advertises when to retry. It rides the shared tail above for + // its body/status, so the header is attached here rather than in a bespoke + // early return — keeping the variant handled in exactly one place. + if matches!(self, AppError::Overloaded(_)) { + resp.headers_mut().insert( + axum::http::header::RETRY_AFTER, + axum::http::HeaderValue::from_static("1"), + ); + } + resp } } @@ -182,4 +201,14 @@ mod tests { StatusCode::INTERNAL_SERVER_ERROR ); } + + #[test] + fn overloaded_maps_to_503_with_retry_after() { + let resp = AppError::Overloaded("x".into()).into_response(); + assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + resp.headers().get("retry-after").unwrap().to_str().unwrap(), + "1" + ); + } } diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index aa0483db..ee36c783 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -378,6 +378,7 @@ async fn main() -> Result<()> { sync_trigger_rate_limiter, peer_write_rate_limiter, shutdown_tx: shutdown_tx.clone(), + git_semaphore: Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_git_ops)), }; // Periodic peer-count poll for the metrics gauge. If p2p is disabled diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 6a84b3c2..e6e69ea6 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -88,6 +88,10 @@ pub struct AppState { /// * the libp2p swarm task /// * the gossip, sync, operator heartbeat, and rate-limit cleanup loops pub shutdown_tx: tokio::sync::watch::Sender, + /// Bounds concurrent served git operations. A handler acquires a permit + /// before spawning git and holds it for the op; when none are free the + /// request is shed with a 503 rather than exhausting the PID/thread table. + pub git_semaphore: Arc, } impl AppState { diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 4fe52f64..cc322a67 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -82,6 +82,8 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), shutdown_tx: tokio::sync::watch::channel(false).0, + // Generous — no test drives the handler-level shed (git_permit is unit-tested). + git_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), } } @@ -187,6 +189,123 @@ mod tests { ); } + /// PR3 (#62): the served-git concurrency cap sheds at the HTTP layer. One test + /// per git handler drives the `let _permit = git_permit(...)` wiring end to end + /// (this one plus the git_upload_pack / git_receive_pack siblings below); the + /// git_permit unit test covers the helper in isolation. DB-free: an exhausted + /// semaphore sheds before any DB/disk access, so a lazy state works. Remove the + /// permit line from git_info_refs and this goes red (the request falls through + /// to the DB and returns something other than 503). + #[tokio::test] + async fn git_info_refs_sheds_with_503_when_semaphore_exhausted() { + let mut state = test_state_lazy(); + state.git_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + + let router = Router::new() + .route( + "/{owner}/{repo}/info/refs", + axum::routing::get(crate::api::repos::git_info_refs), + ) + .with_state(state); + let resp = router + .oneshot(anon_get( + "/alice/repo.git/info/refs?service=git-upload-pack", + )) + .await + .unwrap(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted git semaphore must shed info/refs with 503 before touching the DB" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + + /// PR3 (#62) sibling of the info/refs shed test: git-upload-pack also acquires a + /// permit at the top, so an exhausted semaphore must shed it with a 503 before + /// any DB/disk work. Anonymous-reachable, so no auth injection is needed. Remove + /// the permit line from git_upload_pack and this goes red. + #[tokio::test] + async fn git_upload_pack_sheds_with_503_when_semaphore_exhausted() { + let mut state = test_state_lazy(); + state.git_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + + let router = Router::new() + .route( + "/{owner}/{repo}/git-upload-pack", + axum::routing::post(crate::api::repos::git_upload_pack), + ) + .with_state(state); + let req = Request::builder() + .method(Method::POST) + .uri("/alice/repo.git/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + let resp = router.oneshot(req).await.unwrap(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted git semaphore must shed git-upload-pack with 503 before touching the DB" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + + /// PR3 (#62) sibling for the push path: git-receive-pack requires an + /// AuthenticatedDid extension (production: require_signature injects it), so the + /// request carries one via signed_request_as — without it the Extension + /// extractor 500s before the handler body reaches git_permit. The permit is the + /// first statement, so an exhausted semaphore still sheds 503 before any DB + /// work. Remove the permit line from git_receive_pack and this goes red. + #[tokio::test] + async fn git_receive_pack_sheds_with_503_when_semaphore_exhausted() { + let mut state = test_state_lazy(); + state.git_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + + let router = Router::new() + .route( + "/{owner}/{repo}/git-receive-pack", + axum::routing::post(crate::api::repos::git_receive_pack), + ) + .with_state(state); + let owner = "did:key:zRECVSHEDOWNERAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let resp = router + .oneshot(signed_request_as( + owner, + Method::POST, + "/alice/repo.git/git-receive-pack", + Body::from(&b"0000"[..]), + )) + .await + .unwrap(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted git semaphore must shed git-receive-pack with 503 before touching the DB" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + /// N7: merge_pr is owner-only. A non-owner is rejected by require_repo_owner /// before any git work (so no on-disk repo is needed for the rejection). #[sqlx::test] From db4de844add8e1eeb61f2e5b5118751a677ca6f3 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 10 Jul 2026 12:18:07 -0500 Subject: [PATCH 02/95] feat(node,config): add write-pool and per-caller read-cap knobs (#62) Add max_concurrent_git_pushes (default 32) and max_concurrent_reads_per_caller (default 16), both clap range(1..=1_048_576) so an oversized value is a clean CLI error, not a Semaphore::new boot panic. The per-caller knob documents that per-source-IP keying is only as granular as GITLAWB_TRUSTED_PROXY. Wiring lands in the following commits; these are the config surface for the #174 concurrency-fairness fix. Resolves jatmn P1a/P1b groundwork on #174. --- .env.example | 15 +++++ crates/gitlawb-node/src/config.rs | 96 +++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+) diff --git a/.env.example b/.env.example index bbd9a342..d7c6a30b 100644 --- a/.env.example +++ b/.env.example @@ -113,6 +113,21 @@ GITLAWB_MAX_PACK_BYTES=2147483648 # advertisement or the withheld-blob path, which remain unbounded. Default 600. GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600 +# Max concurrent git-receive-pack (push) operations, in a pool separate from the +# read pool (GITLAWB_MAX_CONCURRENT_GIT_OPS) so anonymous reads cannot shed an +# authenticated push at admission. Over-cap sheds a 503 + Retry-After. Default 32. +GITLAWB_MAX_CONCURRENT_GIT_PUSHES=32 + +# Max concurrent read ops (upload-pack + info/refs advertisements) a single caller +# may hold, so one caller cannot monopolize the read pool. Keyed per-DID when +# signed, else per-source-IP. The per-IP key is only as granular as +# GITLAWB_TRUSTED_PROXY below: left unset, a node behind an edge/NAT keys all +# anonymous callers on the edge IP and this collapses to one global anonymous cap. +# Set GITLAWB_TRUSTED_PROXY for per-client keying; a high-fanout caller (CI behind +# one NAT) should authenticate for a per-DID budget or the operator raises this. +# Default 16. +GITLAWB_MAX_CONCURRENT_READS_PER_CALLER=16 + # ── Push rate limiting (git-receive-pack flood brake) ───────────────────── # Max receive-pack requests (info/refs advertisement + push POST) per client # IP per hour. 0 disables. Default 600. diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 688e99f4..ff2c2556 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -268,6 +268,42 @@ pub struct Config { value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) )] pub max_concurrent_git_ops: usize, + + /// Maximum number of concurrent `git-receive-pack` (push) operations. Pushes + /// draw from this dedicated pool, separate from `max_concurrent_git_ops` + /// (reads), so a flood of anonymous reads cannot shed an authenticated push at + /// admission (#174). Beyond this a push sheds a clean 503 + Retry-After. + /// + /// Default: 32. Must be between 1 and 1_048_576 (the ceiling keeps the value + /// under tokio's `Semaphore` permit limit so an oversized value is a clean CLI + /// error rather than a boot-time panic). + #[arg( + long, + env = "GITLAWB_MAX_CONCURRENT_GIT_PUSHES", + default_value_t = 32, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub max_concurrent_git_pushes: usize, + + /// Maximum concurrent read operations (`upload-pack` and the `info/refs` + /// advertisements) a single caller may hold at once, so one caller cannot + /// monopolize the `max_concurrent_git_ops` read pool (#174). Callers are keyed + /// per-DID when signed, else per-source-IP. IMPORTANT: the per-source-IP key is + /// only as granular as `GITLAWB_TRUSTED_PROXY`. Left unset (the default), a node + /// behind an edge/NAT keys all anonymous callers on the edge IP, so this cap + /// collapses to a single global anonymous cap rather than per-client. Set + /// `GITLAWB_TRUSTED_PROXY` to key on the real client; a known high-fanout caller + /// (a CI fleet behind one NAT) should authenticate for a per-DID budget or the + /// operator raises this. Over-cap for a caller sheds a clean 503 + Retry-After. + /// + /// Default: 16. Must be between 1 and 1_048_576. + #[arg( + long, + env = "GITLAWB_MAX_CONCURRENT_READS_PER_CALLER", + default_value_t = 16, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub max_concurrent_reads_per_caller: usize, } impl Config { @@ -333,4 +369,64 @@ mod tests { 1_048_576 ); } + + #[test] + fn max_concurrent_git_pushes_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).max_concurrent_git_pushes, + 32 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "8"]) + .max_concurrent_git_pushes, + 8 + ); + // 0 permits would shed every push with a 503; clap must reject it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "0"]).is_err() + ); + // Above the ceiling would panic tokio's Semaphore::new at boot; clap rejects it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "1048577"]) + .is_err() + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "1048576"]) + .max_concurrent_git_pushes, + 1_048_576 + ); + } + + #[test] + fn max_concurrent_reads_per_caller_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).max_concurrent_reads_per_caller, + 16 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-reads-per-caller", "4"]) + .max_concurrent_reads_per_caller, + 4 + ); + // 0 would shed every read from a keyed caller; clap must reject it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-reads-per-caller", "0"]) + .is_err() + ); + assert!(Config::try_parse_from([ + "gitlawb-node", + "--max-concurrent-reads-per-caller", + "1048577" + ]) + .is_err()); + assert_eq!( + Config::parse_from([ + "gitlawb-node", + "--max-concurrent-reads-per-caller", + "1048576" + ]) + .max_concurrent_reads_per_caller, + 1_048_576 + ); + } } From 2434dde42f1ac776da08df8186b7f4a3b54073ae Mon Sep 17 00:00:00 2001 From: t Date: Fri, 10 Jul 2026 12:29:30 -0500 Subject: [PATCH 03/95] fix(node,git): isolate authed pushes in a dedicated write pool (#62) git-receive-pack now draws from a separate git_write_semaphore (max_concurrent_git_pushes) instead of the shared pool, so a flood of anonymous reads can no longer shed an authenticated push at admission (jatmn P1a). The shared field is renamed git_read_semaphore and continues to gate upload-pack and both info/refs advertisements. The write permit stays above acquire_write so it precedes the Tigris fresh-acquire (INV-10). Handler-layer tests: write-pool shed (503), and a cross-boundary proof that an exhausted read pool does NOT shed a push; both mutation-checked (routing receive-pack back to the read pool flips each RED). 497 tests pass. Part of #174. --- crates/gitlawb-node/src/api/repos.rs | 12 ++++--- crates/gitlawb-node/src/auth/mod.rs | 3 +- crates/gitlawb-node/src/main.rs | 5 ++- crates/gitlawb-node/src/state.rs | 14 +++++--- crates/gitlawb-node/src/test_support.rs | 47 ++++++++++++++++++++++--- 5 files changed, 65 insertions(+), 16 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 4bc1f737..dd030a36 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -509,7 +509,7 @@ pub async fn git_info_refs( ) -> Result { // Shed with a 503 before spawning git when the concurrency cap is saturated; // held for the whole op (incl. the smart_http call), released on return. - let _permit = git_permit(&state.git_semaphore)?; + let _permit = git_permit(&state.git_read_semaphore)?; let name = smart_http_repo_name(&repo)?; tracing::info!(owner = %owner, repo = %name, "info/refs request"); let record = state @@ -637,7 +637,7 @@ pub async fn git_upload_pack( ) -> Result { // Shed with a 503 before spawning git when the concurrency cap is saturated; // held for the whole op (incl. the smart_http call), released on return. - let _permit = git_permit(&state.git_semaphore)?; + let _permit = git_permit(&state.git_read_semaphore)?; let name = smart_http_repo_name(&repo)?; let record = state .db @@ -876,9 +876,11 @@ pub async fn git_receive_pack( body: Bytes, ) -> Result { // Shed with a 503 before spawning git when the concurrency cap is saturated. - // Acquired at the very top so it wraps the write-guard below; held for the - // whole op (incl. the smart_http call), released on return. - let _permit = git_permit(&state.git_semaphore)?; + // Pushes draw from the dedicated WRITE pool, separate from reads, so a flood of + // anonymous reads cannot shed an authenticated push (#174). Acquired at the very + // top so it wraps the write-guard below (and precedes the Tigris acquire_write, + // bounding concurrent fresh acquires — INV-10); held for the whole op. + let _permit = git_permit(&state.git_write_semaphore)?; let name = smart_http_repo_name(&repo)?; tracing::info!(owner = %owner, repo = %name, "receive-pack request"); let record = state diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index b541b57b..1d169e1c 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -520,7 +520,8 @@ mod tests { sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), shutdown_tx: tokio::sync::watch::channel(false).0, - git_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), } } diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index ee36c783..b8f687b1 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -378,7 +378,10 @@ async fn main() -> Result<()> { sync_trigger_rate_limiter, peer_write_rate_limiter, shutdown_tx: shutdown_tx.clone(), - git_semaphore: Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_git_ops)), + git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_git_ops)), + git_write_semaphore: Arc::new(tokio::sync::Semaphore::new( + config.max_concurrent_git_pushes, + )), }; // Periodic peer-count poll for the metrics gauge. If p2p is disabled diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index e6e69ea6..b43f4de2 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -88,10 +88,16 @@ pub struct AppState { /// * the libp2p swarm task /// * the gossip, sync, operator heartbeat, and rate-limit cleanup loops pub shutdown_tx: tokio::sync::watch::Sender, - /// Bounds concurrent served git operations. A handler acquires a permit - /// before spawning git and holds it for the op; when none are free the - /// request is shed with a 503 rather than exhausting the PID/thread table. - pub git_semaphore: Arc, + /// Bounds concurrent served git READ operations (upload-pack + both info/refs + /// advertisements). A read handler acquires a permit before spawning git and + /// holds it for the op; when none are free the request is shed with a 503. + /// Writes draw from `git_write_semaphore` so a read flood cannot shed an + /// authenticated push at admission (#174). + pub git_read_semaphore: Arc, + /// Bounds concurrent `git-receive-pack` (push) operations, a pool separate + /// from `git_read_semaphore` so anonymous reads can never shed an authenticated + /// push (#174). Sized by `max_concurrent_git_pushes`. + pub git_write_semaphore: Arc, } impl AppState { diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index cc322a67..fd68c8e6 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -83,7 +83,8 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), shutdown_tx: tokio::sync::watch::channel(false).0, // Generous — no test drives the handler-level shed (git_permit is unit-tested). - git_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), } } @@ -199,7 +200,7 @@ mod tests { #[tokio::test] async fn git_info_refs_sheds_with_503_when_semaphore_exhausted() { let mut state = test_state_lazy(); - state.git_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + state.git_read_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); let router = Router::new() .route( @@ -235,7 +236,7 @@ mod tests { #[tokio::test] async fn git_upload_pack_sheds_with_503_when_semaphore_exhausted() { let mut state = test_state_lazy(); - state.git_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + state.git_read_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); let router = Router::new() .route( @@ -273,7 +274,7 @@ mod tests { #[tokio::test] async fn git_receive_pack_sheds_with_503_when_semaphore_exhausted() { let mut state = test_state_lazy(); - state.git_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + state.git_write_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); let router = Router::new() .route( @@ -295,7 +296,7 @@ mod tests { assert_eq!( resp.status(), StatusCode::SERVICE_UNAVAILABLE, - "an exhausted git semaphore must shed git-receive-pack with 503 before touching the DB" + "an exhausted write pool must shed git-receive-pack with 503 before touching the DB" ); assert_eq!( resp.headers() @@ -306,6 +307,42 @@ mod tests { ); } + /// #174 (SC1, load-bearing): a saturated READ pool must NOT shed an + /// authenticated push — the write pool is a separate budget. Read pool at zero, + /// write pool with capacity: the push proceeds PAST admission (it then errors on + /// the placeholder DB, but crucially it is not a 503). Route git-receive-pack + /// back to the read pool and this goes red — that is the isolation proof. + #[tokio::test] + async fn git_receive_pack_not_shed_by_exhausted_read_pool() { + let mut state = test_state_lazy(); + // Read pool exhausted as if a flood of anonymous clones held every slot. + state.git_read_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + // Write pool keeps its default capacity from test_state_lazy. + + let router = Router::new() + .route( + "/{owner}/{repo}/git-receive-pack", + axum::routing::post(crate::api::repos::git_receive_pack), + ) + .with_state(state); + let owner = "did:key:zRECVCROSSBOUNDARYAAAAAAAAAAAAAAAAAAAAA"; + let resp = router + .oneshot(signed_request_as( + owner, + Method::POST, + "/alice/repo.git/git-receive-pack", + Body::from(&b"0000"[..]), + )) + .await + .unwrap(); + + assert_ne!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted READ pool must not shed a push — the write pool is a separate budget (#174)" + ); + } + /// N7: merge_pr is owner-only. A non-owner is rejected by require_repo_owner /// before any git work (so no on-disk repo is needed for the rejection). #[sqlx::test] From fc4c206c3a1ccf38a433b2a4d8255b83910f4357 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 10 Jul 2026 12:50:46 -0500 Subject: [PATCH 04/95] fix(node,git): per-caller concurrency sub-cap on the read pool (#62) Adds PerCallerConcurrency, a bounded-keyed in-flight limiter (distinct from the request-rate RateLimiter) so no single caller monopolizes the served-git read pool. Each caller (per-DID when signed via optional_signature, else per-source-IP via client_key) may hold at most max_concurrent_reads_per_caller concurrent reads; over that it sheds 503. The key map is self-bounding (a key is dropped when its in-flight count hits zero) with a reject-before-insert max_keys backstop so a key farm can't grow it (INV-15). Applied in git_upload_pack and both info/refs advertisements, acquired after the visibility gate so a denied request never consumes a slot (KTD7). Primitive unit-tested (cap + self-bounding + reject-before-insert) and mutation-checked. Handler-layer SC2: same-caller sheds while a different caller passes, proven on BOTH git_info_refs and git_upload_pack with independent mutation probes; plus a None-key bypass test. Per-source-IP keying is trust-config dependent, documented on the config knob. 502 tests pass. Part of #174. --- crates/gitlawb-node/src/api/repos.rs | 210 ++++++++++++++++++++++++ crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/main.rs | 3 + crates/gitlawb-node/src/rate_limit.rs | 142 ++++++++++++++++ crates/gitlawb-node/src/state.rs | 5 + crates/gitlawb-node/src/test_support.rs | 1 + 6 files changed, 362 insertions(+) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index dd030a36..e51b62e6 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -566,6 +566,28 @@ pub async fn git_info_refs( } } + // Per-caller read sub-cap (#174): acquired AFTER the visibility + push-rate + // gates (KTD7) so a denied or rate-limited request never consumes a caller's + // scarce read slot. Applies to BOTH advertisements. Held for the whole op. + let caller_key = read_caller_key( + auth.as_ref().map(|e| e.0 .0.as_str()), + &headers, + peer, + state.push_limiter_trust, + ); + let _caller_permit = match &caller_key { + Some(k) => match state.git_read_per_caller.try_acquire(k) { + Some(p) => Some(p), + None => { + tracing::warn!(repo = %name, caller = %k, "per-caller read cap reached; shedding info/refs with 503"); + return Err(AppError::Overloaded( + "git read service at capacity for this caller, retry shortly".into(), + )); + } + }, + None => None, + }; + // For receive-pack (push), download the latest from Tigris so the client // sees the same refs that acquire_write() will operate on. let disk_path = if service == "git-receive-pack" { @@ -608,6 +630,23 @@ fn git_permit( }) } +/// Resolve the per-caller key for the read sub-cap (#174): the authenticated DID +/// when the caller signed (via `optional_signature`), else the trusted client IP +/// (`client_key`). `None` when neither is available — such a request is bounded by +/// the global read pool only, never a 500. The per-source-IP key is only as +/// granular as `trust`; see the `max_concurrent_reads_per_caller` config doc. +fn read_caller_key( + caller_did: Option<&str>, + headers: &axum::http::HeaderMap, + peer: Option, + trust: crate::rate_limit::TrustedProxy, +) -> Option { + match caller_did { + Some(did) => Some(did.to_string()), + None => crate::rate_limit::client_key(headers, peer, trust), + } +} + /// Map an error from a `smart_http` git service call to the right `AppError`: /// [`smart_http::GitServiceTimeout`] to 504, a malformed client request to 400, /// anything else to a 500 git error. Pure (no logging) so it is unit-testable; @@ -633,6 +672,8 @@ pub async fn git_upload_pack( State(state): State, Path((owner, repo)): Path<(String, String)>, auth: Option>, + crate::rate_limit::PeerAddr(peer): crate::rate_limit::PeerAddr, + headers: axum::http::HeaderMap, body: Bytes, ) -> Result { // Shed with a 503 before spawning git when the concurrency cap is saturated; @@ -658,6 +699,23 @@ pub async fn git_upload_pack( return Err(AppError::RepoNotFound(format!("{owner}/{name}"))); } + // Per-caller read sub-cap (#174): after the visibility gate (KTD7) so a + // visibility-denied caller never consumes a scarce read slot. Keyed per-DID + // when signed, else per-source-IP; no resolvable key -> global read pool only. + let caller_key = read_caller_key(caller, &headers, peer, state.push_limiter_trust); + let _caller_permit = match &caller_key { + Some(k) => match state.git_read_per_caller.try_acquire(k) { + Some(p) => Some(p), + None => { + tracing::warn!(repo = %name, caller = %k, "per-caller read cap reached; shedding upload-pack with 503"); + return Err(AppError::Overloaded( + "git read service at capacity for this caller, retry shortly".into(), + )); + } + }, + None => None, + }; + let disk_path = state .repo_store .acquire(&record.owner_did, &record.name) @@ -2618,6 +2676,158 @@ mod tests { ); } + /// #174 SC2 (info_refs probe): the per-caller read sub-cap sheds a caller that + /// is already at its concurrency budget on the upload-pack advertisement, while + /// a DIFFERENT caller still enters. Remove the sub-cap from `git_info_refs` and + /// the same-caller assertion goes green-not-503 — this is the info_refs half of + /// the two-handler mutation probe. + #[sqlx::test] + async fn info_refs_per_caller_cap_sheds_one_caller_not_others(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6pcadv", "pc", "/tmp/pc-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.31:5000".parse().unwrap(); + // Fill this caller's single read slot (a clone shares the Arc-backed map). + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first slot for this caller"); + + // Same caller (IP) at its cap -> shed 503 before the git/Tigris work. + let router = crate::server::build_router(state.clone()); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6pcadv/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + assert_eq!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a caller already at its per-caller read cap must shed the advertisement with 503" + ); + + // A DIFFERENT caller (IP) has its own budget -> not shed by the per-caller cap. + let other: SocketAddr = "203.0.113.32:5000".parse().unwrap(); + let router2 = crate::server::build_router(state.clone()); + let mut req2 = Request::builder() + .method(Method::GET) + .uri("/z6pcadv/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req2.extensions_mut().insert(ConnectInfo(other)); + assert_ne!( + router2.oneshot(req2).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a different caller must not be shed by another caller's saturated budget" + ); + } + + /// #174 SC2 (upload_pack probe): the same per-caller shed on the POST + /// upload-pack path. Remove the sub-cap from `git_upload_pack` and this goes + /// green-not-503 — the upload_pack half of the two-handler mutation probe. + #[sqlx::test] + async fn upload_pack_per_caller_cap_sheds_one_caller_not_others(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6pcupl", "pc", "/tmp/pc-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.41:5000".parse().unwrap(); + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first slot for this caller"); + + let router = crate::server::build_router(state.clone()); + let mut req = Request::builder() + .method(Method::POST) + .uri("/z6pcupl/pc/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + assert_eq!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a caller already at its per-caller read cap must shed upload-pack with 503" + ); + + let other: SocketAddr = "203.0.113.42:5000".parse().unwrap(); + let router2 = crate::server::build_router(state.clone()); + let mut req2 = Request::builder() + .method(Method::POST) + .uri("/z6pcupl/pc/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + req2.extensions_mut().insert(ConnectInfo(other)); + assert_ne!( + router2.oneshot(req2).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a different caller must not be shed by another caller's saturated budget" + ); + } + + /// #174 SC2 (None-key): a request with no resolvable caller key (no ConnectInfo, + /// no trusted header) must NOT be shed by the per-caller cap even when another + /// caller's budget is full — it is bounded by the global read pool only. A None + /// key never keys into the map, so it never 503s from the per-caller sub-cap. + #[sqlx::test] + async fn info_refs_none_key_bypasses_per_caller_cap(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::http::{Method, Request, StatusCode}; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6pcnone", "pc", "/tmp/pc-nonexistent", None, false) + .await + .unwrap(); + // Saturate an unrelated caller's budget; the None-key request must be + // unaffected because it never keys into the per-caller map. + let _slot = state + .git_read_per_caller + .try_acquire("203.0.113.99") + .expect("hold an unrelated caller's slot"); + + // No ConnectInfo inserted -> PeerAddr is None -> no per-caller key. + let router = crate::server::build_router(state.clone()); + let req = Request::builder() + .method(Method::GET) + .uri("/z6pcnone/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + assert_ne!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a request with no resolvable caller key must not be shed by the per-caller cap" + ); + } + /// Repo creation must be throttled by the per-IP creation limiter BEFORE /// signature verification — otherwise a DID farm (one throwaway did:key per /// repo, each carrying a valid but machine-solved iCaptcha proof) walks past diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 1d169e1c..253b3210 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -522,6 +522,7 @@ mod tests { shutdown_tx: tokio::sync::watch::channel(false).0, git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), } } diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index b8f687b1..c617b434 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -382,6 +382,9 @@ async fn main() -> Result<()> { git_write_semaphore: Arc::new(tokio::sync::Semaphore::new( config.max_concurrent_git_pushes, )), + git_read_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( + config.max_concurrent_reads_per_caller, + ), }; // Periodic peer-count poll for the metrics gauge. If p2p is disabled diff --git a/crates/gitlawb-node/src/rate_limit.rs b/crates/gitlawb-node/src/rate_limit.rs index d40e2691..d505f1bb 100644 --- a/crates/gitlawb-node/src/rate_limit.rs +++ b/crates/gitlawb-node/src/rate_limit.rs @@ -132,6 +132,91 @@ impl RateLimiter { } } +/// A bounded per-caller CONCURRENCY limiter — distinct from [`RateLimiter`], which +/// caps request RATE. Each caller key may hold at most `per_caller` in-flight +/// permits at once; beyond that [`try_acquire`](Self::try_acquire) returns `None` +/// and the caller sheds. Used to stop one caller (a single anonymous source-IP or +/// DID) monopolizing the served-git read pool (#174). +/// +/// The key map is self-bounding: a key is removed the instant its in-flight count +/// reaches zero, so it never holds more keys than there are concurrently-active +/// callers (itself bounded by the read semaphore). A `max_keys` reject-before-insert +/// backstop guarantees a key farm can never grow the map even if that invariant +/// weakened — a NEW key at the cap is rejected WITHOUT allocating an entry (INV-15). +/// +/// Uses a `std::sync::Mutex` (not the file's `tokio::sync::Mutex`) because the +/// permit's `Drop` must release synchronously; the critical section holds no await. +#[derive(Clone)] +pub struct PerCallerConcurrency { + state: Arc>>, + per_caller: usize, + max_keys: usize, +} + +/// RAII permit from [`PerCallerConcurrency::try_acquire`]. On drop it decrements +/// the caller's in-flight count and removes the key when it reaches zero. +pub struct PerCallerPermit { + state: Arc>>, + key: String, +} + +impl PerCallerConcurrency { + pub fn new(per_caller: usize, max_keys: usize) -> Self { + Self { + state: Arc::new(std::sync::Mutex::new(HashMap::new())), + per_caller: per_caller.max(1), + max_keys: max_keys.max(1), + } + } + + /// Convenience constructor with the default key bound. + pub fn with_default_max_keys(per_caller: usize) -> Self { + Self::new(per_caller, DEFAULT_MAX_KEYS) + } + + /// `Some(permit)` when the caller is under its cap and the map has room; + /// `None` (shed) otherwise. Reject-before-insert: a new key at `max_keys` is + /// rejected without allocating. + pub fn try_acquire(&self, key: &str) -> Option { + let mut map = self.state.lock().expect("per-caller mutex poisoned"); + match map.get_mut(key) { + Some(count) => { + if *count >= self.per_caller { + return None; + } + *count += 1; + } + None => { + if map.len() >= self.max_keys { + return None; + } + map.insert(key.to_string(), 1); + } + } + Some(PerCallerPermit { + state: self.state.clone(), + key: key.to_string(), + }) + } + + #[cfg(test)] + pub fn tracked_keys(&self) -> usize { + self.state.lock().unwrap().len() + } +} + +impl Drop for PerCallerPermit { + fn drop(&mut self) { + let mut map = self.state.lock().expect("per-caller mutex poisoned"); + if let Some(count) = map.get_mut(&self.key) { + *count -= 1; + if *count == 0 { + map.remove(&self.key); + } + } + } +} + pub async fn rate_limit_by_did(request: Request, next: Next) -> Response { let limiter = request.extensions().get::().cloned(); @@ -288,6 +373,63 @@ pub async fn rate_limit_by_ip(request: Request, next: Next) -> Response { mod tests { use super::*; + #[test] + fn per_caller_concurrency_caps_one_caller_and_frees_on_drop() { + let lim = PerCallerConcurrency::new(2, 100); + let p1 = lim.try_acquire("did:key:zA").expect("first under cap"); + let p2 = lim.try_acquire("did:key:zA").expect("second under cap"); + assert!( + lim.try_acquire("did:key:zA").is_none(), + "a third in-flight op for the same caller sheds (over the per-caller cap)" + ); + // A DIFFERENT caller is unaffected — the cap is per-caller, not global. + let _other = lim + .try_acquire("did:key:zB") + .expect("a different caller has its own budget"); + drop(p1); + assert!( + lim.try_acquire("did:key:zA").is_some(), + "freeing one in-flight slot lets the same caller back in" + ); + drop(p2); + } + + #[test] + fn per_caller_concurrency_map_is_self_bounding_and_reject_before_insert() { + // Self-bounding: acquire+drop many distinct keys — the map never grows + // because a key is removed the instant its in-flight count hits zero. + let lim = PerCallerConcurrency::new(4, 3); + for i in 0..50 { + let _p = lim.try_acquire(&format!("k{i}")); + } + assert_eq!( + lim.tracked_keys(), + 0, + "keys with zero in-flight ops are removed, so an acquire+drop flood leaves the map empty" + ); + // Reject-before-insert: HOLD max_keys distinct keys, then a new key sheds + // WITHOUT growing the map past the cap (INV-15 — a rejected request never + // allocates an entry). + let held: Vec<_> = (0..3) + .map(|i| lim.try_acquire(&format!("h{i}")).unwrap()) + .collect(); + assert_eq!( + lim.tracked_keys(), + 3, + "three distinct callers held concurrently" + ); + assert!( + lim.try_acquire("h3").is_none(), + "a new key at max_keys is rejected" + ); + assert_eq!( + lim.tracked_keys(), + 3, + "the rejected new key did not allocate an entry (reject-before-insert)" + ); + drop(held); + } + #[tokio::test] async fn allows_within_limit() { let limiter = RateLimiter::new(3, Duration::from_secs(60)); diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index b43f4de2..01007ac4 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -98,6 +98,11 @@ pub struct AppState { /// from `git_read_semaphore` so anonymous reads can never shed an authenticated /// push (#174). Sized by `max_concurrent_git_pushes`. pub git_write_semaphore: Arc, + /// Per-caller concurrency sub-cap on the read pool: each caller (per-DID when + /// signed, else per-source-IP) may hold at most `max_concurrent_reads_per_caller` + /// in-flight read ops, so one caller cannot monopolize `git_read_semaphore` + /// (#174). Applied by `git_upload_pack` and both `info/refs` advertisements. + pub git_read_per_caller: crate::rate_limit::PerCallerConcurrency, } impl AppState { diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index fd68c8e6..7745d233 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -85,6 +85,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { // Generous — no test drives the handler-level shed (git_permit is unit-tested). git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), } } From 2c264e08e391f5675cbfd3edeff0238434e9bb15 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 10 Jul 2026 13:09:34 -0500 Subject: [PATCH 05/95] fix(node,git): bound and reap the info/refs advertisement (#62) info_refs ran a bare Command::output() with no timeout and no process-group teardown, so a hung git pinned its concurrency slot indefinitely and a client disconnect orphaned the child (jatmn P1b). Extract the timeout + process_group(0) + KillGroupOnDrop core from run_git_service into a shared drive_git_child, and route info_refs through it with an injectable git_bin. A hung advertisement now aborts with GitServiceTimeout (mapped to 504); disconnect reaps the group. run_git_service's teardown tests all pass through the shared core (proving the group teardown info_refs inherits), the real-git filter tests cover the advertisement happy path, and a new watchdog-bounded test proves a hung advertisement times out. 503 tests pass. Part of #174. --- crates/gitlawb-node/src/api/repos.rs | 15 +++- crates/gitlawb-node/src/git/smart_http.rs | 87 +++++++++++++++++++---- 2 files changed, 85 insertions(+), 17 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index e51b62e6..c42a6989 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -606,11 +606,20 @@ pub async fn git_info_refs( AppError::Git(e.to_string()) })?; - smart_http::info_refs(&disk_path, &service) + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + smart_http::info_refs("git", &service, &disk_path, git_timeout) .await .map_err(|e| { - tracing::error!(repo = %name, service = %service, err = %e, "info_refs git failed"); - AppError::Git(e.to_string()) + let app = git_service_app_error(&e); + match &app { + AppError::Timeout(_) => { + tracing::warn!(repo = %name, service = %service, "info/refs advertisement timed out") + } + _ => { + tracing::error!(repo = %name, service = %service, err = %e, "info_refs git failed") + } + } + app }) } diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index eeb35b99..b4e05a01 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -14,21 +14,29 @@ use tokio::process::Command; /// or `?service=git-receive-pack` /// /// This is the ref advertisement — the first step of a clone or push. -pub async fn info_refs(repo_path: &Path, service: &str) -> Result { +/// +/// `git_bin` is injectable purely so the process-group teardown can be driven by a +/// fake `git` in tests (production passes `"git"`). `timeout` bounds the whole child +/// interaction: previously the advertisement ran a bare `Command::output()` with no +/// deadline and no teardown, so a hung git pinned its concurrency slot indefinitely +/// and a client disconnect orphaned the child (#174). It now shares +/// [`drive_git_child`]'s timeout + `process_group(0)` + [`KillGroupOnDrop`] teardown. +pub async fn info_refs( + git_bin: &str, + service: &str, + repo_path: &Path, + timeout: Duration, +) -> Result { validate_service(service)?; - let output = Command::new("git") + let mut command = Command::new(git_bin); + command .arg(service_to_command(service)) .arg("--stateless-rpc") .arg("--advertise-refs") - .arg(repo_path) - .output() - .await?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - bail!("git {service} --advertise-refs failed: {stderr}"); - } + .arg(repo_path); + // No request body — advertise-refs does not read stdin. + let stdout = drive_git_child(command, Bytes::new(), timeout, "advertise-refs").await?; let content_type = format!("application/x-{service}-advertisement"); @@ -38,7 +46,7 @@ pub async fn info_refs(repo_path: &Path, service: &str) -> Result { let mut body = Vec::new(); body.extend_from_slice(&pkt_service); body.extend_from_slice(flush); - body.extend_from_slice(&output.stdout); + body.extend_from_slice(&stdout); Ok(Response::builder() .status(StatusCode::OK) @@ -207,7 +215,24 @@ async fn run_git_service( command .arg(service_to_command(service)) .arg("--stateless-rpc") - .arg(repo_path) + .arg(repo_path); + drive_git_child(command, input, timeout, service).await +} + +/// Drive a spawned git child under `timeout` with process-group teardown, returning +/// its stdout. Shared core for [`run_git_service`] and [`info_refs`]: the caller +/// passes a `Command` with its args set; this adds piped stdio and `process_group(0)`. +/// On the deadline the whole group is torn down and reaped before returning +/// [`GitServiceTimeout`]; on a dropped future (client disconnect) the +/// [`KillGroupOnDrop`] guard fires. `input` is written to the child's stdin (empty +/// for the advertise-refs path, which has no request body); `what` labels errors. +async fn drive_git_child( + mut command: Command, + input: Bytes, + timeout: Duration, + what: &str, +) -> Result> { + command .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -294,7 +319,7 @@ async fn run_git_service( // the real cause. if !status.success() { let stderr = String::from_utf8_lossy(&err); - bail!("{service} failed: {stderr}"); + bail!("{what} failed: {stderr}"); } write_result.context("failed to write to git stdin")?; @@ -650,7 +675,9 @@ mod tests { axum::extract::Query(q): axum::extract::Query>, ) -> Response { let service = q.get("service").cloned().unwrap_or_default(); - info_refs(&st.repo, &service).await.unwrap() + info_refs("git", &service, &st.repo, Duration::from_secs(30)) + .await + .unwrap() } async fn pack_handler( @@ -1225,6 +1252,38 @@ mod tests { ); } + // #174 (SC3): the info/refs advertisement is now duration-bounded — previously + // it ran a bare `Command::output()` with no deadline, so a hung git pinned its + // concurrency slot forever. A hung fake git must abort with GitServiceTimeout + // (which the handler maps to 504). The outer watchdog turns a missing timeout + // into a loud failure instead of hanging the suite. info_refs shares + // drive_git_child, so its disconnect/group-teardown is the same code proven by + // run_git_service_tears_down_group_when_future_dropped above. + #[cfg(unix)] + #[tokio::test] + async fn info_refs_times_out_a_hung_advertisement() { + let tmp = tempfile::TempDir::new().unwrap(); + // A fake git that hangs forever instead of advertising refs. + let git_bin = write_fake_git(tmp.path(), "#!/bin/sh\nsleep 300\n"); + + let result = tokio::time::timeout( + Duration::from_secs(10), + info_refs( + git_bin.to_str().unwrap(), + "git-upload-pack", + tmp.path(), + Duration::from_millis(200), + ), + ) + .await + .expect("info_refs must return within the watchdog — its own timeout must fire"); + let err = result.expect_err("a hung advertisement must return an error, not hang"); + assert!( + err.downcast_ref::().is_some(), + "a hung advertisement must abort with GitServiceTimeout (-> 504), got: {err}" + ); + } + // A request that runs to completion must DISARM the guard after reaping, so // no stray group SIGTERM fires. The fake exits non-zero (surfacing as Err) // but leaves a grandchild alive; the grandchild must survive. Goes RED if the From 57c6e01c26ac1e3032cb06b57ebc640e18582a85 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 10 Jul 2026 13:16:41 -0500 Subject: [PATCH 06/95] fix(node,git): bound and reap the withheld-blob pack build (#62) The filtered-pack path ran the whole rev-list + pack-objects build inside a spawn_blocking, so an outer tokio timeout could not cancel the blocking thread and a client disconnect orphaned the git child while the permit freed (jatmn P1b, the second gap path). Split it: rev-list enumeration stays blocking off the runtime (rev_list_keep), but the streaming pack-objects stage now runs under the shared drive_git_child on the async side, so it is duration-bounded (GitServiceTimeout -> 504) and its process group is reaped on disconnect. build_filtered_pack becomes async and takes a git_bin seam + timeout; upload_pack_excluding threads the git_service_timeout through. A watchdog-bounded test proves a hung pack-objects times out (rev-list fast, pack-objects hangs). The refactor's happy path is covered by the existing filtered-pack correctness and real-git partial-clone/fetch tests, all still green; disconnect/group-teardown is the shared drive_git_child code proven by the run_git_service tests. 504 tests pass. Part of #174. --- crates/gitlawb-node/src/api/repos.rs | 2 +- crates/gitlawb-node/src/git/smart_http.rs | 145 ++++++++++++++-------- 2 files changed, 95 insertions(+), 52 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index c42a6989..dba8ea71 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -765,7 +765,7 @@ pub async fn git_upload_pack( smart_http::upload_pack(&disk_path, body, git_timeout).await } else { tracing::info!(repo = %name, caller = ?caller, withheld = withheld.len(), "serving filtered pack"); - smart_http::upload_pack_excluding(&disk_path, body, &withheld).await + smart_http::upload_pack_excluding(&disk_path, body, &withheld, git_timeout).await } } .map_err(|e| { diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index b4e05a01..5586a2c7 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -349,12 +349,17 @@ fn pkt_line(data: &str) -> Vec { format!("{len:04x}{data}").into_bytes() } -/// Build a packfile containing every object reachable from all refs EXCEPT the -/// given blob OIDs. Commits and trees are always included, so SHAs stay intact; -/// only the named blobs are dropped. -pub fn build_filtered_pack(repo_path: &Path, withheld: &HashSet) -> Result> { - // All reachable objects as "oid [path]" lines. - let rev = std::process::Command::new("git") +/// Run `rev-list --objects --all` and return the reachable object ids minus the +/// withheld blobs. Blocking (shells out to git); the caller runs it off the async +/// runtime. Kept separate from the pack-objects stage so that stage can live on the +/// async side under [`drive_git_child`] (`git_bin` is injectable for the same +/// fake-git testing reason as `run_git_service`). +fn rev_list_keep( + git_bin: &str, + repo_path: &Path, + withheld: &HashSet, +) -> Result> { + let rev = std::process::Command::new(git_bin) .args(["rev-list", "--objects", "--all"]) .current_dir(repo_path) .output()?; @@ -372,39 +377,39 @@ pub fn build_filtered_pack(repo_path: &Path, withheld: &HashSet) -> Resu } keep.push(oid.to_string()); } - let mut child = std::process::Command::new("git") - .args(["pack-objects", "--stdout"]) - .current_dir(repo_path) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; - // Feed the object ids on stdin, but always reap the child afterward even if - // the write fails or stdin is missing, so an error can't drop the Child - // unwaited and leak a zombie (#53). - let write_result: std::io::Result<()> = { - use std::io::Write as _; - match child.stdin.take() { - Some(mut stdin) => { - let mut data = keep.join("\n").into_bytes(); - data.push(b'\n'); - stdin.write_all(&data) - } - None => Err(std::io::Error::new( - std::io::ErrorKind::BrokenPipe, - "git pack-objects stdin unavailable", - )), - } + Ok(keep) +} + +/// Build a packfile containing every object reachable from all refs EXCEPT the +/// given blob OIDs. Commits and trees are always included, so SHAs stay intact; +/// only the named blobs are dropped. +/// +/// The rev-list enumeration runs blocking off the async runtime; the streaming +/// `pack-objects` stage runs under [`drive_git_child`], so a hung/slow pack build is +/// duration-bounded and its process group is reaped on client disconnect. An outer +/// `tokio::time::timeout` around a `spawn_blocking` cannot cancel the blocking +/// thread, so the streaming stage MUST live on the async side (#174, KTD5). +pub async fn build_filtered_pack( + git_bin: &str, + repo_path: &Path, + withheld: &HashSet, + timeout: Duration, +) -> Result> { + let keep = { + let git_bin = git_bin.to_string(); + let repo_path = repo_path.to_path_buf(); + let withheld = withheld.clone(); + tokio::task::spawn_blocking(move || rev_list_keep(&git_bin, &repo_path, &withheld)) + .await + .context("rev-list task panicked")?? }; - let out = child.wait_with_output()?; - write_result.context("failed to write object ids to git pack-objects stdin")?; - if !out.status.success() { - bail!( - "git pack-objects failed: {}", - String::from_utf8_lossy(&out.stderr) - ); - } - Ok(out.stdout) + let mut data = keep.join("\n").into_bytes(); + data.push(b'\n'); + let mut command = Command::new(git_bin); + command + .args(["pack-objects", "--stdout"]) + .current_dir(repo_path); + drive_git_child(command, Bytes::from(data), timeout, "pack-objects").await } /// Serve a clone/fetch with the withheld blobs removed from the response pack. @@ -434,17 +439,13 @@ pub async fn upload_pack_excluding( repo_path: &Path, request_body: Bytes, withheld: &HashSet, + timeout: Duration, ) -> Result { - // build_filtered_pack shells out to git (rev-list, pack-objects) with - // blocking std::process I/O; run it off the async worker so a large repo's - // pack build does not stall the tokio runtime. - let pack = { - let repo_path = repo_path.to_path_buf(); - let withheld = withheld.clone(); - tokio::task::spawn_blocking(move || build_filtered_pack(&repo_path, &withheld)) - .await - .context("filtered-pack build task panicked")?? - }; + // The rev-list enumeration runs blocking off the runtime; the streaming + // pack-objects stage is duration-bounded and its process group is reaped on + // disconnect via drive_git_child (#174), so a hung build no longer pins its + // concurrency slot and a client disconnect no longer orphans the git child. + let pack = build_filtered_pack("git", repo_path, withheld, timeout).await?; // The client lists its capabilities on the first `want` line. Honor // side-band-64k when offered (every modern smart-HTTP client offers it); @@ -563,7 +564,9 @@ mod tests { let mut withheld = std::collections::HashSet::new(); withheld.insert(secret.clone()); - let pack = build_filtered_pack(&bare, &withheld).unwrap(); + let pack = build_filtered_pack("git", &bare, &withheld, Duration::from_secs(30)) + .await + .unwrap(); let ids = pack_object_ids(&pack); assert!(ids.contains(&public), "public blob must be in the pack"); assert!( @@ -625,7 +628,9 @@ mod tests { b"0098want 0000000000000000000000000000000000000000 \ side-band-64k ofs-delta agent=git/2\n00000009done\n", ); - let resp = upload_pack_excluding(&bare, req, &withheld).await.unwrap(); + let resp = upload_pack_excluding(&bare, req, &withheld, Duration::from_secs(30)) + .await + .unwrap(); let body = to_bytes(resp.into_body(), usize::MAX).await.unwrap(); let ids = pack_object_ids(&extract_pack(&body)); assert!( @@ -684,7 +689,7 @@ mod tests { axum::extract::State(st): axum::extract::State>, body: Bytes, ) -> Response { - upload_pack_excluding(&st.repo, body, &st.withheld) + upload_pack_excluding(&st.repo, body, &st.withheld, Duration::from_secs(30)) .await .unwrap() } @@ -1284,6 +1289,44 @@ mod tests { ); } + // #174 (SC3, KTD5): the filtered-pack build's streaming pack-objects stage is + // duration-bounded on the ASYNC side. The old build ran the whole thing in a + // spawn_blocking, so an outer tokio timeout could not cancel the blocking thread + // and a disconnect orphaned the git child. A fake git that returns objects fast + // on rev-list but hangs on pack-objects must now abort with GitServiceTimeout; + // the watchdog turns a missing bound into a loud failure. The stage runs under + // drive_git_child, so its disconnect/group-teardown is the same code proven by + // run_git_service_tears_down_group_when_future_dropped. + #[cfg(unix)] + #[tokio::test] + async fn build_filtered_pack_times_out_a_hung_pack_objects() { + let tmp = tempfile::TempDir::new().unwrap(); + // rev-list returns one oid fast; pack-objects hangs forever. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) echo deadbeefdeadbeefdeadbeefdeadbeefdeadbeef ;;\n pack-objects) sleep 300 ;;\n *) exit 1 ;;\nesac\n"; + let git_bin = write_fake_git(tmp.path(), body); + let withheld = HashSet::new(); + + let result = tokio::time::timeout( + Duration::from_secs(10), + build_filtered_pack( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_millis(200), + ), + ) + .await + .expect( + "build_filtered_pack must return within the watchdog — the pack-objects \ + stage must be timeout-bounded, not an uncancellable spawn_blocking", + ); + let err = result.expect_err("a hung pack-objects must return an error, not hang"); + assert!( + err.downcast_ref::().is_some(), + "a hung pack-objects must abort with GitServiceTimeout, got: {err}" + ); + } + // A request that runs to completion must DISARM the guard after reaping, so // no stray group SIGTERM fires. The fake exits non-zero (surfacing as Err) // but leaves a grandchild alive; the grandchild must survive. Goes RED if the From 6336657efcdaf3babc8234224e78e5867d2ebf25 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 10 Jul 2026 13:20:18 -0500 Subject: [PATCH 07/95] docs(node,config): reconcile the concurrency-cap comments with the fix (#62) The max_concurrent_git_ops and git_service_timeout_secs doc-comments (and .env.example) described the info/refs and withheld-blob paths as unbounded follow-up gaps. Both are now closed (#174): the comments reflect the read/write pool split, the per-caller sub-cap, and that every capped path is duration-bounded with process-group teardown. Verified the pattern-doc pre-ship checklist: every git_permit / write-permit / per-caller site holds only a timeout+teardown git path, and all three size knobs are range(1..=1_048_576). Closes the #174 work. No behavior change. --- .env.example | 4 ++-- crates/gitlawb-node/src/config.rs | 32 +++++++++++++++---------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.env.example b/.env.example index d7c6a30b..69865920 100644 --- a/.env.example +++ b/.env.example @@ -109,8 +109,8 @@ GITLAWB_MAX_PACK_BYTES=2147483648 # Max seconds a served git upload-pack / receive-pack (clone / push) may run # before it is aborted with a 504. Bounds a hung git that would otherwise pin a -# worker and, on push, the repo write lock. Does NOT cover the info/refs -# advertisement or the withheld-blob path, which remain unbounded. Default 600. +# worker and, on push, the repo write lock. Also bounds the info/refs +# advertisement and the withheld-blob pack build (#174). Default 600. GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600 # Max concurrent git-receive-pack (push) operations, in a pool separate from the diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index ff2c2556..93c49449 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -172,9 +172,9 @@ pub struct Config { /// its process group torn down, in seconds. Bounds a git that neither /// finishes nor disconnects. Must be positive; set it very large to /// effectively disable the bound. Default: 600s (10 min), generous for large - /// clones. Does not cover the ref advertisement (`info/refs`) or the - /// withheld-blob fetch path (`upload_pack_excluding`, a blocking - /// `spawn_blocking` a tokio timeout cannot cancel); both remain unbounded. + /// clones. Also bounds the ref advertisement (`info/refs`) and the withheld-blob + /// pack build (`upload_pack_excluding`'s pack-objects stage), which now share the + /// same timeout + process-group teardown (#174). #[arg( long, env = "GITLAWB_GIT_SERVICE_TIMEOUT_SECS", @@ -243,20 +243,20 @@ pub struct Config { /// cap is a different axis (500 connections each fan out to git + /// pack-objects + threads). Size below the process budget with headroom. /// - /// A permit is held for the whole op. upload-pack/receive-pack are - /// duration-bounded by `git_service_timeout_secs`, but the `info/refs` - /// advertisement and the withheld-blob (`upload_pack_excluding`) path are - /// not, so a hung git on those two paths holds its slot until it exits, so a - /// stuck advertisement permanently costs one unit of capacity. Bounding - /// those paths' duration is tracked as separate follow-up. + /// This is the READ pool (`git_read_semaphore`): upload-pack and both info/refs + /// advertisements. Authenticated pushes draw from a separate write pool + /// (`max_concurrent_git_pushes`) and each caller is additionally bounded by + /// `max_concurrent_reads_per_caller`, so an anonymous flood can neither shed a + /// push nor monopolize reads (#174). /// - /// The same two paths have the mirror gap on cancellation: the permit frees - /// when the handler future drops (client disconnect), but neither reaps its - /// git child — `info/refs` runs a bare `Command`, and `upload_pack_excluding` - /// a blocking `spawn_blocking` a dropped future cannot cancel — so the child - /// runs to completion after its slot is freed and live git can briefly exceed - /// the cap. The main pack path (`run_git_service`) tears its process group - /// down on drop, so this is confined to the same two follow-up paths. + /// A permit is held for the whole op, and every capped path is now + /// duration-bounded and reaps its process group on disconnect: upload-pack, + /// receive-pack, and both info/refs advertisements run under + /// `git_service_timeout_secs` with `process_group(0)` teardown, and the + /// withheld-blob (`upload_pack_excluding`) pack-objects stage runs on the async + /// side under the same teardown. So a hung git can no longer pin a slot and a + /// client disconnect no longer orphans its git child (#174 closed the two + /// duration/cancellation gaps this comment previously tracked as follow-up). /// /// Default: 128. Must be between 1 and 1_048_576; the ceiling keeps the value /// well under tokio's `Semaphore` permit limit so an oversized value is a From 5069cd13ca18eeaf7f6be59ace288fa30ca35472 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 10 Jul 2026 13:45:38 -0500 Subject: [PATCH 08/95] fix(node,concurrency): resolve #174 code-review findings Review of the served-git concurrency cap found no P0/P1; these are the verified P2 follow-ups. config: the max_concurrent_git_ops doc overclaimed that "every capped path is duration-bounded." The rev-list object enumeration in the withheld-blob path still runs in an uncancellable spawn_blocking, so a stuck rev-list can hold its slot until git exits. Scope the guarantee to the streaming stages and name the residual. Also tighten the fairness claim: the receive-pack advertisement shares the read pool (a shed advertisement is a cheap retryable GET); only the push POST is on the isolated write pool. api/repos: add info_refs_per_caller_cap_keys_on_did_not_ip, the missing handler proof that a signed caller is keyed by its DID, not its source IP. Filling the DID slot sheds a request from a free IP; collapsing read_caller_key to its IP arm turns the assertion green-not-503 (mutation-verified RED). api/repos: extract acquire_read_caller_permit so both read handlers share one shed path instead of a duplicated match block. rate_limit: recover from a poisoned PerCallerConcurrency mutex instead of panicking. The critical section is pure counter arithmetic and cannot poison the lock, but a panic there would brick the limiter for every caller. 505 tests pass; clippy -D warnings and fmt clean. Part of #174. --- crates/gitlawb-node/src/api/repos.rs | 127 +++++++++++++++++++++----- crates/gitlawb-node/src/config.rs | 20 ++-- crates/gitlawb-node/src/rate_limit.rs | 9 +- 3 files changed, 122 insertions(+), 34 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index dba8ea71..f739c04b 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -575,18 +575,12 @@ pub async fn git_info_refs( peer, state.push_limiter_trust, ); - let _caller_permit = match &caller_key { - Some(k) => match state.git_read_per_caller.try_acquire(k) { - Some(p) => Some(p), - None => { - tracing::warn!(repo = %name, caller = %k, "per-caller read cap reached; shedding info/refs with 503"); - return Err(AppError::Overloaded( - "git read service at capacity for this caller, retry shortly".into(), - )); - } - }, - None => None, - }; + let _caller_permit = acquire_read_caller_permit( + &state.git_read_per_caller, + caller_key.as_deref(), + name, + "info/refs", + )?; // For receive-pack (push), download the latest from Tigris so the client // sees the same refs that acquire_write() will operate on. @@ -656,6 +650,30 @@ fn read_caller_key( } } +/// Acquire the per-caller read sub-cap permit (#174), or shed with a 503. `key` is +/// `None` when no caller key resolves — that request is bounded by the global read +/// pool only and is never shed here (returns `Ok(None)`). `handler` labels the shed +/// log line. Shared by both read handlers so the two acquire sites cannot drift. +fn acquire_read_caller_permit( + limiter: &crate::rate_limit::PerCallerConcurrency, + key: Option<&str>, + repo: &str, + handler: &str, +) -> Result> { + match key { + Some(k) => match limiter.try_acquire(k) { + Some(p) => Ok(Some(p)), + None => { + tracing::warn!(repo = %repo, caller = %k, handler, "per-caller read cap reached; shedding with 503"); + Err(AppError::Overloaded( + "git read service at capacity for this caller, retry shortly".into(), + )) + } + }, + None => Ok(None), + } +} + /// Map an error from a `smart_http` git service call to the right `AppError`: /// [`smart_http::GitServiceTimeout`] to 504, a malformed client request to 400, /// anything else to a 500 git error. Pure (no logging) so it is unit-testable; @@ -712,18 +730,12 @@ pub async fn git_upload_pack( // visibility-denied caller never consumes a scarce read slot. Keyed per-DID // when signed, else per-source-IP; no resolvable key -> global read pool only. let caller_key = read_caller_key(caller, &headers, peer, state.push_limiter_trust); - let _caller_permit = match &caller_key { - Some(k) => match state.git_read_per_caller.try_acquire(k) { - Some(p) => Some(p), - None => { - tracing::warn!(repo = %name, caller = %k, "per-caller read cap reached; shedding upload-pack with 503"); - return Err(AppError::Overloaded( - "git read service at capacity for this caller, retry shortly".into(), - )); - } - }, - None => None, - }; + let _caller_permit = acquire_read_caller_permit( + &state.git_read_per_caller, + caller_key.as_deref(), + name, + "upload-pack", + )?; let disk_path = state .repo_store @@ -2798,6 +2810,73 @@ mod tests { ); } + /// #174 SC2 (per-DID key): a SIGNED caller is keyed by its DID, not its source + /// IP. Fill the DID's single read slot, then drive a request carrying that + /// `AuthenticatedDid` from an IP whose own slot is free — it must shed 503, + /// proving the key is the DID. A second request from the SAME IP but a DIFFERENT + /// DID is not shed. Collapse `read_caller_key` to its IP arm and the first + /// assertion goes green-not-503 (the IP slot is free) — this is the per-DID half + /// of the keying proof that the per-IP probes above do not cover. + #[sqlx::test] + async fn info_refs_per_caller_cap_keys_on_did_not_ip(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6pcdid", "pc", "/tmp/pc-nonexistent", None, false) + .await + .unwrap(); + + let did_a = "did:key:z6MkPerCallerKeyingProofDidAAAAAAAAAAAAAAAA"; + let did_b = "did:key:z6MkPerCallerKeyingProofDidBBBBBBBBBBBBBBBB"; + let peer: SocketAddr = "203.0.113.51:5000".parse().unwrap(); + + // Fill DID_A's single read slot; the IP's own slot is left untouched. + let _slot = state + .git_read_per_caller + .try_acquire(did_a) + .expect("first slot for this DID"); + + // Signed as DID_A, from `peer` (whose IP slot is free): keyed by DID -> shed. + let router = crate::server::build_router(state.clone()); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6pcdid/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req.extensions_mut() + .insert(crate::auth::AuthenticatedDid(did_a.to_string())); + assert_eq!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a signed caller at its per-DID read cap must shed with 503 (keyed by DID, not the free IP slot)" + ); + + // Same IP, DIFFERENT DID -> its own budget -> not shed by DID_A's saturation. + let router2 = crate::server::build_router(state.clone()); + let mut req2 = Request::builder() + .method(Method::GET) + .uri("/z6pcdid/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req2.extensions_mut().insert(ConnectInfo(peer)); + req2.extensions_mut() + .insert(crate::auth::AuthenticatedDid(did_b.to_string())); + assert_ne!( + router2.oneshot(req2).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a different DID from the same IP must not be shed by another DID's saturated budget" + ); + } + /// #174 SC2 (None-key): a request with no resolvable caller key (no ConnectInfo, /// no trusted header) must NOT be shed by the per-caller cap even when another /// caller's budget is full — it is bounded by the global read pool only. A None diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 93c49449..90a061e0 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -244,19 +244,23 @@ pub struct Config { /// pack-objects + threads). Size below the process budget with headroom. /// /// This is the READ pool (`git_read_semaphore`): upload-pack and both info/refs - /// advertisements. Authenticated pushes draw from a separate write pool - /// (`max_concurrent_git_pushes`) and each caller is additionally bounded by - /// `max_concurrent_reads_per_caller`, so an anonymous flood can neither shed a - /// push nor monopolize reads (#174). + /// advertisements. The authenticated push POST draws from a separate write pool + /// (`max_concurrent_git_pushes`) that anonymous reads can never reach, and each + /// caller is additionally bounded by `max_concurrent_reads_per_caller`, so an + /// anonymous flood cannot shed the actual push nor monopolize reads (#174). (The + /// receive-pack advertisement itself shares the read pool; a shed advertisement + /// is a cheap retryable GET, and the write POST it precedes always has capacity.) /// - /// A permit is held for the whole op, and every capped path is now + /// A permit is held for the whole op. Every git subprocess that STREAMS is /// duration-bounded and reaps its process group on disconnect: upload-pack, /// receive-pack, and both info/refs advertisements run under /// `git_service_timeout_secs` with `process_group(0)` teardown, and the /// withheld-blob (`upload_pack_excluding`) pack-objects stage runs on the async - /// side under the same teardown. So a hung git can no longer pin a slot and a - /// client disconnect no longer orphans its git child (#174 closed the two - /// duration/cancellation gaps this comment previously tracked as follow-up). + /// side under the same teardown (#174 closed the two duration/cancellation gaps + /// this comment previously tracked). The one remaining blocking stage is the + /// `rev-list` object enumeration in the withheld-blob path — a bounded walk that + /// terminates, run off the async runtime; it is not process-group-reaped on + /// disconnect, so a stuck rev-list can still hold its slot until git exits. /// /// Default: 128. Must be between 1 and 1_048_576; the ceiling keeps the value /// well under tokio's `Semaphore` permit limit so an oversized value is a diff --git a/crates/gitlawb-node/src/rate_limit.rs b/crates/gitlawb-node/src/rate_limit.rs index d505f1bb..22281691 100644 --- a/crates/gitlawb-node/src/rate_limit.rs +++ b/crates/gitlawb-node/src/rate_limit.rs @@ -178,7 +178,12 @@ impl PerCallerConcurrency { /// `None` (shed) otherwise. Reject-before-insert: a new key at `max_keys` is /// rejected without allocating. pub fn try_acquire(&self, key: &str) -> Option { - let mut map = self.state.lock().expect("per-caller mutex poisoned"); + // Recover from a poisoned lock rather than panicking: the critical section + // is pure counter arithmetic and cannot itself panic, so a poisoned mutex + // would only ever come from an unrelated abort, and a slightly-off count + // self-heals as permits drop. A panic here would instead brick the limiter + // for every caller (each subsequent lock re-panics). + let mut map = self.state.lock().unwrap_or_else(|e| e.into_inner()); match map.get_mut(key) { Some(count) => { if *count >= self.per_caller { @@ -207,7 +212,7 @@ impl PerCallerConcurrency { impl Drop for PerCallerPermit { fn drop(&mut self) { - let mut map = self.state.lock().expect("per-caller mutex poisoned"); + let mut map = self.state.lock().unwrap_or_else(|e| e.into_inner()); if let Some(count) = map.get_mut(&self.key) { *count -= 1; if *count == 0 { From 3b1fa54b73c2af9280b89042946cb4c7b26892e4 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 10 Jul 2026 16:05:29 -0500 Subject: [PATCH 09/95] docs(config): add GITLAWB_MAX_CONCURRENT_GIT_OPS to .env.example (#174) The read-pool knob was referenced by the push and per-caller entries' comments but had no example line of its own, so operators couldn't discover it from the template. Add it with the config default (128). --- .env.example | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.env.example b/.env.example index 69865920..cbf3db3a 100644 --- a/.env.example +++ b/.env.example @@ -113,6 +113,13 @@ GITLAWB_MAX_PACK_BYTES=2147483648 # advertisement and the withheld-blob pack build (#174). Default 600. GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600 +# Max concurrent git read ops (upload-pack + info/refs advertisements) served at +# once, a global pool separate from the push pool below. Over-cap sheds a clean +# 503 + Retry-After. Anonymous reads draw from here, so pair it with +# GITLAWB_MAX_CONCURRENT_READS_PER_CALLER (below) so one caller cannot monopolize +# the pool. Default 128. +GITLAWB_MAX_CONCURRENT_GIT_OPS=128 + # Max concurrent git-receive-pack (push) operations, in a pool separate from the # read pool (GITLAWB_MAX_CONCURRENT_GIT_OPS) so anonymous reads cannot shed an # authenticated push at admission. Over-cap sheds a 503 + Retry-After. Default 32. From 362ee3462e84015913c3f57012e29a972e6c1d8d Mon Sep 17 00:00:00 2001 From: t Date: Sat, 11 Jul 2026 13:02:24 -0500 Subject: [PATCH 10/95] fix(node,git): bound the rev-list stage under drive_git_child so a hung enumeration can't pin a git permit (#174) --- crates/gitlawb-node/src/git/smart_http.rs | 108 +++++++++++++++------- 1 file changed, 77 insertions(+), 31 deletions(-) diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index 5586a2c7..206e7b27 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -6,7 +6,7 @@ use bytes::Bytes; use std::collections::HashSet; use std::path::Path; use std::process::Stdio; -use std::time::Duration; +use std::time::{Duration, Instant}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::process::Command; @@ -349,28 +349,27 @@ fn pkt_line(data: &str) -> Vec { format!("{len:04x}{data}").into_bytes() } -/// Run `rev-list --objects --all` and return the reachable object ids minus the -/// withheld blobs. Blocking (shells out to git); the caller runs it off the async -/// runtime. Kept separate from the pack-objects stage so that stage can live on the -/// async side under [`drive_git_child`] (`git_bin` is injectable for the same -/// fake-git testing reason as `run_git_service`). -fn rev_list_keep( +/// Run `rev-list --objects --all` under `timeout` and return the reachable object +/// ids minus the withheld blobs. Runs under [`drive_git_child`] (async, with the +/// `tokio::time::timeout` + `process_group(0)` + [`KillGroupOnDrop`] teardown), so a +/// hung/slow enumeration is duration-bounded and reaped on client disconnect just +/// like the pack-objects stage. A bare blocking `Command::output()` inside a +/// `spawn_blocking` is uncancellable, so a hung rev-list would pin the endpoint's +/// concurrency permit for the whole hang (#174). `git_bin` is injectable for the +/// same fake-git testing reason as `run_git_service`. +async fn rev_list_keep( git_bin: &str, repo_path: &Path, withheld: &HashSet, + timeout: Duration, ) -> Result> { - let rev = std::process::Command::new(git_bin) + let mut command = Command::new(git_bin); + command .args(["rev-list", "--objects", "--all"]) - .current_dir(repo_path) - .output()?; - if !rev.status.success() { - bail!( - "git rev-list failed: {}", - String::from_utf8_lossy(&rev.stderr) - ); - } + .current_dir(repo_path); + let stdout = drive_git_child(command, Bytes::new(), timeout, "rev-list").await?; let mut keep = Vec::new(); - for line in String::from_utf8_lossy(&rev.stdout).lines() { + for line in String::from_utf8_lossy(&stdout).lines() { let oid = line.split_whitespace().next().unwrap_or(""); if oid.is_empty() || withheld.contains(oid) { continue; @@ -384,32 +383,41 @@ fn rev_list_keep( /// given blob OIDs. Commits and trees are always included, so SHAs stay intact; /// only the named blobs are dropped. /// -/// The rev-list enumeration runs blocking off the async runtime; the streaming -/// `pack-objects` stage runs under [`drive_git_child`], so a hung/slow pack build is -/// duration-bounded and its process group is reaped on client disconnect. An outer -/// `tokio::time::timeout` around a `spawn_blocking` cannot cancel the blocking -/// thread, so the streaming stage MUST live on the async side (#174, KTD5). +/// Both git stages — the `rev-list` enumeration and the streaming `pack-objects` +/// build — run under [`drive_git_child`] sharing one deadline, so a hung/slow git at +/// either stage is duration-bounded and its process group is reaped on client +/// disconnect. An outer `tokio::time::timeout` around a `spawn_blocking` cannot +/// cancel the blocking thread, so neither stage may live off the async side +/// (#174, KTD5). pub async fn build_filtered_pack( git_bin: &str, repo_path: &Path, withheld: &HashSet, timeout: Duration, ) -> Result> { - let keep = { - let git_bin = git_bin.to_string(); - let repo_path = repo_path.to_path_buf(); - let withheld = withheld.clone(); - tokio::task::spawn_blocking(move || rev_list_keep(&git_bin, &repo_path, &withheld)) - .await - .context("rev-list task panicked")?? - }; + // One deadline spans both git stages so a slow rev-list eats into the pack + // budget rather than granting each stage a fresh `timeout` (2x the permit hold). + let deadline = Instant::now() + timeout; + let keep = rev_list_keep( + git_bin, + repo_path, + withheld, + deadline.saturating_duration_since(Instant::now()), + ) + .await?; let mut data = keep.join("\n").into_bytes(); data.push(b'\n'); let mut command = Command::new(git_bin); command .args(["pack-objects", "--stdout"]) .current_dir(repo_path); - drive_git_child(command, Bytes::from(data), timeout, "pack-objects").await + drive_git_child( + command, + Bytes::from(data), + deadline.saturating_duration_since(Instant::now()), + "pack-objects", + ) + .await } /// Serve a clone/fetch with the withheld blobs removed from the response pack. @@ -1327,6 +1335,44 @@ mod tests { ); } + // #174 (SC3, KTD5): the filtered-pack build's rev-list ENUMERATION stage is now + // duration-bounded on the ASYNC side too. It previously ran inside a + // spawn_blocking via a bare `Command::output()`, which an outer tokio timeout + // cannot cancel, so a hung/slow rev-list pinned the endpoint's concurrency permit + // for the whole hang. A fake git that hangs on rev-list must now abort with + // GitServiceTimeout; the watchdog turns a missing bound into a loud failure. + // rev-list runs under drive_git_child, so its disconnect/group-teardown is the + // same code proven by run_git_service_tears_down_group_when_future_dropped. + #[cfg(unix)] + #[tokio::test] + async fn build_filtered_pack_times_out_a_hung_rev_list() { + let tmp = tempfile::TempDir::new().unwrap(); + // rev-list hangs forever; pack-objects would return fast if it were reached. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) sleep 300 ;;\n pack-objects) printf '' ;;\n *) exit 1 ;;\nesac\n"; + let git_bin = write_fake_git(tmp.path(), body); + let withheld = HashSet::new(); + + let result = tokio::time::timeout( + Duration::from_secs(10), + build_filtered_pack( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_millis(200), + ), + ) + .await + .expect( + "build_filtered_pack must return within the watchdog — the rev-list \ + stage must be timeout-bounded, not an uncancellable spawn_blocking", + ); + let err = result.expect_err("a hung rev-list must return an error, not hang"); + assert!( + err.downcast_ref::().is_some(), + "a hung rev-list must abort with GitServiceTimeout, got: {err}" + ); + } + // A request that runs to completion must DISARM the guard after reaping, so // no stray group SIGTERM fires. The fake exits non-zero (surfacing as Err) // but leaves a grandchild alive; the grandchild must survive. Goes RED if the From 4983b854d1a25c43354d08f420490555e4bce728 Mon Sep 17 00:00:00 2001 From: t Date: Sat, 11 Jul 2026 23:44:13 -0500 Subject: [PATCH 11/95] fix(node): key the per-caller read cap on source IP, not the signed DID (#174) read_caller_key returned the authenticated DID when a caller signed, dropping the source-IP key. Public read routes accept any valid did:key via optional_signature with no admission step, so one host could mint N disposable DIDs and hold max_concurrent_reads_per_caller slots under each, multiplying its budget N-fold and filling the global read pool, while the same host unsigned was capped on its IP. Key the read sub-cap on the resolved source IP for every caller, signed or not, mirroring the push path's IpRateLimiter which already throttles on source IP for this exact DID-farm reason. Drops the now-unused caller_did parameter at both call sites (git_info_refs, git_upload_pack). Inverts info_refs_per_caller_cap_keys_on_did_not_ip into info_refs_per_caller_cap_keys_on_ip_not_did: fill one source IP's slot, then two requests signed under different DIDs from that same IP both shed 503 (farm defeated), while a signed request from a different IP keeps its own budget. RED on the DID-keyed tree, GREEN after. --- crates/gitlawb-node/src/api/repos.rs | 91 ++++++++++++++++------------ 1 file changed, 53 insertions(+), 38 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index f739c04b..e95910ef 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -569,12 +569,7 @@ pub async fn git_info_refs( // Per-caller read sub-cap (#174): acquired AFTER the visibility + push-rate // gates (KTD7) so a denied or rate-limited request never consumes a caller's // scarce read slot. Applies to BOTH advertisements. Held for the whole op. - let caller_key = read_caller_key( - auth.as_ref().map(|e| e.0 .0.as_str()), - &headers, - peer, - state.push_limiter_trust, - ); + let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); let _caller_permit = acquire_read_caller_permit( &state.git_read_per_caller, caller_key.as_deref(), @@ -633,21 +628,21 @@ fn git_permit( }) } -/// Resolve the per-caller key for the read sub-cap (#174): the authenticated DID -/// when the caller signed (via `optional_signature`), else the trusted client IP -/// (`client_key`). `None` when neither is available — such a request is bounded by -/// the global read pool only, never a 500. The per-source-IP key is only as -/// granular as `trust`; see the `max_concurrent_reads_per_caller` config doc. +/// Resolve the per-caller key for the read sub-cap (#174): always the resolved +/// source IP (`client_key`), never the signed DID. Public read routes accept any +/// valid `did:key` via `optional_signature` with no admission step, so keying on +/// the DID would let one host mint disposable DIDs to multiply its per-source +/// budget; the push path already throttles on the resolved source IP for exactly +/// this DID-farm reason (`rate_limit.rs`, `IpRateLimiter`). `None` when no key +/// resolves (no trusted header and no peer): such a request is bounded by the +/// global read pool only, never a 500. The per-source-IP key is only as granular +/// as `trust`; see the `max_concurrent_reads_per_caller` config doc. fn read_caller_key( - caller_did: Option<&str>, headers: &axum::http::HeaderMap, peer: Option, trust: crate::rate_limit::TrustedProxy, ) -> Option { - match caller_did { - Some(did) => Some(did.to_string()), - None => crate::rate_limit::client_key(headers, peer, trust), - } + crate::rate_limit::client_key(headers, peer, trust) } /// Acquire the per-caller read sub-cap permit (#174), or shed with a 503. `key` is @@ -727,9 +722,10 @@ pub async fn git_upload_pack( } // Per-caller read sub-cap (#174): after the visibility gate (KTD7) so a - // visibility-denied caller never consumes a scarce read slot. Keyed per-DID - // when signed, else per-source-IP; no resolvable key -> global read pool only. - let caller_key = read_caller_key(caller, &headers, peer, state.push_limiter_trust); + // visibility-denied caller never consumes a scarce read slot. Keyed on the + // resolved source IP (never the signed DID, #174 U1); no resolvable key -> + // global read pool only. + let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); let _caller_permit = acquire_read_caller_permit( &state.git_read_per_caller, caller_key.as_deref(), @@ -2810,15 +2806,16 @@ mod tests { ); } - /// #174 SC2 (per-DID key): a SIGNED caller is keyed by its DID, not its source - /// IP. Fill the DID's single read slot, then drive a request carrying that - /// `AuthenticatedDid` from an IP whose own slot is free — it must shed 503, - /// proving the key is the DID. A second request from the SAME IP but a DIFFERENT - /// DID is not shed. Collapse `read_caller_key` to its IP arm and the first - /// assertion goes green-not-503 (the IP slot is free) — this is the per-DID half - /// of the keying proof that the per-IP probes above do not cover. + /// #174 SC2 (per-source key, U1): the per-caller read sub-cap keys on the + /// resolved source IP, NOT the signed DID, so a disposable-DID farm cannot + /// multiply its budget. Fill the source IP's single read slot, then drive two + /// requests signed under DIFFERENT DIDs from that SAME IP: both must shed 503 + /// (keyed by the saturated IP, not their own free DID slots). A signed request + /// from a DIFFERENT source IP keeps its own budget. Revert `read_caller_key` to + /// prefer the DID and the same-IP assertions go green-not-503 (each fresh DID + /// gets a free slot) -- the farm-defeat mutation probe. #[sqlx::test] - async fn info_refs_per_caller_cap_keys_on_did_not_ip(pool: sqlx::PgPool) { + async fn info_refs_per_caller_cap_keys_on_ip_not_did(pool: sqlx::PgPool) { use axum::body::Body; use axum::extract::ConnectInfo; use axum::http::{Method, Request, StatusCode}; @@ -2830,7 +2827,7 @@ mod tests { state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; state .db - .upsert_mirror_repo("z6pcdid", "pc", "/tmp/pc-nonexistent", None, false) + .upsert_mirror_repo("z6pcip", "pc", "/tmp/pc-nonexistent", None, false) .await .unwrap(); @@ -2838,17 +2835,17 @@ mod tests { let did_b = "did:key:z6MkPerCallerKeyingProofDidBBBBBBBBBBBBBBBB"; let peer: SocketAddr = "203.0.113.51:5000".parse().unwrap(); - // Fill DID_A's single read slot; the IP's own slot is left untouched. + // Fill the SOURCE IP's single read slot; both DIDs' own slots stay free. let _slot = state .git_read_per_caller - .try_acquire(did_a) - .expect("first slot for this DID"); + .try_acquire(&peer.ip().to_string()) + .expect("first slot for this source IP"); - // Signed as DID_A, from `peer` (whose IP slot is free): keyed by DID -> shed. + // Signed as DID_A from `peer`: keyed by the saturated source IP -> shed 503. let router = crate::server::build_router(state.clone()); let mut req = Request::builder() .method(Method::GET) - .uri("/z6pcdid/pc/info/refs?service=git-upload-pack") + .uri("/z6pcip/pc/info/refs?service=git-upload-pack") .body(Body::empty()) .unwrap(); req.extensions_mut().insert(ConnectInfo(peer)); @@ -2857,23 +2854,41 @@ mod tests { assert_eq!( router.oneshot(req).await.unwrap().status(), StatusCode::SERVICE_UNAVAILABLE, - "a signed caller at its per-DID read cap must shed with 503 (keyed by DID, not the free IP slot)" + "a signed caller must be keyed by its source IP, not its DID: the saturated IP must shed it 503" ); - // Same IP, DIFFERENT DID -> its own budget -> not shed by DID_A's saturation. + // Same IP, a DIFFERENT DID: still keyed by the same saturated IP -> also shed. + // The farm defeat: minting a fresh DID buys no fresh per-source budget. let router2 = crate::server::build_router(state.clone()); let mut req2 = Request::builder() .method(Method::GET) - .uri("/z6pcdid/pc/info/refs?service=git-upload-pack") + .uri("/z6pcip/pc/info/refs?service=git-upload-pack") .body(Body::empty()) .unwrap(); req2.extensions_mut().insert(ConnectInfo(peer)); req2.extensions_mut() .insert(crate::auth::AuthenticatedDid(did_b.to_string())); - assert_ne!( + assert_eq!( router2.oneshot(req2).await.unwrap().status(), StatusCode::SERVICE_UNAVAILABLE, - "a different DID from the same IP must not be shed by another DID's saturated budget" + "a second DID from the same source IP must also shed 503: a DID farm cannot multiply the per-source budget" + ); + + // A signed caller from a DIFFERENT source IP keeps its own budget -> not shed. + let other: SocketAddr = "203.0.113.52:5000".parse().unwrap(); + let router3 = crate::server::build_router(state.clone()); + let mut req3 = Request::builder() + .method(Method::GET) + .uri("/z6pcip/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req3.extensions_mut().insert(ConnectInfo(other)); + req3.extensions_mut() + .insert(crate::auth::AuthenticatedDid(did_a.to_string())); + assert_ne!( + router3.oneshot(req3).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a signed caller from a different source IP must keep its own per-source budget" ); } From e444bc0ae5f3813c93c7e57b147c6ff974f5f497 Mon Sep 17 00:00:00 2001 From: t Date: Sat, 11 Jul 2026 23:54:21 -0500 Subject: [PATCH 12/95] fix(node): draw the receive-pack advertisement from the write pool (#174) git_info_refs acquired git_read_semaphore for BOTH services, so the push handshake (GET /info/refs?service=git-receive-pack) competed in the global read pool. An anonymous clone flood could exhaust that pool and shed a legitimate push with 503 during its required advertisement phase, before it ever reached git_write_semaphore on the POST. The write pool exists precisely so anonymous reads cannot shed an authenticated push, but only the POST drew from it. Select the pool by service: the receive-pack advertisement (phase one of a push) now draws from git_write_semaphore, like the git-receive-pack POST, so a saturated read pool cannot starve it. The per-IP push_rate_limiter that already brakes the advertisement stays as the anti-flood control, and the advertisement stays reader-visible with no new auth requirement. Because the receive-pack branch is now a write-path op, it no longer consumes a read per-caller slot. Handler-layer proofs: with the read pool at zero the receive-pack advertisement survives while the upload-pack advertisement sheds; with the write pool at zero the receive-pack advertisement sheds while upload-pack is unaffected; and a receive-pack advertisement from an IP whose read per-caller budget is full still gets through (mutation-checked, RED when the skip is neutralized). --- crates/gitlawb-node/src/api/repos.rs | 164 ++++++++++++++++++++++++--- 1 file changed, 148 insertions(+), 16 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index e95910ef..3be94bfc 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -507,10 +507,21 @@ pub async fn git_info_refs( headers: axum::http::HeaderMap, auth: Option>, ) -> Result { - // Shed with a 503 before spawning git when the concurrency cap is saturated; - // held for the whole op (incl. the smart_http call), released on return. - let _permit = git_permit(&state.git_read_semaphore)?; let name = smart_http_repo_name(&repo)?; + let service = query + .service + .ok_or_else(|| AppError::BadRequest("missing ?service= parameter".into()))?; + // Shed with a 503 before spawning git when the concurrency cap is saturated; + // held for the whole op (incl. the smart_http call), released on return. The + // receive-pack advertisement is phase one of a push, so it draws from the WRITE + // pool (like the git-receive-pack POST), not the global read pool an anonymous + // clone flood can exhaust and thereby starve the push handshake (#174 U2). The + // upload-pack advertisement stays on the read pool with its per-caller sub-cap. + let _permit = if service == "git-receive-pack" { + git_permit(&state.git_write_semaphore)? + } else { + git_permit(&state.git_read_semaphore)? + }; tracing::info!(owner = %owner, repo = %name, "info/refs request"); let record = state .db @@ -524,9 +535,6 @@ pub async fn git_info_refs( return Err(AppError::RepoNotFound(format!("{owner}/{name}"))); } - let service = query - .service - .ok_or_else(|| AppError::BadRequest("missing ?service= parameter".into()))?; tracing::debug!(service = %service, repo = %name, "info/refs service"); // Enforce read visibility on the ref advertisement, for BOTH services. The @@ -566,16 +574,23 @@ pub async fn git_info_refs( } } - // Per-caller read sub-cap (#174): acquired AFTER the visibility + push-rate - // gates (KTD7) so a denied or rate-limited request never consumes a caller's - // scarce read slot. Applies to BOTH advertisements. Held for the whole op. - let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); - let _caller_permit = acquire_read_caller_permit( - &state.git_read_per_caller, - caller_key.as_deref(), - name, - "info/refs", - )?; + // Per-caller read sub-cap (#174): upload-pack advertisement only. The + // receive-pack advertisement is a write-path op (it drew from git_write_semaphore + // above and is braked per-IP by push_rate_limiter), so it must NOT consume a + // read caller's scarce slot (#174 U2). Acquired AFTER the visibility + push-rate + // gates (KTD7) so a denied or rate-limited request never consumes a slot. Held + // for the whole op. + let _caller_permit = if service == "git-receive-pack" { + None + } else { + let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); + acquire_read_caller_permit( + &state.git_read_per_caller, + caller_key.as_deref(), + name, + "info/refs", + )? + }; // For receive-pack (push), download the latest from Tigris so the client // sees the same refs that acquire_write() will operate on. @@ -2693,6 +2708,123 @@ mod tests { ); } + /// #174 U2: the receive-pack advertisement (`GET info/refs?service=git-receive-pack`) + /// is phase one of a push, so it draws from the WRITE pool, not the global read + /// pool an anonymous clone flood can exhaust. Proven at the handler by saturating + /// each pool to zero and checking who shares it (INV-10). Revert the branch to the + /// read pool and the read-saturated receive-pack assertion goes 503 (the exact + /// push-handshake starvation jatmn flagged). + #[sqlx::test] + async fn receive_pack_advertisement_draws_from_write_pool(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + // Build a fresh state with the two pools sized independently, then drive one + // info/refs advertisement for `service` and return its handler status. + async fn advert_status( + pool: &sqlx::PgPool, + read_permits: usize, + write_permits: usize, + service: &str, + ) -> StatusCode { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.git_read_semaphore = Arc::new(Semaphore::new(read_permits)); + state.git_write_semaphore = Arc::new(Semaphore::new(write_permits)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6wpadv", "wp", "/tmp/wp-nonexistent", None, false) + .await + .unwrap(); + let peer: SocketAddr = "203.0.113.61:6000".parse().unwrap(); + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/z6wpadv/wp/info/refs?service={service}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + router.oneshot(req).await.unwrap().status() + } + + // Read pool saturated, write pool free: the push handshake SURVIVES. + assert_ne!( + advert_status(&pool, 0, 8, "git-receive-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "receive-pack advertisement must draw from the write pool: a saturated read pool must not shed it" + ); + // Write pool saturated, read pool free: the receive-pack advertisement SHEDS, + // proving it now consumes the write pool (not the read pool). + assert_eq!( + advert_status(&pool, 8, 0, "git-receive-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "receive-pack advertisement draws from the write pool, so a saturated write pool sheds it 503" + ); + // Read pool saturated: the upload-pack advertisement still SHEDS (unchanged). + assert_eq!( + advert_status(&pool, 0, 8, "git-upload-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "upload-pack advertisement stays on the read pool: a saturated read pool sheds it 503" + ); + // Write pool saturated, read pool free: the upload-pack advertisement is + // UNAFFECTED, proving reads never touch the write pool in either direction. + assert_ne!( + advert_status(&pool, 8, 0, "git-upload-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "upload-pack advertisement never touches the write pool" + ); + } + + /// #174 U2: the receive-pack advertisement is a write-path op, so it must not be + /// shed by the READ per-caller sub-cap even when the caller's source IP has + /// exhausted its read budget (e.g. concurrent clones from the same host). Fill + /// the IP's read per-caller slot, then the receive-pack advertisement from that + /// same IP must still get through. Restore the unconditional read-cap acquire on + /// the receive-pack branch and this goes 503. + #[sqlx::test] + async fn receive_pack_advertisement_ignores_read_per_caller_cap(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6wpc", "wp", "/tmp/wp-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.71:6000".parse().unwrap(); + // Exhaust the source IP's single READ per-caller slot, as concurrent clones + // from the same host would. + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("fill the IP's read per-caller slot"); + + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6wpc/wp/info/refs?service=git-receive-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + assert_ne!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "receive-pack advertisement must not be shed by the read per-caller cap: it is a write-path op" + ); + } + /// #174 SC2 (info_refs probe): the per-caller read sub-cap sheds a caller that /// is already at its concurrency budget on the upload-pack advertisement, while /// a DIFFERENT caller still enters. Remove the sub-cap from `git_info_refs` and From 71389e621a233a30c97f959b9d5ccc5c9f1bd57b Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 00:14:39 -0500 Subject: [PATCH 13/95] fix(node): bound the withheld-blob walk at the shared blob_paths seam (#174) The withheld-blob classification walk (blob_paths) fanned out blocking git children with no deadline and no process-group teardown: git for-each-ref, git cat-file, git rev-list, a git ls-tree per commit, and an uncounted git rev-parse (via store::head_commit). A hung or pathologically slow child pinned the caller's served-git permit for the whole hang, and on client disconnect the spawn_blocking task and its git children ran on, orphaned. blob_paths is the shared core of five callers: the upload-pack serve path (holds a read permit) AND, inside git_receive_pack, the post-push replication and encrypt-then-pin walks (hold the write permit U2 reserves for pushes). So the same unbounded walk could pin either pool, and leaving the write-side twin unbounded would have made U2's reservation a claim that does not match behavior. Bound every git child at the blob_paths spawn seam on the blocking side: each child runs in its own process group with a watchdog thread that SIGTERMs (then SIGKILLs) the group on one shared deadline spanning the whole walk, and retains admission until the group is reaped. This is the blocking-side counterpart of smart_http::drive_git_child (spawn_blocking cannot be cancelled by an async timeout). blob_paths stays sync, so all five callers keep their signatures and the 32 classification tests are unchanged; because every caller funnels through blob_paths, one seam bounds both the serve and replication paths. The previously unbounded store::head_commit child becomes a bounded git rev-parse inside the walk. A walk that hits its deadline carries GitServiceTimeout, which the serve handler now maps to 504 rather than a generic 500. Proof: a fake git that hangs on rev-list makes blob_paths return GitServiceTimeout within the watchdog budget (not block on the child) and the recorded process-group leader is reaped, not orphaned; neutralizing the watchdog kill makes it hang past the budget (RED). The 32 real-git classification tests stay green through the refactor, including detached-HEAD, non-standard-ref, and deleted-in-history cases. --- crates/gitlawb-node/src/api/repos.rs | 4 +- .../gitlawb-node/src/git/visibility_pack.rs | 333 +++++++++++++----- 2 files changed, 242 insertions(+), 95 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 3be94bfc..7a8aa5c2 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -781,7 +781,9 @@ pub async fn git_upload_pack( }) .await .map_err(|e| AppError::Git(e.to_string()))? - .map_err(|e| AppError::Git(e.to_string()))? + // A walk that hit its deadline carries GitServiceTimeout; map it to 504 + // like the smart_http paths, not a generic 500 (#174 U3). + .map_err(|e| git_service_app_error(&e))? }; if withheld.is_empty() { diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index cb70e39c..7b9bda69 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -4,11 +4,122 @@ //! content is held back. use crate::db::VisibilityRule; -use crate::git::store; use crate::visibility::{visibility_check, Decision}; use anyhow::{Context, Result}; use std::collections::{BTreeSet, HashMap, HashSet}; use std::path::Path; +use std::process::Stdio; +use std::sync::mpsc; +use std::time::{Duration, Instant}; + +/// Fixed budget bounding the whole withheld-blob classification walk (#174 U3). +/// The walk is fast for a real repo; this bound exists to reap a hung or +/// pathologically slow git child so it cannot pin a served-git permit (the read +/// permit on the upload-pack serve path, the write permit on the receive-pack +/// post-push replication path) past the deadline. Every caller funnels through +/// `blob_paths`, so bounding here bounds both paths at one seam. +const WALK_TIMEOUT: Duration = Duration::from_secs(600); + +/// Run one git child under a shared `deadline` with process-group teardown, +/// BLOCKING, and return its stdout. The child runs in its own process group; a +/// watchdog thread SIGTERMs (lets git clean up its `*.lock` files), then SIGKILLs, +/// the whole group if the deadline passes before the child is reaped, so a hung or +/// slow git can pin neither a served-git permit nor a blocking thread past the +/// deadline (jatmn's "retain admission until they are reaped"). This is the +/// blocking-side counterpart of `smart_http::drive_git_child`, needed because the +/// walk's callers run it inside `spawn_blocking`, which an async timeout cannot +/// cancel. Returns [`crate::git::smart_http::GitServiceTimeout`] on the deadline so +/// the serve handler maps it to 504. `git_bin` is injectable so a fake `git` can +/// drive the teardown in tests without mutating the process-global PATH; +/// `stdin_bytes` feeds children that read stdin (empty for the arg-only children). +#[cfg(unix)] +fn run_bounded_git( + git_bin: &str, + args: &[&str], + repo_path: &Path, + stdin_bytes: &[u8], + deadline: Instant, +) -> Result> { + use std::io::{Read, Write}; + use std::os::unix::process::CommandExt; + use std::sync::mpsc::RecvTimeoutError; + + let label = args.first().copied().unwrap_or("git"); + let mut child = std::process::Command::new(git_bin) + .args(args) + .current_dir(repo_path) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .process_group(0) + .spawn() + .with_context(|| format!("failed to spawn git {label}"))?; + // With process_group(0) the child leads its own group, so pgid == its pid. + let pgid = child.id() as i32; + + // Watchdog: on the deadline, SIGTERM the whole group, poll until it exits, + // escalate to SIGKILL after a grace, and report whether it killed. Signalled to + // stand down when the child is reaped in time. Kept off the main thread because + // the main thread's stdout drain is exactly what blocks until a hung child is + // torn down. + let (done_tx, done_rx) = mpsc::channel::<()>(); + let watchdog = std::thread::spawn(move || -> bool { + let wait = deadline.saturating_duration_since(Instant::now()); + match done_rx.recv_timeout(wait) { + Ok(()) | Err(RecvTimeoutError::Disconnected) => false, + Err(RecvTimeoutError::Timeout) => { + // SAFETY: kill(2) takes only integers and borrows no Rust memory; + // ESRCH on an already-gone group is ignored. + unsafe { libc::kill(-pgid, libc::SIGTERM) }; + for step in 0..200u32 { + if unsafe { libc::kill(-pgid, 0) } != 0 { + return true; // ESRCH: every group member has exited + } + if step == 100 { + unsafe { libc::kill(-pgid, libc::SIGKILL) }; + } + std::thread::sleep(Duration::from_millis(10)); + } + true + } + } + }); + + // Feed stdin on a writer thread and drain stderr on a reader thread so the main + // thread can drain stdout concurrently; writing all of stdin (or draining one + // pipe) before the others can deadlock once a pipe buffer fills. + let mut stdin = child.stdin.take(); + let input = stdin_bytes.to_vec(); + let writer = std::thread::spawn(move || { + if let Some(mut s) = stdin.take() { + let _ = s.write_all(&input); + } + }); + let mut stderr = child.stderr.take().context("git stderr was not piped")?; + let err_reader = std::thread::spawn(move || { + let mut err = Vec::new(); + let _ = stderr.read_to_end(&mut err); + err + }); + let mut stdout = child.stdout.take().context("git stdout was not piped")?; + let mut out = Vec::new(); + let read_result = stdout.read_to_end(&mut out); + let status = child.wait().context("git wait failed")?; + let err = err_reader.join().unwrap_or_default(); + let _ = writer.join(); + + // Reaped: stand the watchdog down and learn whether it killed on the deadline. + let _ = done_tx.send(()); + let killed = watchdog.join().unwrap_or(false); + if killed { + return Err(crate::git::smart_http::GitServiceTimeout.into()); + } + read_result.context("failed to read git stdout")?; + if !status.success() { + anyhow::bail!("git {label} failed: {}", String::from_utf8_lossy(&err)); + } + Ok(out) +} /// Fail closed unless every ref ultimately resolves to a commit (a ref pointing /// directly at a blob or tree, or an annotated tag — even a nested one — of such @@ -21,19 +132,15 @@ use std::path::Path; /// Full peeling is why this is not `for-each-ref %(*objecttype)`, which /// dereferences only one tag level and so misclassifies a tag-of-a-tag-of-a- /// commit as a non-commit. -fn assert_all_refs_are_commits(repo_path: &Path) -> Result<()> { - let refs = std::process::Command::new("git") - .args(["for-each-ref", "--format=%(refname)"]) - .current_dir(repo_path) - .output() - .context("git for-each-ref failed")?; - if !refs.status.success() { - anyhow::bail!( - "git for-each-ref failed: {}", - String::from_utf8_lossy(&refs.stderr) - ); - } - let refs_stdout = String::from_utf8_lossy(&refs.stdout); +fn assert_all_refs_are_commits(repo_path: &Path, git_bin: &str, deadline: Instant) -> Result<()> { + let refs_out = run_bounded_git( + git_bin, + &["for-each-ref", "--format=%(refname)"], + repo_path, + b"", + deadline, + )?; + let refs_stdout = String::from_utf8_lossy(&refs_out); let refnames: Vec<&str> = refs_stdout .lines() .map(str::trim) @@ -43,59 +150,25 @@ fn assert_all_refs_are_commits(repo_path: &Path) -> Result<()> { return Ok(()); } - // Peel every ref in one `git cat-file --batch-check` pass: one - // `^{}` query per line, one output line per input line, in order. - // The stdin write runs on a separate thread so this thread can drain stdout - // concurrently. cat-file echoes the full query on a ` missing` line, - // so output scales with refname length (not a fixed size per ref); writing - // all of stdin before reading any stdout would deadlock both pipes once the - // child's stdout buffer fills. Dropping `stdin` at the end of the closure - // sends EOF. + // Peel every ref in one `git cat-file --batch-check` pass: one `^{}` + // query per line, one output line per input line, in order. cat-file echoes the + // full query on a ` missing` line, so output scales with refname length; + // run_bounded_git drains stdout concurrently with the stdin write, so the pipe + // cannot deadlock, and the whole peel is bounded by the shared walk deadline. let queries = refnames .iter() .map(|r| format!("{r}^{{}}")) .collect::>() .join("\n"); - use std::io::Write; - let mut child = std::process::Command::new("git") - .args(["cat-file", "--batch-check=%(objecttype)"]) - .current_dir(repo_path) - .stdin(std::process::Stdio::piped()) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) - .spawn() - .context("failed to spawn git cat-file")?; - // Feed stdin on a writer thread so this thread can drain stdout via - // wait_with_output concurrently; a None handle (the pipe vanished) becomes a - // broken-pipe write error. wait_with_output reaps the child unconditionally - // before any error is surfaced, so no path drops it unwaited (#53), and the - // writer is joined only after the drain so the join cannot deadlock. - let writer = child - .stdin - .take() - .map(|mut stdin| std::thread::spawn(move || stdin.write_all(queries.as_bytes()))); - let peel_result = child.wait_with_output(); - let write_result = match writer { - Some(handle) => handle - .join() - .map_err(|_| anyhow::anyhow!("git cat-file stdin writer thread panicked"))?, - None => Err(std::io::Error::new( - std::io::ErrorKind::BrokenPipe, - "git cat-file stdin unavailable", - )), - }; - // Surface a write error only if the process didn't already fail with a - // clearer status. - let peel = peel_result.context("git cat-file failed")?; - if !peel.status.success() { - anyhow::bail!( - "git cat-file --batch-check failed: {}", - String::from_utf8_lossy(&peel.stderr) - ); - } - write_result.context("failed to write to git cat-file stdin")?; - - let peel_stdout = String::from_utf8_lossy(&peel.stdout); + let peel_out = run_bounded_git( + git_bin, + &["cat-file", "--batch-check=%(objecttype)"], + repo_path, + queries.as_bytes(), + deadline, + )?; + + let peel_stdout = String::from_utf8_lossy(&peel_out); let types: Vec<&str> = peel_stdout.lines().map(str::trim).collect(); // A short read means at least one ref went unclassified — fail closed. if types.len() != refnames.len() { @@ -147,47 +220,46 @@ fn assert_all_refs_are_commits(repo_path: &Path) -> Result<()> { /// Fails closed: if commit enumeration or any tree walk fails, returns an error so /// the caller aborts the serve/pin rather than producing a partial (under-withheld) /// set. -fn blob_paths(repo_path: &Path) -> Result> { - assert_all_refs_are_commits(repo_path)?; +fn blob_paths(repo_path: &Path, git_bin: &str, timeout: Duration) -> Result> { + // One deadline spans the whole walk (the ref check, the HEAD probe, rev-list, + // and every per-commit ls-tree), so a slow or hung walk is bounded as a unit + // rather than granting each git child a fresh timeout. + let deadline = Instant::now() + timeout; + assert_all_refs_are_commits(repo_path, git_bin, deadline)?; // Enumerate every reachable commit, not just ref tips. `--all` walks all refs; // append HEAD so a detached HEAD (reachable by rev-list/upload-pack but in no // ref) is still classified. When HEAD does not resolve (unborn branch on an - // empty repo) `--all` alone yields nothing, which is correct — no objects exist. - let head = store::head_commit(repo_path).context("resolve HEAD failed")?; + // empty repo) `--all` alone yields nothing, which is correct: no objects exist. + // The HEAD probe is a bounded `git rev-parse --verify HEAD` (a clean exit means + // HEAD resolves), replacing the previously unbounded `store::head_commit` child. + let head_resolves = run_bounded_git( + git_bin, + &["rev-parse", "--verify", "HEAD"], + repo_path, + b"", + deadline, + ) + .is_ok(); let mut rev_args = vec!["rev-list", "--all"]; - if head.is_some() { + if head_resolves { rev_args.push("HEAD"); } - let commits = std::process::Command::new("git") - .args(&rev_args) - .current_dir(repo_path) - .output() - .context("git rev-list --all failed")?; - if !commits.status.success() { - anyhow::bail!( - "git rev-list --all failed: {}", - String::from_utf8_lossy(&commits.stderr) - ); - } - let commits_stdout = String::from_utf8_lossy(&commits.stdout); + let commits_out = run_bounded_git(git_bin, &rev_args, repo_path, b"", deadline)?; + let commits_stdout = String::from_utf8_lossy(&commits_out); let mut out: HashSet<(String, String)> = HashSet::new(); for commit in commits_stdout.lines() { let commit = commit.trim(); if commit.is_empty() { continue; } - let listing = std::process::Command::new("git") - .args(["ls-tree", "-rz", commit]) - .current_dir(repo_path) - .output() - .context("git ls-tree -rz failed")?; - if !listing.status.success() { - anyhow::bail!( - "git ls-tree -rz {commit} failed: {}", - String::from_utf8_lossy(&listing.stderr) - ); - } + let listing_out = run_bounded_git( + git_bin, + &["ls-tree", "-rz", commit], + repo_path, + b"", + deadline, + )?; // `-z` NUL-delimits records and emits paths raw; plain `git ls-tree -r` // C-quotes any path with non-ASCII or special bytes (e.g. café.txt becomes // "secret/caf\303\251.txt"), and that quoted literal would not match a @@ -198,7 +270,7 @@ fn blob_paths(repo_path: &Path) -> Result> { // path (e.g. a non-UTF-8 directory name) with U+FFFD, and the mangled string // would no longer match its deny rule — the same under-withholding class, one // layer down. Fail closed instead so the caller aborts rather than leaks. - let Ok(listing_stdout) = std::str::from_utf8(&listing.stdout) else { + let Ok(listing_stdout) = std::str::from_utf8(&listing_out) else { anyhow::bail!( "git ls-tree -rz {commit} returned a non-UTF-8 path; \ refusing to produce a partial (under-withheld) set" @@ -236,7 +308,7 @@ pub fn withheld_blob_oids( owner_did: &str, caller: Option<&str>, ) -> Result> { - let pairs = blob_paths(repo_path)?; + let pairs = blob_paths(repo_path, "git", WALK_TIMEOUT)?; Ok(withheld_from_pairs( &pairs, rules, is_public, owner_did, caller, )) @@ -332,7 +404,7 @@ pub fn allowed_blob_set_for_caller( owner_did: &str, caller: Option<&str>, ) -> Result> { - let pairs = blob_paths(repo_path)?; + let pairs = blob_paths(repo_path, "git", WALK_TIMEOUT)?; let mut allowed = HashSet::new(); for (oid, path) in &pairs { if visibility_check(rules, is_public, owner_did, caller, path) == Decision::Allow { @@ -372,7 +444,7 @@ pub fn withheld_blob_recipients( owner_did: &str, ) -> Result>> { // One history walk feeds both the withheld set and the recipient mapping. - let pairs = blob_paths(repo_path)?; + let pairs = blob_paths(repo_path, "git", WALK_TIMEOUT)?; let withheld = withheld_from_pairs(&pairs, rules, is_public, owner_did, None); if withheld.is_empty() { return Ok(HashMap::new()); @@ -402,6 +474,79 @@ pub fn withheld_blob_recipients( #[cfg(test)] mod tests { use super::*; + + /// Write an executable fake `git` shell script into `dir` and return its path, + /// so a test can drive the walk's process-group teardown without a real git and + /// without mutating the process-global PATH (the crate's only injection seam). + #[cfg(unix)] + fn write_fake_git(dir: &Path, body: &str) -> String { + use std::os::unix::fs::PermissionsExt; + let p = dir.join("fakegit"); + std::fs::write(&p, body).unwrap(); + let mut perm = std::fs::metadata(&p).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&p, perm).unwrap(); + p.to_str().unwrap().to_string() + } + + /// #174 U3: the withheld-blob walk is bounded at the shared `blob_paths` seam, so + /// a hung git child cannot pin the caller's permit past the deadline. A fake git + /// that hangs on `rev-list` must make `blob_paths` return `GitServiceTimeout` + /// within the watchdog budget (not block for the child's lifetime), and the + /// child's process group must be reaped (its recorded leader PID gone). Every + /// caller (upload-pack serve, receive-pack replication) funnels through + /// `blob_paths`, so this seam-level proof covers both permit pools. Neutralize + /// the watchdog SIGTERM and this hangs past the recv budget (RED). + #[cfg(unix)] + #[test] + fn blob_paths_times_out_and_reaps_a_hung_walk() { + use std::time::Duration; + let tmp = TempDir::new().unwrap(); + // Fast on every stage except rev-list, which records its own (group-leader) + // PID and then hangs. `sleep 30` bounds the worst case if the watchdog is + // ever broken, so a regression cannot wedge the suite for 300s. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) echo $$ > revlist.pid ; sleep 30 ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + + // Run the walk on a thread with a short budget; the recv_timeout succeeding + // is itself proof the walk did not block on the hung child. + let (tx, rx) = mpsc::channel(); + let path = tmp.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(blob_paths(&path, &git_bin, Duration::from_millis(200))); + }); + let result = rx.recv_timeout(Duration::from_secs(10)).expect( + "blob_paths must return within the watchdog budget, not hang on a stuck git child", + ); + let err = result.expect_err("a hung rev-list must abort the walk with an error"); + assert!( + err.downcast_ref::() + .is_some(), + "a hung walk must abort with GitServiceTimeout (mapped to 504), got: {err}" + ); + + // The recorded process-group leader must be gone: the watchdog reaps the + // whole group before blob_paths returns, so no orphaned git lingers. + let pid: i32 = std::fs::read_to_string(tmp.path().join("revlist.pid")) + .expect("the fake git must have recorded its rev-list PID") + .trim() + .parse() + .expect("recorded PID must parse"); + let mut gone = false; + for _ in 0..200 { + // SAFETY: kill(2) with signal 0 only probes existence; ESRCH (-1) means + // the process is gone. Borrows no Rust memory. + if unsafe { libc::kill(pid, 0) } != 0 { + gone = true; + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + assert!( + gone, + "the hung git child (pid {pid}) must be reaped, not orphaned, after the walk aborts" + ); + } use crate::db::VisibilityMode; use chrono::Utc; use std::process::Command; From ab0ea240fcca3087bb7e3c08302885aede32627a Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 00:20:07 -0500 Subject: [PATCH 14/95] docs(node): correct the git-service-timeout coverage note (#174) GITLAWB_GIT_SERVICE_TIMEOUT_SECS bounds the info/refs advertisement too: smart_http::info_refs drives it through drive_git_child under this timeout, with a passing test proving the 504. The old note claimed it does not. It also claimed the withheld-blob path is unbounded; after the blob_paths seam bound (this PR) the walk is bounded and reaped, by a fixed internal deadline rather than this env var, so the line now states that precisely instead of overclaiming this setting covers it. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 57ce2885..d02c4a0b 100644 --- a/README.md +++ b/README.md @@ -343,7 +343,7 @@ Important node settings: | `GITLAWB_REQUIRE_SIGNED_PEER_WRITES` | Require signed peer announce/sync writes. | | `GITLAWB_AUTO_SYNC` | Enable automatic sync from known peers. | | `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. | -| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack/receive-pack may run before it is aborted (504). Default 600. Does not bound `info/refs` or the withheld-blob path. | +| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. The withheld-blob classification walk is also bounded and reaped, by a fixed internal deadline rather than this setting. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | | `GITLAWB_IRYS_URL` | Optional Irys/Arweave permanent anchoring. | From 87a7e40513b018b335d8f3a2e46084d323484798 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 00:36:42 -0500 Subject: [PATCH 15/95] fix(review): close the withheld-walk watchdog reused-pgid race (#174) Code review found run_bounded_git's watchdog could return a spurious 504 and signal a recycled process group. The watchdog runs off a wall clock on its own thread; done_tx.send() only fires after child.wait() reaps the leader, so a walk that finished within microseconds of the deadline took the watchdog's Timeout branch, discarded a fully-captured successful result, and returned GitServiceTimeout (a 504 for a walk that actually completed). Worse, the Timeout branch SIGTERMed -pgid unconditionally after the leader was reaped, so a recycled pgid could be signalled, the exact hazard smart_http guards via disarm-after-wait. Set a reaped AtomicBool the instant the main thread reaps the child; the watchdog checks it before every kill and stands down if the leader is already reaped. Gate the timeout verdict on !status.success(), so a child that exited on its own is never reported as a timeout even if the watchdog fired late. Add the survived-SIGKILL warn smart_http's reap already emits, for operator visibility on a wedged (D-state) git. The hung-walk test stays green (a killed child exits by signal, not success, so it still surfaces GitServiceTimeout and reaps the group) and the 32 real-git classification tests stay green (a fast walk is never spuriously killed). --- .../gitlawb-node/src/git/visibility_pack.rs | 66 +++++++++++++------ 1 file changed, 45 insertions(+), 21 deletions(-) diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index 7b9bda69..0d6661e4 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -63,27 +63,46 @@ fn run_bounded_git( // the main thread's stdout drain is exactly what blocks until a hung child is // torn down. let (done_tx, done_rx) = mpsc::channel::<()>(); - let watchdog = std::thread::spawn(move || -> bool { - let wait = deadline.saturating_duration_since(Instant::now()); - match done_rx.recv_timeout(wait) { - Ok(()) | Err(RecvTimeoutError::Disconnected) => false, - Err(RecvTimeoutError::Timeout) => { - // SAFETY: kill(2) takes only integers and borrows no Rust memory; - // ESRCH on an already-gone group is ignored. - unsafe { libc::kill(-pgid, libc::SIGTERM) }; - for step in 0..200u32 { - if unsafe { libc::kill(-pgid, 0) } != 0 { - return true; // ESRCH: every group member has exited + // Set true the instant the main thread reaps the child, so the watchdog never + // SIGTERMs a pgid whose leader is already reaped and whose pid the kernel may + // recycle (the reused-pgid hazard smart_http guards via disarm-after-wait). + let reaped = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let watchdog = { + let reaped = reaped.clone(); + std::thread::spawn(move || -> bool { + use std::sync::atomic::Ordering; + let wait = deadline.saturating_duration_since(Instant::now()); + match done_rx.recv_timeout(wait) { + Ok(()) | Err(RecvTimeoutError::Disconnected) => false, + Err(RecvTimeoutError::Timeout) => { + // The child was reaped right as the deadline fired: do not signal + // its (possibly-recycled) pgid. + if reaped.load(Ordering::SeqCst) { + return false; } - if step == 100 { - unsafe { libc::kill(-pgid, libc::SIGKILL) }; + // SAFETY: kill(2) takes only integers and borrows no Rust memory; + // ESRCH on an already-gone group is ignored. + unsafe { libc::kill(-pgid, libc::SIGTERM) }; + for step in 0..200u32 { + if reaped.load(Ordering::SeqCst) || unsafe { libc::kill(-pgid, 0) } != 0 { + return true; // reaped, or ESRCH: every group member has exited + } + if step == 100 { + unsafe { libc::kill(-pgid, libc::SIGKILL) }; + } + std::thread::sleep(Duration::from_millis(10)); } - std::thread::sleep(Duration::from_millis(10)); + // Survived SIGKILL past the cap: a wedged (D-state) git, the same + // condition smart_http's reap logs. Warn so an operator sees it. + tracing::warn!( + pgid, + "withheld-walk git survived SIGKILL past the watchdog cap (uninterruptible I/O?)" + ); + true } - true } - } - }); + }) + }; // Feed stdin on a writer thread and drain stderr on a reader thread so the main // thread can drain stdout concurrently; writing all of stdin (or draining one @@ -105,16 +124,21 @@ fn run_bounded_git( let mut out = Vec::new(); let read_result = stdout.read_to_end(&mut out); let status = child.wait().context("git wait failed")?; + // Reaped: bar the watchdog from signalling the now-reaped (possibly recycled) + // pgid before it can fire, then stand it down. + reaped.store(true, std::sync::atomic::Ordering::SeqCst); let err = err_reader.join().unwrap_or_default(); let _ = writer.join(); - - // Reaped: stand the watchdog down and learn whether it killed on the deadline. let _ = done_tx.send(()); let killed = watchdog.join().unwrap_or(false); - if killed { + read_result.context("failed to read git stdout")?; + // The watchdog runs off a wall clock that can race a child finishing right at the + // deadline. A child that exited on its own (success) is not a timeout even if the + // watchdog fired late; only a child that did not exit successfully is a genuine + // timeout, which keeps a walk completing at its budget from a spurious 504. + if killed && !status.success() { return Err(crate::git::smart_http::GitServiceTimeout.into()); } - read_result.context("failed to read git stdout")?; if !status.success() { anyhow::bail!("git {label} failed: {}", String::from_utf8_lossy(&err)); } From e750c3042dff5dc0aa8b1959f5bb72ca7be68137 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 00:56:24 -0500 Subject: [PATCH 16/95] fix(review): cap the receive-pack advertisement per source so anon cannot starve the write pool (#174) U2 moved the receive-pack info/refs advertisement onto git_write_semaphore to keep an anonymous read flood from starving the push handshake. But the advertisement is anon-reachable on public repos and holds its write permit across the slow acquire_fresh Tigris download, and the only per-source brake on it was the push RATE limiter, not a concurrency cap. So a multi-source flood of receive-pack advertisements could hold the write pool's slots across those downloads and shed authenticated pushes (both the advertisement and the owner-gated git-receive-pack POST draw from the same pool). U2 thus introduced the first anonymous consumer of the write pool the state doc promised anon could never reach; the plan's residual note (no worse than the POST) was wrong, because the POST is owner-gated and the advertisement is not. Add git_push_advert_per_caller, a per-source concurrency sub-cap on the receive-pack advertisement keyed on the resolved source IP (the same PerCallerConcurrency mechanism U1 uses for reads), sized to an eighth of the write pool so a single source holds at most that share and saturating the pool takes many distinct source IPs, each also braked by the per-IP push rate limiter. The upload-pack advertisement keeps its read-pool per-caller cap; the owner-gated POST is unchanged. Correct the state doc for git_write_semaphore accordingly. Handler-layer proof: a source at its receive-pack advertisement cap sheds 503 (RED before the acquisition, 500-not-503), while a different source and the upload-pack advertisement are unaffected. Full suite 510 green. --- crates/gitlawb-node/src/api/repos.rs | 101 +++++++++++++++++++++--- crates/gitlawb-node/src/auth/mod.rs | 2 + crates/gitlawb-node/src/main.rs | 7 ++ crates/gitlawb-node/src/state.rs | 22 ++++-- crates/gitlawb-node/src/test_support.rs | 3 + 5 files changed, 121 insertions(+), 14 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 7a8aa5c2..796558cf 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -574,16 +574,24 @@ pub async fn git_info_refs( } } - // Per-caller read sub-cap (#174): upload-pack advertisement only. The - // receive-pack advertisement is a write-path op (it drew from git_write_semaphore - // above and is braked per-IP by push_rate_limiter), so it must NOT consume a - // read caller's scarce slot (#174 U2). Acquired AFTER the visibility + push-rate - // gates (KTD7) so a denied or rate-limited request never consumes a slot. Held - // for the whole op. + // Per-source concurrency sub-cap (#174), keyed on the resolved source IP and + // acquired AFTER the visibility + push-rate gates (KTD7) so a denied or + // rate-limited request never consumes a slot; held for the whole op. The + // upload-pack advertisement is bounded on the read pool (git_read_per_caller). + // The receive-pack advertisement draws from the write pool, so it is bounded per + // source (git_push_advert_per_caller) instead: without this, an anonymous + // multi-source flood of push-handshake advertisements could hold the write pool's + // slots across acquire_fresh and shed authenticated pushes, since the per-IP push + // rate limiter caps rate, not concurrency (#174 review fix). + let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); let _caller_permit = if service == "git-receive-pack" { - None + acquire_read_caller_permit( + &state.git_push_advert_per_caller, + caller_key.as_deref(), + name, + "receive-pack advert", + )? } else { - let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); acquire_read_caller_permit( &state.git_read_per_caller, caller_key.as_deref(), @@ -676,7 +684,7 @@ fn acquire_read_caller_permit( None => { tracing::warn!(repo = %repo, caller = %k, handler, "per-caller read cap reached; shedding with 503"); Err(AppError::Overloaded( - "git read service at capacity for this caller, retry shortly".into(), + "git service at capacity for this caller, retry shortly".into(), )) } }, @@ -2827,6 +2835,81 @@ mod tests { ); } + /// #174 (review fix): the anon-reachable receive-pack advertisement draws from + /// the write pool, so it is bounded per source by `git_push_advert_per_caller` to + /// stop one source from monopolizing the write pool and shedding authenticated + /// pushes. Fill one source IP's advert slot; its next receive-pack advertisement + /// sheds 503, while a different source and the upload-pack advertisement are + /// unaffected. Remove the advert-cap acquisition and the same-source assertion + /// goes green-not-503. + #[sqlx::test] + async fn receive_pack_advertisement_capped_per_source(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_push_advert_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6advcap", "ac", "/tmp/ac-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.81:6000".parse().unwrap(); + // Fill this source IP's single receive-pack-advertisement slot. + let _slot = state + .git_push_advert_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first advert slot for this source IP"); + + // Same source: the receive-pack advertisement sheds 503 (advert cap full). + let router = crate::server::build_router(state.clone()); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6advcap/ac/info/refs?service=git-receive-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + assert_eq!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its receive-pack advertisement cap must shed 503, so it cannot monopolize the write pool" + ); + + // A DIFFERENT source keeps its own advert budget -> not shed. + let other: SocketAddr = "203.0.113.82:6000".parse().unwrap(); + let router2 = crate::server::build_router(state.clone()); + let mut req2 = Request::builder() + .method(Method::GET) + .uri("/z6advcap/ac/info/refs?service=git-receive-pack") + .body(Body::empty()) + .unwrap(); + req2.extensions_mut().insert(ConnectInfo(other)); + assert_ne!( + router2.oneshot(req2).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a different source must keep its own receive-pack advertisement budget" + ); + + // The upload-pack advertisement is NOT bounded by the receive-pack advert cap. + let router3 = crate::server::build_router(state); + let mut req3 = Request::builder() + .method(Method::GET) + .uri("/z6advcap/ac/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req3.extensions_mut().insert(ConnectInfo(peer)); + assert_ne!( + router3.oneshot(req3).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "the upload-pack advertisement must not be shed by the receive-pack advert cap" + ); + } + /// #174 SC2 (info_refs probe): the per-caller read sub-cap sheds a caller that /// is already at its concurrency budget on the upload-pack advertisement, while /// a DIFFERENT caller still enters. Remove the sub-cap from `git_info_refs` and diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 253b3210..b9e28e57 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -523,6 +523,8 @@ mod tests { git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), + git_push_advert_per_caller: + crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), } } diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index c617b434..0f33fde4 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -385,6 +385,13 @@ async fn main() -> Result<()> { git_read_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( config.max_concurrent_reads_per_caller, ), + // Per-source cap on the receive-pack advertisement, sized to an eighth of the + // write pool (min 1): a single source can hold at most this many write-pool + // slots via the anon advertisement, so saturating the pool takes ~8 distinct + // source IPs, each also rate-limited (#174). + git_push_advert_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( + (config.max_concurrent_git_pushes / 8).max(1), + ), }; // Periodic peer-count poll for the metrics gauge. If p2p is disabled diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 01007ac4..f55abeb7 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -95,14 +95,26 @@ pub struct AppState { /// authenticated push at admission (#174). pub git_read_semaphore: Arc, /// Bounds concurrent `git-receive-pack` (push) operations, a pool separate - /// from `git_read_semaphore` so anonymous reads can never shed an authenticated - /// push (#174). Sized by `max_concurrent_git_pushes`. + /// from `git_read_semaphore` so an anonymous READ flood can never shed an + /// authenticated push (#174). Sized by `max_concurrent_git_pushes`. Drawn from + /// by the `git-receive-pack` POST (owner-gated) and the receive-pack `info/refs` + /// advertisement (anon-reachable); the advertisement is additionally bounded per + /// source by `git_push_advert_per_caller` so no single source can monopolize this + /// pool and shed pushes. pub git_write_semaphore: Arc, - /// Per-caller concurrency sub-cap on the read pool: each caller (per-DID when - /// signed, else per-source-IP) may hold at most `max_concurrent_reads_per_caller` + /// Per-caller concurrency sub-cap on the read pool: each caller (keyed on the + /// resolved source IP, #174 U1) may hold at most `max_concurrent_reads_per_caller` /// in-flight read ops, so one caller cannot monopolize `git_read_semaphore` - /// (#174). Applied by `git_upload_pack` and both `info/refs` advertisements. + /// (#174). Applied by `git_upload_pack` and the upload-pack `info/refs` + /// advertisement. pub git_read_per_caller: crate::rate_limit::PerCallerConcurrency, + /// Per-source concurrency sub-cap on the anon-reachable receive-pack `info/refs` + /// advertisement: each source IP may hold at most a small share of the write + /// pool, so a multi-source flood of push-handshake advertisements cannot + /// saturate `git_write_semaphore` and shed authenticated pushes (#174). Sized as + /// a fraction of `max_concurrent_git_pushes`, so filling the write pool takes many + /// distinct source IPs (each also braked by the per-IP push rate limiter). + pub git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency, } impl AppState { diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 7745d233..03d3d5c4 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -86,6 +86,9 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), + git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( + 8, + ), } } From c35452ed41091ba38f1b294a96f44d1d7b827f74 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 02:12:08 -0500 Subject: [PATCH 17/95] test(node): vet the withheld-walk bound end to end; use the configured timeout (#174) Close the reasoned-not-run gaps from the code review by making the walk's git binary and timeout injectable, then driving the missing branches with a real handler and a fake git instead of reasoning about them. - Add state.git_bin and *_bounded variants of the walk entry points taking (git_bin, timeout); the served handlers (upload-pack serve, receive-pack replication and full-scan and encrypt-pin, and the ipfs gate) now pass the operator-configured GITLAWB_GIT_SERVICE_TIMEOUT_SECS, so the whole walk is bounded by the same budget as the other served-git ops rather than a fixed constant. The git_bin-less wrappers stay for the real-git classification tests. Newly vetted by execution (not reasoning): - receive-pack replication path is bounded: replication_withheld_set with an injected hung git returns within the budget and fails closed, so it cannot pin the write permit git_receive_pack holds across it. - a hung withheld-blob walk on the upload-pack POST returns 504 (real handler, real repo on disk, injected hung git), proving the GitServiceTimeout -> git_service_app_error wiring end to end. - the watchdog status-gate: a child that exits successfully is not reported as a timeout even when the watchdog fired (mutation-checked: drop the guard -> RED). - SIGKILL escalation: a SIGTERM-ignoring child is still reaped via SIGKILL and the group is gone; a truly uninterruptible D-state child (unreapable by any signal) is the documented residual, matching the async teardown. - the advertisement per-source cap sizing never derives 0. Full gitlawb-node suite 515 green. --- crates/gitlawb-node/src/api/ipfs.rs | 12 +- crates/gitlawb-node/src/api/repos.rs | 208 +++++++++++++++++- crates/gitlawb-node/src/auth/mod.rs | 1 + .../gitlawb-node/src/git/visibility_pack.rs | 173 ++++++++++++++- crates/gitlawb-node/src/main.rs | 1 + crates/gitlawb-node/src/state.rs | 5 + crates/gitlawb-node/src/test_support.rs | 1 + 7 files changed, 383 insertions(+), 18 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index f3de7570..e921c39f 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -27,7 +27,7 @@ use std::str::FromStr; use crate::auth::AuthenticatedDid; use crate::error::{AppError, Result}; use crate::git::store; -use crate::git::visibility_pack::{allowed_blob_set_for_caller, has_path_scoped_rule}; +use crate::git::visibility_pack::{allowed_blob_set_for_caller_bounded, has_path_scoped_rule}; use crate::state::AppState; use crate::visibility::{visibility_check, Decision}; @@ -142,10 +142,16 @@ pub async fn get_by_cid( let is_public = repo.is_public; let owner = repo.owner_did.clone(); let caller_for_walk = caller_owned.clone(); - // Full-history walk shells out to git — keep it off the async runtime. + let git_bin = state.git_bin.clone(); + let walk_timeout = + std::time::Duration::from_secs(state.config.git_service_timeout_secs); + // Full-history walk shells out to git — keep it off the async runtime, + // bounded and reaped like the served-git ops (#174). let walk = tokio::task::spawn_blocking(move || { - allowed_blob_set_for_caller( + allowed_blob_set_for_caller_bounded( &rp, + &git_bin, + walk_timeout, &r, is_public, &owner, diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 796558cf..1532be58 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -44,6 +44,8 @@ async fn replication_withheld_set( owner_did: &str, is_public: bool, disk_path: std::path::PathBuf, + git_bin: String, + timeout: std::time::Duration, ) -> (bool, Option>) { let announce = match &rules { Some(rules) => crate::visibility::listable_at_root(rules, is_public, owner_did, None), @@ -65,8 +67,8 @@ async fn replication_withheld_set( Some(rules) => { let owner_did = owner_did.to_string(); tokio::task::spawn_blocking(move || { - crate::git::visibility_pack::withheld_blob_oids( - &disk_path, &rules, is_public, &owner_did, None, + crate::git::visibility_pack::withheld_blob_oids_bounded( + &disk_path, &git_bin, timeout, &rules, is_public, &owner_did, None, ) }) .await @@ -106,10 +108,12 @@ async fn fail_closed_full_scan_objects( is_public: bool, owner_did: String, candidates: Vec, + git_bin: String, + timeout: std::time::Duration, ) -> Vec { tokio::task::spawn_blocking(move || -> anyhow::Result> { - let allowed = crate::git::visibility_pack::replicable_blob_set( - &disk_path, &rules, is_public, &owner_did, + let allowed = crate::git::visibility_pack::replicable_blob_set_bounded( + &disk_path, &git_bin, timeout, &rules, is_public, &owner_did, )?; let all_blobs = crate::git::push_delta::all_blob_oids(&disk_path)?; Ok(crate::git::visibility_pack::replicable_objects_fail_closed( @@ -778,9 +782,12 @@ pub async fn git_upload_pack( let owner_did = record.owner_did.clone(); let caller_owned = caller.map(str::to_string); let is_public = record.is_public; + let git_bin = state.git_bin.clone(); tokio::task::spawn_blocking(move || { - visibility_pack::withheld_blob_oids( + visibility_pack::withheld_blob_oids_bounded( &path, + &git_bin, + git_timeout, &rules, is_public, &owner_did, @@ -1185,6 +1192,8 @@ pub async fn git_receive_pack( &record.owner_did, record.is_public, disk_path.clone(), + state.git_bin.clone(), + std::time::Duration::from_secs(state.config.git_service_timeout_secs), ) .await; @@ -1219,6 +1228,8 @@ pub async fn git_receive_pack( record.is_public, record.owner_did.clone(), pin_set.candidates, + state.git_bin.clone(), + std::time::Duration::from_secs(state.config.git_service_timeout_secs), ) .await } else { @@ -1244,6 +1255,8 @@ pub async fn git_receive_pack( let node_did_str = state.node_did.to_string(); let node_seed = state.node_keypair.to_seed(); let repo_name = record.name.clone(); + let enc_git_bin = state.git_bin.clone(); + let enc_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); tokio::spawn(async move { let pinned = crate::ipfs_pin::pin_new_objects( &ipfs_api, @@ -1268,9 +1281,15 @@ pub async fn git_receive_pack( { let p = repo_path_clone.clone(); let owner = owner_did.clone(); + let git_bin = enc_git_bin.clone(); let recip = tokio::task::spawn_blocking(move || { - crate::git::visibility_pack::withheld_blob_recipients( - &p, &rules, is_public, &owner, + crate::git::visibility_pack::withheld_blob_recipients_bounded( + &p, + &git_bin, + enc_timeout, + &rules, + is_public, + &owner, ) }) .await; @@ -2031,16 +2050,39 @@ mod tests { let dummy = std::path::PathBuf::from("/nonexistent"); // Private: no rules at all. - let (announce, _) = replication_withheld_set(None, OWNER_DID, false, dummy.clone()).await; + let (announce, _) = replication_withheld_set( + None, + OWNER_DID, + false, + dummy.clone(), + "git".into(), + std::time::Duration::from_secs(600), + ) + .await; assert!(!announce, "private repo (no rules) must not announce"); // Private: empty rule set, is_public=false → still not listable at root. - let (announce, _) = - replication_withheld_set(Some(vec![]), OWNER_DID, false, dummy.clone()).await; + let (announce, _) = replication_withheld_set( + Some(vec![]), + OWNER_DID, + false, + dummy.clone(), + "git".into(), + std::time::Duration::from_secs(600), + ) + .await; assert!(!announce, "private repo (empty rules) must not announce"); // Public: empty rule set, is_public=true → listable at root, announces. - let (announce, _) = replication_withheld_set(Some(vec![]), OWNER_DID, true, dummy).await; + let (announce, _) = replication_withheld_set( + Some(vec![]), + OWNER_DID, + true, + dummy, + "git".into(), + std::time::Duration::from_secs(600), + ) + .await; assert!(announce, "public repo must announce"); } @@ -2132,6 +2174,150 @@ mod tests { } } + #[cfg(unix)] + fn write_fake_git(dir: &std::path::Path, body: &str) -> String { + use std::os::unix::fs::PermissionsExt; + let p = dir.join("fakegit"); + std::fs::write(&p, body).unwrap(); + let mut perm = std::fs::metadata(&p).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&p, perm).unwrap(); + p.to_str().unwrap().to_string() + } + + /// #174 (write-pool twin, vetted by execution not reasoning): the receive-pack + /// post-push replication walk is bounded. Drive `replication_withheld_set` with an + /// injected fake git that hangs on `rev-list` and a short budget: it must RETURN + /// within the budget (so `git_receive_pack` releases the write permit it holds + /// across this await, rather than pinning it for the hang) AND fail closed + /// (announce suppressed) because the walk could not be vetted. Proves this path + /// funnels through the bounded `blob_paths`, on the write-permit-holding side. + #[cfg(unix)] + #[tokio::test] + async fn replication_walk_is_bounded_and_fails_closed_on_a_hung_git() { + use std::time::Duration; + let tmp = tempfile::TempDir::new().unwrap(); + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) sleep 30 ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + // Public root (announceable) + a path-scoped rule, so the walk actually runs + // rather than taking the has_path_scoped_rule short-circuit. + let rules = Some(vec![vis_rule("/secret/**", &[])]); + + let result = tokio::time::timeout( + Duration::from_secs(10), + replication_withheld_set( + rules, + OWNER_DID, + true, + tmp.path().to_path_buf(), + git_bin, + Duration::from_millis(200), + ), + ) + .await + .expect( + "replication_withheld_set must return within the budget; a hung walk must \ + not pin the write permit git_receive_pack holds across it", + ); + assert_eq!( + result, + (false, None), + "a walk that could not be vetted must suppress the announce (fail closed)" + ); + } + + /// #174 (serve-path 504, vetted by execution): a hung withheld-blob walk on the + /// upload-pack POST maps to 504, not a generic 500. Real repo dir on disk (so + /// acquire's fast path returns it) + a path-scoped rule (so the walk runs) + + /// an injected fake git that hangs on rev-list. The handler must return 504, + /// proving git_upload_pack routes the walk's GitServiceTimeout through + /// git_service_app_error end to end. + #[cfg(unix)] + #[sqlx::test] + async fn upload_pack_hung_withheld_walk_returns_504(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) sleep 30 ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n"; + let fake = write_fake_git(tmp.path(), body); + + let mut state = crate::test_support::test_state(pool).await; + state.git_bin = fake; + let mut cfg = (*state.config).clone(); + cfg.git_service_timeout_secs = 1; + state.config = std::sync::Arc::new(cfg); + state + .db + .upsert_mirror_repo("z6srv504", "sv", "/tmp/z6srv504-sv", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6srv504", "sv").await.unwrap().unwrap(); + // Path-scoped rule so has_path_scoped_rule() is true and the walk runs; the + // public root still lets an anonymous caller past the "/" gate. + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &[], + OWNER_DID, + ) + .await + .unwrap(); + // acquire()'s fast path returns the local path when it exists on disk. + let disk = std::path::Path::new("/tmp/z6srv504/sv.git"); + std::fs::create_dir_all(disk).unwrap(); + + let peer: SocketAddr = "203.0.113.91:7000".parse().unwrap(); + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::POST) + .uri("/z6srv504/sv/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let status = router.oneshot(req).await.unwrap().status(); + let _ = std::fs::remove_dir_all("/tmp/z6srv504"); + assert_eq!( + status, + StatusCode::GATEWAY_TIMEOUT, + "a hung withheld-blob walk must surface as 504, not a generic 500" + ); + } + + /// #174 (F2 sizing edge, vetted by execution): the receive-pack advertisement + /// per-source cap is derived in main.rs as `(max_concurrent_git_pushes / 8).max(1)`, + /// so it is never 0 even at the minimum write-pool size (1). A 0 cap would make + /// PerCallerConcurrency shed EVERY receive-pack advertisement and break all pushes. + #[test] + fn advert_per_caller_cap_sizing_is_never_zero() { + let cap = |pushes: usize| (pushes / 8).max(1); + for pushes in [1usize, 4, 8, 32, 256] { + assert!( + cap(pushes) >= 1, + "advert cap must be >= 1 for pushes={pushes}" + ); + } + assert_eq!(cap(1), 1, "minimum write pool must derive cap 1, not 0"); + assert_eq!( + cap(32), + 4, + "default write pool 32 derives cap 4 (~8 source IPs to fill)" + ); + // A cap of 1 admits one and sheds the second from the same source. + let lim = crate::rate_limit::PerCallerConcurrency::new(cap(1), 100); + let _held = lim.try_acquire("src").expect("first advert admitted"); + assert!( + lim.try_acquire("src").is_none(), + "second advert from the same source is shed" + ); + } + #[test] fn fork_owner_full_did_with_path_rule_allowed() { // Owner reads everything (implicit reader), so nothing is withheld. diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index b9e28e57..5befa7f3 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -525,6 +525,7 @@ mod tests { git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), + git_bin: "git".to_string(), } } diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index 0d6661e4..09a80aba 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -17,7 +17,10 @@ use std::time::{Duration, Instant}; /// pathologically slow git child so it cannot pin a served-git permit (the read /// permit on the upload-pack serve path, the write permit on the receive-pack /// post-push replication path) past the deadline. Every caller funnels through -/// `blob_paths`, so bounding here bounds both paths at one seam. +/// `blob_paths`, so bounding here bounds both paths at one seam. Production callers +/// pass the operator-configured `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` instead; this +/// fixed budget only backs the `git_bin`-less test wrappers. +#[cfg(test)] const WALK_TIMEOUT: Duration = Duration::from_secs(600); /// Run one git child under a shared `deadline` with process-group teardown, @@ -122,6 +125,12 @@ fn run_bounded_git( }); let mut stdout = child.stdout.take().context("git stdout was not piped")?; let mut out = Vec::new(); + // Blocking drain, unblocked by the child closing stdout on exit. The watchdog's + // SIGTERM/SIGKILL is what makes a hung child exit; a git wedged in uninterruptible + // (D-state) I/O survives even SIGKILL, so this drain and the wait below can block + // until the kernel returns, pinning the walk thread and its permit. That residual + // is unreachable in userspace (no signal reaps a D-state process) and matches the + // async `reap_group_on_timeout`, which likewise only warns and gives up there. let read_result = stdout.read_to_end(&mut out); let status = child.wait().context("git wait failed")?; // Reaped: bar the watchdog from signalling the now-reaped (possibly recycled) @@ -325,6 +334,7 @@ fn blob_paths(repo_path: &Path, git_bin: &str, timeout: Duration) -> Result, ) -> Result> { - let pairs = blob_paths(repo_path, "git", WALK_TIMEOUT)?; + withheld_blob_oids_bounded( + repo_path, + "git", + WALK_TIMEOUT, + rules, + is_public, + owner_did, + caller, + ) +} + +/// [`withheld_blob_oids`] with an injectable `git_bin` and walk `timeout`. Served +/// handlers call this with the operator-configured git binary and +/// `GITLAWB_GIT_SERVICE_TIMEOUT_SECS`, so the whole walk is bounded by the same +/// budget as the other served-git ops and a fake `git` can drive its teardown in +/// tests. The `git_bin`-less wrapper above keeps the fixed [`WALK_TIMEOUT`] for the +/// classification tests that run against real git. +pub fn withheld_blob_oids_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> Result> { + let pairs = blob_paths(repo_path, git_bin, timeout)?; Ok(withheld_from_pairs( &pairs, rules, is_public, owner_did, caller, )) @@ -400,6 +436,7 @@ pub fn replicable_objects(all: Vec, withheld: &HashSet) -> Vec Result> { + allowed_blob_set_for_caller_bounded( + repo_path, git_bin, timeout, rules, is_public, owner_did, None, + ) +} + /// Reachable blob OIDs that visibility ALLOWS `caller` at some path. The /// caller-aware generalization of `replicable_blob_set` (which is the anonymous /// `caller = None` case). Used by `GET /ipfs/{cid}` to gate fail-closed against @@ -421,6 +473,7 @@ pub fn replicable_blob_set( /// elsewhere (its content is readable to this caller elsewhere). Trees and /// commits are NOT included here; the caller decides per object type whether /// the allow-set applies (it does not for trees/commits — KTD3). +#[cfg(test)] pub fn allowed_blob_set_for_caller( repo_path: &Path, rules: &[VisibilityRule], @@ -428,7 +481,29 @@ pub fn allowed_blob_set_for_caller( owner_did: &str, caller: Option<&str>, ) -> Result> { - let pairs = blob_paths(repo_path, "git", WALK_TIMEOUT)?; + allowed_blob_set_for_caller_bounded( + repo_path, + "git", + WALK_TIMEOUT, + rules, + is_public, + owner_did, + caller, + ) +} + +/// [`allowed_blob_set_for_caller`] with an injectable `git_bin` and walk `timeout`, +/// for the `GET /ipfs/{cid}` gate. +pub fn allowed_blob_set_for_caller_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> Result> { + let pairs = blob_paths(repo_path, git_bin, timeout)?; let mut allowed = HashSet::new(); for (oid, path) in &pairs { if visibility_check(rules, is_public, owner_did, caller, path) == Decision::Allow { @@ -461,14 +536,28 @@ pub fn replicable_objects_fail_closed( /// owner plus any reader DID that `visibility_check` Allows at some path the /// blob appears at. Least-privilege: a reader of one private subtree is not a /// recipient of a blob that only lives in another. +#[cfg(test)] pub fn withheld_blob_recipients( repo_path: &Path, rules: &[VisibilityRule], is_public: bool, owner_did: &str, +) -> Result>> { + withheld_blob_recipients_bounded(repo_path, "git", WALK_TIMEOUT, rules, is_public, owner_did) +} + +/// [`withheld_blob_recipients`] with an injectable `git_bin` and walk `timeout`, for +/// the receive-pack encrypt-then-pin path. +pub fn withheld_blob_recipients_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, ) -> Result>> { // One history walk feeds both the withheld set and the recipient mapping. - let pairs = blob_paths(repo_path, "git", WALK_TIMEOUT)?; + let pairs = blob_paths(repo_path, git_bin, timeout)?; let withheld = withheld_from_pairs(&pairs, rules, is_public, owner_did, None); if withheld.is_empty() { return Ok(HashMap::new()); @@ -571,6 +660,82 @@ mod tests { "the hung git child (pid {pid}) must be reaped, not orphaned, after the walk aborts" ); } + + /// #174 (F1 status-gate, vetted by execution): a child that exits SUCCESSFULLY is + /// never reported as a timeout even when the watchdog fires, so a walk finishing + /// right at its deadline is not a spurious 504. The fake only exits when signalled + /// and exits 0 on SIGTERM, so with a deadline already elapsed the watchdog always + /// reaches its kill path (killed == true) yet the child's status is success. + /// Drop the `!status.success()` guard and this returns GitServiceTimeout (RED). + #[cfg(unix)] + #[test] + fn run_bounded_git_success_at_the_deadline_is_not_a_timeout() { + use std::time::{Duration, Instant}; + let tmp = TempDir::new().unwrap(); + let body = "#!/bin/sh\ntrap 'exit 0' TERM\nsleep 30 &\nwait\n"; + let git_bin = write_fake_git(tmp.path(), body); + let out = run_bounded_git( + &git_bin, + &["rev-list"], + tmp.path(), + b"", + Instant::now() + Duration::from_millis(100), + ); + assert!( + out.is_ok(), + "a child that exited successfully must not be reported as a timeout even if the watchdog fired: {out:?}" + ); + } + + /// #174 (F3, vetted by execution): a child that IGNORES SIGTERM is still reaped + /// via the watchdog's SIGKILL escalation, so it cannot pin the walk thread or its + /// permit. The fake traps SIGTERM and keeps sleeping; run_bounded_git must still + /// return (via SIGKILL at the grace step) with a timeout error and the group must + /// be gone. (A truly uninterruptible D-state child, which no signal can reap, is + /// the documented residual this teardown, like the async twin, cannot cover.) + #[cfg(unix)] + #[test] + fn run_bounded_git_reaps_a_sigterm_ignoring_child_via_sigkill() { + use std::time::{Duration, Instant}; + let tmp = TempDir::new().unwrap(); + let body = "#!/bin/sh\ntrap '' TERM\necho $$ > pid\nwhile true; do sleep 1; done\n"; + let git_bin = write_fake_git(tmp.path(), body); + let (tx, rx) = std::sync::mpsc::channel(); + let path = tmp.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(run_bounded_git( + &git_bin, + &["rev-list"], + &path, + b"", + Instant::now() + Duration::from_millis(100), + )); + }); + let out = rx + .recv_timeout(Duration::from_secs(10)) + .expect("run_bounded_git must return via SIGKILL even for a SIGTERM-ignoring child"); + assert!( + out.is_err(), + "a SIGTERM-ignoring child killed by SIGKILL is a timeout, not a success: {out:?}" + ); + let pid: i32 = std::fs::read_to_string(tmp.path().join("pid")) + .unwrap() + .trim() + .parse() + .unwrap(); + let mut gone = false; + for _ in 0..300 { + if unsafe { libc::kill(pid, 0) } != 0 { + gone = true; + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + assert!( + gone, + "the SIGTERM-ignoring child (pid {pid}) must be reaped via SIGKILL, not left running" + ); + } use crate::db::VisibilityMode; use chrono::Utc; use std::process::Command; diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 0f33fde4..9b646c2b 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -392,6 +392,7 @@ async fn main() -> Result<()> { git_push_advert_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( (config.max_concurrent_git_pushes / 8).max(1), ), + git_bin: "git".to_string(), }; // Periodic peer-count poll for the metrics gauge. If p2p is disabled diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index f55abeb7..94952120 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -115,6 +115,11 @@ pub struct AppState { /// a fraction of `max_concurrent_git_pushes`, so filling the write pool takes many /// distinct source IPs (each also braked by the per-IP push rate limiter). pub git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency, + /// The `git` executable the served-git withheld-blob walk spawns. Production is + /// `"git"` (resolved via PATH); injectable so a fake `git` can drive the walk's + /// process-group teardown in handler tests without mutating the process-global + /// PATH (#174). + pub git_bin: String, } impl AppState { diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 03d3d5c4..a43a79a0 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -89,6 +89,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( 8, ), + git_bin: "git".to_string(), } } From 454e59ba533535168a6eb763447275ef107564ad Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 02:12:37 -0500 Subject: [PATCH 18/95] docs(node): the withheld-walk is bounded by the service timeout, not a fixed const (#174) Follow-up to threading the configured timeout into the walk: the walk now honors GITLAWB_GIT_SERVICE_TIMEOUT_SECS on both the serve and replication paths, so the README no longer says a fixed internal deadline. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d02c4a0b..0f760449 100644 --- a/README.md +++ b/README.md @@ -343,7 +343,7 @@ Important node settings: | `GITLAWB_REQUIRE_SIGNED_PEER_WRITES` | Require signed peer announce/sync writes. | | `GITLAWB_AUTO_SYNC` | Enable automatic sync from known peers. | | `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. | -| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. The withheld-blob classification walk is also bounded and reaped, by a fixed internal deadline rather than this setting. | +| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths), which is reaped at the deadline. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | | `GITLAWB_IRYS_URL` | Optional Irys/Arweave permanent anchoring. | From 79b9ee5c89070f2c9b8d534d28195803110385a1 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 14:32:24 -0500 Subject: [PATCH 19/95] fix(node): shed the served-git pool before the DB and cap per-source before it (#174) CodeRabbit review: the held git_permit was acquired at the top of git_info_refs and git_upload_pack, before the per-source cap, so one source could occupy the global pool during the DB/visibility window before its sub-cap rejected the excess. Move the held git_permit to after acquire_read_caller_permit (still before acquire_fresh/git, so INV-10's bound on the fresh Tigris acquire holds), and add a cheap pre-DB early shed (available_permits() == 0 -> 503, holds no permit) so the #62 shed-before-DB property is preserved without holding a permit across the DB work. The receive-pack POST is unchanged (owner-only; its top-of-handler git_permit is deliberate for acquire_write bounding). Tests: the two #62 shed-before-DB tests now exercise the explicit early check (load-bearing: disable it and they fall through to the DB); three new *_per_source_cap_sheds_with_global_capacity tests prove the per-source sub-cap sheds a capped source even with global capacity free (load-bearing on the caller cap). Full workspace green. --- crates/gitlawb-node/src/api/repos.rs | 253 ++++++++++++++++++++++-- crates/gitlawb-node/src/test_support.rs | 62 ++++-- 2 files changed, 290 insertions(+), 25 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 1532be58..85eaa227 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -515,17 +515,27 @@ pub async fn git_info_refs( let service = query .service .ok_or_else(|| AppError::BadRequest("missing ?service= parameter".into()))?; - // Shed with a 503 before spawning git when the concurrency cap is saturated; - // held for the whole op (incl. the smart_http call), released on return. The - // receive-pack advertisement is phase one of a push, so it draws from the WRITE - // pool (like the git-receive-pack POST), not the global read pool an anonymous - // clone flood can exhaust and thereby starve the push handshake (#174 U2). The - // upload-pack advertisement stays on the read pool with its per-caller sub-cap. - let _permit = if service == "git-receive-pack" { - git_permit(&state.git_write_semaphore)? - } else { - git_permit(&state.git_read_semaphore)? - }; + // #62 cheap pre-DB load shed: if the pool this service draws from is already + // saturated, shed with 503 before any DB/disk work. Best-effort (holds no + // permit); the authoritative hold is `git_permit` below, after the per-source + // cap. Restores the shed-before-DB property the reordered held acquire alone + // would drop, while the reorder still prevents one source from occupying global + // slots during the DB/visibility window. + { + let pool = if service == "git-receive-pack" { + &state.git_write_semaphore + } else { + &state.git_read_semaphore + }; + if pool.available_permits() == 0 { + tracing::warn!( + "served-git concurrency cap reached; shedding request with 503 (pre-DB)" + ); + return Err(AppError::Overloaded( + "git service at capacity, retry shortly".into(), + )); + } + } tracing::info!(owner = %owner, repo = %name, "info/refs request"); let record = state .db @@ -604,6 +614,22 @@ pub async fn git_info_refs( )? }; + // Shed with a 503 before spawning git when the concurrency cap is saturated; + // held for the whole op (incl. the smart_http call), released on return. Taken + // AFTER the per-source cap above so one source cannot occupy global slots it + // would be sub-cap-denied for during the DB/visibility window and starve other + // sources; still before acquire_fresh/git so it bounds the fresh Tigris acquire + // and git exec (INV-10). The receive-pack advertisement is phase one of a push, + // so it draws from the WRITE pool (like the git-receive-pack POST), not the + // global read pool an anonymous clone flood can exhaust and thereby starve the + // push handshake (#174 U2). The upload-pack advertisement stays on the read + // pool with its per-caller sub-cap. + let _permit = if service == "git-receive-pack" { + git_permit(&state.git_write_semaphore)? + } else { + git_permit(&state.git_read_semaphore)? + }; + // For receive-pack (push), download the latest from Tigris so the client // sees the same refs that acquire_write() will operate on. let disk_path = if service == "git-receive-pack" { @@ -725,9 +751,15 @@ pub async fn git_upload_pack( headers: axum::http::HeaderMap, body: Bytes, ) -> Result { - // Shed with a 503 before spawning git when the concurrency cap is saturated; - // held for the whole op (incl. the smart_http call), released on return. - let _permit = git_permit(&state.git_read_semaphore)?; + // #62 cheap pre-DB load shed (see git_info_refs): shed before DB when the read + // pool is saturated; the authoritative hold is `git_permit` below, after the + // per-source cap. + if state.git_read_semaphore.available_permits() == 0 { + tracing::warn!("served-git concurrency cap reached; shedding request with 503 (pre-DB)"); + return Err(AppError::Overloaded( + "git service at capacity, retry shortly".into(), + )); + } let name = smart_http_repo_name(&repo)?; let record = state .db @@ -760,6 +792,14 @@ pub async fn git_upload_pack( "upload-pack", )?; + // Shed with a 503 before spawning git when the concurrency cap is saturated; + // held for the whole op (incl. the smart_http call), released on return. Taken + // AFTER the per-source cap above so one source cannot occupy global slots it + // would be sub-cap-denied for during the DB/visibility window and starve other + // sources; still before acquire/git so it bounds the Tigris acquire and git + // exec (INV-10). + let _permit = git_permit(&state.git_read_semaphore)?; + let disk_path = state .repo_store .acquire(&record.owner_did, &record.name) @@ -3209,6 +3249,191 @@ mod tests { ); } + /// #174 (review fix): the per-source caller cap is an independent brake that + /// sheds a capped source even when the global pool has free capacity — the + /// sub-cap is not a mere pre-filter for pool exhaustion. Proven by leaving the + /// global read pool with capacity (so the pre-DB early shed passes) AND + /// pre-holding the source's upload-pack read sub-cap: the request reaches the + /// caller cap and sheds there, so its 503 body reads "for this caller". Remove + /// the `acquire_read_caller_permit` call and the capped source falls through to + /// the git op instead of shedding with "for this caller" — this is the + /// caller-cap acquire probe for the info/refs upload-pack branch. + #[sqlx::test] + async fn info_refs_upload_pack_per_source_cap_sheds_with_global_capacity(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + // Global read pool has free capacity (early shed passes); source pre-held at + // its per-caller cap so it sheds on the caller cap, not the global pool. + state.git_read_semaphore = Arc::new(Semaphore::new(4)); + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6ordir", "oi", "/tmp/oi-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.91:5000".parse().unwrap(); + // Pin this source at its single upload-pack read slot. + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first read slot for this source IP"); + + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6ordir/oi/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its read sub-cap must shed 503 even with global pool capacity" + ); + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .expect("body bytes"); + let body = String::from_utf8_lossy(&bytes); + assert!( + body.contains("for this caller"), + "the per-source cap is an independent brake: with global capacity free, the capped source must still shed with the caller-cap body, got {body}" + ); + } + + /// #174 (review fix): same independent-brake guarantee for the receive-pack + /// advertisement branch of info/refs — its per-source cap + /// (`git_push_advert_per_caller`) sheds a capped source even when the global + /// write pool has capacity. Leave the global write pool with capacity (so the + /// pre-DB early shed passes) and pre-hold the source's advert slot: the request + /// reaches the caller cap, so the 503 body reads "for this caller". Remove the + /// caller-cap acquire and the capped source falls through instead of shedding + /// with "for this caller". The push rate limiter is left permissive so the + /// request reaches the caller cap. + #[sqlx::test] + async fn info_refs_receive_pack_per_source_cap_sheds_with_global_capacity(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + // Global write pool has free capacity (early shed passes); source pre-held at + // its advert sub-cap so it sheds on the caller cap, not the global pool. + state.git_write_semaphore = Arc::new(Semaphore::new(4)); + state.git_push_advert_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + // Permissive push rate limiter so the advertisement passes the rate gate and + // reaches the per-source concurrency cap. + state.push_rate_limiter = crate::rate_limit::RateLimiter::new(100, Duration::from_secs(60)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6ordrp", "or", "/tmp/or-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.92:5000".parse().unwrap(); + // Pin this source at its single receive-pack advertisement slot. + let _slot = state + .git_push_advert_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first advert slot for this source IP"); + + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6ordrp/or/info/refs?service=git-receive-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its advert sub-cap must shed 503 even with global write pool capacity" + ); + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .expect("body bytes"); + let body = String::from_utf8_lossy(&bytes); + assert!( + body.contains("for this caller"), + "the per-source advert cap is an independent brake: with global write capacity free, the capped source must still shed with the caller-cap body, got {body}" + ); + } + + /// #174 (review fix): same independent-brake guarantee for the POST upload-pack + /// handler — its per-source read cap sheds a capped source even when the global + /// read pool has capacity. Leave the global read pool with capacity (so the + /// pre-DB early shed passes) and pre-hold the source's read slot: the request + /// reaches the caller cap, so the 503 body reads "for this caller". Remove the + /// caller-cap acquire and the capped source falls through instead of shedding + /// with "for this caller". + #[sqlx::test] + async fn upload_pack_per_source_cap_sheds_with_global_capacity(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + // Global read pool has free capacity (early shed passes); source pre-held at + // its per-caller cap so it sheds on the caller cap, not the global pool. + state.git_read_semaphore = Arc::new(Semaphore::new(4)); + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6ordup", "ou", "/tmp/ou-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.93:5000".parse().unwrap(); + // Pin this source at its single read slot. + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first read slot for this source IP"); + + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::POST) + .uri("/z6ordup/ou/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its read sub-cap must shed 503 even with global pool capacity" + ); + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .expect("body bytes"); + let body = String::from_utf8_lossy(&bytes); + assert!( + body.contains("for this caller"), + "the per-source cap is an independent brake: with global capacity free, the capped source must still shed with the caller-cap body, got {body}" + ); + } + /// #174 SC2 (per-source key, U1): the per-caller read sub-cap keys on the /// resolved source IP, NOT the signed DID, so a disposable-DID farm cannot /// multiply its budget. Fill the source IP's single read slot, then drive two diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index a43a79a0..765545f1 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -195,13 +195,14 @@ mod tests { ); } - /// PR3 (#62): the served-git concurrency cap sheds at the HTTP layer. One test - /// per git handler drives the `let _permit = git_permit(...)` wiring end to end - /// (this one plus the git_upload_pack / git_receive_pack siblings below); the - /// git_permit unit test covers the helper in isolation. DB-free: an exhausted - /// semaphore sheds before any DB/disk access, so a lazy state works. Remove the - /// permit line from git_info_refs and this goes red (the request falls through - /// to the DB and returns something other than 503). + /// PR3 (#62): the served-git concurrency cap sheds at the HTTP layer before the + /// DB. The held `git_permit` acquire now sits after the per-source cap, so the + /// shed-before-DB property is carried by an explicit `available_permits() == 0` + /// early check at the top of the handler (the held permit remains the + /// authoritative bound further down). DB-free: an exhausted semaphore sheds + /// before any DB/disk access, so a lazy state works. Remove the early-shed block + /// from git_info_refs and this goes red (the request falls through to the DB and + /// returns something other than 503). #[tokio::test] async fn git_info_refs_sheds_with_503_when_semaphore_exhausted() { let mut state = test_state_lazy(); @@ -234,10 +235,11 @@ mod tests { ); } - /// PR3 (#62) sibling of the info/refs shed test: git-upload-pack also acquires a - /// permit at the top, so an exhausted semaphore must shed it with a 503 before - /// any DB/disk work. Anonymous-reachable, so no auth injection is needed. Remove - /// the permit line from git_upload_pack and this goes red. + /// PR3 (#62) sibling of the info/refs shed test: git-upload-pack carries the same + /// explicit `available_permits() == 0` early-shed check at the top, so an + /// exhausted semaphore must shed it with a 503 before any DB/disk work. + /// Anonymous-reachable, so no auth injection is needed. Remove the early-shed + /// block from git_upload_pack and this goes red. #[tokio::test] async fn git_upload_pack_sheds_with_503_when_semaphore_exhausted() { let mut state = test_state_lazy(); @@ -270,6 +272,44 @@ mod tests { ); } + /// PR3 (#62) receive-pack sibling of the info/refs shed test: the early shed + /// selects the WRITE pool for a git-receive-pack advertisement (phase one of a + /// push, #174 U2), so an exhausted write pool sheds the advert with 503 before + /// any DB/disk work — even while the read pool is free (only the write pool is + /// zeroed here). Flip the pool selection to the read pool, or remove the + /// early-shed block, and this goes red. + #[tokio::test] + async fn git_info_refs_receive_pack_sheds_with_503_when_write_pool_exhausted() { + let mut state = test_state_lazy(); + state.git_write_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + + let router = Router::new() + .route( + "/{owner}/{repo}/info/refs", + axum::routing::get(crate::api::repos::git_info_refs), + ) + .with_state(state); + let resp = router + .oneshot(anon_get( + "/alice/repo.git/info/refs?service=git-receive-pack", + )) + .await + .unwrap(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted WRITE pool must shed the receive-pack advertisement with 503 before touching the DB" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + /// PR3 (#62) sibling for the push path: git-receive-pack requires an /// AuthenticatedDid extension (production: require_signature injects it), so the /// request carries one via signed_request_as — without it the Extension From f19f2813a2b73c326001e5724e0ae175d0b9d72c Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 09:44:53 -0500 Subject: [PATCH 20/95] fix(node): give the receive-pack advert its own pool so it can't shed authed pushes (#174) jatmn P1: the anon-reachable GET info/refs?service=git-receive-pack drew from the same git_write_semaphore as the authenticated git-receive-pack POST. The per-source advert cap reserves no global slots, so ~8 sources (each within the per-IP push rate limit) could hold all 32 write permits across acquire_fresh + the advertisement, and the next authenticated push then 503s at admission. That is an anonymous caller shedding an authenticated push across the auth boundary (INV-10). Add a dedicated git_push_advert_semaphore, disjoint from the write pool. Both the pre-DB early-shed peek and the held acquire for the receive-pack advertisement now draw from it; the write pool is left exclusively for the POST. An advert flood can at worst exhaust the advert pool (each source still capped by git_push_advert_per_caller and the per-IP push rate limiter); the authenticated push pool is untouched. Test receive_pack_advertisement_draws_from_dedicated_advert_pool proves at the handler layer that the advert sheds when its own pool is saturated AND survives when the write pool is saturated (the reservation). Both flip under revert, verified independently for the pre-DB peek and the held acquire. --- crates/gitlawb-node/src/api/repos.rs | 74 ++++++++++++++++--------- crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/main.rs | 7 +++ crates/gitlawb-node/src/state.rs | 16 ++++-- crates/gitlawb-node/src/test_support.rs | 17 +++--- 5 files changed, 76 insertions(+), 39 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 85eaa227..719246e0 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -522,8 +522,11 @@ pub async fn git_info_refs( // would drop, while the reorder still prevents one source from occupying global // slots during the DB/visibility window. { + // The receive-pack advertisement peeks its DEDICATED advert pool, not the + // write pool the authenticated POST uses (#174) — matching the held acquire + // below, so the pre-DB shed and the authoritative hold agree on the pool. let pool = if service == "git-receive-pack" { - &state.git_write_semaphore + &state.git_push_advert_semaphore } else { &state.git_read_semaphore }; @@ -620,12 +623,14 @@ pub async fn git_info_refs( // would be sub-cap-denied for during the DB/visibility window and starve other // sources; still before acquire_fresh/git so it bounds the fresh Tigris acquire // and git exec (INV-10). The receive-pack advertisement is phase one of a push, - // so it draws from the WRITE pool (like the git-receive-pack POST), not the - // global read pool an anonymous clone flood can exhaust and thereby starve the - // push handshake (#174 U2). The upload-pack advertisement stays on the read - // pool with its per-caller sub-cap. + // but it is ANON-reachable, so it draws from the dedicated advert pool + // (`git_push_advert_semaphore`), NOT the write pool the authenticated POST uses: + // an advert flood can at worst exhaust the advert pool, never a permit a push + // POST needs at admission (#174 U2). A clone flood on the read pool likewise + // can't touch either. The upload-pack advertisement stays on the read pool with + // its per-caller sub-cap. let _permit = if service == "git-receive-pack" { - git_permit(&state.git_write_semaphore)? + git_permit(&state.git_push_advert_semaphore)? } else { git_permit(&state.git_read_semaphore)? }; @@ -2944,14 +2949,19 @@ mod tests { ); } - /// #174 U2: the receive-pack advertisement (`GET info/refs?service=git-receive-pack`) - /// is phase one of a push, so it draws from the WRITE pool, not the global read - /// pool an anonymous clone flood can exhaust. Proven at the handler by saturating - /// each pool to zero and checking who shares it (INV-10). Revert the branch to the - /// read pool and the read-saturated receive-pack assertion goes 503 (the exact - /// push-handshake starvation jatmn flagged). + /// #174 (jatmn P1): the anon-reachable receive-pack advertisement + /// (`GET info/refs?service=git-receive-pack`) draws from a DEDICATED advert pool + /// (`git_push_advert_semaphore`), NOT the write pool the authenticated POST uses. + /// Proven at the handler by saturating each pool to zero and checking who shares + /// it (INV-10, across the auth boundary). The load-bearing pair: + /// * advert pool at 0 -> the advert SHEDS 503 (it is bound to that pool); + /// * write pool at 0 -> the advert SURVIVES (it can NOT consume a permit the + /// authenticated POST needs — the reservation jatmn asked for). + /// Revert the branch to `git_write_semaphore` and BOTH flip: the advert-pool-0 + /// case stops shedding and the write-pool-0 case starts shedding (the exact + /// anon-sheds-authed-push starvation). #[sqlx::test] - async fn receive_pack_advertisement_draws_from_write_pool(pool: sqlx::PgPool) { + async fn receive_pack_advertisement_draws_from_dedicated_advert_pool(pool: sqlx::PgPool) { use axum::body::Body; use axum::extract::ConnectInfo; use axum::http::{Method, Request, StatusCode}; @@ -2960,17 +2970,19 @@ mod tests { use tokio::sync::Semaphore; use tower::ServiceExt; - // Build a fresh state with the two pools sized independently, then drive one + // Build a fresh state with the three pools sized independently, then drive one // info/refs advertisement for `service` and return its handler status. async fn advert_status( pool: &sqlx::PgPool, read_permits: usize, write_permits: usize, + advert_permits: usize, service: &str, ) -> StatusCode { let mut state = crate::test_support::test_state(pool.clone()).await; state.git_read_semaphore = Arc::new(Semaphore::new(read_permits)); state.git_write_semaphore = Arc::new(Semaphore::new(write_permits)); + state.git_push_advert_semaphore = Arc::new(Semaphore::new(advert_permits)); state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; state .db @@ -2988,31 +3000,39 @@ mod tests { router.oneshot(req).await.unwrap().status() } - // Read pool saturated, write pool free: the push handshake SURVIVES. + // Advert pool saturated (read + write free): the receive-pack advert SHEDS, + // proving it is bound to the dedicated advert pool. + assert_eq!( + advert_status(&pool, 8, 8, 0, "git-receive-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "receive-pack advertisement draws from the dedicated advert pool: a saturated advert pool sheds it 503" + ); + // WRITE pool saturated (advert + read free): the advert SURVIVES. This is the + // reservation — an advert flood can never occupy a permit the authenticated + // push POST relies on at admission. assert_ne!( - advert_status(&pool, 0, 8, "git-receive-pack").await, + advert_status(&pool, 8, 0, 8, "git-receive-pack").await, StatusCode::SERVICE_UNAVAILABLE, - "receive-pack advertisement must draw from the write pool: a saturated read pool must not shed it" + "receive-pack advertisement must NOT draw from the write pool: a saturated write pool must not shed it" ); - // Write pool saturated, read pool free: the receive-pack advertisement SHEDS, - // proving it now consumes the write pool (not the read pool). - assert_eq!( - advert_status(&pool, 8, 0, "git-receive-pack").await, + // Read pool saturated (advert + write free): the advert SURVIVES (never on the read pool). + assert_ne!( + advert_status(&pool, 0, 8, 8, "git-receive-pack").await, StatusCode::SERVICE_UNAVAILABLE, - "receive-pack advertisement draws from the write pool, so a saturated write pool sheds it 503" + "receive-pack advertisement must not draw from the read pool" ); // Read pool saturated: the upload-pack advertisement still SHEDS (unchanged). assert_eq!( - advert_status(&pool, 0, 8, "git-upload-pack").await, + advert_status(&pool, 0, 8, 8, "git-upload-pack").await, StatusCode::SERVICE_UNAVAILABLE, "upload-pack advertisement stays on the read pool: a saturated read pool sheds it 503" ); - // Write pool saturated, read pool free: the upload-pack advertisement is - // UNAFFECTED, proving reads never touch the write pool in either direction. + // Write + advert pools saturated, read free: the upload-pack advertisement is + // UNAFFECTED, proving reads never touch either write-side pool. assert_ne!( - advert_status(&pool, 8, 0, "git-upload-pack").await, + advert_status(&pool, 8, 0, 0, "git-upload-pack").await, StatusCode::SERVICE_UNAVAILABLE, - "upload-pack advertisement never touches the write pool" + "upload-pack advertisement never touches the write or advert pool" ); } diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 5befa7f3..5d827794 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -522,6 +522,7 @@ mod tests { shutdown_tx: tokio::sync::watch::channel(false).0, git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 9b646c2b..18019395 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -382,6 +382,13 @@ async fn main() -> Result<()> { git_write_semaphore: Arc::new(tokio::sync::Semaphore::new( config.max_concurrent_git_pushes, )), + // Anon receive-pack advertisements get their OWN pool, same size as the + // write pool but disjoint, so filling it (which takes many source IPs, each + // capped by git_push_advert_per_caller) never occupies a permit the + // authenticated POST needs (#174). + git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new( + config.max_concurrent_git_pushes, + )), git_read_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( config.max_concurrent_reads_per_caller, ), diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 94952120..eb07a3e4 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -97,11 +97,19 @@ pub struct AppState { /// Bounds concurrent `git-receive-pack` (push) operations, a pool separate /// from `git_read_semaphore` so an anonymous READ flood can never shed an /// authenticated push (#174). Sized by `max_concurrent_git_pushes`. Drawn from - /// by the `git-receive-pack` POST (owner-gated) and the receive-pack `info/refs` - /// advertisement (anon-reachable); the advertisement is additionally bounded per - /// source by `git_push_advert_per_caller` so no single source can monopolize this - /// pool and shed pushes. + /// by the `git-receive-pack` POST (owner-gated) ONLY. The anon-reachable + /// receive-pack `info/refs` advertisement draws from the SEPARATE + /// `git_push_advert_semaphore` below, never this pool, so a multi-source flood + /// of push-handshake advertisements can never occupy a permit an authenticated + /// POST needs at admission (#174). pub git_write_semaphore: Arc, + /// Bounds concurrent anon-reachable `git-receive-pack` `info/refs` + /// advertisements — a pool SEPARATE from `git_write_semaphore` so adverts (which + /// hold a permit across `acquire_fresh` + `info/refs`) can never consume a slot + /// the authenticated POST relies on. A per-source flood can at worst exhaust this + /// advert pool (each source also capped by `git_push_advert_per_caller` and the + /// per-IP push rate limiter), and the reserved POST pool is untouched (#174). + pub git_push_advert_semaphore: Arc, /// Per-caller concurrency sub-cap on the read pool: each caller (keyed on the /// resolved source IP, #174 U1) may hold at most `max_concurrent_reads_per_caller` /// in-flight read ops, so one caller cannot monopolize `git_read_semaphore` diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 765545f1..0805fec1 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -85,6 +85,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { // Generous — no test drives the handler-level shed (git_permit is unit-tested). git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( 8, @@ -273,15 +274,15 @@ mod tests { } /// PR3 (#62) receive-pack sibling of the info/refs shed test: the early shed - /// selects the WRITE pool for a git-receive-pack advertisement (phase one of a - /// push, #174 U2), so an exhausted write pool sheds the advert with 503 before - /// any DB/disk work — even while the read pool is free (only the write pool is - /// zeroed here). Flip the pool selection to the read pool, or remove the - /// early-shed block, and this goes red. + /// selects the dedicated ADVERT pool for a git-receive-pack advertisement (#174), + /// so an exhausted advert pool sheds the advert with 503 before any DB/disk work + /// — while the write pool (reserved for authenticated POSTs) is left free here. + /// Flip the pool selection back to the write pool, or remove the early-shed + /// block, and this goes red. #[tokio::test] - async fn git_info_refs_receive_pack_sheds_with_503_when_write_pool_exhausted() { + async fn git_info_refs_receive_pack_sheds_with_503_when_advert_pool_exhausted() { let mut state = test_state_lazy(); - state.git_write_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + state.git_push_advert_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); let router = Router::new() .route( @@ -299,7 +300,7 @@ mod tests { assert_eq!( resp.status(), StatusCode::SERVICE_UNAVAILABLE, - "an exhausted WRITE pool must shed the receive-pack advertisement with 503 before touching the DB" + "an exhausted ADVERT pool must shed the receive-pack advertisement with 503 before touching the DB" ); assert_eq!( resp.headers() From 473924dcfec11c22b8ea5da5f125478bbcd5f845 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 09:49:33 -0500 Subject: [PATCH 21/95] fix(node): compile the bounded visibility git runner on non-Unix targets (#174) jatmn P1: run_bounded_git was gated #[cfg(unix)], but its callers (assert_all_refs_are_commits, blob_paths) and the module are compiled on all targets, so a non-Unix build (the continue-on-error Windows release target) could not resolve the function. Add a #[cfg(not(unix))] twin with the same signature and result semantics. It bounds a single child (no process-group teardown, which is Unix-only): threads feed stdin and drain stderr, the main thread drains stdout, and a watchdog thread kills the child at the deadline. The child is shared with the watchdog behind a mutex the main thread does not hold while draining, so the watchdog can always acquire it to kill, and killing closes stdout to unblock the drain. Best-effort (reaps only the direct child), which is why the hardened group-aware path stays Unix-only. Verified the twin type-checks by compiling its (portable-std) body on the native target; the full non-Unix link is CI's Windows job. --- .../gitlawb-node/src/git/visibility_pack.rs | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index 09a80aba..0eaeed2d 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -154,6 +154,98 @@ fn run_bounded_git( Ok(out) } +/// Non-Unix fallback for [`run_bounded_git`]. Windows and other non-Unix targets +/// have no process-group teardown (`process_group(0)` / `kill(-pgid)` are Unix-only), +/// so this bounds a single child on its own: threads feed stdin and drain stderr +/// while the main thread drains stdout, and a watchdog thread kills the child at the +/// deadline (which closes stdout and unblocks the drain). The child is shared with +/// the watchdog behind a mutex that the main thread does NOT hold while draining, so +/// the watchdog can always acquire it to kill. Best-effort — it reaps only the direct +/// child, not a descendant group — which is why the hardened, group-aware path above +/// is gated to Unix, the only target the served node actually runs on (the Windows +/// release binary is best-effort / `continue-on-error` in CI). Kept in lockstep with +/// the Unix version's signature and result semantics so every caller compiles on all +/// targets (#174). +#[cfg(not(unix))] +fn run_bounded_git( + git_bin: &str, + args: &[&str], + repo_path: &Path, + stdin_bytes: &[u8], + deadline: Instant, +) -> Result> { + use std::io::{Read, Write}; + use std::sync::mpsc::RecvTimeoutError; + + let label = args.first().copied().unwrap_or("git"); + let mut child = std::process::Command::new(git_bin) + .args(args) + .current_dir(repo_path) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .with_context(|| format!("failed to spawn git {label}"))?; + + let mut stdin = child.stdin.take(); + let input = stdin_bytes.to_vec(); + let writer = std::thread::spawn(move || { + if let Some(mut s) = stdin.take() { + let _ = s.write_all(&input); + } + }); + let mut stderr = child.stderr.take().context("git stderr was not piped")?; + let err_reader = std::thread::spawn(move || { + let mut err = Vec::new(); + let _ = stderr.read_to_end(&mut err); + err + }); + let mut stdout = child.stdout.take().context("git stdout was not piped")?; + + // Share the child with the watchdog. The main thread drains stdout WITHOUT + // holding this lock, so the watchdog can always acquire it to kill on timeout; + // killing closes stdout and unblocks the drain below. + let child = std::sync::Arc::new(std::sync::Mutex::new(child)); + let (done_tx, done_rx) = mpsc::channel::<()>(); + let watchdog = { + let child = child.clone(); + std::thread::spawn(move || -> bool { + let wait = deadline.saturating_duration_since(Instant::now()); + match done_rx.recv_timeout(wait) { + Ok(()) | Err(RecvTimeoutError::Disconnected) => false, + Err(RecvTimeoutError::Timeout) => { + if let Ok(mut c) = child.lock() { + let _ = c.kill(); + } + true + } + } + }) + }; + + let mut out = Vec::new(); + let read_result = stdout.read_to_end(&mut out); + // The drain has returned (child exited or was killed), so taking the lock here + // cannot deadlock against the watchdog. + let status = child + .lock() + .expect("git child mutex poisoned") + .wait() + .context("git wait failed")?; + let _ = done_tx.send(()); + let killed = watchdog.join().unwrap_or(false); + let err = err_reader.join().unwrap_or_default(); + let _ = writer.join(); + read_result.context("failed to read git stdout")?; + if killed && !status.success() { + return Err(crate::git::smart_http::GitServiceTimeout.into()); + } + if !status.success() { + anyhow::bail!("git {label} failed: {}", String::from_utf8_lossy(&err)); + } + Ok(out) +} + /// Fail closed unless every ref ultimately resolves to a commit (a ref pointing /// directly at a blob or tree, or an annotated tag — even a nested one — of such /// an object is refused). `git rev-list --all` silently *skips* such refs, but From 7a6bd359eb915285ba4e459ae3339f0ed8716891 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 10:11:41 -0500 Subject: [PATCH 22/95] fix(node): SIGKILL a SIGTERM-ignoring member of the visibility-walk process group (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit jatmn/CodeRabbit P1: the walk watchdog stood down as soon as the group LEADER was reaped. A leader that exits cleanly on SIGTERM while a background member ignores SIGTERM and closes its inherited pipes let the main drain unblock, the leader be reaped, and the watchdog skip its SIGKILL escalation — so the handler returned while the descendant kept running. jatmn's suggested "continue until ESRCH" could not work: the main thread reaping the leader is what frees the pgid, so continuing to kill(-pgid) after that races a recycled group. Restructure the coordination instead: the main thread now defers reaping the leader until the watchdog thread returns. The watchdog always escalates through an unconditional group SIGKILL (no leader-reap short-circuit), and because the leader is still unreaped while every kill(-pgid) fires, the pgid cannot have been recycled. The grace before SIGKILL is a fixed budget (only paid on the exceptional timeout path). Regression run_bounded_git_sigkills_a_sigterm_ignoring_descendant_after_leader_exits: a leader that exits 0 on SIGTERM spawns an sh -c descendant (its own $$, not a ( ) subshell's parent pid) that ignores SIGTERM and closes its pipes; the descendant must be dead when the runner returns. RED on the old teardown (survives), GREEN now. The existing leader-ignores-SIGTERM and hung-walk teardown tests still pass. --- .../gitlawb-node/src/git/visibility_pack.rs | 162 +++++++++++++----- 1 file changed, 118 insertions(+), 44 deletions(-) diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index 0eaeed2d..ec55423e 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -23,6 +23,12 @@ use std::time::{Duration, Instant}; #[cfg(test)] const WALK_TIMEOUT: Duration = Duration::from_secs(600); +/// How long the process-group watchdog waits after SIGTERM before escalating to +/// SIGKILL, giving a well-behaved git child time to clean up its `*.lock` files. Only +/// paid on a timeout (already the exceptional path). +#[cfg(unix)] +const WATCHDOG_TERM_GRACE: Duration = Duration::from_secs(1); + /// Run one git child under a shared `deadline` with process-group teardown, /// BLOCKING, and return its stdout. The child runs in its own process group; a /// watchdog thread SIGTERMs (lets git clean up its `*.lock` files), then SIGKILLs, @@ -60,52 +66,46 @@ fn run_bounded_git( // With process_group(0) the child leads its own group, so pgid == its pid. let pgid = child.id() as i32; - // Watchdog: on the deadline, SIGTERM the whole group, poll until it exits, - // escalate to SIGKILL after a grace, and report whether it killed. Signalled to - // stand down when the child is reaped in time. Kept off the main thread because - // the main thread's stdout drain is exactly what blocks until a hung child is - // torn down. + // Watchdog: on the deadline, tear the WHOLE process group down — SIGTERM, a grace + // for a well-behaved child to clean up its `*.lock` files, then an UNCONDITIONAL + // SIGKILL of the group. It never stands down on leader-reap alone: a group member + // that ignores SIGTERM while the leader exits cleanly would otherwise escape the + // SIGKILL and keep running past the deadline (finding 3, #174). The main thread + // defers reaping the leader until this thread returns (see below), so the leader's + // pid is still unreaped while every `kill(-pgid)` fires and the pgid cannot have + // been recycled — which is why this no longer needs the old `reaped` short-circuit. + // Kept off the main thread because the main thread's stdout drain is exactly what + // blocks until a hung child is torn down. let (done_tx, done_rx) = mpsc::channel::<()>(); - // Set true the instant the main thread reaps the child, so the watchdog never - // SIGTERMs a pgid whose leader is already reaped and whose pid the kernel may - // recycle (the reused-pgid hazard smart_http guards via disarm-after-wait). - let reaped = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); - let watchdog = { - let reaped = reaped.clone(); - std::thread::spawn(move || -> bool { - use std::sync::atomic::Ordering; - let wait = deadline.saturating_duration_since(Instant::now()); - match done_rx.recv_timeout(wait) { - Ok(()) | Err(RecvTimeoutError::Disconnected) => false, - Err(RecvTimeoutError::Timeout) => { - // The child was reaped right as the deadline fired: do not signal - // its (possibly-recycled) pgid. - if reaped.load(Ordering::SeqCst) { - return false; - } - // SAFETY: kill(2) takes only integers and borrows no Rust memory; - // ESRCH on an already-gone group is ignored. - unsafe { libc::kill(-pgid, libc::SIGTERM) }; - for step in 0..200u32 { - if reaped.load(Ordering::SeqCst) || unsafe { libc::kill(-pgid, 0) } != 0 { - return true; // reaped, or ESRCH: every group member has exited - } - if step == 100 { - unsafe { libc::kill(-pgid, libc::SIGKILL) }; - } - std::thread::sleep(Duration::from_millis(10)); - } - // Survived SIGKILL past the cap: a wedged (D-state) git, the same - // condition smart_http's reap logs. Warn so an operator sees it. + let watchdog = std::thread::spawn(move || -> bool { + let wait = deadline.saturating_duration_since(Instant::now()); + match done_rx.recv_timeout(wait) { + Ok(()) | Err(RecvTimeoutError::Disconnected) => false, + Err(RecvTimeoutError::Timeout) => { + // SAFETY: kill(2) takes only integers and borrows no Rust memory; + // ESRCH on an already-gone group is ignored. + unsafe { libc::kill(-pgid, libc::SIGTERM) }; + // Fixed grace: because the main thread defers the leader's reap, a + // fully-exited group still shows a zombie leader here, so polling for + // ESRCH cannot detect early completion — just wait the grace, then + // SIGKILL. On a group of only zombies the SIGKILL is a harmless no-op; + // on a SIGTERM-ignoring member it is what actually kills it. + std::thread::sleep(WATCHDOG_TERM_GRACE); + unsafe { libc::kill(-pgid, libc::SIGKILL) }; + // Brief settle so the SIGKILL is delivered before the main thread + // reaps the leader and frees the pgid. A wedged (D-state) member + // survives even SIGKILL — the documented residual, as in smart_http. + std::thread::sleep(Duration::from_millis(20)); + if unsafe { libc::kill(-pgid, 0) } == 0 { tracing::warn!( pgid, "withheld-walk git survived SIGKILL past the watchdog cap (uninterruptible I/O?)" ); - true } + true } - }) - }; + } + }); // Feed stdin on a writer thread and drain stderr on a reader thread so the main // thread can drain stdout concurrently; writing all of stdin (or draining one @@ -132,14 +132,18 @@ fn run_bounded_git( // is unreachable in userspace (no signal reaps a D-state process) and matches the // async `reap_group_on_timeout`, which likewise only warns and gives up there. let read_result = stdout.read_to_end(&mut out); + // The drain has returned: either the child exited on its own, or the watchdog's + // teardown closed its pipes. Tell the watchdog the drain is done, then WAIT for it + // to return BEFORE reaping the leader. On the timeout path the watchdog runs the + // full SIGTERM -> grace -> SIGKILL teardown; joining it before `child.wait()` keeps + // the leader's pid unreaped while every `kill(-pgid)` fires (so the pgid can't be + // recycled), and guarantees a SIGTERM-ignoring group member has already been + // SIGKILLed rather than left running past the deadline (finding 3, #174). + let _ = done_tx.send(()); + let killed = watchdog.join().unwrap_or(false); let status = child.wait().context("git wait failed")?; - // Reaped: bar the watchdog from signalling the now-reaped (possibly recycled) - // pgid before it can fire, then stand it down. - reaped.store(true, std::sync::atomic::Ordering::SeqCst); let err = err_reader.join().unwrap_or_default(); let _ = writer.join(); - let _ = done_tx.send(()); - let killed = watchdog.join().unwrap_or(false); read_result.context("failed to read git stdout")?; // The watchdog runs off a wall clock that can race a child finishing right at the // deadline. A child that exited on its own (success) is not a timeout even if the @@ -828,6 +832,76 @@ mod tests { "the SIGTERM-ignoring child (pid {pid}) must be reaped via SIGKILL, not left running" ); } + + /// #174 finding 3 (jatmn/CodeRabbit): a group MEMBER that ignores SIGTERM must + /// still be SIGKILLed even when the group LEADER exits cleanly on SIGTERM. The + /// leader traps SIGTERM to exit 0, but first spawns a descendant (`sh -c`, so its + /// `$$` is its OWN pid — a `( )` subshell's `$$` is the parent's) that ignores + /// SIGTERM and closes its inherited stdout/stderr. When the watchdog SIGTERMs the + /// group, the leader exits, its stdout closes, the main drain unblocks, and the + /// leader is reaped — the exact window a `reaped`-gated watchdog stands down in, + /// before escalating to SIGKILL. The descendant must be dead when run_bounded_git + /// returns; a teardown that stands down on leader-reap leaves it running (RED). + #[cfg(unix)] + #[test] + fn run_bounded_git_sigkills_a_sigterm_ignoring_descendant_after_leader_exits() { + use std::time::{Duration, Instant}; + let tmp = TempDir::new().unwrap(); + // Both loops are bounded (~30s) so a broken teardown cannot leak a permanent + // orphan or wedge the suite; the assertion fires well before then. + let body = "#!/bin/sh\n\ +case \"$1\" in\n\ + rev-list)\n\ + sh -c 'trap \"\" TERM; echo $$ > desc.pid; exec 1>&- 2>&-; i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done' &\n\ + trap 'exit 0' TERM\n\ + i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;;\n\ + *) : ;;\n\ +esac\n"; + let git_bin = write_fake_git(tmp.path(), body); + let (tx, rx) = std::sync::mpsc::channel(); + let path = tmp.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(run_bounded_git( + &git_bin, + &["rev-list"], + &path, + b"", + Instant::now() + Duration::from_millis(100), + )); + }); + let _ = rx + .recv_timeout(Duration::from_secs(10)) + .expect("run_bounded_git must return within the watchdog budget"); + + // Wait for the descendant to record its OWN pid, then assert it is gone. + let desc_pid_path = tmp.path().join("desc.pid"); + let mut desc: Option = None; + for _ in 0..200 { + if let Some(p) = std::fs::read_to_string(&desc_pid_path) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + desc = Some(p); + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + let desc = desc.expect("the fake leader must have spawned and recorded a descendant"); + let mut gone = false; + for _ in 0..300 { + if unsafe { libc::kill(desc, 0) } != 0 { + gone = true; + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + // Kill it regardless so a RED run leaks no orphan. + unsafe { libc::kill(desc, libc::SIGKILL) }; + assert!( + gone, + "a SIGTERM-ignoring descendant (pid {desc}) must be SIGKILLed even after the leader exits cleanly, not orphaned" + ); + } use crate::db::VisibilityMode; use chrono::Utc; use std::process::Command; From 9412aa9c5bdc56d5c27cbf5be3dfebea995e7039 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 10:34:44 -0500 Subject: [PATCH 23/95] fix(node): bound and reap push-candidate git children under the write permit (#174) Route push_delta's candidate-discovery git children (cat-file / rev-list) through the bounded run_bounded_git (deadline + process-group SIGKILL teardown) instead of bare Command::output(). A hung scan no longer pins the git-receive-pack write permit past the deadline or a client disconnect. Threads git_bin + a shared per-scan deadline (git_service_timeout_secs) through resolve_candidates_for_push and all_blob_oids. --- crates/gitlawb-node/src/api/repos.rs | 4 +- crates/gitlawb-node/src/git/push_delta.rs | 278 ++++++++++++------ .../gitlawb-node/src/git/visibility_pack.rs | 11 +- 3 files changed, 193 insertions(+), 100 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 719246e0..ea6f126e 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -115,7 +115,7 @@ async fn fail_closed_full_scan_objects( let allowed = crate::git::visibility_pack::replicable_blob_set_bounded( &disk_path, &git_bin, timeout, &rules, is_public, &owner_did, )?; - let all_blobs = crate::git::push_delta::all_blob_oids(&disk_path)?; + let all_blobs = crate::git::push_delta::all_blob_oids(&disk_path, &git_bin, std::time::Instant::now() + timeout)?; Ok(crate::git::visibility_pack::replicable_objects_fail_closed( candidates, &allowed, &all_blobs, )) @@ -1264,6 +1264,8 @@ pub async fn git_receive_pack( disk_path.clone(), new_tips, old_tips, + state.git_bin.clone(), + std::time::Duration::from_secs(state.config.git_service_timeout_secs), ) .await; if pin_set.full_scan { diff --git a/crates/gitlawb-node/src/git/push_delta.rs b/crates/gitlawb-node/src/git/push_delta.rs index 7ab00816..51b6f836 100644 --- a/crates/gitlawb-node/src/git/push_delta.rs +++ b/crates/gitlawb-node/src/git/push_delta.rs @@ -29,9 +29,9 @@ use std::collections::HashSet; use std::path::{Path, PathBuf}; -use std::process::Command; +use std::time::{Duration, Instant}; -use anyhow::{bail, Context, Result}; +use anyhow::Result; /// Env var that forces the push path to always full-scan, bypassing the delta /// optimization (KTD7 kill-switch). Reuses the already-tested fallback branch @@ -64,11 +64,21 @@ pub enum PinCandidates { /// Fail-closed: any condition where the introduced set cannot be safely /// determined returns [`PinCandidates::FullScanRequired`] rather than a partial /// set, so the caller full-scans instead of silently under-pinning. -pub fn resolve_push_delta(repo_path: &Path, new_tips: &[&str], old_tips: &[&str]) -> PinCandidates { - // KTD7 kill-switch: force the (already-tested) full-scan fallback. The env - // read is split out from the pure logic so the resolver stays unit-testable - // without touching process-global state. - resolve_push_delta_inner(repo_path, new_tips, old_tips, force_full_scan()) +pub fn resolve_push_delta( + repo_path: &Path, + new_tips: &[&str], + old_tips: &[&str], + git_bin: &str, + deadline: Instant, +) -> PinCandidates { + resolve_push_delta_inner( + repo_path, + new_tips, + old_tips, + force_full_scan(), + git_bin, + deadline, + ) } /// Whether the force-full-scan kill-switch env var is set. @@ -83,6 +93,8 @@ fn resolve_push_delta_inner( new_tips: &[&str], old_tips: &[&str], force_full_scan: bool, + git_bin: &str, + deadline: Instant, ) -> PinCandidates { if force_full_scan { tracing::debug!("{FORCE_FULL_SCAN_ENV} set — forcing full scan"); @@ -102,7 +114,7 @@ fn resolve_push_delta_inner( // commit). Bare `cat-file -t` returns `tag` for an annotated tag, and // `for-each-ref %(*objecttype)` peels only one level — neither is correct. for tip in new_tips { - match peeled_object_type(repo_path, tip) { + match peeled_object_type(repo_path, tip, git_bin, deadline) { Some(t) if t == "commit" => {} other => { tracing::debug!( @@ -115,7 +127,7 @@ fn resolve_push_delta_inner( } } - match rev_list_delta(repo_path, new_tips, old_tips) { + match rev_list_delta(repo_path, new_tips, old_tips, git_bin, deadline) { Ok(oids) => PinCandidates::Delta(oids), Err(e) => { tracing::debug!(err = %e, "push-delta rev-list failed — forcing full scan"); @@ -126,41 +138,43 @@ fn resolve_push_delta_inner( /// Return the fully-peeled object type of `sha` (e.g. `commit`, `tree`, /// `blob`), or `None` if the object is missing/unpeelable or git errored. -fn peeled_object_type(repo_path: &Path, sha: &str) -> Option { - let output = Command::new("git") - .args(["cat-file", "-t", &format!("{sha}^{{}}")]) - .current_dir(repo_path) - .output() - .ok()?; - if !output.status.success() { - return None; - } - Some(String::from_utf8_lossy(&output.stdout).trim().to_string()) +fn peeled_object_type( + repo_path: &Path, + sha: &str, + git_bin: &str, + deadline: Instant, +) -> Option { + let peel = format!("{sha}^{{}}"); + let out = crate::git::visibility_pack::run_bounded_git( + git_bin, + &["cat-file", "-t", &peel], + repo_path, + b"", + deadline, + ) + .ok()?; + Some(String::from_utf8_lossy(&out).trim().to_string()) } /// Run `git rev-list --objects --no-object-names --not ` and return /// the bare OID set. Decides on `status.success()` *before* parsing stdout, so /// a walk that prints a valid prefix then errors mid-walk is discarded. -fn rev_list_delta(repo_path: &Path, new_tips: &[&str], old_tips: &[&str]) -> Result> { +fn rev_list_delta( + repo_path: &Path, + new_tips: &[&str], + old_tips: &[&str], + git_bin: &str, + deadline: Instant, +) -> Result> { let mut args: Vec<&str> = vec!["rev-list", "--objects", "--no-object-names"]; args.extend_from_slice(new_tips); if !old_tips.is_empty() { args.push("--not"); args.extend_from_slice(old_tips); } - - let output = Command::new("git") - .args(&args) - .current_dir(repo_path) - .output() - .context("failed to run git rev-list for push delta")?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - bail!("git rev-list failed: {stderr}"); - } - - let stdout = String::from_utf8_lossy(&output.stdout); + let out = + crate::git::visibility_pack::run_bounded_git(git_bin, &args, repo_path, b"", deadline)?; + let stdout = String::from_utf8_lossy(&out); Ok(stdout .lines() .map(|l| l.trim().to_string()) @@ -175,23 +189,19 @@ fn rev_list_delta(repo_path: &Path, new_tips: &[&str], old_tips: &[&str]) -> Res /// reconciliation sweep relies on. It returns *all* objects (including /// unreachable/dangling ones), which is what the sweep needs to catch /// stragglers — do not swap it for a reachability walk. -pub fn list_all_objects(repo_path: &Path) -> Result> { - let output = Command::new("git") - .args([ +pub fn list_all_objects(repo_path: &Path, git_bin: &str, deadline: Instant) -> Result> { + let out = crate::git::visibility_pack::run_bounded_git( + git_bin, + &[ "cat-file", "--batch-all-objects", "--batch-check=%(objectname)", - ]) - .current_dir(repo_path) - .output() - .context("failed to run git cat-file")?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - bail!("git cat-file failed: {stderr}"); - } - - let stdout = String::from_utf8_lossy(&output.stdout); + ], + repo_path, + b"", + deadline, + )?; + let stdout = String::from_utf8_lossy(&out); Ok(stdout .lines() .map(|l| l.trim().to_string()) @@ -203,23 +213,23 @@ pub fn list_all_objects(repo_path: &Path) -> Result> { /// `--batch-check='%(objectname) %(objecttype)'`. The pin path's fail-closed /// filter needs to tell blobs (content, withholdable) from commits/trees /// (structural, never withheld) without typing the candidate list itself. -pub fn list_all_objects_with_type(repo_path: &Path) -> Result> { - let output = Command::new("git") - .args([ +pub fn list_all_objects_with_type( + repo_path: &Path, + git_bin: &str, + deadline: Instant, +) -> Result> { + let out = crate::git::visibility_pack::run_bounded_git( + git_bin, + &[ "cat-file", "--batch-all-objects", "--batch-check=%(objectname) %(objecttype)", - ]) - .current_dir(repo_path) - .output() - .context("failed to run git cat-file")?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - bail!("git cat-file failed: {stderr}"); - } - - let stdout = String::from_utf8_lossy(&output.stdout); + ], + repo_path, + b"", + deadline, + )?; + let stdout = String::from_utf8_lossy(&out); Ok(stdout .lines() .filter_map(|l| { @@ -235,8 +245,12 @@ pub fn list_all_objects_with_type(repo_path: &Path) -> Result Result> { - Ok(list_all_objects_with_type(repo_path)? +pub fn all_blob_oids( + repo_path: &Path, + git_bin: &str, + deadline: Instant, +) -> Result> { + Ok(list_all_objects_with_type(repo_path, git_bin, deadline)? .into_iter() .filter(|(_, ty)| ty == "blob") .map(|(oid, _)| oid) @@ -272,18 +286,22 @@ pub async fn resolve_candidates_for_push( repo_path: PathBuf, new_tips: Vec, old_tips: Vec, + git_bin: String, + timeout: Duration, ) -> PinCandidateSet { tokio::task::spawn_blocking(move || { + // ONE shared deadline for the whole scan, per jatmn ("the same deadline"). + let deadline = Instant::now() + timeout; let new_refs: Vec<&str> = new_tips.iter().map(String::as_str).collect(); let old_refs: Vec<&str> = old_tips.iter().map(String::as_str).collect(); - match resolve_push_delta(&repo_path, &new_refs, &old_refs) { + match resolve_push_delta(&repo_path, &new_refs, &old_refs, &git_bin, deadline) { PinCandidates::Delta(objs) => { tracing::info!(delta = objs.len(), repo = %repo_path.display(), "pin candidate set from push delta"); PinCandidateSet { candidates: objs, full_scan: false } } PinCandidates::FullScanRequired => { tracing::warn!(repo = %repo_path.display(), "pin delta unavailable (non-commit tip, git error, or force-full-scan) — full-scan fallback"); - match list_all_objects(&repo_path) { + match list_all_objects(&repo_path, &git_bin, deadline) { Ok(objs) => PinCandidateSet { candidates: objs, full_scan: true }, Err(e) => { tracing::warn!(repo = %repo_path.display(), err = %e, "full-scan fallback failed; pinning nothing this push (reconciliation sweep backstops)"); @@ -304,8 +322,13 @@ pub async fn resolve_candidates_for_push( mod tests { use super::*; use std::collections::HashSet; + use std::process::Command; use tempfile::TempDir; + fn td() -> std::time::Instant { + std::time::Instant::now() + std::time::Duration::from_secs(600) + } + /// Minimal git helper for building test repos. struct Repo { _td: TempDir, @@ -363,9 +386,10 @@ mod tests { let repo = Repo::new(); let c1 = repo.commit_file("a.txt", "one\n"); let c2 = repo.commit_file("b.txt", "two\n"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&c2], &[&c1])) - .into_iter() - .collect(); + let got: HashSet = + delta(resolve_push_delta(&repo.path, &[&c2], &[&c1], "git", td())) + .into_iter() + .collect(); // The new blob b.txt and commit c2 are in the delta; the old blob a.txt // and commit c1 are not. let new_blob = repo.rev("HEAD:b.txt"); @@ -383,7 +407,7 @@ mod tests { // genuinely new objects, never fewer. let repo = Repo::new(); let c1 = repo.commit_file("a.txt", "one\n"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&c1], &[])) + let got: HashSet = delta(resolve_push_delta(&repo.path, &[&c1], &[], "git", td())) .into_iter() .collect(); assert!(got.contains(&c1)); @@ -399,9 +423,15 @@ mod tests { // Rewrite history: reset to base, commit a different file. repo.git(&["reset", "-q", "--hard", &base]); let new_tip = repo.commit_file("c.txt", "three\n"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&new_tip], &[&old_tip])) - .into_iter() - .collect(); + let got: HashSet = delta(resolve_push_delta( + &repo.path, + &[&new_tip], + &[&old_tip], + "git", + td(), + )) + .into_iter() + .collect(); assert!(got.contains(&new_tip), "new tip in delta"); assert!(got.contains(&repo.rev(&format!("{new_tip}:c.txt")))); // No error; force-push computes new-minus-old cleanly. @@ -413,7 +443,7 @@ mod tests { repo.commit_file("a.txt", "one\n"); // All updates were deletions => new_tips empty after the caller strips zeros. assert_eq!( - resolve_push_delta(&repo.path, &[], &[ZERO]), + resolve_push_delta(&repo.path, &[], &[ZERO], "git", td()), PinCandidates::Delta(Vec::new()) ); } @@ -424,7 +454,7 @@ mod tests { repo.commit_file("a.txt", "one\n"); let blob = repo.rev("HEAD:a.txt"); assert_eq!( - resolve_push_delta(&repo.path, &[&blob], &[]), + resolve_push_delta(&repo.path, &[&blob], &[], "git", td()), PinCandidates::FullScanRequired, "a blob tip must force full scan (rev-list would exit 0 and walk it)" ); @@ -436,7 +466,7 @@ mod tests { repo.commit_file("a.txt", "one\n"); let tree = repo.rev("HEAD^{tree}"); assert_eq!( - resolve_push_delta(&repo.path, &[&tree], &[]), + resolve_push_delta(&repo.path, &[&tree], &[], "git", td()), PinCandidates::FullScanRequired ); } @@ -449,7 +479,7 @@ mod tests { repo.git(&["tag", "-a", "treetag", "-m", "x", &tree]); let tag = repo.rev("treetag"); assert_eq!( - resolve_push_delta(&repo.path, &[&tag], &[]), + resolve_push_delta(&repo.path, &[&tag], &[], "git", td()), PinCandidates::FullScanRequired, "annotated tag peeling to a tree must force full scan" ); @@ -465,7 +495,7 @@ mod tests { repo.git(&["tag", "-a", "t2", "-m", "x", &t1]); let t2 = repo.rev("t2"); assert_eq!( - resolve_push_delta(&repo.path, &[&t2], &[]), + resolve_push_delta(&repo.path, &[&t2], &[], "git", td()), PinCandidates::FullScanRequired ); } @@ -480,7 +510,7 @@ mod tests { let c1 = repo.commit_file("a.txt", "one\n"); repo.git(&["tag", "-a", "rel", "-m", "release", &c1]); let tag = repo.rev("rel"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&tag], &[])) + let got: HashSet = delta(resolve_push_delta(&repo.path, &[&tag], &[], "git", td())) .into_iter() .collect(); assert!( @@ -501,7 +531,7 @@ mod tests { let t1 = repo.rev("t1"); repo.git(&["tag", "-a", "t2", "-m", "x", &t1]); let t2 = repo.rev("t2"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&t2], &[])) + let got: HashSet = delta(resolve_push_delta(&repo.path, &[&t2], &[], "git", td())) .into_iter() .collect(); assert!( @@ -516,9 +546,14 @@ mod tests { let repo = Repo::new(); let c1 = repo.commit_file("a.txt", "one\n"); let c2 = repo.commit_file("b.txt", "two\n"); - let set = - resolve_candidates_for_push(repo.path.clone(), vec![c2.clone()], vec![c1.clone()]) - .await; + let set = resolve_candidates_for_push( + repo.path.clone(), + vec![c2.clone()], + vec![c1.clone()], + "git".to_string(), + std::time::Duration::from_secs(600), + ) + .await; assert!(!set.full_scan, "happy-path delta is not a full scan"); let got: HashSet = set.candidates.into_iter().collect(); let new_blob = repo.rev("HEAD:b.txt"); @@ -536,8 +571,18 @@ mod tests { let repo = Repo::new(); repo.commit_file("a.txt", "one\n"); let blob = repo.rev("HEAD:a.txt"); - let all: HashSet = list_all_objects(&repo.path).unwrap().into_iter().collect(); - let set = resolve_candidates_for_push(repo.path.clone(), vec![blob], vec![]).await; + let all: HashSet = list_all_objects(&repo.path, "git", td()) + .unwrap() + .into_iter() + .collect(); + let set = resolve_candidates_for_push( + repo.path.clone(), + vec![blob], + vec![], + "git".to_string(), + std::time::Duration::from_secs(600), + ) + .await; assert!(set.full_scan, "non-commit tip is signalled as a full scan"); let got: HashSet = set.candidates.into_iter().collect(); assert_eq!(got, all, "non-commit tip falls back to full repo scan"); @@ -549,7 +594,7 @@ mod tests { repo.commit_file("a.txt", "one\n"); let bogus = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"; assert_eq!( - resolve_push_delta(&repo.path, &[bogus], &[]), + resolve_push_delta(&repo.path, &[bogus], &[], "git", td()), PinCandidates::FullScanRequired, "a missing/corrupt tip OID must force full scan" ); @@ -564,7 +609,7 @@ mod tests { let c1 = repo.commit_file("a.txt", "one\n"); let c2 = repo.commit_file("b.txt", "two\n"); let old_tree = repo.rev(&format!("{c1}^{{tree}}")); - let result = resolve_push_delta(&repo.path, &[&c2], &[&old_tree]); + let result = resolve_push_delta(&repo.path, &[&c2], &[&old_tree], "git", td()); match result { PinCandidates::FullScanRequired => {} // safe: caller full-scans PinCandidates::Delta(objs) => { @@ -583,10 +628,15 @@ mod tests { // branch2 from base advances independently repo.git(&["checkout", "-q", "-b", "branch2", &base]); let b2 = repo.commit_file("c.txt", "three\n"); - let got: HashSet = - delta(resolve_push_delta(&repo.path, &[&b1, &b2], &[&base, &base])) - .into_iter() - .collect(); + let got: HashSet = delta(resolve_push_delta( + &repo.path, + &[&b1, &b2], + &[&base, &base], + "git", + td(), + )) + .into_iter() + .collect(); assert!(got.contains(&b1), "branch1 new commit"); assert!(got.contains(&b2), "branch2 new commit"); } @@ -595,7 +645,7 @@ mod tests { fn empty_repo_no_tips_yields_empty_delta() { let repo = Repo::new(); assert_eq!( - resolve_push_delta(&repo.path, &[], &[]), + resolve_push_delta(&repo.path, &[], &[], "git", td()), PinCandidates::Delta(Vec::new()) ); } @@ -609,12 +659,12 @@ mod tests { let repo = Repo::new(); let c1 = repo.commit_file("a.txt", "one\n"); assert_eq!( - resolve_push_delta_inner(&repo.path, &[&c1], &[], true), + resolve_push_delta_inner(&repo.path, &[&c1], &[], true, "git", td()), PinCandidates::FullScanRequired ); // And with the flag off, the same push yields a Delta. assert!(matches!( - resolve_push_delta_inner(&repo.path, &[&c1], &[], false), + resolve_push_delta_inner(&repo.path, &[&c1], &[], false, "git", td()), PinCandidates::Delta(_) )); } @@ -624,7 +674,7 @@ mod tests { let repo = Repo::new(); repo.commit_file("a.txt", "one\n"); repo.commit_file("b.txt", "two\n"); - let all = list_all_objects(&repo.path).unwrap(); + let all = list_all_objects(&repo.path, "git", td()).unwrap(); // 2 commits + 2 trees + 2 blobs = 6 objects. assert_eq!(all.len(), 6, "got: {all:?}"); } @@ -642,7 +692,7 @@ mod tests { std::fs::write(repo.path.join("orphan.bin"), b"dangling\n").unwrap(); let dangling = repo.git(&["hash-object", "-w", "orphan.bin"]); - let blobs = all_blob_oids(&repo.path).unwrap(); + let blobs = all_blob_oids(&repo.path, "git", td()).unwrap(); assert!(blobs.contains(&reachable_blob), "reachable blob present"); assert!( blobs.contains(&dangling), @@ -652,7 +702,7 @@ mod tests { assert!(!blobs.contains(&tree), "tree is not a blob"); // The typed lister tags each object; spot-check the dangling blob's type. - let typed = list_all_objects_with_type(&repo.path).unwrap(); + let typed = list_all_objects_with_type(&repo.path, "git", td()).unwrap(); assert!( typed .iter() @@ -660,4 +710,40 @@ mod tests { "dangling object is typed as a blob" ); } + + #[cfg(unix)] + #[test] + fn list_all_objects_times_out_on_a_hung_git_instead_of_blocking() { + use std::os::unix::fs::PermissionsExt; + use std::time::{Duration, Instant}; + let dir = tempfile::TempDir::new().unwrap(); + // Fake git: hang on cat-file (bounded to 30s so a broken test can't wedge). + let fake = dir.path().join("fakegit"); + std::fs::write( + &fake, + "#!/bin/sh\ncase \"$1\" in\n cat-file) i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;;\n *) : ;;\nesac\n", + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + let git_bin = fake.to_str().unwrap().to_string(); + + let (tx, rx) = std::sync::mpsc::channel(); + let path = dir.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(list_all_objects( + &path, + &git_bin, + Instant::now() + Duration::from_millis(150), + )); + }); + let res = rx.recv_timeout(Duration::from_secs(10)).expect( + "list_all_objects must return within the watchdog budget, not block on a hung git", + ); + assert!( + res.is_err(), + "a hung git must make list_all_objects error out, not hang" + ); + } } diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index ec55423e..9f40cd4b 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -42,7 +42,7 @@ const WATCHDOG_TERM_GRACE: Duration = Duration::from_secs(1); /// drive the teardown in tests without mutating the process-global PATH; /// `stdin_bytes` feeds children that read stdin (empty for the arg-only children). #[cfg(unix)] -fn run_bounded_git( +pub(crate) fn run_bounded_git( git_bin: &str, args: &[&str], repo_path: &Path, @@ -171,7 +171,7 @@ fn run_bounded_git( /// the Unix version's signature and result semantics so every caller compiles on all /// targets (#174). #[cfg(not(unix))] -fn run_bounded_git( +pub(crate) fn run_bounded_git( git_bin: &str, args: &[&str], repo_path: &Path, @@ -1235,7 +1235,12 @@ esac\n"; String::from_utf8_lossy(&out.stdout).trim().to_string() }; - let all_blobs = crate::git::push_delta::all_blob_oids(&work).unwrap(); + let all_blobs = crate::git::push_delta::all_blob_oids( + &work, + "git", + std::time::Instant::now() + std::time::Duration::from_secs(600), + ) + .unwrap(); assert!( all_blobs.contains(&dangling_oid), "precondition: the dangling blob is in the all-objects universe" From 4410b834df04163c6121629e3e3b05b227501e27 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 10:46:24 -0500 Subject: [PATCH 24/95] docs(node): correct concurrency + timeout docs to match the final git routing (#174) jatmn P2 (INV-24): the config help, .env.example, and README described routing the handler no longer does. - The read pool serves upload-pack and the UPLOAD-PACK info/refs advertisement only; the anon receive-pack info/refs advertisement draws from its own dedicated pool (sized like the write pool but disjoint), not the read pool and not the write pool. - The write pool is the authenticated push POST only. - The per-caller read cap is keyed on the resolved source IP, never the DID: a signature does not move a caller off it (the docs claimed a per-DID budget, which would tell a NATed authenticated client it escapes the shared cap when it does not). - git_service_timeout_secs now also bounds the push-side pin-candidate discovery (rev-list / cat-file), reaped via process-group teardown, alongside the info/refs advertisements and the withheld-blob walk. --- .env.example | 40 ++++++++++++--------- README.md | 2 +- crates/gitlawb-node/src/config.rs | 60 +++++++++++++++++-------------- 3 files changed, 57 insertions(+), 45 deletions(-) diff --git a/.env.example b/.env.example index cbf3db3a..0ef193f8 100644 --- a/.env.example +++ b/.env.example @@ -109,29 +109,35 @@ GITLAWB_MAX_PACK_BYTES=2147483648 # Max seconds a served git upload-pack / receive-pack (clone / push) may run # before it is aborted with a 504. Bounds a hung git that would otherwise pin a -# worker and, on push, the repo write lock. Also bounds the info/refs -# advertisement and the withheld-blob pack build (#174). Default 600. +# worker and, on push, the repo write lock. Also bounds both info/refs +# advertisements, the withheld-blob pack build, and the push-side candidate +# discovery (rev-list / cat-file), all reaped via process-group teardown (#174). +# Default 600. GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600 -# Max concurrent git read ops (upload-pack + info/refs advertisements) served at -# once, a global pool separate from the push pool below. Over-cap sheds a clean -# 503 + Retry-After. Anonymous reads draw from here, so pair it with -# GITLAWB_MAX_CONCURRENT_READS_PER_CALLER (below) so one caller cannot monopolize -# the pool. Default 128. +# Max concurrent git READ ops (upload-pack + the upload-pack info/refs +# advertisement) served at once, a global pool separate from the push pool below. +# The anon receive-pack info/refs advertisement has its OWN pool (see below), not +# this one. Over-cap sheds a clean 503 + Retry-After. Anonymous reads draw from +# here, so pair it with GITLAWB_MAX_CONCURRENT_READS_PER_CALLER (below) so one +# caller cannot monopolize the pool. Default 128. GITLAWB_MAX_CONCURRENT_GIT_OPS=128 -# Max concurrent git-receive-pack (push) operations, in a pool separate from the -# read pool (GITLAWB_MAX_CONCURRENT_GIT_OPS) so anonymous reads cannot shed an -# authenticated push at admission. Over-cap sheds a 503 + Retry-After. Default 32. +# Max concurrent git-receive-pack (push) POST operations, in a pool separate from +# the read pool (GITLAWB_MAX_CONCURRENT_GIT_OPS) so anonymous reads cannot shed an +# authenticated push at admission. The anon receive-pack info/refs advertisement +# runs in a SEPARATE pool of the same size (disjoint from this one), so an +# advertisement flood cannot shed a push either. Over-cap sheds a 503 + +# Retry-After. Default 32. GITLAWB_MAX_CONCURRENT_GIT_PUSHES=32 -# Max concurrent read ops (upload-pack + info/refs advertisements) a single caller -# may hold, so one caller cannot monopolize the read pool. Keyed per-DID when -# signed, else per-source-IP. The per-IP key is only as granular as -# GITLAWB_TRUSTED_PROXY below: left unset, a node behind an edge/NAT keys all -# anonymous callers on the edge IP and this collapses to one global anonymous cap. -# Set GITLAWB_TRUSTED_PROXY for per-client keying; a high-fanout caller (CI behind -# one NAT) should authenticate for a per-DID budget or the operator raises this. +# Max concurrent read ops (upload-pack + the upload-pack info/refs advertisement) +# a single caller may hold, so one caller cannot monopolize the read pool. Keyed +# on the resolved SOURCE IP, never the DID: a signature does not move a caller off +# this cap. The source-IP key is only as granular as GITLAWB_TRUSTED_PROXY below: +# left unset, a node behind an edge/NAT keys all callers on the edge IP and this +# collapses to one global cap. Set GITLAWB_TRUSTED_PROXY for per-client keying; a +# high-fanout caller (CI behind one NAT) then needs the operator to raise this. # Default 16. GITLAWB_MAX_CONCURRENT_READS_PER_CALLER=16 diff --git a/README.md b/README.md index 0f760449..542a988e 100644 --- a/README.md +++ b/README.md @@ -343,7 +343,7 @@ Important node settings: | `GITLAWB_REQUIRE_SIGNED_PEER_WRITES` | Require signed peer announce/sync writes. | | `GITLAWB_AUTO_SYNC` | Enable automatic sync from known peers. | | `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. | -| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths), which is reaped at the deadline. | +| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | | `GITLAWB_IRYS_URL` | Optional Irys/Arweave permanent anchoring. | diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 90a061e0..d0a4c319 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -243,24 +243,25 @@ pub struct Config { /// cap is a different axis (500 connections each fan out to git + /// pack-objects + threads). Size below the process budget with headroom. /// - /// This is the READ pool (`git_read_semaphore`): upload-pack and both info/refs - /// advertisements. The authenticated push POST draws from a separate write pool - /// (`max_concurrent_git_pushes`) that anonymous reads can never reach, and each - /// caller is additionally bounded by `max_concurrent_reads_per_caller`, so an - /// anonymous flood cannot shed the actual push nor monopolize reads (#174). (The - /// receive-pack advertisement itself shares the read pool; a shed advertisement - /// is a cheap retryable GET, and the write POST it precedes always has capacity.) + /// This is the READ pool (`git_read_semaphore`): upload-pack and the UPLOAD-PACK + /// `info/refs` advertisement only. The authenticated push POST draws from a + /// separate write pool (`max_concurrent_git_pushes`) that anonymous reads can + /// never reach, and each read caller is additionally bounded by + /// `max_concurrent_reads_per_caller`, so an anonymous flood cannot shed the actual + /// push nor monopolize reads (#174). The anon-reachable RECEIVE-PACK `info/refs` + /// advertisement draws from its OWN dedicated pool (sized like the write pool but + /// disjoint), so an advertisement flood can never occupy a permit the + /// authenticated push POST needs at admission (#174). /// /// A permit is held for the whole op. Every git subprocess that STREAMS is /// duration-bounded and reaps its process group on disconnect: upload-pack, /// receive-pack, and both info/refs advertisements run under /// `git_service_timeout_secs` with `process_group(0)` teardown, and the - /// withheld-blob (`upload_pack_excluding`) pack-objects stage runs on the async - /// side under the same teardown (#174 closed the two duration/cancellation gaps - /// this comment previously tracked). The one remaining blocking stage is the - /// `rev-list` object enumeration in the withheld-blob path — a bounded walk that - /// terminates, run off the async runtime; it is not process-group-reaped on - /// disconnect, so a stuck rev-list can still hold its slot until git exits. + /// withheld-blob (`upload_pack_excluding`) pack-objects stage plus the push-side + /// candidate-discovery children (`rev-list` / `cat-file`) now run under the same + /// bounded runner with process-group teardown, so a stuck git child no longer + /// holds its slot indefinitely (#174 closed the duration/cancellation gaps this + /// comment previously tracked). /// /// Default: 128. Must be between 1 and 1_048_576; the ceiling keeps the value /// well under tokio's `Semaphore` permit limit so an oversized value is a @@ -273,10 +274,14 @@ pub struct Config { )] pub max_concurrent_git_ops: usize, - /// Maximum number of concurrent `git-receive-pack` (push) operations. Pushes - /// draw from this dedicated pool, separate from `max_concurrent_git_ops` - /// (reads), so a flood of anonymous reads cannot shed an authenticated push at - /// admission (#174). Beyond this a push sheds a clean 503 + Retry-After. + /// Maximum number of concurrent `git-receive-pack` (push) operations. The + /// authenticated push POST draws from this dedicated pool, separate from + /// `max_concurrent_git_ops` (reads), so a flood of anonymous reads cannot shed an + /// authenticated push at admission (#174). The anon-reachable receive-pack + /// `info/refs` advertisement runs in a SEPARATE pool of the same size (derived + /// from this knob), disjoint from this one, so an advertisement flood cannot + /// occupy a POST's slot either (#174). Beyond this a push sheds a clean 503 + + /// Retry-After. /// /// Default: 32. Must be between 1 and 1_048_576 (the ceiling keeps the value /// under tokio's `Semaphore` permit limit so an oversized value is a clean CLI @@ -289,16 +294,17 @@ pub struct Config { )] pub max_concurrent_git_pushes: usize, - /// Maximum concurrent read operations (`upload-pack` and the `info/refs` - /// advertisements) a single caller may hold at once, so one caller cannot - /// monopolize the `max_concurrent_git_ops` read pool (#174). Callers are keyed - /// per-DID when signed, else per-source-IP. IMPORTANT: the per-source-IP key is - /// only as granular as `GITLAWB_TRUSTED_PROXY`. Left unset (the default), a node - /// behind an edge/NAT keys all anonymous callers on the edge IP, so this cap - /// collapses to a single global anonymous cap rather than per-client. Set - /// `GITLAWB_TRUSTED_PROXY` to key on the real client; a known high-fanout caller - /// (a CI fleet behind one NAT) should authenticate for a per-DID budget or the - /// operator raises this. Over-cap for a caller sheds a clean 503 + Retry-After. + /// Maximum concurrent read operations (`upload-pack` and the upload-pack + /// `info/refs` advertisement) a single caller may hold at once, so one caller + /// cannot monopolize the `max_concurrent_git_ops` read pool (#174). Callers are + /// keyed on the RESOLVED SOURCE IP, never the DID — a signature does not move a + /// caller off this cap, so an authenticated client cannot mint DIDs to escape it. + /// IMPORTANT: the source-IP key is only as granular as `GITLAWB_TRUSTED_PROXY`. + /// Left unset (the default), a node behind an edge/NAT keys all callers on the + /// edge IP, so this cap collapses to a single global cap rather than per-client. + /// Set `GITLAWB_TRUSTED_PROXY` to key on the real client; a high-fanout caller (a + /// CI fleet behind one NAT) then needs the operator to raise this. Over-cap for a + /// caller sheds a clean 503 + Retry-After. /// /// Default: 16. Must be between 1 and 1_048_576. #[arg( From cf97405bc188e67beaff37c6a708694dda04fe94 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 10:55:57 -0500 Subject: [PATCH 25/95] test(node): execution-cover the delta-path push-scan timeout wiring (#174) The finding-2 wiring test covered only list_all_objects (the full-scan fallback). Add delta_path_exec_fns_time_out_on_a_hung_git covering the common push path: peeled_object_type (cat-file), rev_list_delta (rev-list), and list_all_objects_with_type (cat-file, which all_blob_oids delegates to). Each must return within the watchdog budget on a hung git; reverting any to a bare Command::output() blocks past the recv budget (RED-verified). --- crates/gitlawb-node/src/git/push_delta.rs | 56 +++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/crates/gitlawb-node/src/git/push_delta.rs b/crates/gitlawb-node/src/git/push_delta.rs index 51b6f836..962c9c65 100644 --- a/crates/gitlawb-node/src/git/push_delta.rs +++ b/crates/gitlawb-node/src/git/push_delta.rs @@ -746,4 +746,60 @@ mod tests { "a hung git must make list_all_objects error out, not hang" ); } + + /// #174 finding 2: the DELTA-path children — `peeled_object_type` (cat-file), + /// `rev_list_delta` (rev-list) — and `list_all_objects_with_type` (cat-file) are + /// the remaining candidate-discovery execs (the common push path). Each must + /// return within the watchdog budget on a hung git rather than block and pin the + /// write permit. Revert any one to a bare `Command::output()` and its arm blocks + /// past the recv budget (RED). + #[cfg(unix)] + #[test] + fn delta_path_exec_fns_time_out_on_a_hung_git() { + use std::os::unix::fs::PermissionsExt; + use std::time::{Duration, Instant}; + let dir = tempfile::TempDir::new().unwrap(); + // Fake git hangs on BOTH cat-file and rev-list (bounded 30s so a broken + // test can't leak a permanent orphan or wedge the suite). + let fake = dir.path().join("fakegit"); + std::fs::write( + &fake, + "#!/bin/sh\ncase \"$1\" in\n cat-file|rev-list) i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;;\n *) : ;;\nesac\n", + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + let git_bin = fake.to_str().unwrap().to_string(); + let path = dir.path().to_path_buf(); + + // Run `f` on a thread and require it to RETURN (not block) within 10s. + fn returns_within(f: impl FnOnce() -> T + Send + 'static) -> T { + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = tx.send(f()); + }); + rx.recv_timeout(Duration::from_secs(10)).expect( + "a delta-path exec fn must return within the watchdog budget, not block on a hung git", + ) + } + let dl = || Instant::now() + Duration::from_millis(150); + let sha = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"; + + let (p, g, d) = (path.clone(), git_bin.clone(), dl()); + assert!( + returns_within(move || peeled_object_type(&p, sha, &g, d)).is_none(), + "peeled_object_type must time out to None on a hung cat-file" + ); + let (p, g, d) = (path.clone(), git_bin.clone(), dl()); + assert!( + returns_within(move || rev_list_delta(&p, &[sha], &[], &g, d)).is_err(), + "rev_list_delta must error out on a hung rev-list" + ); + let (p, g, d) = (path.clone(), git_bin.clone(), dl()); + assert!( + returns_within(move || list_all_objects_with_type(&p, &g, d)).is_err(), + "list_all_objects_with_type must error out on a hung cat-file" + ); + } } From a030eefb704d68a008e760a01077abc823cbe258 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 16:01:49 -0500 Subject: [PATCH 26/95] test(node): retry the ETXTBSY exec race in build_filtered_pack timeout tests (#174) build_filtered_pack_times_out_a_hung_rev_list and _a_hung_pack_objects spawn a freshly-written fake git directly, bypassing the fake_git_run_with_pids retry the rest of the module uses. Under cargo test fork-storm load a concurrent worker forks while the fake's write fd is still open, so execve returns ETXTBSY and the raw io::Error surfaces where the test asserts GitServiceTimeout, failing test (stable) intermittently. Route both tests through build_filtered_pack_or_exec_race_retry, which retries ONLY errno 26 (ETXTBSY) and keeps the outer 10s watchdog per attempt, so a genuinely-missing async timeout bound still trips the watchdog and a wrong error type still fails at the assertion. Production paths unchanged. --- crates/gitlawb-node/src/git/smart_http.rs | 99 ++++++++++++++++------- 1 file changed, 71 insertions(+), 28 deletions(-) diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index 206e7b27..f885aef3 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -1160,6 +1160,61 @@ mod tests { ); } + /// True when `err` is the transient ETXTBSY exec race a freshly-written fake + /// `git` hits under fork-storm load — a concurrent worker forked while this + /// file's write fd was still open, so `execve` sees it as busy (the same race + /// `fake_git_run_with_pids` retries). Deliberately narrow: only this raw-OS + /// error is retried, so a wrong error *type* (e.g. a missing async bound + /// surfacing as anything other than `GitServiceTimeout`) still fails loudly at + /// the caller's assertion instead of being swallowed. `spawn()?` propagates the + /// `io::Error` with no context, so it survives as the anyhow root. + #[cfg(unix)] + fn is_transient_exec_race(err: &anyhow::Error) -> bool { + // ETXTBSY == 26 on Linux and the BSDs/macOS; std has no stable ErrorKind. + err.downcast_ref::() + .and_then(std::io::Error::raw_os_error) + == Some(26) + } + + /// Drive `build_filtered_pack` under a per-attempt watchdog, retrying ONLY the + /// transient ETXTBSY exec race and returning the terminal error for the caller + /// to classify. Every attempt keeps its own outer `tokio::time::timeout`, so a + /// MISSING async bound — the regression the `build_filtered_pack_times_out_*` + /// tests guard — still trips the watchdog loudly on every attempt: the retry + /// can only absorb a fast exec failure, never a hang (a hang never returns, so + /// it can never reach the retry decision). + #[cfg(unix)] + async fn build_filtered_pack_or_exec_race_retry( + git_bin: &str, + repo_path: &std::path::Path, + withheld: &HashSet, + stage_timeout: Duration, + ) -> anyhow::Error { + for i in 0..FAKE_GIT_RETRY_ATTEMPTS { + let result = tokio::time::timeout( + Duration::from_secs(10), + build_filtered_pack(git_bin, repo_path, withheld, stage_timeout), + ) + .await + .expect( + "build_filtered_pack must return within the watchdog — the git stage \ + must be timeout-bounded, not an uncancellable spawn_blocking", + ); + let err = result.expect_err("a hung git stage must return an error, not hang"); + if is_transient_exec_race(&err) { + // Fresh-fake-git ETXTBSY: back off (growing) so a bursty fork-pressure + // spike subsides before retrying, per fake_git_run_with_pids. + tokio::time::sleep(Duration::from_millis(FAKE_GIT_BACKOFF_STEP_MS * (i + 1))).await; + continue; + } + return err; + } + panic!( + "fake git kept hitting ETXTBSY after {FAKE_GIT_RETRY_ATTEMPTS} attempts \ + (persistent exec failure, not a transient parallel-runner miss)" + ); + } + // Dropping the request future mid-flight (client disconnect) must SIGTERM the // whole group so git AND its pack-objects grandchild die together. Goes RED // if `process_group(0)` or the guard-arming is removed: without its own @@ -1314,21 +1369,15 @@ mod tests { let git_bin = write_fake_git(tmp.path(), body); let withheld = HashSet::new(); - let result = tokio::time::timeout( - Duration::from_secs(10), - build_filtered_pack( - git_bin.to_str().unwrap(), - tmp.path(), - &withheld, - Duration::from_millis(200), - ), + // Retry only the transient ETXTBSY exec race (see the helper); the + // per-attempt watchdog keeps the missing-bound regression loud. + let err = build_filtered_pack_or_exec_race_retry( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_millis(200), ) - .await - .expect( - "build_filtered_pack must return within the watchdog — the pack-objects \ - stage must be timeout-bounded, not an uncancellable spawn_blocking", - ); - let err = result.expect_err("a hung pack-objects must return an error, not hang"); + .await; assert!( err.downcast_ref::().is_some(), "a hung pack-objects must abort with GitServiceTimeout, got: {err}" @@ -1352,21 +1401,15 @@ mod tests { let git_bin = write_fake_git(tmp.path(), body); let withheld = HashSet::new(); - let result = tokio::time::timeout( - Duration::from_secs(10), - build_filtered_pack( - git_bin.to_str().unwrap(), - tmp.path(), - &withheld, - Duration::from_millis(200), - ), + // Retry only the transient ETXTBSY exec race (see the helper); the + // per-attempt watchdog keeps the missing-bound regression loud. + let err = build_filtered_pack_or_exec_race_retry( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_millis(200), ) - .await - .expect( - "build_filtered_pack must return within the watchdog — the rev-list \ - stage must be timeout-bounded, not an uncancellable spawn_blocking", - ); - let err = result.expect_err("a hung rev-list must return an error, not hang"); + .await; assert!( err.downcast_ref::().is_some(), "a hung rev-list must abort with GitServiceTimeout, got: {err}" From 3cc01890fbb3b395a336df951cff0e5b89393530 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 14 Jul 2026 19:27:12 -0500 Subject: [PATCH 27/95] fix(node): keep the visibility-walk watchdog armed until the child is reaped (#174) A group member (or the leader itself) that closes stdout before the deadline made the stdout drain return EOF early, which stood the watchdog down via done_tx before it could ever fire; child.wait() then blocked on the still-live child, pinning the walk thread and its read/write permit past GITLAWB_GIT_SERVICE_TIMEOUT_SECS. This is distinct from the descendant case already covered: there the leader sleeps until the deadline so the watchdog does time out; here the drain-EOF races ahead of it. Stand the watchdog down only once the child has actually terminated, detected without reaping (waitid + WNOWAIT) so the leader's pid stays unreaped and its pgid un-recycled until the watchdog finishes its teardown and is joined. Past the deadline the watchdog still owns the full SIGTERM -> grace -> SIGKILL sequence. Regression: run_bounded_git_reaps_a_leader_that_closes_stdout_then_hangs, RED before (blocks on child.wait past the recv budget), GREEN after; the three existing teardown tests still pass (no recycle-hazard regression). --- .../gitlawb-node/src/git/visibility_pack.rs | 116 ++++++++++++++++-- 1 file changed, 108 insertions(+), 8 deletions(-) diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index 9f40cd4b..9f07020d 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -41,6 +41,31 @@ const WATCHDOG_TERM_GRACE: Duration = Duration::from_secs(1); /// the serve handler maps it to 504. `git_bin` is injectable so a fake `git` can /// drive the teardown in tests without mutating the process-global PATH; /// `stdin_bytes` feeds children that read stdin (empty for the arg-only children). +/// Returns true if `pid` (a process-group leader we spawned) has terminated, WITHOUT +/// reaping it. `waitid(..., WNOWAIT)` reports the exit state but leaves the child +/// waitable, so the caller's later `child.wait()` still collects the status and the +/// pid/pgid stays live until then — which is what keeps the watchdog's `kill(-pgid)` +/// teardown from ever racing a recycled pgid. Used to distinguish "the child actually +/// exited" from "the child merely closed stdout" after the drain returns (#174 P1-a). +#[cfg(unix)] +fn child_terminated_without_reaping(pid: i32) -> bool { + // SAFETY: waitid writes only into the zeroed siginfo and borrows no Rust memory; + // WNOWAIT leaves the child unreaped, WNOHANG makes the probe non-blocking. + let mut info: libc::siginfo_t = unsafe { std::mem::zeroed() }; + let rc = unsafe { + libc::waitid( + libc::P_PID, + pid as libc::id_t, + &mut info, + libc::WEXITED | libc::WNOHANG | libc::WNOWAIT, + ) + }; + // rc == 0 with si_pid == 0 means "no state change yet" (still running); a non-zero + // si_pid means the child has entered a waitable, exited state. EINTR/other errors + // (rc != 0) are treated as "not yet terminated" and the caller re-polls. + rc == 0 && unsafe { info.si_pid() } != 0 +} + #[cfg(unix)] pub(crate) fn run_bounded_git( git_bin: &str, @@ -132,14 +157,28 @@ pub(crate) fn run_bounded_git( // is unreachable in userspace (no signal reaps a D-state process) and matches the // async `reap_group_on_timeout`, which likewise only warns and gives up there. let read_result = stdout.read_to_end(&mut out); - // The drain has returned: either the child exited on its own, or the watchdog's - // teardown closed its pipes. Tell the watchdog the drain is done, then WAIT for it - // to return BEFORE reaping the leader. On the timeout path the watchdog runs the - // full SIGTERM -> grace -> SIGKILL teardown; joining it before `child.wait()` keeps - // the leader's pid unreaped while every `kill(-pgid)` fires (so the pgid can't be - // recycled), and guarantees a SIGTERM-ignoring group member has already been - // SIGKILLed rather than left running past the deadline (finding 3, #174). - let _ = done_tx.send(()); + // The drain has returned, but that only means all stdout write ends are closed — + // NOT that the child has exited. A group member, or the leader itself, can close + // stdout and keep running; standing the watchdog down on the drain alone (as the + // old code did) would then let `child.wait()` block forever on that live child, + // past the deadline, pinning the walk thread and its permit (finding P1-a, #174). + // So stand the watchdog down only once the child has ACTUALLY terminated, detected + // WITHOUT reaping (waitid + WNOWAIT) so the leader's pid stays unreaped and its + // pgid un-recycled until the watchdog finishes and we join it below. Past the + // deadline the watchdog owns the teardown, so we stop polling and let it run the + // full SIGTERM -> grace -> SIGKILL; joining it before `child.wait()` keeps every + // `kill(-pgid)` firing while the pid is still unreaped and guarantees a + // stdout-closing-then-hanging member has been SIGKILLed rather than left running. + loop { + if child_terminated_without_reaping(pgid) { + let _ = done_tx.send(()); + break; + } + if Instant::now() >= deadline { + break; + } + std::thread::sleep(Duration::from_millis(2)); + } let killed = watchdog.join().unwrap_or(false); let status = child.wait().context("git wait failed")?; let err = err_reader.join().unwrap_or_default(); @@ -902,6 +941,67 @@ esac\n"; "a SIGTERM-ignoring descendant (pid {desc}) must be SIGKILLed even after the leader exits cleanly, not orphaned" ); } + + /// #174 U1 (P1-a, RED-before/GREEN-after): the group LEADER closes its own + /// stdout/stderr BEFORE the deadline and then keeps running. On the pre-fix code + /// the stdout drain returns EOF early, `done_tx.send` stands the watchdog down + /// before it ever fires (`recv` gets `Ok` -> `false`, no kill), and `child.wait()` + /// then blocks on the still-alive leader — pinning the walk thread and its read/ + /// write permit past the deadline, bypassing GITLAWB_GIT_SERVICE_TIMEOUT_SECS. + /// This is distinct from the descendant case above: there the leader sleeps until + /// the deadline so the watchdog DOES time out; here the drain-EOF races ahead of + /// the deadline. The fix keeps the watchdog armed until the child is actually + /// reaped, so the deadline SIGTERM still fires and the call returns within budget. + /// A pre-fix build blocks on `child.wait()` past the recv budget (RED). + #[cfg(unix)] + #[test] + fn run_bounded_git_reaps_a_leader_that_closes_stdout_then_hangs() { + use std::time::{Duration, Instant}; + let tmp = TempDir::new().unwrap(); + // rev-list records its (leader) pid, closes stdout+stderr so the drain EOFs + // immediately, then sleeps without trapping TERM. `sleep 30` bounds the worst + // case so a RED run cannot wedge the suite; the recv budget fires first. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) echo $$ > leader.pid; exec 1>&- 2>&-; sleep 30 ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + let (tx, rx) = std::sync::mpsc::channel(); + let path = tmp.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(run_bounded_git( + &git_bin, + &["rev-list"], + &path, + b"", + Instant::now() + Duration::from_millis(100), + )); + }); + let out = rx.recv_timeout(Duration::from_secs(10)).expect( + "run_bounded_git must return within the watchdog budget when the leader closes stdout then hangs, not block on child.wait()", + ); + assert!( + out.is_err(), + "a leader killed at the deadline (no TERM trap) is a timeout, not a success: {out:?}" + ); + let pid: i32 = std::fs::read_to_string(tmp.path().join("leader.pid")) + .unwrap() + .trim() + .parse() + .unwrap(); + let mut gone = false; + for _ in 0..300 { + if unsafe { libc::kill(pid, 0) } != 0 { + gone = true; + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + // Kill it regardless so a RED run leaks no orphan. + unsafe { libc::kill(pid, libc::SIGKILL) }; + assert!( + gone, + "the hung leader (pid {pid}) must be killed and reaped at the deadline, not left running" + ); + } + use crate::db::VisibilityMode; use chrono::Utc; use std::process::Command; From ac59bd7e01bf42b7d42b7adb57ccb927c0309993 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 14 Jul 2026 19:50:41 -0500 Subject: [PATCH 28/95] fix(node): reap the served-git process group on disconnect, not just SIGTERM (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a client disconnect KillGroupOnDrop sent a lone SIGTERM with no escalation or reap, so a group member that ignores SIGTERM (a wedged pack-objects, a malicious helper) survived while the handler released its concurrency permit — letting disconnect-spam accumulate live git processes and defeat the load shed (P1-c). The timeout path already ran the full teardown. Make the disconnect teardown as strong as the timeout path: the guard now owns the child and, on drop, launches a detached reaper that runs the same TERM -> grace -> SIGKILL -> reap-the-group sequence (reap_group_on_timeout). Owning the tokio Child in the reaper keeps it the sole reaper of the leader, so it never races tokio's own orphan reaper (a raw waitpid(-pgid) would). Regression: run_git_service_sigkills_a_sigterm_ignoring_child_on_disconnect, RED before (the descendant survives the lone SIGTERM), GREEN after; all existing teardown/timeout/disarm tests still pass (the three hand-built-guard tests were updated to the owned-child shape). Residual: capacity (the handler's semaphore permit) still releases at disconnect rather than strictly after the reaper confirms the group dead, so accumulation is now bounded to ~one teardown window (<=4s) rather than eliminated; gating the permit on the reap needs threading it through the serve handlers and is left for a follow-up (it also touches the receive-pack permit lifetime U4 changes). --- crates/gitlawb-node/src/git/smart_http.rs | 235 +++++++++++++++++----- 1 file changed, 188 insertions(+), 47 deletions(-) diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index f885aef3..a5649a42 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -108,24 +108,60 @@ pub async fn receive_pack( /// `wait_with_output()` returns, so a request that completed cleanly never signals. #[cfg(unix)] struct KillGroupOnDrop { + // Holds the child + its pgid while armed. The interaction drives the child through + // `child_mut()`; the success/timeout paths call `disarm` once they have reaped it. + // On drop — a client disconnect that drops the whole request future — the guard + // launches a detached reaper that OWNS the child and runs the full + // SIGTERM -> grace -> SIGKILL -> reap sequence (`reap_group_on_timeout`), as strong + // as the timeout path, so a SIGTERM-ignoring group member is SIGKILLed and reaped + // rather than left running until EPIPE to accumulate past the concurrency cap + // (#174 P1-c). Owning the tokio `Child` in the reaper (rather than a raw + // `waitpid(-pgid)`) keeps a single reaper of the leader, so it never races tokio's + // own SIGCHLD-driven orphan reaper. + child: Option, pgid: Option, } #[cfg(unix)] impl KillGroupOnDrop { + /// The child, while armed. The interaction drives stdin/stdout/`wait` through this. + fn child_mut(&mut self) -> &mut tokio::process::Child { + self.child.as_mut().expect("child present while armed") + } + + /// Disarm on the success/timeout path: the body has already reaped the child (its + /// `wait()` returned, or `reap_group_on_timeout` ran), so drop the handle and clear + /// the pgid, leaving the guard's Drop a no-op. fn disarm(&mut self) { self.pgid = None; + self.child = None; } } #[cfg(unix)] impl Drop for KillGroupOnDrop { fn drop(&mut self) { - if let Some(pgid) = self.pgid { - // SAFETY: kill(2) takes only integer arguments and borrows no Rust - // memory. Signalling a stale group just returns ESRCH, which we ignore. - unsafe { - libc::kill(-pgid, libc::SIGTERM); + let (Some(mut child), Some(pgid)) = (self.child.take(), self.pgid) else { + return; + }; + // A sync Drop cannot await, so launch a detached reaper that owns the child and + // runs the same teardown as the timeout path (TERM -> grace -> SIGKILL -> reap + // the whole group). Owning the tokio `Child` means this task is the sole reaper + // of the leader, so it cannot race tokio's orphan reaper. Prefer the current + // runtime handle; if dropped outside a runtime, fall back to a best-effort + // synchronous SIGTERM so the group is at least signalled. + match tokio::runtime::Handle::try_current() { + Ok(handle) => { + handle.spawn(async move { + reap_group_on_timeout(&mut child).await; + }); + } + Err(_) => { + // SAFETY: kill(2) takes only integers and borrows no Rust memory; + // ESRCH on an already-gone group is ignored. + unsafe { + libc::kill(-pgid, libc::SIGTERM); + } } } } @@ -244,20 +280,28 @@ async fn drive_git_child( let mut child = command.spawn()?; - // Arm the group-kill guard for the lifetime of the request. With - // process_group(0) the child is its own group leader, so pgid == its pid. - // This fires on a client disconnect (the whole future is dropped mid-request). + // Own the pipes so the child stays reap-able: wait_with_output would consume it, + // but on a timeout we must actively reap the group first (see + // reap_group_on_timeout), and on a disconnect the guard's detached reaper does. + // Take the pipes before the child moves into the guard below. + let mut stdin = child.stdin.take(); + let mut stdout = child.stdout.take().context("git stdout was not piped")?; + let mut stderr = child.stderr.take().context("git stderr was not piped")?; + + // Arm the group-kill guard for the lifetime of the request. With process_group(0) + // the child is its own group leader, so pgid == its pid. On a client disconnect + // (the whole future is dropped mid-request) the guard's Drop launches a detached + // reaper that OWNS the child and runs the full TERM/grace/SIGKILL/reap — as strong + // as the timeout path below (#174 P1-c). The guard owns the child; the interaction + // drives it through `child_mut()`. + #[cfg(unix)] + let pgid = child.id().map(|id| id as i32); #[cfg(unix)] let mut group_guard = KillGroupOnDrop { - pgid: child.id().map(|id| id as i32), + child: Some(child), + pgid, }; - // Own the pipes so `child` stays reap-able after a timeout: wait_with_output - // would consume it, but on timeout we must actively reap the group before - // returning (see reap_group_on_timeout). - let mut stdin = child.stdin.take(); - let mut stdout = child.stdout.take().context("git stdout was not piped")?; - let mut stderr = child.stderr.take().context("git stderr was not piped")?; let mut out = Vec::new(); let mut err = Vec::new(); @@ -274,11 +318,15 @@ async fn drive_git_child( None => Ok(()), } }; + #[cfg(unix)] + let child_ref = group_guard.child_mut(); + #[cfg(not(unix))] + let child_ref = &mut child; let (write_result, r_out, r_err, status) = tokio::join!( write, stdout.read_to_end(&mut out), stderr.read_to_end(&mut err), - child.wait(), + child_ref.wait(), ); r_out?; r_err?; @@ -290,7 +338,7 @@ async fn drive_git_child( Ok(result) => { // The join runs all arms to completion, so the child is reaped: disarm // before surfacing any interaction error (a read/wait error), else the - // guard's drop would fire SIGTERM on the reaped, possibly-reused pgid. + // guard's drop would reap an already-reaped child / signal a reused pgid. #[cfg(unix)] group_guard.disarm(); result? @@ -301,7 +349,7 @@ async fn drive_git_child( // the (now redundant) guard so its drop can't hit a reused pgid. #[cfg(unix)] { - reap_group_on_timeout(&mut child).await; + reap_group_on_timeout(group_guard.child_mut()).await; group_guard.disarm(); } #[cfg(not(unix))] @@ -960,23 +1008,36 @@ mod tests { #[cfg(unix)] #[tokio::test] async fn kill_group_guard_terminates_child_on_drop() { - let mut child = tokio::process::Command::new("sleep") + let child = tokio::process::Command::new("sleep") .arg("300") .process_group(0) .spawn() .unwrap(); - let pgid = child.id().map(|id| id as i32); + let pid = child.id().unwrap() as i32; { - let _guard = KillGroupOnDrop { pgid }; - } // guard drops here -> SIGTERM to the group - - use std::os::unix::process::ExitStatusExt; - let status = child.wait().await.unwrap(); - assert_eq!( - status.signal(), - Some(libc::SIGTERM), - "child must be terminated by SIGTERM via its process group" + // The guard owns the child; on drop its detached reaper TERM/grace/KILL/ + // reaps the group (a plain sleep dies on the first SIGTERM). + let _guard = KillGroupOnDrop { + child: Some(child), + pgid: Some(pid), + }; + } + + let mut gone = false; + for _ in 0..300 { + if !alive(pid) { + gone = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + unsafe { + libc::kill(pid, libc::SIGKILL); + } + assert!( + gone, + "child must be terminated and reaped via its process group on guard drop" ); } @@ -994,9 +1055,10 @@ mod tests { .process_group(0) .spawn() .unwrap(); - let pgid = child.id().map(|id| id as i32); + let pid = child.id().unwrap() as i32; - // Read the backgrounded grandchild's pid from the first stdout line. + // Read the backgrounded grandchild's pid from the first stdout line, before + // the child moves into the guard. let mut stdout = child.stdout.take().unwrap(); let mut buf = Vec::new(); loop { @@ -1011,20 +1073,25 @@ mod tests { assert!(alive(grandchild), "grandchild should be running"); { - let _guard = KillGroupOnDrop { pgid }; - } // group SIGTERM reaches sh AND the sleep grandchild - - let _ = child.wait().await; // reap sh + // The guard owns the child; on drop the detached reaper group-kills sh AND + // the sleep grandchild. + let _guard = KillGroupOnDrop { + child: Some(child), + pgid: Some(pid), + }; + } - // The grandchild reparents to init and is reaped; poll until it's gone. let mut gone = false; - for _ in 0..200 { + for _ in 0..300 { if !alive(grandchild) { gone = true; break; } tokio::time::sleep(std::time::Duration::from_millis(10)).await; } + unsafe { + libc::kill(grandchild, libc::SIGKILL); + } assert!( gone, "grandchild must be terminated by the group signal (#53)" @@ -1036,27 +1103,31 @@ mod tests { async fn kill_group_guard_disarmed_does_not_kill() { // A request that completed cleanly disarms the guard; dropping it must not // signal anything. - let mut child = tokio::process::Command::new("sleep") + let child = tokio::process::Command::new("sleep") .arg("300") .process_group(0) .spawn() .unwrap(); + let pid = child.id().unwrap() as i32; { let mut guard = KillGroupOnDrop { - pgid: child.id().map(|id| id as i32), + child: Some(child), + pgid: Some(pid), }; guard.disarm(); - } // disarmed -> no kill + } // disarmed -> no reaper, no kill - assert!( - child.try_wait().unwrap().is_none(), - "disarmed guard must not kill the child" - ); + // Give any erroneously-spawned reaper a chance to run, then assert alive. + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + assert!(alive(pid), "disarmed guard must not kill the child"); - // Clean up the still-running child. - let _ = child.kill().await; - let _ = child.wait().await; + // Clean up the still-running child (disarm dropped the handle, so reap by pid). + unsafe { + libc::kill(pid, libc::SIGKILL); + let mut status = 0; + libc::waitpid(pid, &mut status, 0); + } } // ── #62 PR1: end-to-end teardown wiring through run_git_service ───────── @@ -1320,6 +1391,76 @@ mod tests { ); } + /// #174 U2 (P1-c, RED-before/GREEN-after): on a client disconnect the teardown must + /// be as strong as the timeout path — a group member that IGNORES SIGTERM is still + /// SIGKILLed and reaped, not left running to accumulate past the concurrency cap. + /// The old `KillGroupOnDrop::drop` sent a lone SIGTERM with no escalation or reap, + /// so a SIGTERM-ignoring descendant survived the disconnect (RED). The fix launches + /// a detached reaper owning the child that runs the full TERM/grace/SIGKILL/reap. + /// This is distinct from the well-behaved-grandchild test above, whose `sleep` dies + /// on the first SIGTERM. + #[cfg(unix)] + #[tokio::test] + async fn run_git_service_sigkills_a_sigterm_ignoring_child_on_disconnect() { + let tmp = tempfile::TempDir::new().unwrap(); + let descfile = tmp.path().join("desc.pid"); + // Leader (dies on the group SIGTERM) spawns a descendant that traps SIGTERM, + // records its own pid, and loops ~30s (bounded so a RED run leaks no permanent + // orphan; the assertion fires well before then). + let body = format!( + "#!/bin/sh\n\ + sh -c 'trap \"\" TERM; echo $$ > \"{}\"; i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done' &\n\ + wait\n", + descfile.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + + let mut fut = Box::pin(run_git_service( + git_bin.to_str().unwrap(), + "git-upload-pack", + tmp.path(), + Bytes::new(), + Duration::from_secs(60), + )); + // Drive the future a slice at a time until the descendant records its pid. + let mut desc: Option = None; + for _ in 0..500 { + let _ = tokio::time::timeout(Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&descfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + desc = Some(p); + break; + } + } + let desc = desc.expect("the fake leader must have spawned and recorded a descendant"); + let _cleanup = ReapOnPanic(vec![desc]); + assert!(alive(desc), "descendant should be running before the drop"); + + // Client disconnect: drop the request future. The guard's detached reaper must + // escalate SIGTERM -> SIGKILL and reap the SIGTERM-ignoring descendant. + drop(fut); + + let mut gone = false; + for _ in 0..500 { + if !alive(desc) { + gone = true; + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + // Kill regardless so a RED run leaks no orphan. + unsafe { + libc::kill(desc, libc::SIGKILL); + } + assert!( + gone, + "a SIGTERM-ignoring descendant (pid {desc}) must be SIGKILLed and reaped on \ + disconnect, not left running (old KillGroupOnDrop sent SIGTERM only)" + ); + } + // #174 (SC3): the info/refs advertisement is now duration-bounded — previously // it ran a bare `Command::output()` with no deadline, so a hung git pinned its // concurrency slot forever. A hung fake git must abort with GitServiceTimeout From be0cdd6a9eae1b39618f16fdf4e28eae6e83ca51 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 14 Jul 2026 20:09:40 -0500 Subject: [PATCH 29/95] fix(node): hold upload-pack admission through the walk on disconnect (#174) On the path-scoped upload-pack path the per-source and global read permits were handler locals, so a client disconnect dropped the handler future and released both permits while the awaited spawn_blocking withheld-blob walk kept running (spawn_blocking can't be cancelled). A permitless caller could disconnect-spam to exceed the read cap while real git work continued (P1-b). Move both permits into the blocking task and return them out: on success they flow back so the serve phase keeps them; on a dropped future the returned tuple (with the permits) is discarded only when the blocking task completes, so admission tracks the walk's real duration. Regression: upload_pack_permit_held_through_walk_after_disconnect drives the handler to mid-walk (fake git hangs on rev-list), disconnects, and asserts the global read slot stays held (available_permits == 0). RED before (the slot frees to 1 the instant the future drops), GREEN after. The test roots the repo store at its own TempDir so it is isolated per run. --- crates/gitlawb-node/src/api/repos.rs | 166 +++++++++++++++++++++++++-- 1 file changed, 158 insertions(+), 8 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index ea6f126e..8612709c 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -819,9 +819,16 @@ pub async fn git_upload_pack( let resp = if !visibility_pack::has_path_scoped_rule(&rules) { smart_http::upload_pack(&disk_path, body, git_timeout).await } else { - // withheld_blob_oids walks every ref with blocking `git ls-tree`; keep - // that off the async worker thread. - let withheld = { + // withheld_blob_oids walks every ref with blocking `git ls-tree`; keep that + // off the async worker thread. Move BOTH admission permits INTO the blocking + // task so they are held for the walk's real duration: spawn_blocking cannot be + // cancelled, so on a client disconnect the handler future drops but the walk + // keeps running — and now so do its permits, released only when the walk + // finishes rather than the instant the future drops (#174 P1-b). On success the + // task hands the permits back so the serve phase below keeps them; on a + // dropped future the returned tuple (with the permits) is discarded only when + // the blocking task completes, so admission tracks the real git work. + let (withheld, _permit, _caller_permit) = { let path = disk_path.clone(); let rules = rules.clone(); let owner_did = record.owner_did.clone(); @@ -829,7 +836,7 @@ pub async fn git_upload_pack( let is_public = record.is_public; let git_bin = state.git_bin.clone(); tokio::task::spawn_blocking(move || { - visibility_pack::withheld_blob_oids_bounded( + let withheld = visibility_pack::withheld_blob_oids_bounded( &path, &git_bin, git_timeout, @@ -837,14 +844,15 @@ pub async fn git_upload_pack( is_public, &owner_did, caller_owned.as_deref(), - ) + ); + (withheld, _permit, _caller_permit) }) .await .map_err(|e| AppError::Git(e.to_string()))? - // A walk that hit its deadline carries GitServiceTimeout; map it to 504 - // like the smart_http paths, not a generic 500 (#174 U3). - .map_err(|e| git_service_app_error(&e))? }; + // A walk that hit its deadline carries GitServiceTimeout; map it to 504 like + // the smart_http paths, not a generic 500 (#174 U3). + let withheld = withheld.map_err(|e| git_service_app_error(&e))?; if withheld.is_empty() { smart_http::upload_pack(&disk_path, body, git_timeout).await @@ -3456,6 +3464,148 @@ mod tests { ); } + /// #174 U3 (P1-b, RED-before/GREEN-after): a client disconnect during the + /// path-scoped withheld-blob walk must NOT release the read admission while the + /// uncancellable `spawn_blocking` walk is still running. The handler takes the + /// global read permit, enters the walk (a fake git hangs on rev-list), then the + /// request future is dropped mid-walk. With both permits moved into the blocking + /// task the global slot stays occupied until the walk finishes; on the pre-fix code + /// the handler-local permits drop on future-drop and the slot frees instantly (RED), + /// letting disconnect-spam exceed the cap while real git work keeps running. + #[sqlx::test] + async fn upload_pack_permit_held_through_walk_after_disconnect(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + let revlist_pid = tmp.path().join("revlist.pid"); + // Fake git: resolve refs fast, hang on rev-list (recording its pid first). The + // ~6s sleep bounds the walk so a broken fix cannot wedge the suite. + let body = format!( + "#!/bin/sh\ncase \"$1\" in\n rev-list) echo $$ > \"{}\" ; sleep 6 ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n", + revlist_pid.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + // Root the repo store at this test's TempDir so the bare repo is isolated per + // run (the default for_testing store uses a fixed /tmp path that would collide + // across runs). + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.git_read_semaphore = Arc::new(Semaphore::new(1)); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let owner = "z6up3rd"; + let name = "up3"; + state + .db + .upsert_mirror_repo(owner, name, "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + // Real bare repo at the path acquire() computes, so the handler reaches the walk. + state + .repo_store + .init(&rec.owner_did, &rec.name) + .await + .unwrap(); + // A path-scoped rule so has_path_scoped_rule() is true (the walk path) without + // denying the "/" gate for the public repo. + state + .db + .set_visibility_rule( + &rec.id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3ReaderAAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + + let sem = state.git_read_semaphore.clone(); + assert_eq!( + sem.available_permits(), + 1, + "one read slot before the request" + ); + + let router = crate::server::build_router(state); + let peer: SocketAddr = "203.0.113.77:5000".parse().unwrap(); + let mut req = Request::builder() + .method(Method::POST) + .uri(format!("/{owner}/{name}/git-upload-pack")) + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + + let mut fut = Box::pin(router.oneshot(req)); + // Drive until the walk's rev-list starts (its pidfile appears) — i.e. the + // request is inside the spawn_blocking walk, holding the global read permit. + let mut in_walk = false; + for _ in 0..500 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if revlist_pid.exists() { + in_walk = true; + break; + } + } + assert!( + in_walk, + "the walk's rev-list must start (request reached the spawn_blocking walk)" + ); + assert_eq!( + sem.available_permits(), + 0, + "the read slot is held while the walk runs" + ); + + // Client disconnect: drop the request future mid-walk. + drop(fut); + + // Load-bearing: the slot must STAY held while the uncancellable walk runs. On + // the pre-fix code the handler-local permits drop here and the slot frees at + // once (RED). + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + assert_eq!( + sem.available_permits(), + 0, + "on disconnect the read admission must be held until the spawn_blocking walk \ + finishes, not released the instant the future drops (P1-b)" + ); + + // Cleanup: let the walk finish so the slot releases and no blocking task leaks. + for _ in 0..400 { + if sem.available_permits() == 1 { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + if let Some(p) = std::fs::read_to_string(&revlist_pid) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + unsafe { + libc::kill(p, libc::SIGKILL); + } + } + } + /// #174 SC2 (per-source key, U1): the per-caller read sub-cap keys on the /// resolved source IP, NOT the signed DID, so a disposable-DID farm cannot /// multiply its budget. Fill the source IP's single read slot, then drive two From 12643573f6c9f4805bd1876008141edda58f9192 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 14 Jul 2026 20:20:49 -0500 Subject: [PATCH 30/95] fix(node): cap concurrent receive-pack pushes per source IP (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The authenticated git-receive-pack POST acquired only the global write semaphore — no per-source sub-cap, unlike the read and anon-advert pools. Owner enforcement defaults off, so one host minting disposable did:key identities could open max_concurrent_git_pushes slow POSTs from a single source IP and 503 every other source's push; the 600/hour push limiter bounds arrival rate, not in-flight concurrency (P1-d). Add git_write_per_caller (a PerCallerConcurrency sized like the advert cap, max_concurrent_git_pushes/8), and acquire it before the global write permit, keyed on the resolved source IP via read_caller_key — never the signed DID (a DID farm defeats a DID key). This required adding the PeerAddr + HeaderMap extractors to git_receive_pack, which it lacked (without them the key is None and the cap is inert). Neutralized the shared acquire helper's log wording now that it serves both read and write paths. Regression: receive_pack_per_source_write_cap_sheds_capped_source_not_others — a source at its write sub-cap sheds Overloaded/503 (proving the extractors resolve a key), while a different source is not shed. RED before (the capped source proceeds past admission to a git error instead of shedding), GREEN after; the existing receive-pack advert cap test still passes. --- crates/gitlawb-node/src/api/repos.rs | 103 ++++++++++++++++++++++-- crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/main.rs | 7 ++ crates/gitlawb-node/src/state.rs | 7 ++ crates/gitlawb-node/src/test_support.rs | 1 + 5 files changed, 114 insertions(+), 5 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 8612709c..67c6c82a 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -717,7 +717,7 @@ fn acquire_read_caller_permit( Some(k) => match limiter.try_acquire(k) { Some(p) => Ok(Some(p)), None => { - tracing::warn!(repo = %repo, caller = %k, handler, "per-caller read cap reached; shedding with 503"); + tracing::warn!(repo = %repo, caller = %k, handler, "per-caller cap reached; shedding with 503"); Err(AppError::Overloaded( "git service at capacity for this caller, retry shortly".into(), )) @@ -1033,15 +1033,32 @@ pub async fn git_receive_pack( State(state): State, Path((owner, repo)): Path<(String, String)>, Extension(auth): Extension, + crate::rate_limit::PeerAddr(peer): crate::rate_limit::PeerAddr, + headers: axum::http::HeaderMap, body: Bytes, ) -> Result { + let name = smart_http_repo_name(&repo)?; + // Per-source write sub-cap (#174 P1-d): before the global write permit so one + // source IP cannot occupy the whole write pool via many slow authenticated pushes + // and 503 every other source. Owner enforcement defaults off, so any valid did:key + // is accepted (auth != authz), and the 600/hour push limiter bounds arrival RATE, + // not in-flight concurrency — so without this a single host minting disposable DIDs + // saturates the pool. Keyed on the resolved source IP, NEVER the signed DID (a DID + // farm defeats a DID key); no resolvable key -> global write pool only. + let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); + let _caller_permit = acquire_read_caller_permit( + &state.git_write_per_caller, + caller_key.as_deref(), + name, + "receive-pack", + )?; // Shed with a 503 before spawning git when the concurrency cap is saturated. // Pushes draw from the dedicated WRITE pool, separate from reads, so a flood of - // anonymous reads cannot shed an authenticated push (#174). Acquired at the very - // top so it wraps the write-guard below (and precedes the Tigris acquire_write, - // bounding concurrent fresh acquires — INV-10); held for the whole op. + // anonymous reads cannot shed an authenticated push (#174). Taken after the + // per-source cap above so one source cannot occupy global slots it would be + // sub-cap-denied for; still before the Tigris acquire_write, bounding concurrent + // fresh acquires (INV-10); held for the whole op. let _permit = git_permit(&state.git_write_semaphore)?; - let name = smart_http_repo_name(&repo)?; tracing::info!(owner = %owner, repo = %name, "receive-pack request"); let record = state .db @@ -3606,6 +3623,82 @@ mod tests { } } + /// #174 U4 (P1-d, RED-before/GREEN-after): the authenticated receive-pack POST + /// carries a per-source WRITE sub-cap so one source IP cannot monopolize the write + /// pool with many slow pushes (owner enforcement defaults off, so disposable DIDs + /// are free). Global write pool has capacity; the source is pre-held at its single + /// write slot. A push from THAT source sheds (Overloaded/503) — which also proves + /// the PeerAddr+HeaderMap extractors resolve a key (without them the key is None and + /// the cap is inert, never shedding). A push from a DIFFERENT source is NOT shed by + /// the cap. Called directly so the test needs no signed request; the handler is + /// where the cap lives. Remove the `git_write_per_caller` acquire and the capped + /// source no longer sheds (RED). + #[sqlx::test] + async fn receive_pack_per_source_write_cap_sheds_capped_source_not_others(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + let mut state = crate::test_support::test_state(pool).await; + // Global write pool has capacity; the per-source cap is 1. + state.git_write_semaphore = Arc::new(Semaphore::new(4)); + state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6rp4wr", "rp4", "/tmp/rp4-nonexistent", None, false) + .await + .unwrap(); + + let did = "did:key:z6MkReceivePackWriteCapProofDidAAAAAAAAAA"; + let capped: SocketAddr = "203.0.113.44:5000".parse().unwrap(); + let other: SocketAddr = "203.0.113.45:5000".parse().unwrap(); + + // Pin the capped source at its single write slot. + let _slot = state + .git_write_per_caller + .try_acquire(&capped.ip().to_string()) + .expect("first write slot for the capped source IP"); + + // A push from the capped source must shed on the per-source write cap even with + // global write capacity free. The shed also proves the source-IP key resolved + // via the extractors (an inert None key would fall through to Ok(None)). + let capped_result = git_receive_pack( + State(state.clone()), + Path(("z6rp4wr".to_string(), "rp4".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(capped)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await; + assert!( + matches!(capped_result, Err(AppError::Overloaded(_))), + "a source at its per-source write cap must shed (Overloaded/503) with global \ + pool capacity free; got {capped_result:?}" + ); + + // A push from a DIFFERENT source must NOT be shed by the per-source cap — it + // proceeds past admission (and fails later on the nonexistent repo, which is not + // an Overloaded error). + let other_result = git_receive_pack( + State(state.clone()), + Path(("z6rp4wr".to_string(), "rp4".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(other)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await; + assert!( + !matches!(other_result, Err(AppError::Overloaded(_))), + "a different source must not be shed by the per-source write cap while the \ + capped source holds its slot; got {other_result:?}" + ); + } + /// #174 SC2 (per-source key, U1): the per-caller read sub-cap keys on the /// resolved source IP, NOT the signed DID, so a disposable-DID farm cannot /// multiply its budget. Fill the source IP's single read slot, then drive two diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 5d827794..5597cd8d 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -526,6 +526,7 @@ mod tests { git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), + git_write_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), git_bin: "git".to_string(), } } diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 18019395..74504f90 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -399,6 +399,13 @@ async fn main() -> Result<()> { git_push_advert_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( (config.max_concurrent_git_pushes / 8).max(1), ), + // Per-source cap on the authenticated receive-pack POST, sized like the advert + // cap: one source IP can hold at most this many write-pool slots, so + // monopolizing the pool takes ~8 distinct source IPs, each also rate-limited + // (#174 P1-d). + git_write_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( + (config.max_concurrent_git_pushes / 8).max(1), + ), git_bin: "git".to_string(), }; diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index eb07a3e4..6ce158e3 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -123,6 +123,13 @@ pub struct AppState { /// a fraction of `max_concurrent_git_pushes`, so filling the write pool takes many /// distinct source IPs (each also braked by the per-IP push rate limiter). pub git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency, + /// Per-source concurrency sub-cap on the authenticated `git-receive-pack` POST: + /// each source IP may hold at most a small share of `git_write_semaphore`, so one + /// host minting disposable `did:key` identities cannot open enough slow pushes to + /// monopolize the write pool and 503 every other source's push (#174 P1-d). Keyed + /// on the resolved source IP (never the DID — a DID farm defeats a DID key). Sized + /// like `git_push_advert_per_caller`, a fraction of `max_concurrent_git_pushes`. + pub git_write_per_caller: crate::rate_limit::PerCallerConcurrency, /// The `git` executable the served-git withheld-blob walk spawns. Production is /// `"git"` (resolved via PATH); injectable so a fake `git` can drive the walk's /// process-group teardown in handler tests without mutating the process-global diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 0805fec1..9241b518 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -90,6 +90,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( 8, ), + git_write_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), git_bin: "git".to_string(), } } From 2a54c159e6984032205545816e0e44e0881f996c Mon Sep 17 00:00:00 2001 From: t Date: Tue, 14 Jul 2026 20:28:52 -0500 Subject: [PATCH 31/95] fix(node): bound concurrent post-push encryption walks with an admission pool (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After a successful path-scoped push the handler released its write permit and then ran a DETACHED tokio task whose spawn_blocking(withheld_blob_recipients_bounded) performed another full-history git walk under no admission. N fast completed pushes spawned N concurrent full-history walks past GITLAWB_MAX_CONCURRENT_GIT_PUSHES (which bounds only the in-handler phase), and the per-child bounding from 71389e6 caps each walk's duration but not their count (P1-e). Add git_encrypt_semaphore, a pool of its own (sized from max_concurrent_git_pushes, no new knob — Q1) so a long background walk never holds a foreground write slot and a handler holding a write permit can't self-deadlock. Route the walk through a new withheld_recipients_gated helper that acquires the pool before the walk and DEFERS (blocks) when full rather than shedding — dropping the walk would lose the withheld-blob recovery copy, so durability stays fail-closed. Regression: encrypt_walk_defers_when_pool_exhausted drives the gated helper with the pool exhausted and asserts the walk blocks and does not run its rev-list, then runs once a permit frees. RED before (the walk runs regardless of the pool), GREEN after. The detached push task calls this exact helper. --- crates/gitlawb-node/src/api/repos.rs | 138 +++++++++++++++++++++--- crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/main.rs | 6 ++ crates/gitlawb-node/src/state.rs | 11 ++ crates/gitlawb-node/src/test_support.rs | 1 + 5 files changed, 144 insertions(+), 13 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 67c6c82a..430b2f7f 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -727,6 +727,37 @@ fn acquire_read_caller_permit( } } +/// Acquire an encryption-walk admission permit, then run the bounded withheld-blob +/// recipients walk. Blocks (defers) when `git_encrypt_semaphore` is full rather than +/// shedding — the walk is background so added latency is fine, and dropping it would +/// lose the withheld-blob recovery copy (#174 P1-e). Bounds the number of concurrent +/// post-push encryption walks so N fast completed pushes cannot spawn N concurrent +/// full-history git walks. Mirrors the original `spawn_blocking(...).await` return +/// shape so the caller's `Ok(Ok(recipients))` match is unchanged. +async fn withheld_recipients_gated( + encrypt_sem: std::sync::Arc, + repo_path: std::path::PathBuf, + git_bin: String, + timeout: std::time::Duration, + rules: Vec, + is_public: bool, + owner_did: String, +) -> std::result::Result< + anyhow::Result>>, + tokio::task::JoinError, +> { + let _permit = encrypt_sem + .acquire_owned() + .await + .expect("git_encrypt_semaphore is never closed"); + tokio::task::spawn_blocking(move || { + crate::git::visibility_pack::withheld_blob_recipients_bounded( + &repo_path, &git_bin, timeout, &rules, is_public, &owner_did, + ) + }) + .await +} + /// Map an error from a `smart_http` git service call to the right `AppError`: /// [`smart_http::GitServiceTimeout`] to 504, a malformed client request to 400, /// anything else to a 500 git error. Pure (no logging) so it is unit-testable; @@ -1329,6 +1360,7 @@ pub async fn git_receive_pack( let repo_name = record.name.clone(); let enc_git_bin = state.git_bin.clone(); let enc_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let encrypt_sem = state.git_encrypt_semaphore.clone(); tokio::spawn(async move { let pinned = crate::ipfs_pin::pin_new_objects( &ipfs_api, @@ -1351,19 +1383,18 @@ pub async fn git_receive_pack( // the has_path_scoped_rule gate on the other two withheld-walk sites. if let Some(rules) = rules_for_enc.filter(|r| visibility_pack::has_path_scoped_rule(r)) { - let p = repo_path_clone.clone(); - let owner = owner_did.clone(); - let git_bin = enc_git_bin.clone(); - let recip = tokio::task::spawn_blocking(move || { - crate::git::visibility_pack::withheld_blob_recipients_bounded( - &p, - &git_bin, - enc_timeout, - &rules, - is_public, - &owner, - ) - }) + // Bound the number of concurrent post-push encryption walks (#174 P1-e): + // acquire an admission permit before the full-history walk, deferring + // when the pool is full rather than shedding the recovery pin. + let recip = withheld_recipients_gated( + encrypt_sem.clone(), + repo_path_clone.clone(), + enc_git_bin.clone(), + enc_timeout, + rules, + is_public, + owner_did.clone(), + ) .await; if let Ok(Ok(recipients)) = recip { let delta = crate::encrypted_pin::encrypt_and_pin( @@ -3699,6 +3730,87 @@ mod tests { ); } + /// #174 U5 (P1-e, RED-before/GREEN-after): the post-push encryption walk acquires a + /// `git_encrypt_semaphore` permit before running, so completed pushes cannot spawn + /// unbounded concurrent full-history walks. With the pool exhausted the gated walk + /// must DEFER (block on admission) and NOT run its rev-list; on the pre-fix code + /// (no acquire) the walk runs regardless of the pool (RED). It defers rather than + /// sheds — releasing the permit lets the SAME walk run and pin (durability stays + /// fail-closed). Exercises the gating seam directly; the detached push task calls + /// this exact helper. + #[tokio::test] + async fn encrypt_walk_defers_when_pool_exhausted() { + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let marker = tmp.path().join("revlist.ran"); + // Fake git records when rev-list runs (the walk's first git call). + let body = format!( + "#!/bin/sh\ncase \"$1\" in\n rev-list) echo ran > \"{}\" ;;\n *) : ;;\nesac\nexit 0\n", + marker.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + let git_bin = git_path.to_str().unwrap().to_string(); + let owner = "did:key:z6MkEncWalkOwnerAAAAAAAAAAAAAAAAAAAAAAAA".to_string(); + + // Exhaust the pool: hold its only permit so a gated walk must defer. + let sem = Arc::new(Semaphore::new(1)); + let held = sem.clone().acquire_owned().await.unwrap(); + + // Blocked: the gated walk must NOT complete or run rev-list while exhausted. + let blocked = tokio::time::timeout( + Duration::from_millis(500), + withheld_recipients_gated( + sem.clone(), + tmp.path().to_path_buf(), + git_bin.clone(), + Duration::from_secs(5), + Vec::new(), + true, + owner.clone(), + ), + ) + .await; + assert!( + blocked.is_err(), + "the encryption walk must defer (block on admission) when the pool is exhausted" + ); + assert!( + !marker.exists(), + "the walk's rev-list must not run while its admission permit is unavailable (P1-e)" + ); + + // Release admission: the SAME walk now runs (defer, not shed) — rev-list fires. + drop(held); + let ran = withheld_recipients_gated( + sem, + tmp.path().to_path_buf(), + git_bin, + Duration::from_secs(5), + Vec::new(), + true, + owner, + ) + .await; + assert!( + ran.is_ok(), + "with a permit the walk runs and joins: {ran:?}" + ); + assert!( + marker.exists(), + "once admission is available the deferred walk runs its rev-list" + ); + } + /// #174 SC2 (per-source key, U1): the per-caller read sub-cap keys on the /// resolved source IP, NOT the signed DID, so a disposable-DID farm cannot /// multiply its budget. Fill the source IP's single read slot, then drive two diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 5597cd8d..f18b4963 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -523,6 +523,7 @@ mod tests { git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 74504f90..380ab5f4 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -389,6 +389,12 @@ async fn main() -> Result<()> { git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new( config.max_concurrent_git_pushes, )), + // Bounds concurrent detached post-push encryption walks, sized from the push + // pool (no separate knob — Q1): completed pushes cannot outnumber active + // encryption walks past this (#174 P1-e). + git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new( + config.max_concurrent_git_pushes, + )), git_read_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( config.max_concurrent_reads_per_caller, ), diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 6ce158e3..3ff86923 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -110,6 +110,17 @@ pub struct AppState { /// advert pool (each source also capped by `git_push_advert_per_caller` and the /// per-IP push rate limiter), and the reserved POST pool is untouched (#174). pub git_push_advert_semaphore: Arc, + /// Bounds concurrent post-push encrypt-then-pin history walks. Each successful + /// path-scoped push releases its handler write permit and then runs a DETACHED + /// full-history walk (`withheld_blob_recipients_bounded`) to seal withheld blobs; + /// without a cap, N fast pushes spawn N concurrent full-history git walks past + /// `max_concurrent_git_pushes` (which only bounds the in-handler phase). The walk + /// acquires a permit here and DEFERS (blocks) when the pool is full rather than + /// shedding — the work is background and dropping it would lose the recovery copy + /// (#174 P1-e). A pool of its own, not `git_write_semaphore`: a long background + /// walk must not hold a foreground write slot, and a handler already holding a + /// write permit that needed a second would self-deadlock at pool size 1. + pub git_encrypt_semaphore: Arc, /// Per-caller concurrency sub-cap on the read pool: each caller (keyed on the /// resolved source IP, #174 U1) may hold at most `max_concurrent_reads_per_caller` /// in-flight read ops, so one caller cannot monopolize `git_read_semaphore` diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 9241b518..a27be946 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -86,6 +86,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( 8, From 5749df6e67be0d06fd99b69f0a64156ba3df29ca Mon Sep 17 00:00:00 2001 From: t Date: Tue, 14 Jul 2026 20:33:01 -0500 Subject: [PATCH 32/95] test(node): INV-22 completeness guard for the served-git concurrency surface (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A source-scan tripwire that fails when a new site reintroduces the concurrency-cap class PR #174 fixed, or when one of the five gates (U1-U5) is removed. INV-22: a permit held per op recovers only if every path is duration-bounded and reaps the group before releasing admission, and every detached git task carries admission. Lives in tests/ (a separate crate) on purpose — a guard scanning the file it lives in would match its own identifier literals and pass vacuously; scanning src/ from here keeps every check load-bearing. Checks: run_bounded_git confirms child exit via child_terminated_without_reaping (U1); KillGroupOnDrop launches the reaper via Handle::try_current (U2); git_receive_pack acquires state.git_write_per_caller (U4); the encryption walk runs through withheld_recipients_gated over git_encrypt_semaphore (U5); and the bounded recipients walk is invoked exactly once — only inside the gated helper — so a new spawn_blocking bypass (count > 1) fails (P1-e non-bypass). Proven load-bearing: a synthetic second call site turns the count tripwire RED, and each named gate maps to a fix whose own regression already goes RED on revert. --- crates/gitlawb-node/tests/inv22_gates.rs | 80 ++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 crates/gitlawb-node/tests/inv22_gates.rs diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs new file mode 100644 index 00000000..5f576300 --- /dev/null +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -0,0 +1,80 @@ +//! #174 U6 — INV-22 completeness guard (rung-raising). +//! +//! INV-22: a permit held per op recovers only if every path that holds it is also +//! duration-bounded and reaps the process group before releasing admission, and every +//! detached git/blocking task carries its own admission. PR #174 fixed five paths +//! (U1-U5) that violated this. Each fix has a per-unit RED/GREEN regression; together +//! those form the five-revert matrix. This guard adds the missing piece: a source-scan +//! tripwire that fails when a NEW site reintroduces the class, or when one of the five +//! gates is removed. +//! +//! It lives in `tests/` (a separate crate) on purpose: a guard that scanned the same +//! file it lives in would match its own identifier literals and pass vacuously. Here +//! the scanned `src/` files never contain this file's literals, so each check is +//! load-bearing — reverting the named gate turns the assertion red. +//! +//! These are deliberately coarse structural checks, not a parser. They cannot prove a +//! gate is *correct* (the per-unit tests do that); they prove a gate is *present and +//! not bypassed*, which is what stops the class from silently regressing. + +use std::path::Path; + +fn src(rel: &str) -> String { + let p = Path::new(env!("CARGO_MANIFEST_DIR")).join("src").join(rel); + std::fs::read_to_string(&p).unwrap_or_else(|e| panic!("read {}: {e}", p.display())) +} + +#[test] +fn inv22_concurrency_gates_present_and_not_bypassed() { + let repos = src("api/repos.rs"); + let smart_http = src("git/smart_http.rs"); + let vis = src("git/visibility_pack.rs"); + + // U1 / P1-a: run_bounded_git stands the watchdog down only after confirming the + // child actually terminated (WNOWAIT), not on the raw stdout-drain EOF — otherwise + // a child that closes stdout then hangs pins the permit past the deadline. The + // probe is defined and called, so >= 2 occurrences; reverting the fix removes both. + assert!( + vis.matches("child_terminated_without_reaping").count() >= 2, + "U1/P1-a gate missing: run_bounded_git must confirm child exit via \ + child_terminated_without_reaping before signalling the watchdog" + ); + + // U2 / P1-c: on client disconnect KillGroupOnDrop must launch a detached reaper + // that runs the full TERM/grace/KILL/reap, not a lone SIGTERM. The reaper is spawned + // via a runtime handle in Drop; `Handle::try_current` is unique to that launch (the + // timeout path already has an async context and never calls it). + assert!( + smart_http.contains("Handle::try_current"), + "U2/P1-c gate missing: KillGroupOnDrop::drop must launch the full \ + TERM/grace/KILL reaper on disconnect (Handle::try_current), not a lone SIGTERM" + ); + + // U4 / P1-d: git_receive_pack must acquire the per-source write sub-cap before the + // global write permit. The acquire reads `state.git_write_per_caller`; comments name + // the field without the `state.` prefix, so this targets the real acquire site. + assert!( + repos.contains("state.git_write_per_caller"), + "U4/P1-d gate missing: git_receive_pack must acquire the per-source write cap \ + (state.git_write_per_caller) before the global write permit" + ); + + // U5 / P1-e: the detached post-push encryption walk must run through the + // admission-gated helper, which is wired to the shared encrypt pool. + assert!( + repos.contains("fn withheld_recipients_gated") + && repos.contains("state.git_encrypt_semaphore"), + "U5/P1-e gate missing: the encryption walk must run through \ + withheld_recipients_gated, which acquires git_encrypt_semaphore" + ); + + // P1-e non-bypass tripwire: the bounded recipients walk is spawn_blocking'd nowhere + // but inside withheld_recipients_gated. A second call site (count > 1) is a new + // detached git walk that skips the admission gate — exactly the class U5 closed. + assert_eq!( + repos.matches("withheld_blob_recipients_bounded").count(), + 1, + "P1-e bypass: the bounded recipients walk must be invoked only inside \ + withheld_recipients_gated; a new call site bypasses the encrypt-walk admission cap" + ); +} From 72e899dae022dbc757be4198161629dad5245a92 Mon Sep 17 00:00:00 2001 From: t Date: Wed, 15 Jul 2026 11:33:20 -0500 Subject: [PATCH 33/95] fix(node): hold served-git admission until the process group is reaped on the plain spawn paths (#174) On the plain (non-path-scoped) info_refs / upload-pack / receive-pack paths the global + per-source admission permits were handler-locals that dropped the instant the handler future dropped on a client disconnect, while KillGroupOnDrop reaps the git process group in a DETACHED task (SIGTERM -> grace -> SIGKILL -> reap). So a disconnect-spammer admitted replacements while prior groups were still alive, defeating the process cap and the per-source cap during teardown. be0cdd6 already fixed this for the path-scoped upload-pack walk; this closes the residual plain paths (P1-a). Introduce AdmissionGuard, a move-only holder of the permits, threaded through info_refs/upload_pack/receive_pack/run_git_service into drive_git_child's KillGroupOnDrop. On disconnect the guard moves into the detached reaper and drops only after the group is ESRCH-confirmed reaped; on success/timeout disarm() returns it for the earliest provably-free drop. The rev-list/pack-objects visibility-walk callers hold no admission and pass None. Thread git_bin through upload_pack/receive_pack (they hardcoded "git") so the plain POST path is drivable by an injected fake git, matching the existing injectable seam on info_refs/run_git_service/upload_pack_excluding. Handler-layer regression upload_pack_plain_permit_held_through_group_reap_after_disconnect: isolates the global read pool (size 1, per-source + rate limiter permissive), a SIGTERM-ignoring descendant keeps the group alive across the reap window, and asserts available_permits()==0 held on disconnect + a cross-source replacement sheds 503, then frees after ESRCH. Reverting the guard-into-reaper move (release immediately) turns it RED at the held-through-reap assertion. Plus a None-key arm. --- crates/gitlawb-node/src/api/repos.rs | 269 +++++++++++++++++++++- crates/gitlawb-node/src/git/smart_http.rs | 130 +++++++++-- 2 files changed, 382 insertions(+), 17 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 430b2f7f..16a8ed0d 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -653,8 +653,14 @@ pub async fn git_info_refs( AppError::Git(e.to_string()) })?; + // Move the admission permits into the guard so they release only after the spawned + // git process group is confirmed reaped, on complete/timeout/disconnect — not the + // instant a disconnect drops this future while the detached reaper is still tearing + // the group down (#174 P1-a). The handler keeps no copy: `_permit`/`_caller_permit` + // are moved in, so admission tracks the real process lifetime. + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); - smart_http::info_refs("git", &service, &disk_path, git_timeout) + smart_http::info_refs("git", &service, &disk_path, git_timeout, Some(admission)) .await .map_err(|e| { let app = git_service_app_error(&e); @@ -848,7 +854,12 @@ pub async fn git_upload_pack( // withheld walk and serve the pack directly. let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); let resp = if !visibility_pack::has_path_scoped_rule(&rules) { - smart_http::upload_pack(&disk_path, body, git_timeout).await + // Plain (non-path-scoped) serve: move both admission permits into the guard so + // they release only after the spawned git group is reaped, on + // complete/timeout/disconnect — not the instant a disconnect drops this future + // (#174 P1-a). The handler keeps no copy. + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + smart_http::upload_pack(&state.git_bin, &disk_path, body, git_timeout, Some(admission)).await } else { // withheld_blob_oids walks every ref with blocking `git ls-tree`; keep that // off the async worker thread. Move BOTH admission permits INTO the blocking @@ -886,9 +897,17 @@ pub async fn git_upload_pack( let withheld = withheld.map_err(|e| git_service_app_error(&e))?; if withheld.is_empty() { - smart_http::upload_pack(&disk_path, body, git_timeout).await + // No blobs to withhold: serve the plain pack, moving the permits returned by + // the walk into the guard so admission tracks the served git group's reap + // (the walk already held them per be0cdd6; this hands them to the serve). + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + smart_http::upload_pack(&state.git_bin, &disk_path, body, git_timeout, Some(admission)).await } else { tracing::info!(repo = %name, caller = ?caller, withheld = withheld.len(), "serving filtered pack"); + // upload_pack_excluding runs its own rev-list/pack-objects (both pass `None` + // admission internally); the walk's permits stay handler-locals held across + // this serve, as be0cdd6 established, and drop when the handler returns. + let _hold = (_permit, _caller_permit); smart_http::upload_pack_excluding(&disk_path, body, &withheld, git_timeout).await } } @@ -1172,7 +1191,14 @@ pub async fn git_receive_pack( tracing::debug!(repo = %name, path = %disk_path.display(), "running git receive-pack"); let body_len = body.len(); let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); - let receive_result = smart_http::receive_pack(&disk_path, body, git_timeout).await; + // Move both admission permits into the guard so they release only after the spawned + // receive-pack process group is reaped, on complete/timeout/disconnect — not the + // instant a disconnect drops this future while the detached reaper runs (#174 P1-a). + // The handler keeps no copy. This is independent of the write-lock `guard.release` + // below: admission tracks the git process lifetime, the write lock tracks the repo. + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + let receive_result = + smart_http::receive_pack(&state.git_bin, &disk_path, body, git_timeout, Some(admission)).await; // Always release the advisory lock — even on error — to prevent stale locks // from blocking subsequent pushes. Only upload to Tigris when the push @@ -3654,6 +3680,241 @@ mod tests { } } + /// #174 U1 (P1-a, plain-spawn residual, RED-before/GREEN-after): on the PLAIN + /// (non-path-scoped) upload-pack path a client disconnect must NOT release the + /// global read admission while the detached process-group reaper is still tearing + /// down a git group that ignores SIGTERM. The `be0cdd6` fix moved permits into the + /// path-scoped `spawn_blocking` walk; this closes the residual plain path, where the + /// permits were handler-locals that dropped the instant the future was dropped. + /// + /// Isolate the GLOBAL pool: read pool = 1, per-source cap + rate limiter permissive, + /// so the only thing that can shed a replacement is the leaked global permit. Drive + /// the handler until git spawns, disconnect, then assert the global slot stays held + /// (`available_permits() == 0`) AND a replacement sheds 503 while the group is alive; + /// after the reaper SIGKILLs+reaps the group the slot frees and a replacement is no + /// longer shed by the global cap. On the pre-fix code the handler-local permit drops + /// on future-drop and the slot frees at once (RED). + #[cfg(unix)] + #[sqlx::test] + async fn upload_pack_plain_permit_held_through_group_reap_after_disconnect(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + let descfile = tmp.path().join("desc.pid"); + // Fake git for the plain upload-pack path (invoked as `git upload-pack + // --stateless-rpc `). It forks a descendant that TRAPS SIGTERM, records its + // pid, and loops ~20s, then `wait`s — so on disconnect the group leader dies on + // the reaper's SIGTERM but the descendant survives until the reaper escalates to + // SIGKILL, keeping the group alive (ESRCH not reached) across the observation + // window. Bounded so a broken fix leaks no permanent orphan. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + upload-pack)\n\ + sh -c 'trap \"\" TERM; echo $$ > \"{}\"; i=0; while [ $i -lt 20 ]; do sleep 1; i=$((i+1)); done' &\n\ + wait ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + descfile.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + // Isolate the global read pool: size 1; per-source cap + rate limiter permissive + // so only the leaked global permit can shed the replacement. + state.git_read_semaphore = Arc::new(Semaphore::new(1)); + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let owner = "z6up1st"; + let name = "up1"; + state + .db + .upsert_mirror_repo(owner, name, "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + // Real bare repo at the path acquire() computes, so the handler reaches the + // spawn. No path-scoped rule -> the PLAIN serve branch (this test's target). + state + .repo_store + .init(&rec.owner_did, &rec.name) + .await + .unwrap(); + + let sem = state.git_read_semaphore.clone(); + assert_eq!(sem.available_permits(), 1, "one read slot before the request"); + + let router = crate::server::build_router(state); + let make_req = |peer: SocketAddr| { + let mut req = Request::builder() + .method(Method::POST) + .uri(format!("/{owner}/{name}/git-upload-pack")) + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req + }; + + let peer: SocketAddr = "203.0.113.71:5000".parse().unwrap(); + let mut fut = Box::pin(router.clone().oneshot(make_req(peer))); + // Drive until git spawns (the descendant records its pid) — the request is + // inside the plain serve, holding the global read permit. Stop polling the + // instant the future completes (re-polling a completed oneshot panics); read the + // descfile first so a spawn that recorded its pid then returned is still caught. + let mut spawned: Option = None; + let mut early = None; + for _ in 0..500 { + let done = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&descfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + spawned = Some(p); + break; + } + if let Ok(resp) = done { + early = Some(resp.map(|r| r.status())); + break; + } + } + let desc = spawned + .unwrap_or_else(|| panic!("the fake git must have spawned; early finish: {early:?}")); + // Kill the descendant regardless of outcome so a RED run leaks no orphan. + struct ReapOnDrop(i32); + impl Drop for ReapOnDrop { + fn drop(&mut self) { + unsafe { + libc::kill(self.0, libc::SIGKILL); + } + } + } + let _cleanup = ReapOnDrop(desc); + assert!( + unsafe { libc::kill(desc, 0) == 0 }, + "descendant should be running before the disconnect" + ); + assert_eq!( + sem.available_permits(), + 0, + "the read slot is held while the git op runs" + ); + + // Client disconnect: drop the request future. The detached reaper now owns the + // AdmissionGuard and will not drop it until the group is ESRCH-confirmed reaped. + drop(fut); + + // Load-bearing: the slot must STAY held while the SIGTERM-ignoring group is still + // alive. On the pre-fix code the handler-local permit drops here and the slot + // frees at once (RED). Check quickly (before the reaper's ~2s SIGKILL escalation). + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + assert!( + unsafe { libc::kill(desc, 0) == 0 }, + "the SIGTERM-ignoring descendant must still be alive during the hold window" + ); + assert_eq!( + sem.available_permits(), + 0, + "on disconnect the read admission must be HELD until the process group is \ + reaped, not released the instant the future drops (P1-a)" + ); + // A replacement request from a DIFFERENT source must shed 503 — the only pool + // that can shed it is the leaked global permit (per-source cap is permissive). + let peer2: SocketAddr = "203.0.113.72:5000".parse().unwrap(); + let resp = router.clone().oneshot(make_req(peer2)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "while the prior group is still alive the held global permit must shed a \ + replacement with 503" + ); + + // After the reaper SIGKILLs + reaps the group the AdmissionGuard drops and the + // slot frees. Poll for recovery. + let mut freed = false; + for _ in 0..400 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + assert!( + freed, + "once the reaper confirms the group gone the admission guard must drop and \ + free the global slot" + ); + // A replacement is now no longer shed by the global cap (it proceeds past + // admission; it then fails downstream on the fake git, which is not a 503). + let peer3: SocketAddr = "203.0.113.73:5000".parse().unwrap(); + let resp = router.oneshot(make_req(peer3)).await.unwrap(); + assert_ne!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "after the group is reaped the freed slot must admit a replacement" + ); + } + + /// #174 U1 (P1-a): the `None`-key arm — a request with no resolvable source key + /// (no trusted-proxy header, no peer) is bounded by the GLOBAL read pool only, never + /// a per-source cap. With the global read pool exhausted such a request still sheds + /// 503, proving the plain path admits/sheds on the global pool for the `None` arm + /// (the counterpart to the `Some(ip)` arm above). Complements the resolver-arm rule: + /// neither arm is vacuous. + #[tokio::test] + async fn upload_pack_plain_none_key_arm_sheds_on_global_pool() { + use axum::body::Body; + use axum::http::{Method, Request, StatusCode}; + use axum::Router; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state_lazy(); + // Global read pool exhausted; per-source cap permissive so only the global pool + // can shed. No ConnectInfo + no trusted header -> read_caller_key resolves None. + state.git_read_semaphore = Arc::new(Semaphore::new(0)); + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let router = Router::new() + .route( + "/{owner}/{repo}/git-upload-pack", + axum::routing::post(crate::api::repos::git_upload_pack), + ) + .with_state(state); + // No ConnectInfo extension and no XFF header: the caller key is None. + let req = Request::builder() + .method(Method::POST) + .uri("/alice/repo.git/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a None-key request must still shed 503 on the exhausted GLOBAL read pool" + ); + } + /// #174 U4 (P1-d, RED-before/GREEN-after): the authenticated receive-pack POST /// carries a per-source WRITE sub-cap so one source IP cannot monopolize the write /// pool with many slow pushes (owner enforcement defaults off, so disposable DIDs diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index a5649a42..64d3cb7f 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -10,6 +10,40 @@ use std::time::{Duration, Instant}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::process::Command; +/// Owns the served-git admission permits (global + per-source) for the lifetime of +/// the work they admitted, so admission is released only when that work is truly +/// done — not the instant the handler future drops on a client disconnect. +/// +/// A move-only wrapper: no methods beyond construction and `Drop`. The handler +/// MOVEs its permits in and keeps no copy (a retained copy would drop early and +/// release admission the moment the future is dropped, defeating the guard). It is +/// threaded into `drive_git_child`, whose [`KillGroupOnDrop`] moves it into the +/// detached reaper on disconnect, so both permits drop only after the process group +/// is confirmed reaped (`kill(-pgid,0)==ESRCH`) rather than while the group is still +/// alive holding PIDs past the concurrency cap (#174 P1-a, plain-spawn residual). +/// +/// The `be0cdd6` path-scoped upload-pack walk already applies this discipline by +/// moving its permits into the `spawn_blocking`; this generalizes it to the plain +/// (non-path-scoped) `info_refs` / `run_git_service` spawn paths. +pub struct AdmissionGuard { + // Boxed so the concrete permit types stay private to the caller (repos.rs) — the + // guard only needs to hold them and drop them, never inspect them. `Send + + // 'static` so the guard can move into the detached reaper task. + _global: Option>, + _caller: Option>, +} + +impl AdmissionGuard { + /// Take ownership of the global permit and an optional per-caller permit. Both are + /// erased to `Box` — the guard's only job is to hold them until it drops. + pub fn new(global: impl Send + 'static, caller: Option) -> Self { + Self { + _global: Some(Box::new(global)), + _caller: caller.map(|c| Box::new(c) as Box), + } + } +} + /// Handle `GET /:owner/:repo/info/refs?service=git-upload-pack` /// or `?service=git-receive-pack` /// @@ -26,6 +60,7 @@ pub async fn info_refs( service: &str, repo_path: &Path, timeout: Duration, + admission: Option, ) -> Result { validate_service(service)?; @@ -36,7 +71,8 @@ pub async fn info_refs( .arg("--advertise-refs") .arg(repo_path); // No request body — advertise-refs does not read stdin. - let stdout = drive_git_child(command, Bytes::new(), timeout, "advertise-refs").await?; + let stdout = + drive_git_child(command, Bytes::new(), timeout, "advertise-refs", admission).await?; let content_type = format!("application/x-{service}-advertisement"); @@ -61,12 +97,21 @@ pub async fn info_refs( /// Serves pack data for a clone or fetch. This is stateless — the entire /// negotiation happens in a single request/response. pub async fn upload_pack( + git_bin: &str, repo_path: &Path, request_body: Bytes, timeout: Duration, + admission: Option, ) -> Result { - let output = - run_git_service("git", "git-upload-pack", repo_path, request_body, timeout).await?; + let output = run_git_service( + git_bin, + "git-upload-pack", + repo_path, + request_body, + timeout, + admission, + ) + .await?; Ok(Response::builder() .status(StatusCode::OK) @@ -80,12 +125,21 @@ pub async fn upload_pack( /// Accepts a push. The caller MUST verify HTTP Signature auth before /// calling this function. pub async fn receive_pack( + git_bin: &str, repo_path: &Path, request_body: Bytes, timeout: Duration, + admission: Option, ) -> Result { - let output = - run_git_service("git", "git-receive-pack", repo_path, request_body, timeout).await?; + let output = run_git_service( + git_bin, + "git-receive-pack", + repo_path, + request_body, + timeout, + admission, + ) + .await?; Ok(Response::builder() .status(StatusCode::OK) @@ -120,6 +174,12 @@ struct KillGroupOnDrop { // own SIGCHLD-driven orphan reaper. child: Option, pgid: Option, + // The global + per-source admission permits for this op, if any. On the plain + // (non-path-scoped) spawn paths repos.rs moves its permits in here so admission is + // released only when the group is confirmed reaped, on every exit — complete, + // timeout, or client-disconnect (#174 P1-a). The rev-list/pack-objects + // visibility-walk callers hold no admission and pass `None`. + admission: Option, } #[cfg(unix)] @@ -131,17 +191,25 @@ impl KillGroupOnDrop { /// Disarm on the success/timeout path: the body has already reaped the child (its /// `wait()` returned, or `reap_group_on_timeout` ran), so drop the handle and clear - /// the pgid, leaving the guard's Drop a no-op. - fn disarm(&mut self) { + /// the pgid, leaving the guard's Drop a no-op. Returns the admission guard so the + /// caller drops it at the earliest provably-free point (the group is already reaped + /// on both callers of this) rather than at end-of-bookkeeping. + fn disarm(&mut self) -> Option { self.pgid = None; self.child = None; + self.admission.take() } } #[cfg(unix)] impl Drop for KillGroupOnDrop { fn drop(&mut self) { + // Move any admission guard out first so it travels with the reaper (or drops + // here on the no-child / no-runtime fallback), never before the group is gone. + let admission = self.admission.take(); let (Some(mut child), Some(pgid)) = (self.child.take(), self.pgid) else { + // Nothing to reap (already disarmed); dropping `admission` here is correct — + // the group is already gone. return; }; // A sync Drop cannot await, so launch a detached reaper that owns the child and @@ -154,6 +222,13 @@ impl Drop for KillGroupOnDrop { Ok(handle) => { handle.spawn(async move { reap_group_on_timeout(&mut child).await; + // Release admission only now: the group is ESRCH-confirmed gone (or + // hit the ~4s D-state hard cap, past which nothing in userspace can + // free the PIDs anyway). Holding the permits until here is what + // stops disconnect-spam from admitting replacements while the prior + // group is still alive (#174 P1-a). `admission` is moved in and + // dropped when this closure ends. + drop(admission); }); } Err(_) => { @@ -162,6 +237,9 @@ impl Drop for KillGroupOnDrop { unsafe { libc::kill(-pgid, libc::SIGTERM); } + // No runtime to await the reap; drop admission best-effort after the + // synchronous signal (the fallback path is not on the hot request path). + drop(admission); } } } @@ -246,13 +324,14 @@ async fn run_git_service( repo_path: &Path, input: Bytes, timeout: Duration, + admission: Option, ) -> Result> { let mut command = Command::new(git_bin); command .arg(service_to_command(service)) .arg("--stateless-rpc") .arg(repo_path); - drive_git_child(command, input, timeout, service).await + drive_git_child(command, input, timeout, service, admission).await } /// Drive a spawned git child under `timeout` with process-group teardown, returning @@ -267,6 +346,7 @@ async fn drive_git_child( input: Bytes, timeout: Duration, what: &str, + admission: Option, ) -> Result> { command .stdin(Stdio::piped()) @@ -300,7 +380,12 @@ async fn drive_git_child( let mut group_guard = KillGroupOnDrop { child: Some(child), pgid, + admission, }; + // On non-unix there is no process-group teardown, so hold the admission guard here + // for the child's whole interaction; it drops when this function returns. + #[cfg(not(unix))] + let _admission = admission; let mut out = Vec::new(); let mut err = Vec::new(); @@ -339,18 +424,21 @@ async fn drive_git_child( // The join runs all arms to completion, so the child is reaped: disarm // before surfacing any interaction error (a read/wait error), else the // guard's drop would reap an already-reaped child / signal a reused pgid. + // Dropping the returned admission guard here releases the permits at the + // earliest provably-free point on the success path (the op finished). #[cfg(unix)] - group_guard.disarm(); + drop(group_guard.disarm()); result? } Err(_elapsed) => { // Timeout: tear the whole group down and reap it before returning so a // caller releasing a write lock can't race a still-live git. Then disarm - // the (now redundant) guard so its drop can't hit a reused pgid. + // the (now redundant) guard so its drop can't hit a reused pgid. The + // returned admission guard drops here, AFTER the group is confirmed reaped. #[cfg(unix)] { reap_group_on_timeout(group_guard.child_mut()).await; - group_guard.disarm(); + drop(group_guard.disarm()); } #[cfg(not(unix))] { @@ -415,7 +503,9 @@ async fn rev_list_keep( command .args(["rev-list", "--objects", "--all"]) .current_dir(repo_path); - let stdout = drive_git_child(command, Bytes::new(), timeout, "rev-list").await?; + // The visibility-walk callers intentionally hold no admission permit (their + // admission is governed elsewhere), so pass `None` (#174 KTD2). + let stdout = drive_git_child(command, Bytes::new(), timeout, "rev-list", None).await?; let mut keep = Vec::new(); for line in String::from_utf8_lossy(&stdout).lines() { let oid = line.split_whitespace().next().unwrap_or(""); @@ -464,6 +554,8 @@ pub async fn build_filtered_pack( Bytes::from(data), deadline.saturating_duration_since(Instant::now()), "pack-objects", + // Visibility-walk pack build: no admission permit here (#174 KTD2). + None, ) .await } @@ -736,7 +828,7 @@ mod tests { axum::extract::Query(q): axum::extract::Query>, ) -> Response { let service = q.get("service").cloned().unwrap_or_default(); - info_refs("git", &service, &st.repo, Duration::from_secs(30)) + info_refs("git", &service, &st.repo, Duration::from_secs(30), None) .await .unwrap() } @@ -1021,6 +1113,7 @@ mod tests { let _guard = KillGroupOnDrop { child: Some(child), pgid: Some(pid), + admission: None, }; } @@ -1078,6 +1171,7 @@ mod tests { let _guard = KillGroupOnDrop { child: Some(child), pgid: Some(pid), + admission: None, }; } @@ -1114,6 +1208,7 @@ mod tests { let mut guard = KillGroupOnDrop { child: Some(child), pgid: Some(pid), + admission: None, }; guard.disarm(); } // disarmed -> no reaper, no kill @@ -1320,6 +1415,7 @@ mod tests { tmp.path(), Bytes::new(), Duration::from_secs(60), + None, )); // Advance the future a slice at a time until the fake records its @@ -1421,6 +1517,7 @@ mod tests { tmp.path(), Bytes::new(), Duration::from_secs(60), + None, )); // Drive the future a slice at a time until the descendant records its pid. let mut desc: Option = None; @@ -1482,6 +1579,7 @@ mod tests { "git-upload-pack", tmp.path(), Duration::from_millis(200), + None, ), ) .await @@ -1589,6 +1687,7 @@ mod tests { tmp.path(), Bytes::new(), Duration::from_secs(60), + None, ) }) .await; @@ -1633,6 +1732,7 @@ mod tests { tmp.path(), Bytes::new(), Duration::from_secs(60), + None, ) }) .await; @@ -1683,6 +1783,7 @@ mod tests { tmp.path(), Bytes::new(), git_timeout, + None, ), ) }) @@ -1757,6 +1858,7 @@ mod tests { tmp.path(), Bytes::new(), git_timeout, + None, ), ) }) @@ -1794,6 +1896,7 @@ mod tests { tmp.path(), Bytes::new(), Duration::from_millis(200), + None, ), ) .await; @@ -1834,6 +1937,7 @@ mod tests { tmp.path(), big, Duration::from_secs(60), + None, ) .await; From 8d17038e5bc1e37cc76c931288c4d993a17b39c3 Mon Sep 17 00:00:00 2001 From: t Date: Wed, 15 Jul 2026 12:00:02 -0500 Subject: [PATCH 34/95] fix(node): bound the served-git storage-acquisition phase with a release-on-expiry deadline (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The served-git admission permit is acquired, then RepoStore::acquire*/advisory-lock runs before the bounded git runner starts — with no local deadline. acquire/acquire_fresh await Tigris HEAD/GET, and acquire_write's advisory-lock loop awaits a per-iteration pg_try_advisory_lock that can block indefinitely on a hung Postgres pool. Since GITLAWB_GIT_SERVICE_TIMEOUT_SECS only starts once git spawns, a stalled backend pins the permit and drains the pool until every later request 503s (P1-2). Wrap each git-path acquire (git_info_refs read/advert, git_upload_pack, git_receive_pack) and the /ipfs per-repo acquire loop in tokio::time::timeout(git_acquire_timeout_secs, ..). On expiry the git paths return AppError::Overloaded (503 + Retry-After); the permit is a handler-local at that point (moved into the AdmissionGuard only after acquire) so the early return frees the slot. The /ipfs per-repo acquire keeps its fail-closed `continue` on timeout (never serves an un-acquired repo). No repo_store.rs signature change: the outer timeout cancels a mid-sleep/mid-fetch_one future, so no in-loop deadline is needed. New knob GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS (default 30, range 1..), modeled on git_service_timeout_secs, kept separate because acquisition and git execution are distinct cost centers. Handler-layer regression receive_pack_acquire_deadline_sheds_and_releases_permit: holds the same pg advisory lock acquire_write derives on a second connection so the real loop must retry, drives git-receive-pack with the deadline at 2s and write pool size 1, and asserts 503 at ~2s + the permit recovers + a follow-up admits once the lock frees. Removing the timeout wrapper turns it RED (blocks ~31s to the test ceiling on the ~59s advisory loop). --- crates/gitlawb-node/src/api/ipfs.rs | 21 +- crates/gitlawb-node/src/api/repos.rs | 307 ++++++++++++++++++++++++--- crates/gitlawb-node/src/config.rs | 21 ++ 3 files changed, 315 insertions(+), 34 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index e921c39f..284c9398 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -114,9 +114,24 @@ pub async fn get_by_cid( continue; } - let repo_path = match state.repo_store.acquire(&repo.owner_did, &repo.name).await { - Ok(p) => p, - Err(_) => continue, + // Bound the per-repo acquire under `git_acquire_timeout_secs`: this loop shares + // the P1-2 stall vector (a hung Tigris HEAD/GET on one repo would otherwise + // block the whole /ipfs request). On expiry keep the existing fail-closed skip — + // never serve an un-acquired repo; a public copy (if any) still gets its turn. + let acquire_deadline = + std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); + let repo_path = match tokio::time::timeout( + acquire_deadline, + state.repo_store.acquire(&repo.owner_did, &repo.name), + ) + .await + { + Ok(Ok(p)) => p, + Ok(Err(_)) => continue, + Err(_elapsed) => { + tracing::warn!(repo = %repo.name, "repo acquire timed out during /ipfs walk; skipping repo"); + continue; + } }; // Check whether the object exists in this repo before any expensive diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 16a8ed0d..5b11404d 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -637,21 +637,37 @@ pub async fn git_info_refs( // For receive-pack (push), download the latest from Tigris so the client // sees the same refs that acquire_write() will operate on. - let disk_path = if service == "git-receive-pack" { - state - .repo_store - .acquire_fresh(&record.owner_did, &record.name) - .await - } else { - state - .repo_store - .acquire(&record.owner_did, &record.name) - .await - } - .map_err(|e| { - tracing::error!(repo = %name, service = %service, err = %e, "repo acquire failed"); - AppError::Git(e.to_string()) - })?; + // + // Bound the acquire under `git_acquire_timeout_secs`: the concurrency permit is + // already held above, and `git_service_timeout_secs` only starts once git spawns, + // so an un-deadlined acquire (a hung Tigris HEAD/GET here) pins the permit until + // the pool drains (#174 P1-2). On expiry the handler-local `_permit`/`_caller_permit` + // drop on the early return (the AdmissionGuard is not built until after acquire), + // so the shed frees the slot; return a bounded 503. + let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); + let acquire_fut = async { + if service == "git-receive-pack" { + state + .repo_store + .acquire_fresh(&record.owner_did, &record.name) + .await + } else { + state + .repo_store + .acquire(&record.owner_did, &record.name) + .await + } + }; + let disk_path = tokio::time::timeout(acquire_deadline, acquire_fut) + .await + .map_err(|_elapsed| { + tracing::warn!(repo = %name, service = %service, "repo acquire timed out; shedding with 503"); + AppError::Overloaded("git service acquisition timed out, retry shortly".into()) + })? + .map_err(|e| { + tracing::error!(repo = %name, service = %service, err = %e, "repo acquire failed"); + AppError::Git(e.to_string()) + })?; // Move the admission permits into the guard so they release only after the spawned // git process group is confirmed reaped, on complete/timeout/disconnect — not the @@ -842,11 +858,21 @@ pub async fn git_upload_pack( // exec (INV-10). let _permit = git_permit(&state.git_read_semaphore)?; - let disk_path = state - .repo_store - .acquire(&record.owner_did, &record.name) - .await - .map_err(|e| AppError::Git(e.to_string()))?; + // Bound the acquire under `git_acquire_timeout_secs` so a hung Tigris HEAD/GET + // cannot pin the read permit indefinitely (#174 P1-2). The permit is a handler + // local here (moved into the AdmissionGuard only below, once git is spawned), so + // the early return on timeout drops it and frees the slot; shed a bounded 503. + let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); + let disk_path = tokio::time::timeout( + acquire_deadline, + state.repo_store.acquire(&record.owner_did, &record.name), + ) + .await + .map_err(|_elapsed| { + tracing::warn!(repo = %name, "repo acquire timed out; shedding with 503"); + AppError::Overloaded("git service acquisition timed out, retry shortly".into()) + })? + .map_err(|e| AppError::Git(e.to_string()))?; let body_len = body.len(); // No path-scoped rule can withhold an individual blob, and the whole-repo @@ -1179,14 +1205,31 @@ pub async fn git_receive_pack( } tracing::debug!(repo = %name, "acquiring write lock"); - let guard = state - .repo_store - .acquire_write(&record.owner_did, &record.name) - .await - .map_err(|e| { - tracing::error!(repo = %name, err = %e, "acquire_write failed"); - AppError::Git(e.to_string()) - })?; + // Bound the write acquire under `git_acquire_timeout_secs`. acquire_write's + // advisory-lock loop already caps at ~60s, but its per-iteration + // `pg_try_advisory_lock().fetch_one(&pool)` can block indefinitely on a hung / + // exhausted Postgres pool (so the 60-count never advances) — and the write permit + // is held the whole time, draining the pool (#174 P1-2). The outer + // `tokio::time::timeout` cancels a mid-sleep/mid-`fetch_one` future, so it bounds + // both the loop and a hung iteration without any repo_store.rs change (KTD3). The + // permit is a handler local here (moved into the AdmissionGuard only after this), + // so the early return on timeout drops it and frees the slot; shed a bounded 503. + let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); + let guard = tokio::time::timeout( + acquire_deadline, + state + .repo_store + .acquire_write(&record.owner_did, &record.name), + ) + .await + .map_err(|_elapsed| { + tracing::warn!(repo = %name, "acquire_write timed out; shedding with 503"); + AppError::Overloaded("git service acquisition timed out, retry shortly".into()) + })? + .map_err(|e| { + tracing::error!(repo = %name, err = %e, "acquire_write failed"); + AppError::Git(e.to_string()) + })?; let disk_path = guard.path().to_path_buf(); tracing::debug!(repo = %name, path = %disk_path.display(), "running git receive-pack"); let body_len = body.len(); @@ -1197,8 +1240,14 @@ pub async fn git_receive_pack( // The handler keeps no copy. This is independent of the write-lock `guard.release` // below: admission tracks the git process lifetime, the write lock tracks the repo. let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); - let receive_result = - smart_http::receive_pack(&state.git_bin, &disk_path, body, git_timeout, Some(admission)).await; + let receive_result = smart_http::receive_pack( + &state.git_bin, + &disk_path, + body, + git_timeout, + Some(admission), + ) + .await; // Always release the advisory lock — even on error — to prevent stale locks // from blocking subsequent pushes. Only upload to Tigris when the push @@ -3761,7 +3810,11 @@ mod tests { .unwrap(); let sem = state.git_read_semaphore.clone(); - assert_eq!(sem.available_permits(), 1, "one read slot before the request"); + assert_eq!( + sem.available_permits(), + 1, + "one read slot before the request" + ); let router = crate::server::build_router(state); let make_req = |peer: SocketAddr| { @@ -3991,6 +4044,198 @@ mod tests { ); } + /// #174 U2 (P1-2, RED-before/GREEN-after): the storage-acquisition phase is bounded + /// by `git_acquire_timeout_secs`, so a stalled backend releases the admission permit + /// and sheds a 503 instead of pinning the pool. The permit is taken BEFORE + /// `acquire_write`, whose advisory-lock loop can spin ~60s (and whose per-iteration + /// `pg_try_advisory_lock` can block indefinitely on a hung pool), so without the + /// `tokio::time::timeout` wrapper the permit is held far past the deadline. + /// + /// Real stall (no `RepoStore` trait to fake): hold the SAME session-level advisory + /// lock `acquire_write` derives (`advisory_lock_key(owner_slug, repo_name)`, where + /// `owner_slug = owner_did.replace([':','/'], "_")`) on a second pooled connection, + /// so the handler's `pg_try_advisory_lock` returns false every iteration and the loop + /// must retry against the deadline. `git_acquire_timeout_secs = 2`; the request must + /// return 503 (Overloaded) at ~2s (NOT ~59s), and the write permit must be released + /// (`available_permits()` recovers to full once the shed returns). Covers R2. + /// + /// Load-bearing / mutation: remove the `tokio::time::timeout` wrapper on + /// `acquire_write` and the loop runs to ~59s with the permit held the whole time — + /// the `< DEADLINE_CEILING` timing assertion goes RED (observed ~59s) and the permit + /// stays pinned past the deadline. Restore to return GREEN. + #[sqlx::test] + async fn receive_pack_acquire_deadline_sheds_and_releases_permit(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + // Reproduce acquire_write's session-level advisory-lock key exactly so the + // second-connection lock collides with the handler's pg_try_advisory_lock + // (repo_store.rs: advisory_lock_key over owner_slug then repo_name). + fn advisory_lock_key(owner_slug: &str, repo_name: &str) -> i64 { + use std::hash::{Hash, Hasher}; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + owner_slug.hash(&mut hasher); + repo_name.hash(&mut hasher); + hasher.finish() as i64 + } + + let owner = "z6acqdead"; + let name = "acq1"; + // owner_slug as local_path() computes it from the record's owner_did. The + // mirror row stores the short owner as owner_did, so slug == owner (no ':'/'/'). + let owner_slug = owner.replace([':', '/'], "_"); + let lock_key = advisory_lock_key(&owner_slug, name); + + let mut state = crate::test_support::test_state(pool.clone()).await; + // Isolate the write pool at size 1 so available_permits() cleanly reports + // held (0) vs released (1). Per-source cap + trust permissive so only the + // write pool / acquire path can gate. + state.git_write_semaphore = Arc::new(Semaphore::new(1)); + state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Short acquire deadline: the fix must shed here, well before acquire_write's + // ~59s advisory-lock loop would bail on its own. + const ACQUIRE_TIMEOUT_SECS: u64 = 2; + let mut cfg = (*state.config).clone(); + cfg.git_acquire_timeout_secs = ACQUIRE_TIMEOUT_SECS; + // Keep the git-service timeout large so the deadline under test is the acquire + // one, not git execution (which is never reached on the stalled path anyway). + cfg.git_service_timeout_secs = 600; + state.config = std::sync::Arc::new(cfg); + + state + .db + .upsert_mirror_repo(owner, name, "/tmp/z6acqdead-acq1", None, false) + .await + .unwrap(); + + // Hold the advisory lock on a dedicated pooled connection (a distinct session), + // so the handler's pg_try_advisory_lock($lock_key) returns false every iteration + // and acquire_write's real loop must retry against the deadline. Released when + // this connection drops at end of test. + let mut lock_conn = pool + .acquire() + .await + .expect("second connection for the lock"); + sqlx::query("SELECT pg_advisory_lock($1)") + .bind(lock_key) + .execute(&mut *lock_conn) + .await + .expect("hold the advisory lock on the second connection"); + + let did = "did:key:z6MkAcquireDeadlineProofDidAAAAAAAAAAAAAAAA"; + let peer: SocketAddr = "203.0.113.61:5000".parse().unwrap(); + + let sem = state.git_write_semaphore.clone(); + assert_eq!( + sem.available_permits(), + 1, + "one write slot before the request" + ); + + // Drive the authenticated push in the background so we can observe the permit is + // held while acquire_write stalls, then that it is released on the shed. + let state_for_task = state.clone(); + let start = std::time::Instant::now(); + let handle = tokio::spawn(async move { + git_receive_pack( + State(state_for_task), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(peer)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await + }); + + // The handler takes the write permit BEFORE acquire_write, so once it is stalled + // in the advisory-lock loop the pool reports 0 available. Wait for that to prove + // the permit is genuinely held during the stall (and the request really reached + // acquire_write, not an earlier reject). + let mut held = false; + for _ in 0..200 { + if sem.available_permits() == 0 { + held = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert!( + held, + "the write permit must be held while acquire_write stalls on the advisory lock" + ); + + // The bounded acquire deadline must shed with 503 (Overloaded), NOT wait out the + // ~59s advisory-lock loop. Ceiling is comfortably above the 2s deadline + task + // scheduling but far below 59s, so a RED run (no wrapper -> ~59s) fails here. + const DEADLINE_CEILING: std::time::Duration = std::time::Duration::from_secs(20); + let result = tokio::time::timeout( + DEADLINE_CEILING + std::time::Duration::from_secs(10), + handle, + ) + .await + .expect("the handler must return within the ceiling — a hang means the acquire deadline is missing (RED)") + .expect("the receive-pack task must not panic"); + let elapsed = start.elapsed(); + + assert!( + matches!(result, Err(AppError::Overloaded(_))), + "a stalled acquire_write must shed with Overloaded/503 at the acquire deadline; \ + got {result:?}" + ); + assert!( + elapsed < DEADLINE_CEILING, + "the shed must land at ~{ACQUIRE_TIMEOUT_SECS}s (the acquire deadline), not ~59s \ + (the advisory-lock loop). Observed {elapsed:?}; without the timeout wrapper this \ + is ~59s (RED)" + ); + + // Permit release on expiry: the Overloaded return drops the handler-local permit, + // so the isolated write pool must recover to full. A leaked permit here means the + // pool drains under a stalled backend (the #174 P1-2 bug). + let mut freed = false; + for _ in 0..200 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert!( + freed, + "on the acquire-deadline shed the write permit must be released; the pool did \ + not recover to full (permit leaked)" + ); + + // Follow-up admits once the contended lock is released: release the second-conn + // lock, then a fresh push proceeds PAST admission (it fails later on the + // nonexistent on-disk repo, which is NOT an Overloaded/503). Proves the freed + // slot is usable, not merely counted. + sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(lock_key) + .execute(&mut *lock_conn) + .await + .expect("release the advisory lock"); + let followup = git_receive_pack( + State(state.clone()), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some("203.0.113.62:5000".parse().unwrap())), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await; + assert!( + !matches!(followup, Err(AppError::Overloaded(_))), + "once the lock frees, a follow-up push must admit past the (recovered) write \ + pool and acquire; got {followup:?}" + ); + } + /// #174 U5 (P1-e, RED-before/GREEN-after): the post-push encryption walk acquires a /// `git_encrypt_semaphore` permit before running, so completed pushes cannot spawn /// unbounded concurrent full-history walks. With the pool exhausted the gated walk diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index d0a4c319..68125364 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -183,6 +183,27 @@ pub struct Config { )] pub git_service_timeout_secs: u64, + /// Maximum wall-clock time the storage-acquisition phase of a served git + /// operation may run before the request is shed with a 503, in seconds. This + /// bounds `RepoStore::{acquire,acquire_fresh,acquire_write}` — the Tigris + /// HEAD/GET on a read/advert acquire and the advisory-lock retry loop (incl. a + /// per-iteration `pg_try_advisory_lock` that can block on a hung Postgres pool) + /// on a write acquire. A concurrency permit is taken BEFORE this phase, and + /// `git_service_timeout_secs` only starts once git spawns, so without this the + /// acquire phase is unbounded: a stalled backend pins the permit and drains the + /// pool until every later request 503s. On expiry the permit is released and a + /// bounded 503 + Retry-After is returned (fail-closed). Kept separate from + /// `git_service_timeout_secs` because acquisition and git execution are distinct + /// cost centers — one shared budget would let a slow acquire starve git. Must be + /// positive; set it very large to effectively disable the bound. Default: 30s. + #[arg( + long, + env = "GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS", + default_value_t = 30, + value_parser = clap::value_parser!(u64).range(1..) + )] + pub git_acquire_timeout_secs: u64, + /// Maximum connections in the PostgreSQL pool. This is a cap, not a floor /// (connections open lazily). Size against the database server's /// max_connections, remembering admin tooling opens its own pool. From f424ccbbcde49260932d776f3e8b560ec2a674c8 Mon Sep 17 00:00:00 2001 From: t Date: Wed, 15 Jul 2026 12:42:55 -0500 Subject: [PATCH 35/95] fix(node): gate the /ipfs walk with bounded concurrency admission + per-source cap + route rate limit (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GET /ipfs/{cid} (get_by_cid, publicly reachable via optional_signature) ran allowed_blob_set_for_caller_bounded in spawn_blocking inside a per-repo loop with NO global/per-source admission and NO route rate limit — the surface had zero concurrency or rate control. A permissionless caller could fan out concurrent full-history git walks (each up to the walk timeout) exhausting blocking-pool threads and PIDs outside every served-git pool (P1-3). Acquire a global git_ipfs_walk permit (try_acquire_owned, shed 503) plus a per-source sub-cap (with_default_max_keys, reject-before-insert, keyed via client_key/TrustedProxy so XFF can't spoof it) ONCE after CID validation, held as handler locals across the whole request — including every spawn_blocking walk, so the slot reflects real blocking-thread occupancy (a tokio timeout cannot cancel spawn_blocking). None key -> global pool only. Cap repos walked per request so one request can't serialize N full-history walks. Attach rate_limit_by_ip + IpRateLimiter to the /ipfs route (mirrors the git/create/peer routers; the extension is required or the middleware is a silent no-op). Visibility/authz semantics unchanged. New knobs: GITLAWB_MAX_CONCURRENT_IPFS_WALKS (32), GITLAWB_IPFS_WALK_PER_SOURCE (4), GITLAWB_IPFS_MAX_REPOS_WALKED (64), GITLAWB_IPFS_RATE_LIMIT (600/hr, 0 disables) — all range-validated and enforced on-path. Handler-layer regressions (all mutation-verified RED->GREEN): shed-at-capacity 503 (delete acquire -> falls through), per-source cap sheds same source / admits another, None-key arm sheds on the global pool, map self-bounds reject-before-insert, the walk permit is held while the spawn_blocking walk runs (available_permits()==0; dropping it before the loop -> RED), repos-walked cap (remove break -> 2 walks), and the route IP rate limit actually fires 429 (drop the Extension -> RED no-op). --- crates/gitlawb-node/src/api/ipfs.rs | 691 ++++++++++++++++++++++++ crates/gitlawb-node/src/auth/mod.rs | 4 + crates/gitlawb-node/src/config.rs | 120 ++++ crates/gitlawb-node/src/main.rs | 17 + crates/gitlawb-node/src/server.rs | 11 + crates/gitlawb-node/src/state.rs | 27 + crates/gitlawb-node/src/test_support.rs | 6 + 7 files changed, 876 insertions(+) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 284c9398..ac8e615c 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -58,6 +58,11 @@ pub async fn get_by_cid( Path(cid_str): Path, State(state): State, auth: Option>, + // Per-source keying for the walk concurrency sub-cap. Infallible extractors + // (mirror the git handlers in `repos.rs`): `PeerAddr` yields `None` under + // `oneshot` with no `ConnectInfo`, and the header map falls back per `client_key`. + crate::rate_limit::PeerAddr(peer): crate::rate_limit::PeerAddr, + req_headers: HeaderMap, ) -> Result { // 1. Decode the CID and extract the SHA-256 digest let cid = CidGeneric::<64>::from_str(&cid_str) @@ -76,6 +81,36 @@ pub async fn get_by_cid( let caller = auth.as_ref().map(|e| e.0 .0.as_str()); let caller_owned = caller.map(|c| c.to_string()); + // Bounded walk admission (#174 P1-3), taken before any DB/git work so a flood sheds + // cheaply. The per-repo `spawn_blocking` walk below is a full-history git walk with + // no served-git admission of its own; a permissionless caller could otherwise fan + // out concurrent walks past every git pool, exhausting the blocking pool + PIDs. + // Acquire the global permit (and, for a resolvable source, the per-source + // sub-permit) ONCE here and hold BOTH for the whole request — across every + // `spawn_blocking` walk in the loop below — so the slot reflects real blocking-thread + // occupancy (a tokio walk-timeout cannot free it while the blocking work still runs) + // and one request cannot open more than its share of concurrent walks. On + // unavailability shed a clean 503. The per-source key is the resolved source IP + // (`client_key`), never the DID (`/ipfs` admits any `did:key` unthrottled, so a DID + // key would be free to mint around); a `None` key (no trusted header, no peer) is + // bounded by the global pool only, never the per-source sub-cap. + let _ipfs_walk_permit = state + .git_ipfs_walk_semaphore + .clone() + .try_acquire_owned() + .map_err(|_| { + tracing::warn!("/ipfs walk concurrency cap reached; shedding request with 503"); + AppError::Overloaded("ipfs service at capacity, retry shortly".into()) + })?; + let source_key = crate::rate_limit::client_key(&req_headers, peer, state.push_limiter_trust); + let _ipfs_caller_permit = match &source_key { + Some(ip) => Some(state.git_ipfs_walk_per_caller.try_acquire(ip).ok_or_else(|| { + tracing::warn!(key = %ip, "/ipfs per-source walk cap reached; shedding request with 503"); + AppError::Overloaded("ipfs service at capacity for this source, retry shortly".into()) + })?), + None => None, + }; + // 2. Search all repos for an object with this SHA-256 let repos = state .db @@ -104,6 +139,11 @@ pub async fn get_by_cid( // deny entry (#126). let mut allowed_memo: HashMap> = HashMap::new(); + // Cap the number of candidate repos one request walks (it already short-circuits on + // serve): a CID present in — or path-gated out of — many repos must not serialize an + // unbounded number of full-history walks inside the single held admission slot. + let mut repos_walked: usize = 0; + for repo in &repos { // Repo-level read gate against THIS row's own rules (KTD2a). let rules: &[crate::db::VisibilityRule] = rules_by_repo @@ -114,6 +154,19 @@ pub async fn get_by_cid( continue; } + // Loop bound (#174 P1-3): once this request has walked its cap of candidate + // repos (each a git subprocess, up to a full-history walk), stop and fall + // through to the opaque 404 rather than serialize an unbounded number under the + // single held admission slot. + if repos_walked >= state.config.ipfs_max_repos_walked { + tracing::warn!( + cap = state.config.ipfs_max_repos_walked, + "/ipfs request hit the per-request repo-walk cap; stopping the scan" + ); + break; + } + repos_walked += 1; + // Bound the per-repo acquire under `git_acquire_timeout_secs`: this loop shares // the P1-2 stall vector (a hung Tigris HEAD/GET on one repo would otherwise // block the whole /ipfs request). On expiry keep the existing fail-closed skip — @@ -249,3 +302,641 @@ pub async fn list_pins(State(state): State) -> Result Router { + Router::new() + .route( + "/ipfs/{cid}", + axum::routing::get(crate::api::ipfs::get_by_cid), + ) + .layer(axum::middleware::from_fn(crate::auth::optional_signature)) + .with_state(state) + } + + /// A syntactically valid CIDv1(raw, sha2-256) string the handler decodes past its + /// CID/hash-code validation, so the request reaches the walk admission (not a 400). + fn valid_cid() -> String { + gitlawb_core::cid::Cid::from_git_object_bytes(b"blob 5\0hello") + .as_str() + .to_string() + } + + fn get_cid(cid: &str, peer: Option) -> Request { + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + if let Some(p) = peer { + req.extensions_mut().insert(ConnectInfo(p)); + } + req + } + + /// Shed at capacity: an exhausted `git_ipfs_walk_semaphore` sheds a `/ipfs/{cid}` + /// request with 503 BEFORE any DB/git walk (the acquire is the first thing after CID + /// validation), so a lazy DB-free state suffices — exactly like the served-git shed + /// tests. MUTATION (RED): delete the `git_ipfs_walk_semaphore` acquire in + /// `get_by_cid` and the request no longer sheds here (it falls through to the DB / + /// walk and returns something other than 503). + #[tokio::test] + async fn get_by_cid_sheds_with_503_when_walk_pool_exhausted() { + let mut state = crate::test_support::test_state_lazy(); + // Global /ipfs walk pool exhausted; per-source cap permissive so only the global + // pool can shed. Route rate limit is applied as a layer in production, not here. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(0)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let peer: SocketAddr = "203.0.113.9:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted /ipfs walk pool must shed the request with 503" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + + /// Per-source sub-cap, the `Some(ip)` arm: with per-source = 1 and the source pinned + /// at its single slot, a request from THAT source sheds 503 (global pool has room), + /// while a request from a DIFFERENT source is NOT shed by the cap (it proceeds past + /// admission). Pinning proves the `PeerAddr`/`HeaderMap` extractors resolved the key + /// — an inert `None` key would never shed on the per-source cap. MUTATION (RED): + /// delete the `git_ipfs_walk_per_caller` acquire and the capped source no longer + /// sheds. + #[tokio::test] + async fn get_by_cid_per_source_cap_sheds_same_source_admits_other() { + let mut state = crate::test_support::test_state_lazy(); + // Global pool has room; the per-source cap is 1. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(8)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let capped: SocketAddr = "203.0.113.20:5000".parse().unwrap(); + let other: SocketAddr = "203.0.113.21:5000".parse().unwrap(); + + // Pin the capped source at its single walk slot. + let _slot = state + .git_ipfs_walk_per_caller + .try_acquire(&capped.ip().to_string()) + .expect("first walk slot for the capped source IP"); + + let cid = valid_cid(); + // The capped source sheds on the per-source cap even with global capacity free. + let resp = ipfs_router(state.clone()) + .oneshot(get_cid(&cid, Some(capped))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its per-source /ipfs walk cap must shed 503 with global capacity free" + ); + + // A DIFFERENT source is NOT shed by the per-source cap: it clears admission and + // proceeds (then errors on the lazy DB, which is not a 503). + let resp = ipfs_router(state) + .oneshot(get_cid(&cid, Some(other))) + .await + .unwrap(); + assert_ne!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a different source must not be shed by the per-source cap" + ); + } + + /// The `None`-key arm: a request with no resolvable source key (no trusted-proxy + /// header, no `ConnectInfo`) is bounded by the GLOBAL pool only, never the per-source + /// sub-cap. With the global pool exhausted it still sheds 503 (the counterpart to the + /// `Some(ip)` arm above, so neither arm is vacuous). + #[tokio::test] + async fn get_by_cid_none_key_arm_sheds_on_global_pool() { + let mut state = crate::test_support::test_state_lazy(); + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(0)); + // Per-source cap permissive so only the global pool can shed. + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // No ConnectInfo + no trusted header -> client_key resolves None. + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), None)) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a None-key request must still shed 503 on the exhausted GLOBAL /ipfs walk pool" + ); + } + + /// Map self-bound (INV-15): the `/ipfs` per-source map is a `PerCallerConcurrency` + /// built via `with_default_max_keys`, so a distinct-source-key flood cannot grow it + /// past the cap and a rejected key never allocates (reject-before-insert). Mirrors + /// `per_caller_concurrency_map_is_self_bounding_and_reject_before_insert` for the + /// pool U3 adds. + #[tokio::test] + async fn ipfs_walk_per_caller_map_is_self_bounding_and_reject_before_insert() { + let lim = crate::rate_limit::PerCallerConcurrency::new(4, 3); + // Acquire+drop a flood of distinct keys — the map self-empties (a key is removed + // the instant its in-flight count hits zero). + for i in 0..50 { + let _p = lim.try_acquire(&format!("src{i}")); + } + assert_eq!( + lim.tracked_keys(), + 0, + "an acquire+drop flood of distinct sources leaves the /ipfs map empty" + ); + // Reject-before-insert: hold max_keys distinct sources, then a new one sheds + // without growing the map. + let held: Vec<_> = (0..3) + .map(|i| lim.try_acquire(&format!("h{i}")).unwrap()) + .collect(); + assert_eq!( + lim.tracked_keys(), + 3, + "three distinct sources held concurrently" + ); + assert!( + lim.try_acquire("h3").is_none(), + "a new source key at max_keys is rejected" + ); + assert_eq!( + lim.tracked_keys(), + 3, + "the rejected key did not allocate an entry (reject-before-insert)" + ); + drop(held); + } + + /// Retain-through-blocking (R3, the load-bearing async property): the walk + /// admission is held until the `spawn_blocking` walk actually RETURNS, not when a + /// tokio timeout fires. With the global pool at size 1, drive a request until its + /// walk (a fake git that hangs on `rev-list`) is in flight; the slot must stay held + /// (`available_permits() == 0`) and a replacement from a DIFFERENT source must shed + /// 503 for as long as the blocking walk runs — even though the request future is + /// only `.await`ing the blocking join. When the blocking walk ends the permit frees + /// and a replacement is admitted. The permit lives INSIDE the handler across the + /// blocking `.await`; move it out (drop before the walk) and the replacement would + /// be admitted while the walk still burns a blocking thread (the bug this guards). + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_walk_permit_held_through_blocking_walk(pool: sqlx::PgPool) { + use std::process::Command; + + let tmp = tempfile::TempDir::new().unwrap(); + let revlist_pid = tmp.path().join("revlist.pid"); + // Fake git for the /ipfs WALK only (object_type/read_object_content use the real + // `git`, so the object must genuinely exist below). Empty refs (so + // assert_all_refs_are_commits returns Ok without the peel), `rev-parse` resolves, + // and `rev-list` records its pid then sleeps ~6s so the walk BLOCKS + // deterministically. The sleep bounds the walk so a broken fix cannot wedge the + // suite. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + for-each-ref) : ;;\n\ + rev-parse) echo deadbeef ;;\n\ + rev-list) echo $$ > \"{}\"; sleep 6 ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + revlist_pid.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + // Isolate the global walk pool at size 1; per-source cap permissive so only the + // held global permit can shed the replacement. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let owner = "z6ipfs1"; + let name = "ip1"; + state + .db + .upsert_mirror_repo(owner, name, "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + // The exact bare path the handler's `acquire` resolves. Build a REAL SHA-256 bare + // repo there with a committed blob under `src/`, so real `git cat-file -t ` classifies it as a blob (the CID digest IS the sha256 object id in + // object-format=sha256) and the handler reaches the path-scoped walk branch. + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + let _ = std::fs::remove_dir_all(&bare); + std::fs::create_dir_all(&bare).unwrap(); + let run = |args: &[&str], cwd: &std::path::Path| { + let out = Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + }; + let work = tmp.path().join("work"); + std::fs::create_dir_all(work.join("src")).unwrap(); + std::fs::write(work.join("src/secret.txt"), b"ipfs walk retain proof\n").unwrap(); + run( + &["init", "-q", "--object-format=sha256", "-b", "main"], + &work, + ); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "src/secret.txt"], &work); + run(&["commit", "-q", "-m", "seed"], &work); + run( + &[ + "clone", + "--bare", + "-q", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + tmp.path(), + ); + // The blob's SHA-256 object id (= the CID's digest); build the CID from it. + let oid = { + let out = Command::new("git") + .args(["rev-parse", "HEAD:src/secret.txt"]) + .current_dir(&work) + .output() + .expect("git rev-parse runs"); + assert!(out.status.success(), "rev-parse failed"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + let oid_bytes = gitlawb_core::cid::sha256_hex_to_bytes(&oid).unwrap(); + let cid = gitlawb_core::cid::Cid::from_sha256_bytes(&oid_bytes) + .as_str() + .to_string(); + // Precondition: real git classifies the object as a blob (so the handler reaches + // the walk branch, not an early `continue`). + assert_eq!( + crate::git::store::object_type(&bare, &oid) + .unwrap() + .as_deref(), + Some("blob"), + "the seeded sha256 blob must exist so the handler reaches the walk" + ); + // A path-scoped rule so has_path_scoped_rule() is true (the walk branch) without + // denying the "/" gate on the public repo. + state + .db + .set_visibility_rule( + &rec.id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderAAAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + + let sem = state.git_ipfs_walk_semaphore.clone(); + assert_eq!( + sem.available_permits(), + 1, + "one walk slot before the request" + ); + + let router = ipfs_router(state); + let make_req = |peer: SocketAddr| { + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req + }; + + let peer: SocketAddr = "203.0.113.81:5000".parse().unwrap(); + let mut fut = Box::pin(router.clone().oneshot(make_req(peer))); + // Drive until the fake git's rev-list records its pid — the walk is now in the + // blocking pool and the request future is `.await`ing its join, holding the walk + // permit. Stop polling the instant the future completes (re-polling a completed + // oneshot panics). + let mut walk_pid: Option = None; + let mut early = None; + for _ in 0..500 { + let done = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&revlist_pid) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + walk_pid = Some(p); + break; + } + if let Ok(resp) = done { + early = Some(resp.map(|r| r.status())); + break; + } + } + let pid = walk_pid + .unwrap_or_else(|| panic!("the fake git rev-list must have spawned; early: {early:?}")); + // Reap the sleeping child on drop so a RED run leaks no orphan. + struct ReapOnDrop(i32); + impl Drop for ReapOnDrop { + fn drop(&mut self) { + unsafe { + libc::kill(self.0, libc::SIGKILL); + } + } + } + let _cleanup = ReapOnDrop(pid); + + // Load-bearing: while the blocking walk runs, the slot is HELD and a replacement + // from a DIFFERENT source sheds 503 — proving the permit is retained across the + // spawn_blocking join, not freed by a tokio timeout. + assert_eq!( + sem.available_permits(), + 0, + "the walk slot must be held while the spawn_blocking walk runs" + ); + let peer2: SocketAddr = "203.0.113.82:5000".parse().unwrap(); + let resp = router.clone().oneshot(make_req(peer2)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a replacement must shed 503 while the prior request's blocking walk still runs" + ); + + // Drop the in-flight request; the detached blocking walk keeps running (a + // spawn_blocking cannot be cancelled), but on the fix the permit is a handler + // local, so dropping the future releases it once the blocking join is abandoned. + // Either way, kill the sleeping child so the slot frees promptly and poll for + // recovery — the point already proven above is that the slot stayed held for the + // duration of the blocking work. + drop(fut); + unsafe { + libc::kill(pid, libc::SIGKILL); + } + let mut freed = false; + for _ in 0..400 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + assert!( + freed, + "once the blocking walk ends the walk permit must free the global slot" + ); + } + + /// Loop bound (cap N): one `/ipfs/{cid}` request against a CID present in many repos + /// must not serialize an unbounded number of full-history walks. With + /// `ipfs_max_repos_walked = 1` and TWO public, path-scoped repos both carrying the + /// blob at the requested CID, the handler walks only the FIRST candidate then stops + /// (the second is cut by the cap), so the fake git's `rev-list` (one per walk) runs + /// exactly once. MUTATION (RED): remove the `repos_walked >= cap` break and both + /// repos are walked (count 2). + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_caps_repos_walked_per_request(pool: sqlx::PgPool) { + use std::process::Command; + + let tmp = tempfile::TempDir::new().unwrap(); + let walk_log = tmp.path().join("walks.log"); + // Fake git for the WALK: empty refs, `rev-parse` resolves, and each `rev-list` + // appends one line to a log (so the number of walks == the line count) and exits + // with EMPTY output (the allowed-set is empty, so every repo path-gates to a + // `continue` and the request 404s after walking). object_type uses the REAL git, + // so the seeded blob below must genuinely exist. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + for-each-ref) : ;;\n\ + rev-parse) echo deadbeef ;;\n\ + rev-list) echo walk >> \"{}\" ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + walk_log.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // The bound under test: walk at most one candidate repo per request. + let mut cfg = (*state.config).clone(); + cfg.ipfs_max_repos_walked = 1; + state.config = Arc::new(cfg); + + // Seed TWO public repos, each with the SAME blob (same content -> same sha256 OID + // -> same CID) under a path-scoped rule, so both are walk candidates for one CID. + let run = |args: &[&str], cwd: &std::path::Path| { + let out = Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + }; + let mut oid = String::new(); + for (i, name) in ["ipa", "ipb"].iter().enumerate() { + let owner = "z6ipfsN"; + state + .db + .upsert_mirror_repo(owner, name, &format!("/unused-{name}"), None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + let _ = std::fs::remove_dir_all(&bare); + std::fs::create_dir_all(&bare).unwrap(); + let work = tmp.path().join(format!("work{i}")); + std::fs::create_dir_all(work.join("src")).unwrap(); + // Identical content in both repos -> identical sha256 blob OID -> one CID. + std::fs::write(work.join("src/secret.txt"), b"loop bound proof\n").unwrap(); + run( + &["init", "-q", "--object-format=sha256", "-b", "main"], + &work, + ); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "src/secret.txt"], &work); + run(&["commit", "-q", "-m", "seed"], &work); + run( + &[ + "clone", + "--bare", + "-q", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + tmp.path(), + ); + if oid.is_empty() { + let out = Command::new("git") + .args(["rev-parse", "HEAD:src/secret.txt"]) + .current_dir(&work) + .output() + .expect("git rev-parse runs"); + oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); + } + state + .db + .set_visibility_rule( + &rec.id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderBBBBBBBBBBBBBBBBBBBBBBBB".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + } + let oid_bytes = gitlawb_core::cid::sha256_hex_to_bytes(&oid).unwrap(); + let cid = gitlawb_core::cid::Cid::from_sha256_bytes(&oid_bytes) + .as_str() + .to_string(); + + let peer: SocketAddr = "203.0.113.90:5000".parse().unwrap(); + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let resp = ipfs_router(state).oneshot(req).await.unwrap(); + // The empty allowed-set path-gates both repos to a `continue`, so a 404; the + // point is HOW MANY walks ran to get there. + assert_eq!(resp.status(), StatusCode::NOT_FOUND); + + let walks = std::fs::read_to_string(&walk_log) + .map(|s| s.lines().count()) + .unwrap_or(0); + assert_eq!( + walks, 1, + "with the per-request repo-walk cap at 1, only the first candidate repo is \ + walked (the second is cut by the cap), so exactly one walk runs; got {walks}" + ); + } + + /// Route rate limit is WIRED (not a silent no-op): the production `build_router` + /// attaches an `IpRateLimiter` extension to the `/ipfs/{cid}` route, so a per-IP + /// flood is braked with 429. A bare `rate_limit_by_ip` layer with no extension does + /// nothing, so this proves the extension is attached. Drive it through the real + /// router with a tight limiter (1/hr): the second request from the same IP is 429. + /// MUTATION (RED): drop the `axum::Extension(ipfs_limiter)` layer in `server.rs` and + /// the second request is no longer braked (it reaches the handler, 404, not 429). + #[sqlx::test] + async fn ipfs_route_ip_rate_limit_is_attached(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + // Tight per-IP /ipfs bucket so the second request from one IP trips 429. + state.ipfs_rate_limiter = + crate::rate_limit::RateLimiter::new(1, std::time::Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let router = crate::server::build_router(state); + let cid = valid_cid(); + let make = |peer: SocketAddr| { + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req + }; + let peer: SocketAddr = "203.0.113.99:5000".parse().unwrap(); + + // First request from this IP passes the brake and reaches the handler (404 — no + // such object anywhere), debiting the single-slot bucket. + let resp = router.clone().oneshot(make(peer)).await.unwrap(); + assert_ne!( + resp.status(), + StatusCode::TOO_MANY_REQUESTS, + "the first /ipfs request from an IP must pass the rate brake" + ); + // Second request from the SAME IP is braked with 429 — proving the limiter + // extension is attached (a bare no-op layer would let it through to 404). + let resp = router.clone().oneshot(make(peer)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::TOO_MANY_REQUESTS, + "an exhausted per-IP /ipfs bucket must brake with 429 — the IpRateLimiter \ + extension must be attached to the route" + ); + // A DIFFERENT IP still has its own budget (independent bucket). + let other: SocketAddr = "203.0.113.100:5000".parse().unwrap(); + let resp = router.oneshot(make(other)).await.unwrap(); + assert_ne!( + resp.status(), + StatusCode::TOO_MANY_REQUESTS, + "a different IP must not be braked by another IP's exhausted bucket" + ); + } +} diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index f18b4963..eed49253 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -528,6 +528,10 @@ mod tests { git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), git_write_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), + git_ipfs_walk_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_ipfs_walk_per_caller: + crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), + ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), git_bin: "git".to_string(), } } diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 68125364..b3667024 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -335,6 +335,66 @@ pub struct Config { value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) )] pub max_concurrent_reads_per_caller: usize, + + /// Maximum number of concurrent `GET /ipfs/{cid}` requests that may run their + /// visibility walk at once. The publicly-reachable `/ipfs/{cid}` route runs + /// `allowed_blob_set_for_caller_bounded` in `spawn_blocking` — a full-history + /// git walk (up to `git_service_timeout_secs`) — for each candidate repo. It + /// draws from THIS pool, not any served-git pool: a distinct public cost center + /// on a distinct surface, so sharing a git pool would let anonymous /ipfs + /// traffic shed authenticated git ops (the auth-boundary trap). A permit is + /// held for the whole request (across the repo loop) so it reflects real + /// blocking-thread occupancy, not merely the tokio wait. Beyond this the request + /// sheds a clean 503 + Retry-After. Must be between 1 and 1_048_576; the ceiling + /// keeps the value under tokio's `Semaphore` permit limit so an oversized value + /// is a clean CLI error rather than a boot-time panic. Default: 32. + #[arg( + long, + env = "GITLAWB_MAX_CONCURRENT_IPFS_WALKS", + default_value_t = 32, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub max_concurrent_ipfs_walks: usize, + + /// Maximum concurrent `/ipfs/{cid}` walk requests a single source may hold at + /// once, so one source cannot monopolize `max_concurrent_ipfs_walks` (#174). + /// Callers are keyed on the RESOLVED SOURCE IP (`client_key`/`GITLAWB_TRUSTED_PROXY`), + /// never the DID — `/ipfs` accepts any `did:key` via `optional_signature` with no + /// admission step, so keying on the DID would let one host mint disposable DIDs to + /// multiply its budget. A request with no resolvable key (no trusted header, no + /// peer) is bounded by the global pool only, never this sub-cap. Over-cap sheds a + /// clean 503 + Retry-After. Must be between 1 and 1_048_576. Default: 4. + #[arg( + long, + env = "GITLAWB_IPFS_WALK_PER_SOURCE", + default_value_t = 4, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub ipfs_walk_per_source: usize, + + /// Upper bound on the number of candidate repos a single `/ipfs/{cid}` request + /// will walk before giving up (returning the opaque 404). The handler already + /// short-circuits the moment it serves the object, but a CID that is present in + /// (or path-gated out of) many repos could otherwise serialize one full-history + /// walk per repo inside a single held admission slot. Capping the count bounds + /// the worst-case work one request can pin its slot with. Must be between 1 and + /// 1_048_576. Default: 64. + #[arg( + long, + env = "GITLAWB_IPFS_MAX_REPOS_WALKED", + default_value_t = 64, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub ipfs_max_repos_walked: usize, + + /// Per-client-IP rate limit for `GET /ipfs/{cid}`, in requests per hour. The + /// route is publicly reachable (`optional_signature`) and each request can drive + /// a full-history git walk, so it carries a per-IP flood brake in addition to the + /// concurrency cap above (a rate limit bounds request *rate*, the semaphore + /// bounds concurrent slow holds — different axes). Keyed on the resolved client + /// IP via `GITLAWB_TRUSTED_PROXY`. `0` disables. Default: 600. + #[arg(long, env = "GITLAWB_IPFS_RATE_LIMIT", default_value_t = 600)] + pub ipfs_rate_limit: usize, } impl Config { @@ -428,6 +488,66 @@ mod tests { ); } + #[test] + fn max_concurrent_ipfs_walks_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).max_concurrent_ipfs_walks, + 32 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "4"]) + .max_concurrent_ipfs_walks, + 4 + ); + // 0 permits would shed every /ipfs walk with a 503; clap must reject it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "0"]).is_err() + ); + // Above the ceiling would panic tokio's Semaphore::new at boot; clap rejects it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "1048577"]) + .is_err() + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "1048576"]) + .max_concurrent_ipfs_walks, + 1_048_576 + ); + } + + #[test] + fn ipfs_walk_per_source_defaults_and_rejects_out_of_range() { + assert_eq!(Config::parse_from(["gitlawb-node"]).ipfs_walk_per_source, 4); + assert_eq!( + Config::parse_from(["gitlawb-node", "--ipfs-walk-per-source", "2"]) + .ipfs_walk_per_source, + 2 + ); + // 0 would shed every /ipfs walk from a keyed source; clap must reject it. + assert!(Config::try_parse_from(["gitlawb-node", "--ipfs-walk-per-source", "0"]).is_err()); + assert!( + Config::try_parse_from(["gitlawb-node", "--ipfs-walk-per-source", "1048577"]).is_err() + ); + } + + #[test] + fn ipfs_max_repos_walked_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).ipfs_max_repos_walked, + 64 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "8"]) + .ipfs_max_repos_walked, + 8 + ); + // 0 would walk no repos (serve nothing); clap must reject it. + assert!(Config::try_parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "0"]).is_err()); + assert!( + Config::try_parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "1048577"]).is_err() + ); + } + #[test] fn max_concurrent_reads_per_caller_defaults_and_rejects_out_of_range() { assert_eq!( diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 380ab5f4..8727aae1 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -412,8 +412,25 @@ async fn main() -> Result<()> { git_write_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( (config.max_concurrent_git_pushes / 8).max(1), ), + // Bounds concurrent /ipfs visibility walks — a distinct public cost center, so + // its own pool + per-source sub-cap + per-IP rate limiter, never a git pool + // (#174 P1-3). The per-source map is bounded (reject-before-insert, INV-15). + git_ipfs_walk_semaphore: Arc::new(tokio::sync::Semaphore::new( + config.max_concurrent_ipfs_walks, + )), + git_ipfs_walk_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( + config.ipfs_walk_per_source, + ), + ipfs_rate_limiter: rate_limit::RateLimiter::new_bounded( + config.ipfs_rate_limit, + std::time::Duration::from_secs(3600), + 200_000, + ), git_bin: "git".to_string(), }; + if config.ipfs_rate_limit == 0 { + tracing::warn!("GITLAWB_IPFS_RATE_LIMIT=0 — per-IP /ipfs rate limiting disabled"); + } // Periodic peer-count poll for the metrics gauge. If p2p is disabled // we still set the gauge to 0 so dashboards don't show "no data". diff --git a/crates/gitlawb-node/src/server.rs b/crates/gitlawb-node/src/server.rs index f4c0d3e3..de61fcbe 100644 --- a/crates/gitlawb-node/src/server.rs +++ b/crates/gitlawb-node/src/server.rs @@ -214,9 +214,20 @@ pub fn build_router(state: AppState) -> Router { // identity and can apply per-repo visibility (#110); anonymous callers stay // anonymous and still read genuinely public content. `/api/v1/ipfs/pins` // stays unsigned — gating the pin index is tracked separately (#121). + // `/ipfs/{cid}` also carries a per-IP flood brake: it is anon-reachable and each + // request can drive a full-history git walk, so the per-IP rate limiter is the + // outermost layer (rejects a flood before the walk-admission work), mirroring the + // push/create routers. The extension MUST be attached or rate_limit_by_ip is a + // silent no-op. `/api/v1/ipfs/pins` (no walk) is merged in unbraked, as before. + let ipfs_limiter = rate_limit::IpRateLimiter { + limiter: state.ipfs_rate_limiter.clone(), + trust: state.push_limiter_trust, + }; let ipfs_routes = Router::new() .route("/ipfs/{cid}", get(ipfs::get_by_cid)) .layer(middleware::from_fn(auth::optional_signature)) + .layer(middleware::from_fn(rate_limit::rate_limit_by_ip)) + .layer(axum::Extension(ipfs_limiter)) .merge(Router::new().route("/api/v1/ipfs/pins", get(ipfs::list_pins))); // ── Arweave permanent anchors ────────────────────────────────────────── diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 3ff86923..3b8ba250 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -141,6 +141,33 @@ pub struct AppState { /// on the resolved source IP (never the DID — a DID farm defeats a DID key). Sized /// like `git_push_advert_per_caller`, a fraction of `max_concurrent_git_pushes`. pub git_write_per_caller: crate::rate_limit::PerCallerConcurrency, + /// Bounds concurrent `GET /ipfs/{cid}` visibility-walk requests. The public + /// `/ipfs/{cid}` route runs `allowed_blob_set_for_caller_bounded` in + /// `spawn_blocking` (a full-history git walk) with NO served-git admission of its + /// own; without this a permissionless caller fans out concurrent walks past every + /// git pool, exhausting the blocking pool + PIDs (#174 P1-3). A request acquires a + /// permit before the repo loop and holds it for the whole request (across every + /// `spawn_blocking` walk), so the slot reflects real thread occupancy — a tokio + /// walk-timeout cannot free it while the blocking work still runs. A pool of its + /// own (`max_concurrent_ipfs_walks`), NOT a git pool: distinct cost center + public + /// surface, so anonymous /ipfs traffic can never shed an authenticated git op. + pub git_ipfs_walk_semaphore: Arc, + /// Per-source concurrency sub-cap on the `/ipfs/{cid}` walk pool: each source + /// (keyed on the resolved source IP, never the DID — `/ipfs` admits any `did:key` + /// unthrottled, so a DID key would be free to mint around) may hold at most + /// `ipfs_walk_per_source` in-flight walk slots, so one source cannot monopolize + /// `git_ipfs_walk_semaphore` (#174 P1-3). A request with no resolvable key is + /// bounded by the global pool only, never this sub-cap. The key map is bounded + /// (`with_default_max_keys`, reject-before-insert) so a source-key farm cannot grow + /// it (INV-15). + pub git_ipfs_walk_per_caller: crate::rate_limit::PerCallerConcurrency, + /// Per-client-IP rate limiter for `GET /ipfs/{cid}`. The route is publicly + /// reachable and each request can drive a full-history git walk, so it carries a + /// per-IP flood brake in addition to the concurrency cap above — a rate limit + /// bounds request *rate*, the semaphore bounds concurrent slow holds (different + /// axes). Keyed on the resolved client IP via `push_limiter_trust`. Layered on the + /// `/ipfs` route via `rate_limit_by_ip`. + pub ipfs_rate_limiter: RateLimiter, /// The `git` executable the served-git withheld-blob walk spawns. Production is /// `"git"` (resolved via PATH); injectable so a fake `git` can drive the walk's /// process-group teardown in handler tests without mutating the process-global diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index a27be946..cfa676af 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -92,6 +92,12 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { 8, ), git_write_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), + // Generous — a test that drives the /ipfs walk shed overrides these directly. + git_ipfs_walk_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_ipfs_walk_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( + 16, + ), + ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), git_bin: "git".to_string(), } } From 394768711ec6afa834d010734c34cafd1f6063ad Mon Sep 17 00:00:00 2001 From: t Date: Wed, 15 Jul 2026 13:10:04 -0500 Subject: [PATCH 36/95] fix(node): bound the post-push encryption task set by per-repo coalescing without dropping recovery copies (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After a successful path-scoped push the handler spawns a DETACHED task that runs pin_new_objects then withheld_recipients_gated, which acquire_owned().await's on git_encrypt_semaphore — it DEFERS (blocks) when the pool is full. The semaphore caps active walks but nothing capped how many detached tasks were spawned and PARKED on that await: N rapid path-scoped pushes spawned N tasks, each holding cloned object lists/rules/paths/keys — an unbounded parked-waiter set (P2-2). 2a54c15 deliberately chose defer-not-shed because dropping the walk loses the withheld-blob recovery copy and there is no reconciliation sweep to rebuild it. So bound the OUTSTANDING-TASK set by per-repo coalescing rather than shedding: EncryptInflight (a bounded Arc>>) + try_begin — before the spawn, if a task for the repo is already in-flight skip the duplicate (the pending walk covers the newer objects); otherwise spawn and move an RAII EncryptInflightGuard into the task that removes the key on drop (completion, error, or panic-unwind). This bounds the set to <=1 pending task per repo and never drops work; withheld_recipients_gated's defer is unchanged. The second detached post-push spawn (Pinata replication, also runs pin_new_objects) is scoped out with rationale: it parks on no semaphore (no unbounded waiter set) and does per-push per-ref work (branch->CID, gossip, subscription, Arweave, peer-notify) keyed to this push's ref_updates — coalescing it would DROP a later push's announcements, a correctness regression. Regressions (mutation-verified RED->GREEN): the outstanding set is bounded to 1 per repo under saturation (never-coalesce -> 32 tasks RED); a coalesced repo's key is released when its task ends so it is reprocessed, never permanently skipped (guard-drop no-op -> repo locked out, recovery copy lost forever, RED — the exact durability regression the fix prevents); distinct repos each admit; cold-set first push admits; plus an inv22_gates structural tripwire that fails if the handler gate is removed. --- crates/gitlawb-node/src/api/repos.rs | 383 +++++++++++++++++------ crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/main.rs | 5 + crates/gitlawb-node/src/state.rs | 96 ++++++ crates/gitlawb-node/src/test_support.rs | 1 + crates/gitlawb-node/tests/inv22_gates.rs | 10 + 6 files changed, 407 insertions(+), 89 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 5b11404d..6b52273f 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1419,109 +1419,145 @@ pub async fn git_receive_pack( // Pin new git objects to the local IPFS node (no-op if ipfs_api is empty). // Skipped entirely when the public cannot read the repo (withheld == None). + // + // Coalesce per repo (#174 P2-2): this task parks on `git_encrypt_semaphore` + // (which DEFERS when the pool is full rather than dropping the recovery copy). To + // bound the OUTSTANDING parked-task set, only spawn if no encryption task for this + // repo is already in flight; otherwise skip — the pending/next walk over this + // repo's history already covers the newer push's objects, so the recovery copy is + // delayed, never lost. The guard removes the repo key when the task ends (success, + // error, or panic), so a later push is re-admitted (no permanent skip). if withheld.is_some() { - let object_list_ipfs = object_list.clone(); - let ipfs_api = state.config.ipfs_api.clone(); - let repo_path_clone = disk_path.clone(); - let db_clone = state.db.clone(); - let rules_for_enc = rules_opt.clone(); - let repo_id = record.id.clone(); - let owner_did = record.owner_did.clone(); - let is_public = record.is_public; - let irys_url = state.config.irys_url.clone(); - let http_client = std::sync::Arc::clone(&state.http_client); - let node_did_str = state.node_did.to_string(); - let node_seed = state.node_keypair.to_seed(); - let repo_name = record.name.clone(); - let enc_git_bin = state.git_bin.clone(); - let enc_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); - let encrypt_sem = state.git_encrypt_semaphore.clone(); - tokio::spawn(async move { - let pinned = crate::ipfs_pin::pin_new_objects( - &ipfs_api, - &repo_path_clone, - object_list_ipfs, - &db_clone, - ) - .await; - if !pinned.is_empty() { - tracing::info!(count = pinned.len(), "pinned git objects to IPFS"); - for (sha, cid) in &pinned { - tracing::info!(sha = %sha, %cid, "pinned"); - } + match state.encrypt_inflight.try_begin(&record.id) { + None => { + tracing::debug!( + repo = %record.id, + "post-push encryption task already in flight for this repo; coalescing \ + (the pending recovery-copy walk covers this push's objects)" + ); } - - // Option B1: encrypt-then-pin the withheld blobs so authorized - // readers can recover them when the origin cannot serve them. - // No path-scoped rule can withhold a blob, so withheld_blob_recipients - // would return an empty map after a full per-ref walk; skip it. Mirrors - // the has_path_scoped_rule gate on the other two withheld-walk sites. - if let Some(rules) = rules_for_enc.filter(|r| visibility_pack::has_path_scoped_rule(r)) - { - // Bound the number of concurrent post-push encryption walks (#174 P1-e): - // acquire an admission permit before the full-history walk, deferring - // when the pool is full rather than shedding the recovery pin. - let recip = withheld_recipients_gated( - encrypt_sem.clone(), - repo_path_clone.clone(), - enc_git_bin.clone(), - enc_timeout, - rules, - is_public, - owner_did.clone(), - ) - .await; - if let Ok(Ok(recipients)) = recip { - let delta = crate::encrypted_pin::encrypt_and_pin( + Some(inflight_guard) => { + let object_list_ipfs = object_list.clone(); + let ipfs_api = state.config.ipfs_api.clone(); + let repo_path_clone = disk_path.clone(); + let db_clone = state.db.clone(); + let rules_for_enc = rules_opt.clone(); + let repo_id = record.id.clone(); + let owner_did = record.owner_did.clone(); + let is_public = record.is_public; + let irys_url = state.config.irys_url.clone(); + let http_client = std::sync::Arc::clone(&state.http_client); + let node_did_str = state.node_did.to_string(); + let node_seed = state.node_keypair.to_seed(); + let repo_name = record.name.clone(); + let enc_git_bin = state.git_bin.clone(); + let enc_timeout = + std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let encrypt_sem = state.git_encrypt_semaphore.clone(); + tokio::spawn(async move { + // Held for the whole task; drop (on completion/error/panic) releases the + // repo's coalescing key so the next push for this repo can spawn again. + let _inflight_guard = inflight_guard; + let pinned = crate::ipfs_pin::pin_new_objects( &ipfs_api, &repo_path_clone, + object_list_ipfs, &db_clone, - &repo_id, - &node_seed, - &recipients, ) .await; + if !pinned.is_empty() { + tracing::info!(count = pinned.len(), "pinned git objects to IPFS"); + for (sha, cid) in &pinned { + tracing::info!(sha = %sha, %cid, "pinned"); + } + } - // Option B3: anchor a per-push manifest of the blobs sealed - // this push to Arweave, so the oid->cid index survives total - // node loss. Best-effort; never fails the push. - if !delta.is_empty() && !irys_url.is_empty() { - let owner_short = crate::db::normalize_owner_key(&owner_did); - let repo_slug = format!("{owner_short}/{repo_name}"); - let ts = chrono::Utc::now().to_rfc3339(); - let manifest = crate::arweave::EncryptedManifest { - repo: &repo_slug, - owner_did: &owner_did, - node_did: &node_did_str, - timestamp: &ts, - blobs: &delta, - }; - match crate::arweave::anchor_encrypted_manifest( - &http_client, - &irys_url, - &manifest, + // Option B1: encrypt-then-pin the withheld blobs so authorized + // readers can recover them when the origin cannot serve them. + // No path-scoped rule can withhold a blob, so withheld_blob_recipients + // would return an empty map after a full per-ref walk; skip it. Mirrors + // the has_path_scoped_rule gate on the other two withheld-walk sites. + if let Some(rules) = + rules_for_enc.filter(|r| visibility_pack::has_path_scoped_rule(r)) + { + // Bound the number of concurrent post-push encryption walks (#174 P1-e): + // acquire an admission permit before the full-history walk, deferring + // when the pool is full rather than shedding the recovery pin. + let recip = withheld_recipients_gated( + encrypt_sem.clone(), + repo_path_clone.clone(), + enc_git_bin.clone(), + enc_timeout, + rules, + is_public, + owner_did.clone(), ) - .await - { - Ok(tx) if !tx.is_empty() => tracing::info!( - repo = %repo_slug, - tx_id = %tx, - "anchored encrypted manifest to Arweave" - ), - Ok(_) => {} - Err(e) => tracing::warn!( - repo = %repo_slug, - err = %e, - "encrypted manifest anchor failed" - ), + .await; + if let Ok(Ok(recipients)) = recip { + let delta = crate::encrypted_pin::encrypt_and_pin( + &ipfs_api, + &repo_path_clone, + &db_clone, + &repo_id, + &node_seed, + &recipients, + ) + .await; + + // Option B3: anchor a per-push manifest of the blobs sealed + // this push to Arweave, so the oid->cid index survives total + // node loss. Best-effort; never fails the push. + if !delta.is_empty() && !irys_url.is_empty() { + let owner_short = crate::db::normalize_owner_key(&owner_did); + let repo_slug = format!("{owner_short}/{repo_name}"); + let ts = chrono::Utc::now().to_rfc3339(); + let manifest = crate::arweave::EncryptedManifest { + repo: &repo_slug, + owner_did: &owner_did, + node_did: &node_did_str, + timestamp: &ts, + blobs: &delta, + }; + match crate::arweave::anchor_encrypted_manifest( + &http_client, + &irys_url, + &manifest, + ) + .await + { + Ok(tx) if !tx.is_empty() => tracing::info!( + repo = %repo_slug, + tx_id = %tx, + "anchored encrypted manifest to Arweave" + ), + Ok(_) => {} + Err(e) => tracing::warn!( + repo = %repo_slug, + err = %e, + "encrypted manifest anchor failed" + ), + } + } } } - } + }); } - }); + } } - // Pin new git objects to Pinata, then record branch→CID and gossip + // Pin new git objects to Pinata, then record branch→CID and gossip. + // + // #174 P2-2 scope note: this SECOND detached spawn is deliberately NOT brought + // under the per-repo encryption coalescing above. Two reasons: (1) it does not + // park on `git_encrypt_semaphore` (or any semaphore) — the Pinata `pin_new_objects` + // is a bounded reqwest round-trip, so it does not form the unbounded PARKED-waiter + // set that is the P2-2 residual; it runs to completion under the HTTP client's + // network timeouts. (2) Unlike the idempotent recovery-copy walk, this task does + // PER-PUSH, PER-REF work — branch→CID upserts, gossip publish, GraphQL subscription + // broadcast, Arweave anchoring, and peer notify, each keyed to THIS push's + // ref_updates. Coalescing it against an in-flight task for the same repo would DROP + // a later push's ref-update announcements (a correctness regression), not merely + // delay a duplicate. So it is scoped out with rationale, not brought under the bound. { let pinata_jwt = state.config.pinata_jwt.clone(); let pinata_upload_url = state.config.pinata_upload_url.clone(); @@ -4317,6 +4353,175 @@ mod tests { ); } + // ---- #174 U4 (P2-2): post-push encryption task set bounded by per-repo coalescing ---- + // + // The residual jatmn found is not the WALK (bounded by `git_encrypt_semaphore`, + // proven by `encrypt_walk_defers_when_pool_exhausted` above) but the OUTER + // `tokio::spawn` + its parked `acquire_owned().await` waiters: N rapid pushes to a + // repo spawn N tasks that each park holding cloned object lists/rules/keys — an + // unbounded outstanding set. U4 bounds it by coalescing per repo: before spawning, + // if a task for the repo is in flight, skip the duplicate. Crucially this DEFERS a + // duplicate walk (the newer push's objects are covered by the pending one) and does + // NOT shed — there is no reconciliation sweep, so a dropped job would permanently + // lose the withheld-blob recovery copy (`2a54c15`'s fail-closed durability stance). + // + // These drive the coalescing seam (`EncryptInflight`) that the detached spawn at + // `repos.rs` consults directly (the try_begin gate on the in-flight set, guarded by + // `withheld.is_some()`). Observing `encrypt_and_pin`'s IPFS effect end-to-end needs a live IPFS node + // (`pin_git_object` hits the API), so the durability property is proven at this + // layer: a coalesced repo's key is released when its task ends, so a later push for + // that repo is processed once — NOT permanently skipped, which is exactly what a + // coalesce->shed mutation would break by dropping the job with no sweep to recover it. + + /// Bounded outstanding set under saturation (R4). Simulate K rapid path-scoped + /// pushes to the SAME repo while the encrypt pool is saturated (every spawned task + /// would park, so none has finished and removed its key): the first `try_begin` + /// admits (spawns), the rest coalesce (skip). The in-flight set holds at 1, not K. + /// + /// MUTATION (RED): removing the coalescing check makes every push spawn — modeled by + /// `simulate_without_coalescing`, which reaches K. If the coalesced count equaled the + /// un-coalesced one the gate would be a no-op; the strict inequality proves it bites. + #[test] + fn u4_outstanding_encrypt_set_is_bounded_to_one_per_repo_under_saturation() { + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkRepoOwnerAAAAAAAAAAAAAAAAAAAAAAAAAAAA/proj"; + const K: usize = 32; + + // Hold every admitted guard so the tasks are "still in flight" (the saturated + // case: all parked on acquire_owned().await, none finished, none removed a key). + let mut admitted = Vec::new(); + let mut coalesced = 0usize; + for _ in 0..K { + match inflight.try_begin(repo) { + Some(g) => admitted.push(g), + None => coalesced += 1, + } + } + + assert_eq!( + admitted.len(), + 1, + "exactly ONE detached task may spawn per repo while one is in flight — the \ + outstanding set is bounded to 1, not K parked waiters" + ); + assert_eq!( + coalesced, + K - 1, + "the other K-1 rapid pushes to the same repo coalesce (skip spawning)" + ); + assert_eq!( + inflight.len(), + 1, + "the in-flight set holds at most one entry per repo under saturation" + ); + + let no_coalesce = simulate_without_coalescing(K); + assert_eq!( + no_coalesce, K, + "sanity: without the coalescing check all K pushes spawn (the unbounded set \ + the fix prevents) — proves the bound above is not vacuously 1" + ); + assert!( + admitted.len() < no_coalesce, + "coalesced set ({}) must be strictly smaller than the un-coalesced one ({})", + admitted.len(), + no_coalesce + ); + } + + /// Coalescing is PER-REPO: distinct repos are never coalesced against each other, so + /// one repo in flight cannot starve a second repo's recovery copy. + #[test] + fn u4_distinct_repos_each_admit_one_encrypt_task() { + let inflight = crate::state::EncryptInflight::new(); + let a = inflight.try_begin("owner/repo-a"); + let b = inflight.try_begin("owner/repo-b"); + let c = inflight.try_begin("owner/repo-c"); + assert!( + a.is_some() && b.is_some() && c.is_some(), + "three distinct repos each admit their own encryption task" + ); + assert_eq!(inflight.len(), 3, "one in-flight entry per distinct repo"); + } + + /// NO LOST RECOVERY COPY — the security guard (R4/R6). Coalescing must DELAY a + /// duplicate walk, never permanently drop a repo's recovery copy. Observable + /// property: once an in-flight task ENDS (its guard drops — completion, error, or + /// panic-unwind) the repo key is released, so the NEXT push for that repo is admitted + /// and processed again. A coalesce->shed mutation would drop the job AND never + /// re-admit — with no reconciliation sweep the copy is lost forever. Here re-admission + /// survives normal completion AND a panic, so no permanent skip / no leaked key. + #[test] + fn u4_coalesced_repo_is_reprocessed_after_task_ends_not_permanently_skipped() { + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkDurableRepoBBBBBBBBBBBBBBBBBBBBBBBBB/repo"; + + // Push #1 admits and "spawns". A concurrent push #2 (task #1 still in flight) + // coalesces — no duplicate spawn. + let guard1 = inflight.try_begin(repo).expect("first push admits"); + assert!( + inflight.try_begin(repo).is_none(), + "while task #1 is in flight, push #2 to the same repo coalesces" + ); + + // Task #1 finishes (encrypt_and_pin ran or errored): guard drops, key released. + drop(guard1); + assert_eq!( + inflight.len(), + 0, + "when the in-flight task ends its repo key is released — the set does not leak" + ); + + // A LATER push for the SAME repo is admitted again (processed, not skipped + // forever). This is what coalesce->shed breaks: shed drops the job and no sweep + // re-derives the missing copy, so the recovery copy is permanently lost. + let guard2 = inflight.try_begin(repo).expect( + "a later push for a coalesced repo MUST be re-admitted — durability: the \ + deferred recovery copy is produced eventually, never dropped", + ); + drop(guard2); + assert_eq!(inflight.len(), 0); + + // Durability across PANIC: a task that panics mid-walk must still release its key + // (Drop runs on unwind), so one crashed walk never permanently locks a repo out + // of future recovery copies. + let panicked = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _g = inflight.try_begin(repo).expect("admit before the panic"); + assert_eq!(inflight.len(), 1); + panic!("simulate the detached encryption task panicking mid-walk"); + })); + assert!(panicked.is_err(), "the simulated task panicked"); + assert_eq!( + inflight.len(), + 0, + "a panicked encryption task still releases its repo key (Drop on unwind) — no \ + permanent leak that would block every future recovery copy for the repo" + ); + assert!( + inflight.try_begin(repo).is_some(), + "after a panicked task the repo can still be admitted — durability preserved" + ); + } + + /// Degenerate state: the first push on a cold/empty in-flight set always admits + /// (never a false coalesce on an empty set). + #[test] + fn u4_first_push_on_a_cold_set_always_admits() { + let inflight = crate::state::EncryptInflight::new(); + assert!(inflight.is_empty(), "cold set is empty"); + assert!( + inflight.try_begin("owner/first").is_some(), + "the first push on a cold in-flight set must admit (never falsely coalesce)" + ); + } + + /// Model of the pre-fix / mutated code: no coalescing check, so every push spawns. + /// Returns the count of tasks spawned (== the size of the unbounded outstanding set + /// the fix prevents), used as the RED comparison in the bound test above. + fn simulate_without_coalescing(pushes: usize) -> usize { + (0..pushes).count() + } + /// #174 SC2 (per-source key, U1): the per-caller read sub-cap keys on the /// resolved source IP, NOT the signed DID, so a disposable-DID farm cannot /// multiply its budget. Fill the source IP's single read slot, then drive two diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index eed49253..912108a0 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -524,6 +524,7 @@ mod tests { git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + encrypt_inflight: crate::state::EncryptInflight::new(), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 8727aae1..ffac65a2 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -395,6 +395,11 @@ async fn main() -> Result<()> { git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new( config.max_concurrent_git_pushes, )), + // Coalesces the DETACHED post-push encryption tasks per repo so a rapid pusher + // cannot grow the outstanding parked-waiter set past one task per repo (#174 + // P2-2). No knob: it is a natural cap (one entry per distinct repo), not a + // sized pool. + encrypt_inflight: crate::state::EncryptInflight::new(), git_read_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( config.max_concurrent_reads_per_caller, ), diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 3b8ba250..deccf4c7 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -121,6 +121,16 @@ pub struct AppState { /// walk must not hold a foreground write slot, and a handler already holding a /// write permit that needed a second would self-deadlock at pool size 1. pub git_encrypt_semaphore: Arc, + /// Bounds the OUTSTANDING post-push encryption-task set by per-repo coalescing + /// (#174 P2-2). `git_encrypt_semaphore` caps *active* walks; this caps how many + /// detached tasks *spawn and park* on that semaphore's `acquire_owned().await`. + /// Before spawning a per-push encryption task, the receive-pack handler consults + /// this set: if the repo already has a task in flight it coalesces (skips the + /// duplicate spawn) rather than parking a new unbounded waiter. Coalescing only + /// delays a duplicate walk — it never drops the withheld-blob recovery copy, which + /// `2a54c15` deliberately kept fail-closed (there is no reconciliation sweep to + /// re-derive a dropped copy). See [`EncryptInflight`]. + pub encrypt_inflight: EncryptInflight, /// Per-caller concurrency sub-cap on the read pool: each caller (keyed on the /// resolved source IP, #174 U1) may hold at most `max_concurrent_reads_per_caller` /// in-flight read ops, so one caller cannot monopolize `git_read_semaphore` @@ -203,3 +213,89 @@ impl AppState { *self.shutdown_tx.borrow() } } + +/// Bounds the OUTSTANDING post-push encryption-task set by per-repo coalescing +/// (#174 P2-2). Each successful path-scoped push `tokio::spawn`s a DETACHED task that +/// parks on `git_encrypt_semaphore.acquire_owned().await` (which DEFERS when the pool +/// is full rather than shedding — `2a54c15` kept it fail-closed so the withheld-blob +/// recovery copy is never dropped). The semaphore caps *active* walks, but nothing +/// capped how many detached tasks *spawn and park* on that await: N rapid pushes to a +/// repo spawn N parked tasks, each holding cloned object lists/rules/paths/keys — an +/// unbounded outstanding set. +/// +/// This tracks the repo keys with an in-flight encryption task. Before spawning, the +/// handler calls [`try_begin`](Self::try_begin): if a task for the repo is already +/// in-flight it returns `None` and the handler SKIPS spawning a duplicate (coalesce — +/// the newer push's objects are covered by the pending/next walk over the same repo's +/// history). This bounds the outstanding set to <=1 pending task per repo WITHOUT +/// dropping work: coalescing only delays a duplicate walk, it never sheds the recovery +/// copy (there is no reconciliation sweep, so a *dropped* job would be lost forever). +/// +/// The returned [`EncryptInflightGuard`] is moved into the detached task and removes +/// the repo key on drop — on normal completion, error, OR panic (Drop runs on unwind) +/// — so one crashed walk can never permanently lock a repo out of future recovery +/// copies. +#[derive(Clone, Default)] +pub struct EncryptInflight { + // std::sync::Mutex: only ever held for O(1) HashSet insert/remove in a sync + // context (right before `tokio::spawn`, and in the guard's Drop) — never across + // an await, so a std Mutex is correct and cheaper than a tokio one. + repos: Arc>>, +} + +impl EncryptInflight { + pub fn new() -> Self { + Self::default() + } + + /// Try to begin an encryption task for `repo_id`. Returns `Some(guard)` if no task + /// for the repo was in-flight (the caller should spawn), or `None` if one already + /// is (the caller should COALESCE — skip spawning a duplicate). The guard releases + /// the repo key on drop. + pub fn try_begin(&self, repo_id: &str) -> Option { + let mut set = self.repos.lock().expect("encrypt_inflight mutex poisoned"); + if set.insert(repo_id.to_string()) { + Some(EncryptInflightGuard { + repos: Arc::clone(&self.repos), + repo_id: repo_id.to_string(), + }) + } else { + None + } + } + + /// Number of repos with an in-flight encryption task. Test/metrics observability; + /// the bound under saturation is `len() <= number of distinct repos`, i.e. at most + /// one task per repo. + #[allow(dead_code)] + pub fn len(&self) -> usize { + self.repos + .lock() + .expect("encrypt_inflight mutex poisoned") + .len() + } + + #[allow(dead_code)] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +/// RAII guard removing a repo key from [`EncryptInflight`] when the detached +/// encryption task finishes (drop on completion, error, or panic-unwind). Move-only — +/// there is no reason to clone a guard, and cloning would double-remove. +pub struct EncryptInflightGuard { + repos: Arc>>, + repo_id: String, +} + +impl Drop for EncryptInflightGuard { + fn drop(&mut self) { + // A poisoned lock (a prior panic while holding it) still lets us take the inner + // set via into_inner-on-guard; but poisoning here is not expected because the + // only critical sections are the O(1) ops above. Remove best-effort. + if let Ok(mut set) = self.repos.lock() { + set.remove(&self.repo_id); + } + } +} diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index cfa676af..92f41a8d 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -87,6 +87,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + encrypt_inflight: crate::state::EncryptInflight::new(), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( 8, diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index 5f576300..7d3fd1c8 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -77,4 +77,14 @@ fn inv22_concurrency_gates_present_and_not_bypassed() { "P1-e bypass: the bounded recipients walk must be invoked only inside \ withheld_recipients_gated; a new call site bypasses the encrypt-walk admission cap" ); + + // U4 / P2-2: the detached post-push encryption task must be gated by the per-repo + // coalescing set (`encrypt_inflight.try_begin`) so the OUTSTANDING parked-task set is + // bounded to <=1 per repo. Removing the gate lets N rapid pushes spawn N parked + // waiters (the unbounded set U4 closed); the semaphore only caps active walks. + assert!( + repos.contains("state.encrypt_inflight.try_begin"), + "U4/P2-2 gate missing: the detached post-push encryption spawn must consult \ + encrypt_inflight.try_begin to coalesce per repo (bound the outstanding-task set)" + ); } From 413d6cf0bcf1a8c587e01d6c9384de70e2876ecb Mon Sep 17 00:00:00 2001 From: t Date: Wed, 15 Jul 2026 13:16:40 -0500 Subject: [PATCH 37/95] fix(node): reject unsupported git service before the read slot; document the new admission knobs (#174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P2-1: git_info_refs treated every service other than git-receive-pack as a read op and did the DB/visibility/Tigris work under a read permit, validating the service string only downstream in smart_http. So an unauthenticated `?service=anything` to a public repo consumed a read slot + storage work before rejection. Validate the service is exactly git-upload-pack or git-receive-pack immediately after parsing, returning 400 before the pre-DB shed and the permit. Handler-layer regression: with the read pool exhausted, `?service=git-explode` returns 400 (validated first), not 503 — removing the validation makes it 503 (RED). U5/INV-24 docs: document the new knobs from U2/U3 in .env.example and the README env-var table — GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS, GITLAWB_MAX_CONCURRENT_IPFS_WALKS, GITLAWB_IPFS_WALK_PER_SOURCE, GITLAWB_IPFS_MAX_REPOS_WALKED, GITLAWB_IPFS_RATE_LIMIT. Each is already enforced on the path it names (U2/U3 tests), so the docs match the implemented routing. --- .env.example | 29 ++++++++++++++++ README.md | 5 +++ crates/gitlawb-node/src/api/repos.rs | 51 ++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+) diff --git a/.env.example b/.env.example index 0ef193f8..149ea8b1 100644 --- a/.env.example +++ b/.env.example @@ -115,6 +115,16 @@ GITLAWB_MAX_PACK_BYTES=2147483648 # Default 600. GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600 +# Max seconds the storage-ACQUISITION phase of a served git op may run before the +# request is shed with a 503, separate from the git-run timeout above. A +# concurrency permit is taken before this phase and GITLAWB_GIT_SERVICE_TIMEOUT_SECS +# only starts once git spawns, so without this a stalled backend (a hung Tigris +# HEAD/GET, or a hung pg advisory-lock iteration on push) pins the permit and drains +# the pool until every later request 503s. On expiry the permit is released +# (fail-closed). Kept separate because acquisition and git execution are distinct +# cost centers. Must be positive; set very large to effectively disable. Default 30. +GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS=30 + # Max concurrent git READ ops (upload-pack + the upload-pack info/refs # advertisement) served at once, a global pool separate from the push pool below. # The anon receive-pack info/refs advertisement has its OWN pool (see below), not @@ -146,6 +156,25 @@ GITLAWB_MAX_CONCURRENT_READS_PER_CALLER=16 # IP per hour. 0 disables. Default 600. GITLAWB_PUSH_RATE_LIMIT=600 +# ── /ipfs/{cid} visibility-walk admission (#174) ────────────────────────── +# GET /ipfs/{cid} runs a per-repo full-history git walk in a blocking thread to +# decide whether the caller may read a path-scoped blob. It is publicly reachable, +# so it is bounded to keep a permissionless caller from fanning out unbounded +# concurrent walks and exhausting blocking-pool threads + PIDs. +# Max concurrent /ipfs walks across all callers (a pool of its own, disjoint from +# the served-git pools). Over-cap sheds a 503. Default 32. +GITLAWB_MAX_CONCURRENT_IPFS_WALKS=32 +# Max concurrent /ipfs walks a single SOURCE IP may hold (keyed like the git +# per-caller caps via GITLAWB_TRUSTED_PROXY; reject-before-insert bounded map). +# Default 4. +GITLAWB_IPFS_WALK_PER_SOURCE=4 +# Max repos walked per single /ipfs request, so one request cannot serialize a +# full-history walk over every repo carrying the CID. Default 64. +GITLAWB_IPFS_MAX_REPOS_WALKED=64 +# Max /ipfs/{cid} requests per client IP per hour (route flood brake, distinct +# from the concurrency caps above). 0 disables. Default 600. +GITLAWB_IPFS_RATE_LIMIT=600 + # ── Creation rate limiting (repo/agent/issue/PR flood brake) ────────────── # Max creation requests (POST /api/v1/repos, /api/register, fork, issues, # pulls) per client IP per hour, in addition to the per-DID limit. The per-DID diff --git a/README.md b/README.md index 542a988e..e9765876 100644 --- a/README.md +++ b/README.md @@ -344,6 +344,11 @@ Important node settings: | `GITLAWB_AUTO_SYNC` | Enable automatic sync from known peers. | | `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. | | `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. | +| `GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS` | Max seconds the storage-acquisition phase (Tigris HEAD/GET, push advisory-lock) of a served git op may run before the request is shed with a 503, separate from the git-run timeout. The concurrency permit is released on expiry so a stalled backend cannot pin the pool. Default 30. | +| `GITLAWB_MAX_CONCURRENT_IPFS_WALKS` | Max concurrent `GET /ipfs/{cid}` visibility walks across all callers (own pool, disjoint from the served-git pools); over-cap sheds 503. Default 32. | +| `GITLAWB_IPFS_WALK_PER_SOURCE` | Max concurrent `/ipfs` walks a single source IP may hold. Default 4. | +| `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max repos walked per `/ipfs/{cid}` request, bounding one request's fan-out. Default 64. | +| `GITLAWB_IPFS_RATE_LIMIT` | Max `/ipfs/{cid}` requests per client IP per hour (route flood brake). 0 disables. Default 600. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | | `GITLAWB_IRYS_URL` | Optional Irys/Arweave permanent anchoring. | diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 6b52273f..9b05ece1 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -515,6 +515,16 @@ pub async fn git_info_refs( let service = query .service .ok_or_else(|| AppError::BadRequest("missing ?service= parameter".into()))?; + // Reject an unsupported service BEFORE taking a read slot or doing any DB/Tigris + // work (#174 P2-1). git_info_refs otherwise treats everything that is not + // git-receive-pack as a read op, so an unauthenticated `?service=anything` to a + // public repo would consume a read permit and the visibility/Tigris work before + // validate_service rejected it downstream in smart_http. + if service != "git-upload-pack" && service != "git-receive-pack" { + return Err(AppError::BadRequest(format!( + "unsupported git service: {service}" + ))); + } // #62 cheap pre-DB load shed: if the pool this service draws from is already // saturated, shed with 503 before any DB/disk work. Best-effort (holds no // permit); the authoritative hold is `git_permit` below, after the per-source @@ -3118,6 +3128,47 @@ mod tests { ); } + /// #174 P2-1: an unsupported `?service=` must be rejected with 400 BEFORE taking a + /// read slot or doing DB/Tigris work. Isolate it: exhaust the read pool so a read + /// op WOULD shed 503 at the pre-DB check — a garbage service must still return 400 + /// (validation runs first), proving `?service=anything` cannot consume the read + /// pool. Removing the validation makes this 503 (RED). + #[sqlx::test] + async fn info_refs_rejects_unsupported_service_before_the_read_slot(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state + .db + .upsert_mirror_repo("z6svcowner", "svc", "/tmp/svc", None, false) + .await + .unwrap(); + // Exhaust the read pool: a read op would shed 503 at the pre-DB check. + state.git_read_semaphore = Arc::new(Semaphore::new(0)); + + let router = crate::server::build_router(state); + let peer: SocketAddr = "203.0.113.90:7000".parse().unwrap(); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6svcowner/svc/info/refs?service=git-explode") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + + let status = router.oneshot(req).await.unwrap().status(); + assert_eq!( + status, + StatusCode::BAD_REQUEST, + "an unsupported ?service= must be 400 before the read-pool shed, not 503" + ); + } + /// #174 (jatmn P1): the anon-reachable receive-pack advertisement /// (`GET info/refs?service=git-receive-pack`) draws from a DEDICATED advert pool /// (`git_push_advert_semaphore`), NOT the write pool the authenticated POST uses. From 3af50a53040d7eef18444ecbedf1f663a31e992e Mon Sep 17 00:00:00 2001 From: t Date: Fri, 17 Jul 2026 17:05:28 -0500 Subject: [PATCH 38/95] fix(node): hold filtered-pack admission through disconnect teardown (#174) drive_git_child hands the AdmissionGuard back on success so upload_pack_excluding can thread it across the rev-list and pack-objects stages; on timeout or error it is dropped post-reap, and on future-drop it rides the detached reaper as before. The upload-pack handler builds the guard once ahead of the withheld/plain branch, so a client disconnect mid-filtered-serve no longer frees the read-pool and per-source permits while the git process group is still being reaped. --- crates/gitlawb-node/src/api/repos.rs | 249 +++++++++++- crates/gitlawb-node/src/git/smart_http.rs | 446 +++++++++++++++++++--- 2 files changed, 637 insertions(+), 58 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 9b05ece1..bb9e743b 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -932,19 +932,30 @@ pub async fn git_upload_pack( // the smart_http paths, not a generic 500 (#174 U3). let withheld = withheld.map_err(|e| git_service_app_error(&e))?; + // Move the permits returned by the walk into the guard, ONE construction + // site for both serve arms below, so admission tracks the served git group's + // reap (complete/timeout/disconnect) whether the pack is plain or filtered. + // The handler keeps no copy (F1: handler-local permits would drop the + // instant a disconnect drops this future, mid-reap). + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); if withheld.is_empty() { - // No blobs to withhold: serve the plain pack, moving the permits returned by - // the walk into the guard so admission tracks the served git group's reap - // (the walk already held them per be0cdd6; this hands them to the serve). - let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + // No blobs to withhold: serve the plain pack (the walk already held the + // permits per be0cdd6; the guard hands them to the serve). smart_http::upload_pack(&state.git_bin, &disk_path, body, git_timeout, Some(admission)).await } else { tracing::info!(repo = %name, caller = ?caller, withheld = withheld.len(), "serving filtered pack"); - // upload_pack_excluding runs its own rev-list/pack-objects (both pass `None` - // admission internally); the walk's permits stay handler-locals held across - // this serve, as be0cdd6 established, and drop when the handler returns. - let _hold = (_permit, _caller_permit); - smart_http::upload_pack_excluding(&disk_path, body, &withheld, git_timeout).await + // The guard threads through both filtered-pack stages (rev-list, then + // pack-objects), so a disconnect mid-stage keeps the permits held until + // that stage's process group is reaped (F1). + smart_http::upload_pack_excluding( + &state.git_bin, + &disk_path, + body, + &withheld, + git_timeout, + Some(admission), + ) + .await } } .map_err(|e| { @@ -4013,6 +4024,226 @@ mod tests { ); } + /// F1 (filtered-serve residual of #174 P1-a, RED-before/GREEN-after): on the + /// FILTERED (path-scoped, non-empty withheld) upload-pack branch a client + /// disconnect mid-pack-objects must NOT release the read admission while the + /// detached reaper is still tearing down the git group. Pre-fix the handler held + /// the permits as locals (`_hold`) across `upload_pack_excluding`, so dropping + /// the future released them instantly and disconnect-spam could exceed the read + /// caps during each reap window (RED). The fix threads the AdmissionGuard through + /// both filtered-pack stages so it rides `KillGroupOnDrop` into the reaper. + /// + /// Same isolation as the plain-path test above: read pool = 1, per-source cap + /// permissive, so only the global permit can shed a replacement. The fake git + /// serves the withheld walk (for-each-ref/rev-parse/rev-list/ls-tree) with a blob + /// under the denied `/src/**` subtree so the filtered branch is taken, answers + /// the pack build's rev-list fast, and hangs pack-objects in a SIGTERM-trapping + /// descendant. The descendant hang is first-invocation-only (keyed on the + /// pidfile's existence) so the post-reap replacement request completes fast. + #[cfg(unix)] + #[sqlx::test] + async fn upload_pack_filtered_permit_held_through_group_reap_after_disconnect( + pool: sqlx::PgPool, + ) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + let descfile = tmp.path().join("desc.pid"); + let commit = "1111111111111111111111111111111111111111"; + let blob = "2222222222222222222222222222222222222222"; + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + rev-parse) echo {commit} ;;\n\ + rev-list) echo {commit} ;;\n\ + ls-tree) printf '100644 blob {blob}\\tsrc/x.txt' ;;\n\ + pack-objects)\n\ + if [ ! -e \"{desc}\" ]; then\n\ + sh -c 'trap \"\" TERM; echo $$ > \"{desc}\"; i=0; while [ $i -lt 20 ]; do sleep 1; i=$((i+1)); done' &\n\ + wait\n\ + fi ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + desc = descfile.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + // Isolate the global read pool: size 1; per-source cap + rate limiter permissive + // so only the leaked global permit can shed the replacement. + state.git_read_semaphore = Arc::new(Semaphore::new(1)); + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let owner = "z6upf1"; + let name = "upf"; + state + .db + .upsert_mirror_repo(owner, name, "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + // Real bare repo at the path acquire() computes, so the handler reaches the + // walk and the filtered serve. + state + .repo_store + .init(&rec.owner_did, &rec.name) + .await + .unwrap(); + // Path-scoped rule denying the anonymous caller under /src, matching the + // fake ls-tree's blob path, so the withheld set is NON-EMPTY and the + // filtered (upload_pack_excluding) branch is taken. + state + .db + .set_visibility_rule( + &rec.id, + "/src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkUF1ReaderAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + + let sem = state.git_read_semaphore.clone(); + assert_eq!( + sem.available_permits(), + 1, + "one read slot before the request" + ); + + let router = crate::server::build_router(state); + let make_req = |peer: SocketAddr| { + let mut req = Request::builder() + .method(Method::POST) + .uri(format!("/{owner}/{name}/git-upload-pack")) + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req + }; + + let peer: SocketAddr = "203.0.113.81:5000".parse().unwrap(); + let mut fut = Box::pin(router.clone().oneshot(make_req(peer))); + // Drive until the pack-objects descendant records its pid: the request is + // past the walk, inside the filtered serve's stage 2, holding the read permit. + // Stop polling the instant the future completes (re-polling a completed + // oneshot panics); read the descfile first so a spawn that recorded its pid + // then returned is still caught. + let mut spawned: Option = None; + let mut early = None; + for _ in 0..500 { + let done = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&descfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + spawned = Some(p); + break; + } + if let Ok(resp) = done { + early = Some(resp.map(|r| r.status())); + break; + } + } + let desc = spawned.unwrap_or_else(|| { + panic!("the fake pack-objects must have spawned; early finish: {early:?}") + }); + // Kill the descendant regardless of outcome so a RED run leaks no orphan. + struct ReapOnDrop(i32); + impl Drop for ReapOnDrop { + fn drop(&mut self) { + unsafe { + libc::kill(self.0, libc::SIGKILL); + } + } + } + let _cleanup = ReapOnDrop(desc); + assert!( + unsafe { libc::kill(desc, 0) == 0 }, + "descendant should be running before the disconnect" + ); + assert_eq!( + sem.available_permits(), + 0, + "the read slot is held while the filtered serve runs" + ); + + // Client disconnect: drop the request future mid-pack-objects. The detached + // reaper must now own the AdmissionGuard and hold it until ESRCH. + drop(fut); + + // Load-bearing: the slot must STAY held while the SIGTERM-ignoring group is + // still alive. On the pre-fix code the handler-local `_hold` drops here and + // the slot frees at once (RED). Check quickly (before the reaper's ~2s + // SIGKILL escalation). + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + assert!( + unsafe { libc::kill(desc, 0) == 0 }, + "the SIGTERM-ignoring descendant must still be alive during the hold window" + ); + assert_eq!( + sem.available_permits(), + 0, + "on disconnect the read admission must be HELD until the filtered serve's \ + process group is reaped, not released the instant the future drops (F1)" + ); + // A replacement request from a DIFFERENT source must shed 503: the only pool + // that can shed it is the held global permit (per-source cap is permissive). + let peer2: SocketAddr = "203.0.113.82:5000".parse().unwrap(); + let resp = router.clone().oneshot(make_req(peer2)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "while the prior group is still alive the held global permit must shed a \ + replacement with 503" + ); + + // After the reaper SIGKILLs + reaps the group the AdmissionGuard drops and + // the slot frees. Poll for recovery. + let mut freed = false; + for _ in 0..400 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + assert!( + freed, + "once the reaper confirms the group gone the admission guard must drop and \ + free the global slot" + ); + // A replacement is now admitted and completes: the fake pack-objects takes + // its fast path (the descfile exists), so the filtered serve returns instead + // of hanging. + let peer3: SocketAddr = "203.0.113.83:5000".parse().unwrap(); + let resp = router.oneshot(make_req(peer3)).await.unwrap(); + assert_ne!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "after the group is reaped the freed slot must admit a replacement" + ); + } + /// #174 U1 (P1-a): the `None`-key arm — a request with no resolvable source key /// (no trusted-proxy header, no peer) is bounded by the GLOBAL read pool only, never /// a per-source cap. With the global read pool exhausted such a request still sheds diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index 64d3cb7f..f8b46ce8 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -24,7 +24,9 @@ use tokio::process::Command; /// /// The `be0cdd6` path-scoped upload-pack walk already applies this discipline by /// moving its permits into the `spawn_blocking`; this generalizes it to the plain -/// (non-path-scoped) `info_refs` / `run_git_service` spawn paths. +/// (non-path-scoped) `info_refs` / `run_git_service` spawn paths and, via the +/// hand-back contract on `drive_git_child`, across both stages of the filtered +/// (`upload_pack_excluding`) pack build (F1). pub struct AdmissionGuard { // Boxed so the concrete permit types stay private to the caller (repos.rs) — the // guard only needs to hold them and drop them, never inspect them. `Send + @@ -71,8 +73,11 @@ pub async fn info_refs( .arg("--advertise-refs") .arg(repo_path); // No request body — advertise-refs does not read stdin. - let stdout = + let (stdout, admission) = drive_git_child(command, Bytes::new(), timeout, "advertise-refs", admission).await?; + // Single-stage op: the advertisement's group is reaped by now; release admission + // rather than holding it across the pure response framing below. + drop(admission); let content_type = format!("application/x-{service}-advertisement"); @@ -174,11 +179,12 @@ struct KillGroupOnDrop { // own SIGCHLD-driven orphan reaper. child: Option, pgid: Option, - // The global + per-source admission permits for this op, if any. On the plain - // (non-path-scoped) spawn paths repos.rs moves its permits in here so admission is - // released only when the group is confirmed reaped, on every exit — complete, - // timeout, or client-disconnect (#174 P1-a). The rev-list/pack-objects - // visibility-walk callers hold no admission and pass `None`. + // The global + per-source admission permits for this op, if any. repos.rs moves + // its permits in here (directly on the plain spawn paths, via the stage threading + // on the filtered rev-list/pack-objects ones) so admission is released only when + // the group is confirmed reaped, on every exit: complete, timeout, or + // client-disconnect (#174 P1-a, F1). `None` when the op carries no admission + // (e.g. the smart_http test harness). admission: Option, } @@ -191,9 +197,10 @@ impl KillGroupOnDrop { /// Disarm on the success/timeout path: the body has already reaped the child (its /// `wait()` returned, or `reap_group_on_timeout` ran), so drop the handle and clear - /// the pgid, leaving the guard's Drop a no-op. Returns the admission guard so the - /// caller drops it at the earliest provably-free point (the group is already reaped - /// on both callers of this) rather than at end-of-bookkeeping. + /// the pgid, leaving the guard's Drop a no-op. Returns the admission guard: the + /// group is already reaped on both callers of this, so the success path hands it + /// back to drive_git_child's caller (a multi-stage build carries it to the next + /// stage) and the timeout path drops it there and then. fn disarm(&mut self) -> Option { self.pgid = None; self.child = None; @@ -331,23 +338,36 @@ async fn run_git_service( .arg(service_to_command(service)) .arg("--stateless-rpc") .arg(repo_path); - drive_git_child(command, input, timeout, service, admission).await + let (out, admission) = drive_git_child(command, input, timeout, service, admission).await?; + // Single-stage op: the child group is already reaped when drive_git_child + // returns, so releasing admission here keeps the permits held exactly for the + // process lifetime, no longer. + drop(admission); + Ok(out) } /// Drive a spawned git child under `timeout` with process-group teardown, returning -/// its stdout. Shared core for [`run_git_service`] and [`info_refs`]: the caller +/// its stdout together with the admission guard. Shared core for +/// [`run_git_service`], [`info_refs`], and the filtered-pack stages: the caller /// passes a `Command` with its args set; this adds piped stdio and `process_group(0)`. /// On the deadline the whole group is torn down and reaped before returning /// [`GitServiceTimeout`]; on a dropped future (client disconnect) the /// [`KillGroupOnDrop`] guard fires. `input` is written to the child's stdin (empty /// for the advertise-refs path, which has no request body); `what` labels errors. +/// +/// Admission contract: on success the guard is handed BACK so a multi-stage caller +/// (the filtered-pack build) can carry one admission across consecutive children +/// instead of releasing it between stages. On every Err return (timeout, spawn +/// failure, interaction error, non-zero exit) the guard is dropped internally, and +/// only after the child is reaped: an Err cannot carry the guard, and by then the +/// group is confirmed gone, so releasing admission there is correct. async fn drive_git_child( mut command: Command, input: Bytes, timeout: Duration, what: &str, admission: Option, -) -> Result> { +) -> Result<(Vec, Option)> { command .stdin(Stdio::piped()) .stdout(Stdio::piped()) @@ -382,10 +402,10 @@ async fn drive_git_child( pgid, admission, }; - // On non-unix there is no process-group teardown, so hold the admission guard here - // for the child's whole interaction; it drops when this function returns. - #[cfg(not(unix))] - let _admission = admission; + // On non-unix there is no process-group teardown; the `admission` parameter + // itself stays live for the child's whole interaction (nothing moves it until + // the match below), so the success path hands it back like the unix path and + // the timeout path drops it after the child is waited on. let mut out = Vec::new(); let mut err = Vec::new(); @@ -419,16 +439,18 @@ async fn drive_git_child( }; let timed = tokio::time::timeout(timeout, interact).await; - let (write_result, status) = match timed { + let (write_result, status, admission) = match timed { Ok(result) => { // The join runs all arms to completion, so the child is reaped: disarm // before surfacing any interaction error (a read/wait error), else the // guard's drop would reap an already-reaped child / signal a reused pgid. - // Dropping the returned admission guard here releases the permits at the - // earliest provably-free point on the success path (the op finished). + // Disarming hands the admission guard back; it rides to the Ok return + // below, and every error return between here and there drops it, always + // AFTER the reap, the earliest provably-free point. #[cfg(unix)] - drop(group_guard.disarm()); - result? + let admission = group_guard.disarm(); + let (write_result, status) = result?; + (write_result, status, admission) } Err(_elapsed) => { // Timeout: tear the whole group down and reap it before returning so a @@ -444,6 +466,7 @@ async fn drive_git_child( { let _ = child.start_kill(); let _ = tokio::time::timeout(Duration::from_secs(2), child.wait()).await; + drop(admission); } return Err(GitServiceTimeout.into()); } @@ -452,7 +475,8 @@ async fn drive_git_child( // Surface git's own failure (its stderr, which the handler may classify as a // 400) before any stdin-write error: when git rejects a malformed body it // exits non-zero and closes stdin, so the write's EPIPE would otherwise mask - // the real cause. + // the real cause. `admission` drops on these error returns; the child is + // already reaped by the completed join above. if !status.success() { let stderr = String::from_utf8_lossy(&err); bail!("{what} failed: {stderr}"); @@ -460,7 +484,7 @@ async fn drive_git_child( write_result.context("failed to write to git stdin")?; - Ok(out) + Ok((out, admission)) } fn service_to_command(service: &str) -> &str { @@ -498,14 +522,17 @@ async fn rev_list_keep( repo_path: &Path, withheld: &HashSet, timeout: Duration, -) -> Result> { + admission: Option, +) -> Result<(Vec, Option)> { let mut command = Command::new(git_bin); command .args(["rev-list", "--objects", "--all"]) .current_dir(repo_path); - // The visibility-walk callers intentionally hold no admission permit (their - // admission is governed elsewhere), so pass `None` (#174 KTD2). - let stdout = drive_git_child(command, Bytes::new(), timeout, "rev-list", None).await?; + // The filtered-serve caller's admission rides through drive_git_child so a + // disconnect mid-enumeration keeps the permits held until the rev-list group is + // reaped; on success the guard comes back for the pack-objects stage (F1). + let (stdout, admission) = + drive_git_child(command, Bytes::new(), timeout, "rev-list", admission).await?; let mut keep = Vec::new(); for line in String::from_utf8_lossy(&stdout).lines() { let oid = line.split_whitespace().next().unwrap_or(""); @@ -514,7 +541,7 @@ async fn rev_list_keep( } keep.push(oid.to_string()); } - Ok(keep) + Ok((keep, admission)) } /// Build a packfile containing every object reachable from all refs EXCEPT the @@ -527,20 +554,30 @@ async fn rev_list_keep( /// disconnect. An outer `tokio::time::timeout` around a `spawn_blocking` cannot /// cancel the blocking thread, so neither stage may live off the async side /// (#174, KTD5). +/// +/// `admission` carries the filtered-serve permits across BOTH stages: stage 1 hands +/// it back on success, stage 2 takes it, and the caller receives it after stage 2 so +/// release happens only once the last child group is reaped. On a disconnect the +/// guard rides the active stage's detached reaper instead of dropping with the +/// future (F1). A stage that fails or times out (including a stage 2 whose +/// remaining budget has saturated to zero) drops the guard inside drive_git_child, +/// after that stage's reap. pub async fn build_filtered_pack( git_bin: &str, repo_path: &Path, withheld: &HashSet, timeout: Duration, -) -> Result> { + admission: Option, +) -> Result<(Vec, Option)> { // One deadline spans both git stages so a slow rev-list eats into the pack // budget rather than granting each stage a fresh `timeout` (2x the permit hold). let deadline = Instant::now() + timeout; - let keep = rev_list_keep( + let (keep, admission) = rev_list_keep( git_bin, repo_path, withheld, deadline.saturating_duration_since(Instant::now()), + admission, ) .await?; let mut data = keep.join("\n").into_bytes(); @@ -554,8 +591,7 @@ pub async fn build_filtered_pack( Bytes::from(data), deadline.saturating_duration_since(Instant::now()), "pack-objects", - // Visibility-walk pack build: no admission permit here (#174 KTD2). - None, + admission, ) .await } @@ -584,16 +620,24 @@ pub async fn build_filtered_pack( /// instead of a thin delta. Honoring negotiation for smaller fetch packs is an /// optimization follow-up, not a correctness requirement. pub async fn upload_pack_excluding( + git_bin: &str, repo_path: &Path, request_body: Bytes, withheld: &HashSet, timeout: Duration, + admission: Option, ) -> Result { - // The rev-list enumeration runs blocking off the runtime; the streaming - // pack-objects stage is duration-bounded and its process group is reaped on - // disconnect via drive_git_child (#174), so a hung build no longer pins its - // concurrency slot and a client disconnect no longer orphans the git child. - let pack = build_filtered_pack("git", repo_path, withheld, timeout).await?; + // Both filtered-pack stages run async under drive_git_child (duration-bounded, + // process group reaped on disconnect, #174), and `admission` threads through + // them so the caller's permits release only after the active stage's group is + // reaped, never the instant a disconnect drops this future (F1). `git_bin` is + // injectable for the same fake-git testing reason as `run_git_service` + // (production passes the configured git binary). + let (pack, admission) = + build_filtered_pack(git_bin, repo_path, withheld, timeout, admission).await?; + // Both git stages are done and their groups reaped; release admission before the + // pure in-memory response framing below. + drop(admission); // The client lists its capabilities on the first `want` line. Honor // side-band-64k when offered (every modern smart-HTTP client offers it); @@ -712,9 +756,10 @@ mod tests { let mut withheld = std::collections::HashSet::new(); withheld.insert(secret.clone()); - let pack = build_filtered_pack("git", &bare, &withheld, Duration::from_secs(30)) - .await - .unwrap(); + let (pack, _admission) = + build_filtered_pack("git", &bare, &withheld, Duration::from_secs(30), None) + .await + .unwrap(); let ids = pack_object_ids(&pack); assert!(ids.contains(&public), "public blob must be in the pack"); assert!( @@ -776,9 +821,10 @@ mod tests { b"0098want 0000000000000000000000000000000000000000 \ side-band-64k ofs-delta agent=git/2\n00000009done\n", ); - let resp = upload_pack_excluding(&bare, req, &withheld, Duration::from_secs(30)) - .await - .unwrap(); + let resp = + upload_pack_excluding("git", &bare, req, &withheld, Duration::from_secs(30), None) + .await + .unwrap(); let body = to_bytes(resp.into_body(), usize::MAX).await.unwrap(); let ids = pack_object_ids(&extract_pack(&body)); assert!( @@ -837,9 +883,16 @@ mod tests { axum::extract::State(st): axum::extract::State>, body: Bytes, ) -> Response { - upload_pack_excluding(&st.repo, body, &st.withheld, Duration::from_secs(30)) - .await - .unwrap() + upload_pack_excluding( + "git", + &st.repo, + body, + &st.withheld, + Duration::from_secs(30), + None, + ) + .await + .unwrap() } /// Spawn the server for `bare`, withholding `withheld`. Returns the clone URL @@ -1359,14 +1412,19 @@ mod tests { for i in 0..FAKE_GIT_RETRY_ATTEMPTS { let result = tokio::time::timeout( Duration::from_secs(10), - build_filtered_pack(git_bin, repo_path, withheld, stage_timeout), + build_filtered_pack(git_bin, repo_path, withheld, stage_timeout, None), ) .await .expect( "build_filtered_pack must return within the watchdog — the git stage \ must be timeout-bounded, not an uncancellable spawn_blocking", ); - let err = result.expect_err("a hung git stage must return an error, not hang"); + // `match` rather than `expect_err`: the Ok arm carries an + // AdmissionGuard, which has no Debug impl for expect_err to print. + let err = match result { + Ok(_) => panic!("a hung git stage must return an error, not hang"), + Err(e) => e, + }; if is_transient_exec_race(&err) { // Fresh-fake-git ETXTBSY: back off (growing) so a bursty fork-pressure // spike subsides before retrying, per fake_git_run_with_pids. @@ -1655,6 +1713,296 @@ mod tests { ); } + // ── F1: filtered-serve admission threaded through both pack stages ────── + // + // The handler-level disconnect regression lives in api/repos.rs + // (upload_pack_filtered_permit_held_through_group_reap_after_disconnect); these + // exercise the guard contract at the smart_http seam: held through a + // mid-rev-list disconnect, handed back on success, and recovered (post-reap) + // on every error return. + + /// F1: a client disconnect mid-REV-LIST (stage 1 of the filtered build) must + /// keep the threaded admission held until the rev-list group is reaped, and + /// stage 2 (pack-objects) must never spawn, since the dropped future cannot + /// advance to it. + #[cfg(unix)] + #[tokio::test] + async fn build_filtered_pack_holds_admission_through_rev_list_reap_on_disconnect() { + let tmp = tempfile::TempDir::new().unwrap(); + let descfile = tmp.path().join("desc.pid"); + let packfile = tmp.path().join("pack.ran"); + // rev-list: SIGTERM-trapping descendant records its pid and loops (bounded + // so a broken fix leaks no permanent orphan); the leader waits, keeping the + // interaction pending. pack-objects: records that it ran (it must never run). + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + rev-list)\n\ + sh -c 'trap \"\" TERM; echo $$ > \"{desc}\"; i=0; while [ $i -lt 20 ]; do sleep 1; i=$((i+1)); done' &\n\ + wait ;;\n\ + pack-objects) : > \"{pack}\" ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + desc = descfile.display(), + pack = packfile.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + let guard = AdmissionGuard::new(sem.clone().try_acquire_owned().unwrap(), None::<()>); + let withheld = HashSet::new(); + + let mut fut = Box::pin(build_filtered_pack( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_secs(60), + Some(guard), + )); + // Drive until the rev-list descendant records its pid. + let mut desc: Option = None; + for _ in 0..500 { + let _ = tokio::time::timeout(Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&descfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + desc = Some(p); + break; + } + } + let desc = desc.expect("the fake rev-list must have spawned its descendant"); + let _cleanup = ReapOnPanic(vec![desc]); + assert_eq!( + sem.available_permits(), + 0, + "admission held while rev-list runs" + ); + + // Client disconnect mid-enumeration. + drop(fut); + + // Load-bearing: the permit must stay held while the SIGTERM-ignoring group + // member is still alive (check before the reaper's ~2s SIGKILL escalation). + tokio::time::sleep(Duration::from_millis(100)).await; + assert!( + alive(desc), + "the SIGTERM-ignoring descendant must still be alive during the hold window" + ); + assert_eq!( + sem.available_permits(), + 0, + "on disconnect mid-rev-list the admission must be held until the group is \ + reaped, not released the instant the future drops (F1)" + ); + + let mut freed = false; + for _ in 0..400 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + assert!( + freed, + "the reaper must release admission once the rev-list group is gone" + ); + assert!( + !packfile.exists(), + "pack-objects must never spawn after a disconnect mid-rev-list" + ); + } + + /// F1 success path: a filtered serve that completes hands the guard back + /// through both stages and releases admission when the serve returns. No leak, + /// no double-release. + #[tokio::test] + async fn upload_pack_excluding_releases_admission_after_success() { + let td = TempDir::new().unwrap(); + let (_work, bare, secret_oid, _public_oid) = fixture_with_secret(&td); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + let guard = AdmissionGuard::new(sem.clone().try_acquire_owned().unwrap(), None::<()>); + let withheld = HashSet::from([secret_oid]); + let resp = upload_pack_excluding( + "git", + &bare, + Bytes::from_static(b"0000"), + &withheld, + Duration::from_secs(30), + Some(guard), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!( + sem.available_permits(), + 1, + "admission must be released once the filtered serve returns" + ); + } + + /// F1 error path: a rev-list that exits non-zero fails the build AND recovers + /// the threaded admission: drive_git_child drops the guard only after the + /// completed join has reaped the child, and the error return must not leak it. + #[cfg(unix)] + #[tokio::test] + async fn build_filtered_pack_releases_admission_after_rev_list_failure() { + let tmp = tempfile::TempDir::new().unwrap(); + let body = + "#!/bin/sh\ncase \"$1\" in\n rev-list) echo boom >&2; exit 2 ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + let withheld = HashSet::new(); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + + for i in 0..FAKE_GIT_RETRY_ATTEMPTS { + let guard = AdmissionGuard::new(sem.clone().try_acquire_owned().unwrap(), None::<()>); + let result = build_filtered_pack( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_secs(30), + Some(guard), + ) + .await; + let err = match result { + Ok(_) => panic!("a non-zero rev-list exit must fail the build"), + Err(e) => e, + }; + // Load-bearing on EVERY attempt: even a transient spawn-failure return + // must recover the permit (the guard drops before the Err surfaces). + assert_eq!( + sem.available_permits(), + 1, + "the admission permit must be recovered on the error return, not leaked" + ); + if is_transient_exec_race(&err) { + tokio::time::sleep(Duration::from_millis(FAKE_GIT_BACKOFF_STEP_MS * (i + 1))).await; + continue; + } + assert!( + err.to_string().contains("rev-list failed"), + "the failure must surface rev-list's own error, got: {err}" + ); + return; + } + panic!( + "fake git kept hitting ETXTBSY after {FAKE_GIT_RETRY_ATTEMPTS} attempts \ + (persistent exec failure, not a transient parallel-runner miss)" + ); + } + + /// F1: a pack-objects stage that times out (stage 2 of the filtered build, + /// guard threaded through) recovers the admission after the reap: the timeout + /// Err cannot carry the guard, so drive_git_child must drop it post-reap. + #[cfg(unix)] + #[tokio::test] + async fn build_filtered_pack_recovers_admission_after_pack_objects_timeout() { + let tmp = tempfile::TempDir::new().unwrap(); + // rev-list returns one oid fast (guard handed back); pack-objects hangs. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) echo deadbeefdeadbeefdeadbeefdeadbeefdeadbeef ;;\n pack-objects) sleep 300 ;;\n *) exit 1 ;;\nesac\n"; + let git_bin = write_fake_git(tmp.path(), body); + let withheld = HashSet::new(); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + + for i in 0..FAKE_GIT_RETRY_ATTEMPTS { + let guard = AdmissionGuard::new(sem.clone().try_acquire_owned().unwrap(), None::<()>); + let result = tokio::time::timeout( + Duration::from_secs(10), + build_filtered_pack( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_millis(200), + Some(guard), + ), + ) + .await + .expect( + "build_filtered_pack must return within the watchdog; the git stage \ + must be timeout-bounded", + ); + let err = match result { + Ok(_) => panic!("a hung pack-objects must fail the build"), + Err(e) => e, + }; + assert_eq!( + sem.available_permits(), + 1, + "admission must be recovered after the stage-2 timeout reap, not leaked" + ); + if is_transient_exec_race(&err) { + tokio::time::sleep(Duration::from_millis(FAKE_GIT_BACKOFF_STEP_MS * (i + 1))).await; + continue; + } + assert!( + err.downcast_ref::().is_some(), + "a hung pack-objects must abort with GitServiceTimeout, got: {err}" + ); + return; + } + panic!( + "fake git kept hitting ETXTBSY after {FAKE_GIT_RETRY_ATTEMPTS} attempts \ + (persistent exec failure, not a transient parallel-runner miss)" + ); + } + + /// F1 zero-budget contract: build_filtered_pack hands pack-objects + /// `deadline.saturating_duration_since(now)`, which saturates to ZERO once + /// rev-list has consumed the whole budget. A zero-budget stage must still route + /// the guard through the timeout's reap-then-drop rather than leak it: the + /// child spawns, the deadline fires immediately, the group is reaped, and only + /// then does the guard drop. + #[cfg(unix)] + #[tokio::test] + async fn drive_git_child_zero_budget_drops_admission_after_reap() { + let tmp = tempfile::TempDir::new().unwrap(); + let git_bin = write_fake_git(tmp.path(), "#!/bin/sh\nsleep 300\n"); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + + for i in 0..FAKE_GIT_RETRY_ATTEMPTS { + let guard = AdmissionGuard::new(sem.clone().try_acquire_owned().unwrap(), None::<()>); + let mut command = tokio::process::Command::new(git_bin.to_str().unwrap()); + command + .args(["pack-objects", "--stdout"]) + .current_dir(tmp.path()); + let result = tokio::time::timeout( + Duration::from_secs(10), + drive_git_child( + command, + Bytes::new(), + Duration::ZERO, + "pack-objects", + Some(guard), + ), + ) + .await + .expect("a zero-budget stage must abort via its own deadline, not hang"); + let err = match result { + Ok(_) => panic!("a zero-budget stage must return an error"), + Err(e) => e, + }; + assert_eq!( + sem.available_permits(), + 1, + "the guard must drop after the zero-budget timeout reap, not leak" + ); + if is_transient_exec_race(&err) { + tokio::time::sleep(Duration::from_millis(FAKE_GIT_BACKOFF_STEP_MS * (i + 1))).await; + continue; + } + assert!( + err.downcast_ref::().is_some(), + "a zero-budget stage must abort with GitServiceTimeout, got: {err}" + ); + return; + } + panic!( + "fake git kept hitting ETXTBSY after {FAKE_GIT_RETRY_ATTEMPTS} attempts \ + (persistent exec failure, not a transient parallel-runner miss)" + ); + } + // A request that runs to completion must DISARM the guard after reaping, so // no stray group SIGTERM fires. The fake exits non-zero (surfacing as Err) // but leaves a grandchild alive; the grandchild must survive. Goes RED if the From 138b48210a1ae949edc798a93afd1ee38412cfb9 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 17 Jul 2026 17:35:01 -0500 Subject: [PATCH 39/95] fix(node): make /ipfs scan verdicts honest: walk-scoped cap, visit ceiling, 503 on truncation (#174) A request-scoped taint list replaces the silent folds: any repo skipped without a verdict (acquire timeout, probe or read failure, walk failure, cap or ceiling exhaustion) turns the terminal into a retryable 503 with Retry-After, and 404 is reserved for a scan in which every candidate reached a verdict. The walk cap now counts only expensive allowed-set walks and skip-continues on exhaustion so a plain public copy past the cap still serves; a new GITLAWB_IPFS_MAX_REPO_VISITS knob (default 1024) bounds the acquire+probe cost class and stops the scan, documented with its worst-case object-store fetch count. --- .env.example | 12 +- README.md | 3 +- crates/gitlawb-node/src/api/ipfs.rs | 706 ++++++++++++++++++++++-- crates/gitlawb-node/src/config.rs | 54 +- crates/gitlawb-node/src/git/tigris.rs | 19 + crates/gitlawb-node/src/test_support.rs | 25 +- 6 files changed, 765 insertions(+), 54 deletions(-) diff --git a/.env.example b/.env.example index 149ea8b1..144108ba 100644 --- a/.env.example +++ b/.env.example @@ -168,9 +168,17 @@ GITLAWB_MAX_CONCURRENT_IPFS_WALKS=32 # per-caller caps via GITLAWB_TRUSTED_PROXY; reject-before-insert bounded map). # Default 4. GITLAWB_IPFS_WALK_PER_SOURCE=4 -# Max repos walked per single /ipfs request, so one request cannot serialize a -# full-history walk over every repo carrying the CID. Default 64. +# Max EXPENSIVE path-scope visibility walks per single /ipfs request (only a +# blob in a path-scoped repo costs a full-history walk). Over-cap repos are +# skipped without a verdict and the scan continues; if the object is then found +# nowhere the request sheds a retryable 503 instead of a false 404. Default 64. GITLAWB_IPFS_MAX_REPOS_WALKED=64 +# Ceiling on repos one /ipfs request may VISIT past the visibility gate. Each +# visit costs a repo acquire — on a Tigris cache miss a full archive download, +# so this is also the worst-case object-store fetch count per request — plus a +# cat-file probe. On exhaustion the scan stops and sheds a retryable 503. +# Default 1024. +GITLAWB_IPFS_MAX_REPO_VISITS=1024 # Max /ipfs/{cid} requests per client IP per hour (route flood brake, distinct # from the concurrency caps above). 0 disables. Default 600. GITLAWB_IPFS_RATE_LIMIT=600 diff --git a/README.md b/README.md index e9765876..f3fa361b 100644 --- a/README.md +++ b/README.md @@ -347,7 +347,8 @@ Important node settings: | `GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS` | Max seconds the storage-acquisition phase (Tigris HEAD/GET, push advisory-lock) of a served git op may run before the request is shed with a 503, separate from the git-run timeout. The concurrency permit is released on expiry so a stalled backend cannot pin the pool. Default 30. | | `GITLAWB_MAX_CONCURRENT_IPFS_WALKS` | Max concurrent `GET /ipfs/{cid}` visibility walks across all callers (own pool, disjoint from the served-git pools); over-cap sheds 503. Default 32. | | `GITLAWB_IPFS_WALK_PER_SOURCE` | Max concurrent `/ipfs` walks a single source IP may hold. Default 4. | -| `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max repos walked per `/ipfs/{cid}` request, bounding one request's fan-out. Default 64. | +| `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max expensive path-scope visibility walks per `/ipfs/{cid}` request; over-cap repos are skipped and the scan continues, shedding a retryable 503 (not a false 404) if the object is then found nowhere. Default 64. | +| `GITLAWB_IPFS_MAX_REPO_VISITS` | Ceiling on repos one `/ipfs/{cid}` request may visit (acquire + probe) past the visibility gate — also the worst-case per-request Tigris fetch count. On exhaustion the scan stops with a retryable 503. Default 1024. | | `GITLAWB_IPFS_RATE_LIMIT` | Max `/ipfs/{cid}` requests per client IP per hour (route flood brake). 0 disables. Default 600. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index ac8e615c..d4a6c7a8 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -51,6 +51,15 @@ use crate::visibility::{visibility_check, Decision}; /// fail-closed 404'd under path-scoped rules (#126). Denial and genuine /// not-found both fall through to an opaque 404. /// +/// Scan completeness (F2): the 404 above is returned ONLY when every candidate +/// repo reached a VERDICT — visibility deny, probe-says-absent, walk-gate deny, +/// or served. A candidate skipped WITHOUT a verdict (acquire failure/timeout, +/// probe error, walk failure/panic, content-read error, or truncation by +/// `ipfs_max_repos_walked` / `ipfs_max_repo_visits`) taints the scan, and a +/// tainted scan that found nothing sheds a retryable 503 + Retry-After naming +/// the truncation sources — existing content is never misreported absent +/// because of unrelated repos or transient faults. +/// /// Scope: this closes the direct unauthenticated scan, including the dangling /// case. A stale-public mirror row still serves withheld content (tracked /// separately, #124). @@ -139,13 +148,35 @@ pub async fn get_by_cid( // deny entry (#126). let mut allowed_memo: HashMap> = HashMap::new(); - // Cap the number of candidate repos one request walks (it already short-circuits on - // serve): a CID present in — or path-gated out of — many repos must not serialize an - // unbounded number of full-history walks inside the single held admission slot. + // Verdict-or-taint bookkeeping (F2): a candidate repo the loop cannot bring to a + // VERDICT (visibility deny / probe-says-absent / walk-gate deny / served) marks + // the scan truncated with its source. A truncated scan that finds nothing must + // NOT report 404 — the object may sit in a repo we skipped — so the terminal arm + // sheds a retryable 503 naming the sources, keyed so the operator can tell which + // knob (or backend) to look at. + let mut truncated_by: Vec<&'static str> = Vec::new(); + fn taint(truncated_by: &mut Vec<&'static str>, source: &'static str) { + if !truncated_by.contains(&source) { + truncated_by.push(source); + } + } + + // Cap on EXPENSIVE walks only (F2): counts the repos that actually require the + // full-history `allowed_blob_set_for_caller_bounded` walk (a path-scoped blob), + // checked immediately before the spawn_blocking below. Cheap probe-only visits + // are bounded by `repos_visited` — counting them here starved later-ordered + // repos out of a plain 200 on nodes with more readable repos than the cap. let mut repos_walked: usize = 0; + // Ceiling on VISITS (F2): every repo past the visibility gate costs an acquire + // (worst case a full Tigris archive download on a cache miss) plus a cat-file + // probe, so one request can trigger at most `ipfs_max_repo_visits` object-store + // fetches. On exhaustion the scan STOPS — there is no cheaper way to continue. + let mut repos_visited: usize = 0; for repo in &repos { - // Repo-level read gate against THIS row's own rules (KTD2a). + // Repo-level read gate against THIS row's own rules (KTD2a). Deny is a + // VERDICT: this repo would never serve the caller, so skipping it cannot + // hide content from them. let rules: &[crate::db::VisibilityRule] = rules_by_repo .get(&repo.id) .map(Vec::as_slice) @@ -154,23 +185,24 @@ pub async fn get_by_cid( continue; } - // Loop bound (#174 P1-3): once this request has walked its cap of candidate - // repos (each a git subprocess, up to a full-history walk), stop and fall - // through to the opaque 404 rather than serialize an unbounded number under the - // single held admission slot. - if repos_walked >= state.config.ipfs_max_repos_walked { + // Visit ceiling (F2): bound the acquire+probe cost class. Stopping here + // leaves the remaining candidates unproven, so the scan is truncated. + if repos_visited >= state.config.ipfs_max_repo_visits { tracing::warn!( - cap = state.config.ipfs_max_repos_walked, - "/ipfs request hit the per-request repo-walk cap; stopping the scan" + ceiling = state.config.ipfs_max_repo_visits, + "/ipfs request hit the per-request repo-visit ceiling \ + (GITLAWB_IPFS_MAX_REPO_VISITS); stopping the scan without a verdict" ); + taint(&mut truncated_by, "visit-ceiling"); break; } - repos_walked += 1; + repos_visited += 1; // Bound the per-repo acquire under `git_acquire_timeout_secs`: this loop shares // the P1-2 stall vector (a hung Tigris HEAD/GET on one repo would otherwise - // block the whole /ipfs request). On expiry keep the existing fail-closed skip — - // never serve an un-acquired repo; a public copy (if any) still gets its turn. + // block the whole /ipfs request). On expiry keep the fail-closed skip — never + // serve an un-acquired repo; a public copy (if any) still gets its turn — but + // the repo got no verdict, so the skip taints the scan. let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); let repo_path = match tokio::time::timeout( @@ -180,21 +212,28 @@ pub async fn get_by_cid( .await { Ok(Ok(p)) => p, - Ok(Err(_)) => continue, + Ok(Err(e)) => { + tracing::warn!(repo = %repo.name, err = %e, "repo acquire failed during /ipfs scan; skipping repo without a verdict"); + taint(&mut truncated_by, "acquire"); + continue; + } Err(_elapsed) => { - tracing::warn!(repo = %repo.name, "repo acquire timed out during /ipfs walk; skipping repo"); + tracing::warn!(repo = %repo.name, "repo acquire timed out during /ipfs scan; skipping repo without a verdict"); + taint(&mut truncated_by, "acquire"); continue; } }; // Check whether the object exists in this repo before any expensive // reachability walk. This prevents random-CID spray from triggering - // full-history git walks on repos that don't carry the object. + // full-history git walks on repos that don't carry the object. Absent + // (`Ok(None)`) is a VERDICT; a probe that could not run is not. let obj_type = match store::object_type(&repo_path, &sha256_hex) { Ok(Some(t)) => t, Ok(None) => continue, Err(e) => { - tracing::warn!(repo = %repo.name, err = %e, "error checking git object type"); + tracing::warn!(repo = %repo.name, err = %e, "object-type probe failed during /ipfs scan; skipping repo without a verdict"); + taint(&mut truncated_by, "probe"); continue; } }; @@ -205,6 +244,21 @@ pub async fn get_by_cid( let path_scoped = has_path_scoped_rule(rules); if path_scoped && obj_type == "blob" { if !allowed_memo.contains_key(&repo.id) { + // Walk cap (F2), checked at the one site that actually spends a walk: + // on exhaustion skip THIS repo without a verdict and KEEP scanning — + // later candidates may still reach a cheap probe-only verdict (a plain + // public copy serves its 200 with no walk at all). + if repos_walked >= state.config.ipfs_max_repos_walked { + tracing::warn!( + cap = state.config.ipfs_max_repos_walked, + repo = %repo.name, + "/ipfs request hit the per-request walk cap \ + (GITLAWB_IPFS_MAX_REPOS_WALKED); skipping repo without a verdict" + ); + taint(&mut truncated_by, "walk-cap"); + continue; + } + repos_walked += 1; let rp = repo_path.clone(); let r = rules.to_vec(); let is_public = repo.is_public; @@ -229,20 +283,25 @@ pub async fn get_by_cid( .await; // Fail closed on EITHER a task panic (JoinError) or a walk error: // we cannot prove the caller may read here, so skip this repo and - // let a public copy (if any) serve. Never serve on an unproven gate. + // let a public copy (if any) serve. Never serve on an unproven gate + // — and never report absent on one either (no verdict, taint). let set = match walk { Ok(Ok(set)) => set, Ok(Err(e)) => { - tracing::warn!(repo = %repo.name, err = %e, "allowed-blob walk failed; skipping repo"); + tracing::warn!(repo = %repo.name, err = %e, "allowed-blob walk failed during /ipfs scan; skipping repo without a verdict"); + taint(&mut truncated_by, "walk-failure"); continue; } Err(e) => { - tracing::warn!(repo = %repo.name, err = %e, "allowed-blob walk task panicked; skipping repo"); + tracing::warn!(repo = %repo.name, err = %e, "allowed-blob walk task panicked during /ipfs scan; skipping repo without a verdict"); + taint(&mut truncated_by, "walk-failure"); continue; } }; allowed_memo.insert(repo.id.clone(), set); } + // Not in the caller's reachable allowed-set: a VERDICT (deny), the walk + // proved this repo would never serve the blob to this caller. let in_allowed = allowed_memo .get(&repo.id) .is_some_and(|set| set.contains(&sha256_hex)); @@ -251,11 +310,14 @@ pub async fn get_by_cid( } } - // Now that we've passed the gate, read the content. + // Now that we've passed the gate, read the content. A failed read after a + // passed gate is not an absence verdict — the probe just said the object + // exists here — so the skip taints the scan. let content = match store::read_object_content(&repo_path, &sha256_hex, &obj_type) { Ok(c) => c, Err(e) => { - tracing::warn!(repo = %repo.name, err = %e, "error reading git object content"); + tracing::warn!(repo = %repo.name, err = %e, "object content read failed during /ipfs scan; skipping repo without a verdict"); + taint(&mut truncated_by, "read"); continue; } }; @@ -279,7 +341,20 @@ pub async fn get_by_cid( return Ok((StatusCode::OK, headers, content).into_response()); } - // Not found in any repo + // Truncated scan (F2): at least one candidate repo yielded no verdict, so the + // object is not proven absent. A 404 here would misreport existing content, so + // shed retryable instead — Overloaded is the single 503 + Retry-After site in + // error.rs, and the message names the truncation sources so the operator can + // map the shed to the right knob or backend. + if !truncated_by.is_empty() { + return Err(AppError::Overloaded(format!( + "ipfs scan incomplete ({}) for CID {cid_str}; retry shortly", + truncated_by.join("+") + ))); + } + + // Complete scan: every candidate reached a verdict and none served, so the + // object is definitively absent (or denied) for this caller. Err(AppError::RepoNotFound(format!( "no git object found for CID {cid_str}" ))) @@ -353,6 +428,554 @@ mod tests { req } + /// Run real git, asserting success. Shared by the F2 scan-verdict tests. + fn run_git(args: &[&str], cwd: &std::path::Path) { + let out = std::process::Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + } + + /// Seed a repo row plus a REAL sha256 bare repo at its acquired path holding one + /// committed blob (`src/secret.txt` = `content`). Returns `(repo_id, blob_oid)`. + /// Same recipe as `get_by_cid_walk_permit_held_through_blocking_walk`: the CID + /// digest IS the sha256 object id under `--object-format=sha256`, so the real + /// `cat-file` probe finds the blob. + async fn seed_repo_with_blob( + state: &crate::state::AppState, + tmp: &std::path::Path, + owner: &str, + name: &str, + content: &[u8], + ) -> (String, String) { + state + .db + .upsert_mirror_repo(owner, name, &format!("/unused-{name}"), None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + let _ = std::fs::remove_dir_all(&bare); + std::fs::create_dir_all(&bare).unwrap(); + let work = tmp.join(format!("work-{owner}-{name}")); + std::fs::create_dir_all(work.join("src")).unwrap(); + std::fs::write(work.join("src/secret.txt"), content).unwrap(); + run_git( + &["init", "-q", "--object-format=sha256", "-b", "main"], + &work, + ); + run_git(&["config", "user.email", "t@t"], &work); + run_git(&["config", "user.name", "t"], &work); + run_git(&["add", "src/secret.txt"], &work); + run_git(&["commit", "-q", "-m", "seed"], &work); + run_git( + &[ + "clone", + "--bare", + "-q", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + tmp, + ); + let out = std::process::Command::new("git") + .args(["rev-parse", "HEAD:src/secret.txt"]) + .current_dir(&work) + .output() + .expect("git rev-parse runs"); + assert!(out.status.success(), "rev-parse failed"); + let oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); + (rec.id, oid) + } + + /// CIDv1(raw, sha2-256) for a sha256 object id, as the handler resolves it. + fn cid_for_oid(oid: &str) -> String { + let oid_bytes = gitlawb_core::cid::sha256_hex_to_bytes(oid).unwrap(); + gitlawb_core::cid::Cid::from_sha256_bytes(&oid_bytes) + .as_str() + .to_string() + } + + /// Fake git for the WALK only (`state.git_bin`): empty refs, `rev-parse` + /// resolves, and each `rev-list` appends one line to `log` and prints nothing — + /// every walked repo yields an EMPTY allowed-set (path-gate deny verdict) and + /// the log's line count == the number of expensive walks run. The probe and the + /// content read shell to the real `git`, so seeded objects must genuinely exist. + #[cfg(unix)] + fn walk_logging_fake_git(dir: &std::path::Path, log: &std::path::Path) -> String { + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + for-each-ref) : ;;\n\ + rev-parse) echo deadbeef ;;\n\ + rev-list) echo walk >> \"{}\" ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + log.display() + ); + let git_path = dir.join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + git_path.to_str().unwrap().to_string() + } + + /// F2 buried-row repro: with more readable repos than `ipfs_max_repos_walked`, + /// existing PUBLIC content past the cap must still serve. The cap counts + /// EXPENSIVE walks only — this request has no path-scoped rules anywhere, so it + /// runs ZERO walks (the fake-git walk log stays empty) and the cap can never cut + /// the scan: the blob buried in the OLDER-updated repo (iterated last under + /// `list_all_repos`' updated_at DESC) serves its 200. Before F2 the cap counted + /// visibility-passing VISITS and broke the loop into the opaque 404 — existing + /// content misreported absent because of unrelated repos. MUTATION (RED): count + /// visits against the cap again (re-add the check+increment at the visibility + /// gate) and the buried row 503s instead of serving. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_buried_public_row_past_walk_cap_still_serves(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let walk_log = tmp.path().join("walks.log"); + state.git_bin = walk_logging_fake_git(tmp.path(), &walk_log); + // Tighter than the repo count: the old visit-counting cap cut the scan here. + let mut cfg = (*state.config).clone(); + cfg.ipfs_max_repos_walked = 1; + state.config = Arc::new(cfg); + + // Seed the blob-carrying repo FIRST so its updated_at is OLDER: the empty + // repo is iterated first and the blob row sits past the old visit budget. + let (_, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6f2buried", + "buried", + b"buried row proof\n", + ) + .await; + seed_repo_with_blob( + &state, + tmp.path(), + "z6f2buried", + "fresh", + b"unrelated content\n", + ) + .await; + + let peer: SocketAddr = "203.0.113.60:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&cid_for_oid(&oid), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::OK, + "a public blob in a repo past the walk cap must still serve — the cap \ + counts expensive walks and this scan needs none" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + assert_eq!(&body[..], b"buried row proof\n"); + let walks = std::fs::read_to_string(&walk_log) + .map(|s| s.lines().count()) + .unwrap_or(0); + assert_eq!( + walks, 0, + "a request with no path-scoped rules anywhere must run zero expensive walks" + ); + } + + /// F2 walk-cap skip-and-continue: exhausting `ipfs_max_repos_walked` skips the + /// walk-NEEDING repo without a verdict but keeps the scan alive. Three public + /// repos carry the same blob, newest first: the first (path-scoped) consumes the + /// cap-of-1 walk and denies (empty allowed-set — a verdict); the second + /// (path-scoped) needs a walk the cap forbids and is skipped WITHOUT one (taint); + /// the third is plain public and serves the 200 from a cheap probe — found beats + /// taint, and exactly one expensive walk ran. Before F2 the cap broke the loop at + /// the second repo and the request 404'd despite the public copy. MUTATION (RED): + /// turn the walk-cap skip back into a `break` and the public copy never serves. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_walk_cap_skip_continues_to_later_public_copy(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let walk_log = tmp.path().join("walks.log"); + state.git_bin = walk_logging_fake_git(tmp.path(), &walk_log); + let mut cfg = (*state.config).clone(); + cfg.ipfs_max_repos_walked = 1; + state.config = Arc::new(cfg); + + // Insert order = oldest first, so iteration (updated_at DESC) is reversed: + // gatedwalk, then gatedskip, then pubcopy. Identical content -> one CID. + let content = b"skip and continue proof\n"; + let (_, oid) = + seed_repo_with_blob(&state, tmp.path(), "z6f2skip", "pubcopy", content).await; + let (skip_id, _) = + seed_repo_with_blob(&state, tmp.path(), "z6f2skip", "gatedskip", content).await; + let (walk_id, _) = + seed_repo_with_blob(&state, tmp.path(), "z6f2skip", "gatedwalk", content).await; + for id in [&walk_id, &skip_id] { + state + .db + .set_visibility_rule( + id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderCCCCCCCCCCCCCCCCCCCCCCCC".to_string()], + "z6f2skip", + ) + .await + .unwrap(); + } + + let peer: SocketAddr = "203.0.113.61:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&cid_for_oid(&oid), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::OK, + "the walk-cap skip must continue the scan so the plain public copy serves" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + assert_eq!(&body[..], content.as_slice()); + let walks = std::fs::read_to_string(&walk_log) + .map(|s| s.lines().count()) + .unwrap_or(0); + assert_eq!( + walks, 1, + "cap honored exactly: the first path-scoped repo walks, the second is cut" + ); + } + + /// F2 visit ceiling: `ipfs_max_repo_visits` bounds the acquire+probe cost class + /// (each visit can be a full Tigris archive fetch on a cache miss). Unlike the + /// walk cap there is no cheap way to keep scanning, so exhaustion STOPS the scan + /// — and the stop is a truncation, not an absence: with ceiling 1 the newer + /// empty repo consumes the only visit and the blob-carrying older repo is never + /// probed, so the request sheds a retryable 503 + Retry-After, never a false + /// 404. MUTATION (RED): drop the ceiling check and the blob serves (200); drop + /// only the taint on the break and the 503 decays to a 404. + #[sqlx::test] + async fn get_by_cid_visit_ceiling_stops_scan_with_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let mut cfg = (*state.config).clone(); + cfg.ipfs_max_repo_visits = 1; + state.config = Arc::new(cfg); + + // Blob repo first (older, iterated second); empty repo second (newer, + // consumes the single visit). + let (_, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6f2visit", + "buried", + b"visit ceiling proof\n", + ) + .await; + seed_repo_with_blob(&state, tmp.path(), "z6f2visit", "fresh", b"unrelated\n").await; + + let peer: SocketAddr = "203.0.113.62:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&cid_for_oid(&oid), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a visit-ceiling truncation must shed a retryable 503, not report absent" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + } + + /// F2 negative arm: a COMPLETE scan that finds nothing keeps its definitive 404 + /// — the truncation 503 must never fire when every candidate reached a verdict. + /// Two public repos both probe clean (the requested CID is nowhere), no rules, + /// no cap or ceiling hit: 404 with no Retry-After. MUTATION (RED): taint the + /// scan unconditionally and this decays into a 503. + #[sqlx::test] + async fn get_by_cid_complete_scan_keeps_definitive_404(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + seed_repo_with_blob(&state, tmp.path(), "z6f2clean", "one", b"content one\n").await; + seed_repo_with_blob(&state, tmp.path(), "z6f2clean", "two", b"content two\n").await; + + // valid_cid() is the "hello" blob — present in neither repo. + let peer: SocketAddr = "203.0.113.63:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::NOT_FOUND, + "a complete clean scan is a definitive absence — 404, never the 503 shed" + ); + assert!( + resp.headers().get("retry-after").is_none(), + "a definitive 404 must not advertise a retry" + ); + } + + /// F2 acquire taint: a repo row with NO local copy over a Tigris backend that + /// stalls (non-routable endpoint — the connect just hangs) hits the 1s acquire + /// timeout at the read-acquire site. The skip carries no verdict, so the scan is + /// truncated: retryable 503 + Retry-After, never the old silent-skip 404. + /// MUTATION (RED): drop the taint on the acquire-timeout arm and this decays to + /// a 404. + #[sqlx::test] + async fn get_by_cid_acquire_timeout_taints_scan_to_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + // Endpoint-pinned test client (no AWS_* env reads — env is racy under a + // parallel test run); 10.255.255.1 is non-routable so the HEAD hangs. + let tigris = crate::git::tigris::TigrisClient::for_testing_with_endpoint( + "test-bucket", + "http://10.255.255.1:9000", + ) + .await; + state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let mut cfg = (*state.config).clone(); + cfg.git_acquire_timeout_secs = 1; + state.config = Arc::new(cfg); + + // Row exists in the DB but has no local copy, so the read acquire must + // consult Tigris (local-miss path) and stall until the timeout. + state + .db + .upsert_mirror_repo("z6f2acq", "ghost", "/unused-ghost", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.64:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an acquire timeout leaves the repo unproven — the scan must shed 503, not 404" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + } + + /// F2 probe taint: a repo row whose local dir does not exist (no Tigris) — + /// `RepoStore::acquire` returns the path anyway (local passthrough), and the + /// `cat-file -t` probe cannot even spawn (missing working dir), so + /// `object_type` is Err. That is not an absence verdict, so the scan is + /// truncated: 503, never 404. A second, real repo probes clean (absent verdict) + /// — the one bad row is what taints. NOTE: the probe shells to the real `git` + /// (not `state.git_bin`) and maps a NONZERO cat-file exit to `Ok(None)` (an + /// absent verdict), so a fake git exiting nonzero cannot reach this arm; only a + /// spawn failure is Err, hence the missing-dir recipe. MUTATION (RED): drop the + /// taint on the probe-error arm and this decays to a 404. + #[sqlx::test] + async fn get_by_cid_probe_error_taints_scan_to_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // Older row: a real repo that probes clean. Newer row: no dir on disk. + seed_repo_with_blob(&state, tmp.path(), "z6f2probe", "real", b"probe clean\n").await; + state + .db + .upsert_mirror_repo("z6f2probe", "ghost", "/unused-ghost", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.65:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a failed probe leaves the repo unproven — the scan must shed 503, not 404" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + } + + /// F2 read taint: the gate passes (the probe reads the truncated loose object's + /// intact "blob 64" header) but the content read fails (`cat-file blob` dies on + /// the deflate stream cut mid-content) — the probe just said the object EXISTS + /// here, so the failed read is no absence verdict: 503, never 404. The loose + /// object is hand-rolled: zlib header + one stored deflate block declaring 72 + /// bytes ("blob 64\0" + 64), truncated after the header NUL + 4 content bytes, + /// no adler trailer. MUTATION (RED): drop the taint on the read-error arm and + /// this decays to a 404. + #[sqlx::test] + async fn get_by_cid_read_error_taints_scan_to_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + state + .db + .upsert_mirror_repo("z6f2read", "corrupt", "/unused-corrupt", None, false) + .await + .unwrap(); + let rec = state + .db + .get_repo("z6f2read", "corrupt") + .await + .unwrap() + .unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + std::fs::create_dir_all(&bare).unwrap(); + run_git(&["init", "-q", "--bare", "--object-format=sha256"], &bare); + // Hand-rolled truncated loose object (dangling is fine: no path-scoped rules, + // so the "/" gate is the whole story and the read follows the probe). + let oid = "6bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459c"; + let mut corrupt: Vec = vec![0x78, 0x01, 0x01, 0x48, 0x00, 0xb7, 0xff]; + corrupt.extend_from_slice(b"blob 64\0AAAA"); + let obj_dir = bare.join("objects").join(&oid[..2]); + std::fs::create_dir_all(&obj_dir).unwrap(); + std::fs::write(obj_dir.join(&oid[2..]), &corrupt).unwrap(); + // Preconditions: the probe classifies it as a blob, the full read fails — + // otherwise the test would pass vacuously via some other arm. + assert_eq!( + crate::git::store::object_type(&bare, oid) + .unwrap() + .as_deref(), + Some("blob"), + "the truncated loose object's header must still probe as a blob" + ); + assert!( + crate::git::store::read_object_content(&bare, oid, "blob").is_err(), + "the truncated loose object's content read must fail" + ); + + let peer: SocketAddr = "203.0.113.66:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&cid_for_oid(oid), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a failed read after a passed gate leaves the repo unproven — 503, not 404" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + } + + /// F2 denied-is-a-verdict: repos that DENY the caller at the visibility gate + /// are settled, not skipped — an all-denied scan is COMPLETE: 404, zero visits. + /// The private rows deliberately have no local dirs: if the deny didn't + /// short-circuit before the visit, the missing-dir probe would taint the scan + /// into a 503, which the 404 assertion rules out — so the 404 also proves zero + /// acquires, probes, or walks ran for denied rows. + #[sqlx::test] + async fn get_by_cid_all_denied_is_complete_scan_404(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + for name in ["priv-a", "priv-b"] { + let now = chrono::Utc::now(); + state + .db + .create_repo(&crate::db::RepoRecord { + id: uuid::Uuid::new_v4().to_string(), + name: name.to_string(), + owner_did: "did:key:z6MkF2DenyOwnerAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + description: None, + is_public: false, + default_branch: "main".to_string(), + created_at: now, + updated_at: now, + disk_path: format!("/nonexistent/{name}"), + forked_from: None, + machine_id: None, + }) + .await + .unwrap(); + } + + let peer: SocketAddr = "203.0.113.67:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::NOT_FOUND, + "an anonymous caller denied by every repo gets a complete-scan 404 — a deny \ + is a verdict and must not visit, taint, or 503" + ); + } + /// Shed at capacity: an exhausted `git_ipfs_walk_semaphore` sheds a `/ipfs/{cid}` /// request with 503 BEFORE any DB/git walk (the acquire is the first thing after CID /// validation), so a lazy DB-free state suffices — exactly like the served-git shed @@ -736,13 +1359,16 @@ mod tests { ); } - /// Loop bound (cap N): one `/ipfs/{cid}` request against a CID present in many repos - /// must not serialize an unbounded number of full-history walks. With - /// `ipfs_max_repos_walked = 1` and TWO public, path-scoped repos both carrying the - /// blob at the requested CID, the handler walks only the FIRST candidate then stops - /// (the second is cut by the cap), so the fake git's `rev-list` (one per walk) runs - /// exactly once. MUTATION (RED): remove the `repos_walked >= cap` break and both - /// repos are walked (count 2). + /// Loop bound (cap N) + F2 truncation verdict: one `/ipfs/{cid}` request against a + /// CID present in many path-scoped repos must not serialize an unbounded number of + /// full-history walks — and cutting a candidate WITHOUT a verdict must not report + /// the object absent. With `ipfs_max_repos_walked = 1` and TWO public, path-scoped + /// repos both carrying the blob, the first candidate is walked (empty allowed-set → + /// a deny VERDICT) and the second is cut by the cap (no verdict), so the fake git's + /// `rev-list` runs exactly once and the request sheds a retryable 503 + Retry-After + /// — never the old false 404 (the blob genuinely sits in the second repo). + /// MUTATION (RED): remove the `repos_walked >= cap` skip and both repos are walked + /// (count 2); drop the truncation taint on the skip and the 503 decays to a 404. #[cfg(unix)] #[sqlx::test] async fn get_by_cid_caps_repos_walked_per_request(pool: sqlx::PgPool) { @@ -871,9 +1497,21 @@ mod tests { .unwrap(); req.extensions_mut().insert(ConnectInfo(peer)); let resp = ipfs_router(state).oneshot(req).await.unwrap(); - // The empty allowed-set path-gates both repos to a `continue`, so a 404; the - // point is HOW MANY walks ran to get there. - assert_eq!(resp.status(), StatusCode::NOT_FOUND); + // The first repo's walk yields the empty allowed-set (deny verdict); the second + // repo NEEDS a walk the cap forbids, so the scan is truncated without a verdict + // on it: retryable 503, never a false 404 for the blob it genuinely carries. + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a walk-cap truncation must shed a retryable 503, not report the object absent" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); let walks = std::fs::read_to_string(&walk_log) .map(|s| s.lines().count()) diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index b3667024..3e0a8e50 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -372,13 +372,17 @@ pub struct Config { )] pub ipfs_walk_per_source: usize, - /// Upper bound on the number of candidate repos a single `/ipfs/{cid}` request - /// will walk before giving up (returning the opaque 404). The handler already - /// short-circuits the moment it serves the object, but a CID that is present in - /// (or path-gated out of) many repos could otherwise serialize one full-history - /// walk per repo inside a single held admission slot. Capping the count bounds - /// the worst-case work one request can pin its slot with. Must be between 1 and - /// 1_048_576. Default: 64. + /// Upper bound on the number of EXPENSIVE visibility walks + /// (`allowed_blob_set_for_caller_bounded`, a full-history git walk in a + /// blocking thread) a single `/ipfs/{cid}` request may run. Only a blob in a + /// path-scoped repo costs a walk, so the cap counts exactly those candidates + /// — cheap probe-only visits are bounded by `ipfs_max_repo_visits` instead + /// (counting them here would starve a plain public copy past the cap out of + /// its 200). On exhaustion the walk-needing repo is skipped WITHOUT a verdict + /// and the scan continues; if the request then finds the object nowhere it + /// sheds a retryable 503 + Retry-After rather than misreport existing content + /// absent with a 404. The handler still short-circuits the moment it serves. + /// Must be between 1 and 1_048_576. Default: 64. #[arg( long, env = "GITLAWB_IPFS_MAX_REPOS_WALKED", @@ -387,6 +391,24 @@ pub struct Config { )] pub ipfs_max_repos_walked: usize, + /// Ceiling on the number of repos a single `/ipfs/{cid}` request may VISIT — + /// pass the repo-level visibility gate into the acquire + `cat-file` probe. + /// Each visit costs a `RepoStore::acquire` (on a Tigris cache miss that is a + /// full repo-archive download from object storage, so the worst-case + /// object-store fetch count for one request equals this ceiling) plus a git + /// probe subprocess. On exhaustion the scan STOPS — unlike + /// `ipfs_max_repos_walked`, which skips just the walk-needing repo, there is + /// no cheaper way to keep scanning — and the request sheds a retryable 503 + + /// Retry-After rather than a false 404. Must be between 1 and 1_048_576. + /// Default: 1024. + #[arg( + long, + env = "GITLAWB_IPFS_MAX_REPO_VISITS", + default_value_t = 1024, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub ipfs_max_repo_visits: usize, + /// Per-client-IP rate limit for `GET /ipfs/{cid}`, in requests per hour. The /// route is publicly reachable (`optional_signature`) and each request can drive /// a full-history git walk, so it carries a per-IP flood brake in addition to the @@ -548,6 +570,24 @@ mod tests { ); } + #[test] + fn ipfs_max_repo_visits_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).ipfs_max_repo_visits, + 1024 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--ipfs-max-repo-visits", "8"]) + .ipfs_max_repo_visits, + 8 + ); + // 0 would visit no repos (serve nothing); clap must reject it. + assert!(Config::try_parse_from(["gitlawb-node", "--ipfs-max-repo-visits", "0"]).is_err()); + assert!( + Config::try_parse_from(["gitlawb-node", "--ipfs-max-repo-visits", "1048577"]).is_err() + ); + } + #[test] fn max_concurrent_reads_per_caller_defaults_and_rejects_out_of_range() { assert_eq!( diff --git a/crates/gitlawb-node/src/git/tigris.rs b/crates/gitlawb-node/src/git/tigris.rs index ad26ddc5..cf7abfd5 100644 --- a/crates/gitlawb-node/src/git/tigris.rs +++ b/crates/gitlawb-node/src/git/tigris.rs @@ -31,6 +31,25 @@ impl TigrisClient { }) } + /// Test-only constructor with an explicit S3 endpoint, region, and static + /// credentials — no env-var reads, so parallel tests cannot race each other's + /// `AWS_*` environment the way the env-based `new` would. Lets a test point + /// the client at a non-routable endpoint to exercise acquire-stall paths. + #[cfg(test)] + pub(crate) async fn for_testing_with_endpoint(bucket: &str, endpoint_url: &str) -> Self { + let creds = aws_sdk_s3::config::Credentials::new("test", "test", None, None, "test"); + let config = aws_config::defaults(aws_config::BehaviorVersion::latest()) + .endpoint_url(endpoint_url) + .region(aws_config::Region::new("auto")) + .credentials_provider(creds) + .load() + .await; + Self { + s3: S3Client::new(&config), + bucket: bucket.to_string(), + } + } + /// S3 key for a given repo: `repos/v1/{owner_slug}/{repo_name}.tar.zst` fn repo_key(owner_slug: &str, repo_name: &str) -> String { format!("repos/v1/{owner_slug}/{repo_name}.tar.zst") diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 92f41a8d..df095b44 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -2409,7 +2409,10 @@ mod tests { /// the handler skips the whole repo rather than serving. Asserts no leak of the /// withheld blob AND that even the *public* blob in that repo is withheld — the /// latter distinguishes fail-closed-skip from normal per-blob withholding and - /// would serve 200 if the error arm wrongly proceeded. + /// would serve 200 if the error arm wrongly proceeded. The skip carries no + /// VERDICT (F2), so the response is the retryable truncation 503, not a 404 + /// claiming the object is absent — never-serve-unproven and never-404-unproven + /// hold together. #[sqlx::test] async fn ipfs_cid_walk_error_fails_closed(pool: PgPool) { use crate::db::VisibilityMode; @@ -2454,7 +2457,8 @@ mod tests { .await .expect("deny rule"); - // Withheld secret CID under a walk error → 404, no leak. + // Withheld secret CID under a walk error → the repo is skipped without a + // verdict, so the scan is truncated (503), and nothing leaks. let (st, body) = cid_parts( cid_router(&state) .oneshot(cid_anon(&secret_cid)) @@ -2464,17 +2468,17 @@ mod tests { .await; assert_eq!( st, - StatusCode::NOT_FOUND, - "walk error must not serve the withheld blob" + StatusCode::SERVICE_UNAVAILABLE, + "walk error must not serve the withheld blob — the unproven skip sheds 503" ); assert!( !body.contains("TOP SECRET"), - "walk-error 404 must not leak the secret" + "walk-error 503 must not leak the secret" ); - // The PUBLIC blob in the same repo is also 404: the walk error fails closed - // by skipping the whole repo, not by serving. Without the fail-closed arm - // this would serve 200, so this assertion is the load-bearing discriminator. + // The PUBLIC blob in the same repo is also not served: the walk error fails + // closed by skipping the whole repo. Without the fail-closed arm this would + // serve 200, so this assertion is the load-bearing discriminator. let (st, _) = cid_parts( cid_router(&state) .oneshot(cid_anon(&public_cid)) @@ -2484,8 +2488,9 @@ mod tests { .await; assert_eq!( st, - StatusCode::NOT_FOUND, - "walk error fails closed: repo skipped, even the public blob is not served" + StatusCode::SERVICE_UNAVAILABLE, + "walk error fails closed: repo skipped without a verdict, even the public \ + blob is not served and the scan sheds 503" ); } From 39d30dc3c11d08b1c816b38c108eb56f8111d11d Mon Sep 17 00:00:00 2001 From: t Date: Fri, 17 Jul 2026 18:03:49 -0500 Subject: [PATCH 40/95] fix(node): bound an admitted /ipfs request's total lifetime with a shared budget (#174) GITLAWB_IPFS_REQUEST_BUDGET_SECS (default 600) is captured as one deadline at handler entry: every acquire and expensive walk is clamped to the remaining budget, every stage checks remaining before starting, and exhaustion taints the scan into the retryable 503. A walk in flight is never aborted at the tokio layer (that would free the walk permit while the blocking thread still runs); its clamped git deadline plus group teardown ends it. Docs state the residual overshoot: the in-flight stage's kill/reap slack plus the unclamped probe subprocesses. --- .env.example | 7 + README.md | 1 + crates/gitlawb-node/src/api/ipfs.rs | 500 +++++++++++++++++++++++++++- crates/gitlawb-node/src/config.rs | 44 +++ 4 files changed, 546 insertions(+), 6 deletions(-) diff --git a/.env.example b/.env.example index 144108ba..94685069 100644 --- a/.env.example +++ b/.env.example @@ -179,6 +179,13 @@ GITLAWB_IPFS_MAX_REPOS_WALKED=64 # cat-file probe. On exhaustion the scan stops and sheds a retryable 503. # Default 1024. GITLAWB_IPFS_MAX_REPO_VISITS=1024 +# Absolute wall-clock budget for one admitted /ipfs request's acquire+walk +# lifetime (all stages of the whole scan). Bounds how LONG one request may hold +# its walk slot: per-stage timeouts are clamped to the remaining budget and no +# stage starts once it is exhausted (the scan then sheds a retryable 503). +# Residual overshoot: the in-flight clamped stage's kill/reap slack plus one +# unclamped cat-file probe subprocess. Default 600. +GITLAWB_IPFS_REQUEST_BUDGET_SECS=600 # Max /ipfs/{cid} requests per client IP per hour (route flood brake, distinct # from the concurrency caps above). 0 disables. Default 600. GITLAWB_IPFS_RATE_LIMIT=600 diff --git a/README.md b/README.md index f3fa361b..bc338339 100644 --- a/README.md +++ b/README.md @@ -349,6 +349,7 @@ Important node settings: | `GITLAWB_IPFS_WALK_PER_SOURCE` | Max concurrent `/ipfs` walks a single source IP may hold. Default 4. | | `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max expensive path-scope visibility walks per `/ipfs/{cid}` request; over-cap repos are skipped and the scan continues, shedding a retryable 503 (not a false 404) if the object is then found nowhere. Default 64. | | `GITLAWB_IPFS_MAX_REPO_VISITS` | Ceiling on repos one `/ipfs/{cid}` request may visit (acquire + probe) past the visibility gate — also the worst-case per-request Tigris fetch count. On exhaustion the scan stops with a retryable 503. Default 1024. | +| `GITLAWB_IPFS_REQUEST_BUDGET_SECS` | Absolute wall-clock budget for one admitted `/ipfs/{cid}` request's acquire+walk lifetime. Per-stage timeouts are clamped to the remaining budget and no stage starts once it is exhausted; the scan then stops with a retryable 503. Overshoot is bounded by the in-flight clamped stage's kill/reap slack plus one unclamped `cat-file` probe. Default 600. | | `GITLAWB_IPFS_RATE_LIMIT` | Max `/ipfs/{cid}` requests per client IP per hour (route flood brake). 0 disables. Default 600. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index d4a6c7a8..28b9cc67 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -55,11 +55,19 @@ use crate::visibility::{visibility_check, Decision}; /// repo reached a VERDICT — visibility deny, probe-says-absent, walk-gate deny, /// or served. A candidate skipped WITHOUT a verdict (acquire failure/timeout, /// probe error, walk failure/panic, content-read error, or truncation by -/// `ipfs_max_repos_walked` / `ipfs_max_repo_visits`) taints the scan, and a +/// `ipfs_max_repos_walked` / `ipfs_max_repo_visits` / +/// `ipfs_request_budget_secs`) taints the scan, and a /// tainted scan that found nothing sheds a retryable 503 + Retry-After naming /// the truncation sources — existing content is never misreported absent /// because of unrelated repos or transient faults. /// +/// Request budget (F3): one absolute clock (`ipfs_request_budget_secs`) spans +/// the whole admitted request. No stage (acquire, probe, walk, content read) +/// starts once it is exhausted, and the acquire wait and walk deadline are +/// clamped to the remainder, so an admitted request cannot hold its scarce walk +/// slot past the budget plus the in-flight stage's kill/reap slack and one +/// unclamped probe subprocess. +/// /// Scope: this closes the direct unauthenticated scan, including the dangling /// case. A stale-public mirror row still serves withheld content (tracked /// separately, #124). @@ -90,6 +98,19 @@ pub async fn get_by_cid( let caller = auth.as_ref().map(|e| e.0 .0.as_str()); let caller_owned = caller.map(|c| c.to_string()); + // One absolute budget bounds this request's whole acquire+walk lifetime (F3), + // captured before admission so the clock covers everything the walk permit + // holds. Each stage below (acquire, probe, walk, read) starts only while + // budget remains, and the acquire wait + walk deadline run clamped to the + // remainder, so an admitted request cannot hold its scarce walk slot for + // hours by drawing a fresh per-stage timeout every iteration. The budget + // NEVER aborts a running spawn_blocking walk: the clamped git deadline + // inside the walk is what ends it (a tokio timeout around the walk future + // would free the walk permit while the blocking thread still runs, the + // exact hole the held permit closes). + let request_deadline = std::time::Instant::now() + + std::time::Duration::from_secs(state.config.ipfs_request_budget_secs); + // Bounded walk admission (#174 P1-3), taken before any DB/git work so a flood sheds // cheaply. The per-repo `spawn_blocking` walk below is a full-history git walk with // no served-git admission of its own; a permissionless caller could otherwise fan @@ -161,6 +182,16 @@ pub async fn get_by_cid( } } + // Remaining request budget (F3), or None once exhausted. A stage is never + // started with zero remaining; the probe and read subprocesses carry no + // internal duration clamp, so the check before them is their entire bound + // (documented overshoot: at most one unclamped cat-file plus the in-flight + // clamped stage's kill/reap slack). + fn budget_remaining(deadline: std::time::Instant) -> Option { + let left = deadline.saturating_duration_since(std::time::Instant::now()); + (!left.is_zero()).then_some(left) + } + // Cap on EXPENSIVE walks only (F2): counts the repos that actually require the // full-history `allowed_blob_set_for_caller_bounded` walk (a path-scoped blob), // checked immediately before the spawn_blocking below. Cheap probe-only visits @@ -185,6 +216,21 @@ pub async fn get_by_cid( continue; } + // Budget gate for the acquire stage (F3): once the request budget is + // exhausted the scan STOPS, leaving this and every later candidate + // unproven (taint, never a false 404). Checked ahead of the visit + // bookkeeping so an unstarted acquire is not counted as a visit. + let Some(budget_left) = budget_remaining(request_deadline) else { + tracing::warn!( + repo = %repo.name, + budget_secs = state.config.ipfs_request_budget_secs, + "/ipfs request budget exhausted before the repo acquire \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); stopping the scan without a verdict" + ); + taint(&mut truncated_by, "budget"); + break; + }; + // Visit ceiling (F2): bound the acquire+probe cost class. Stopping here // leaves the remaining candidates unproven, so the scan is truncated. if repos_visited >= state.config.ipfs_max_repo_visits { @@ -202,9 +248,13 @@ pub async fn get_by_cid( // the P1-2 stall vector (a hung Tigris HEAD/GET on one repo would otherwise // block the whole /ipfs request). On expiry keep the fail-closed skip — never // serve an un-acquired repo; a public copy (if any) still gets its turn — but - // the repo got no verdict, so the skip taints the scan. - let acquire_deadline = - std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); + // the repo got no verdict, so the skip taints the scan. Clamped to the + // remaining request budget (F3) so per-repo acquires cannot each draw a + // fresh full timeout past it. + let acquire_deadline = std::cmp::min( + std::time::Duration::from_secs(state.config.git_acquire_timeout_secs), + budget_left, + ); let repo_path = match tokio::time::timeout( acquire_deadline, state.repo_store.acquire(&repo.owner_did, &repo.name), @@ -224,6 +274,18 @@ pub async fn get_by_cid( } }; + // Budget gate for the probe stage (F3). The probe subprocess has no + // internal duration clamp, so this pre-start check is its entire bound. + if budget_remaining(request_deadline).is_none() { + tracing::warn!( + repo = %repo.name, + "/ipfs request budget exhausted before the object-type probe; \ + stopping the scan without a verdict" + ); + taint(&mut truncated_by, "budget"); + break; + } + // Check whether the object exists in this repo before any expensive // reachability walk. This prevents random-CID spray from triggering // full-history git walks on repos that don't carry the object. Absent @@ -244,6 +306,21 @@ pub async fn get_by_cid( let path_scoped = has_path_scoped_rule(rules); if path_scoped && obj_type == "blob" { if !allowed_memo.contains_key(&repo.id) { + // Budget gate for the walk stage (F3): a walk is never STARTED + // with zero remaining (probed-present is not a serve), and a + // started walk runs its git children under a deadline clamped + // to the remainder (the min below), so a walk can never + // complete past the budget. + let Some(budget_left) = budget_remaining(request_deadline) else { + tracing::warn!( + repo = %repo.name, + budget_secs = state.config.ipfs_request_budget_secs, + "/ipfs request budget exhausted before the visibility walk \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); stopping the scan without a verdict" + ); + taint(&mut truncated_by, "budget"); + break; + }; // Walk cap (F2), checked at the one site that actually spends a walk: // on exhaustion skip THIS repo without a verdict and KEEP scanning — // later candidates may still reach a cheap probe-only verdict (a plain @@ -265,8 +342,10 @@ pub async fn get_by_cid( let owner = repo.owner_did.clone(); let caller_for_walk = caller_owned.clone(); let git_bin = state.git_bin.clone(); - let walk_timeout = - std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let walk_timeout = std::cmp::min( + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + budget_left, + ); // Full-history walk shells out to git — keep it off the async runtime, // bounded and reaped like the served-git ops (#174). let walk = tokio::task::spawn_blocking(move || { @@ -310,6 +389,21 @@ pub async fn get_by_cid( } } + // Budget gate for the content-read stage (F3): the read subprocess is + // unclamped, so it never starts past the budget. Tainting instead of + // serving keeps the terminal arm honest and the stop unconditional; the + // retryable 503 tells the caller to come back rather than letting an + // over-budget request keep spending. + if budget_remaining(request_deadline).is_none() { + tracing::warn!( + repo = %repo.name, + "/ipfs request budget exhausted before the content read; \ + stopping the scan without a verdict" + ); + taint(&mut truncated_by, "budget"); + break; + } + // Now that we've passed the gate, read the content. A failed read after a // passed gate is not an absence verdict — the probe just said the object // exists here — so the skip taints the scan. @@ -976,6 +1070,400 @@ mod tests { ); } + /// F3 budget expiry mid-loop: one absolute request budget + /// (`ipfs_request_budget_secs`) bounds the whole admitted scan; per-repo + /// stages may not each draw a fresh timeout past it. Budget 1s, per-iteration + /// acquire timeout 2s; the NEWER row is a Tigris-backed ghost (no local copy, + /// non-routable endpoint) whose acquire stalls, the OLDER row is a plain + /// public repo carrying the blob. The ghost's acquire runs clamped to the ~1s + /// remainder and times out; at the next repo the budget gate sees zero + /// remaining, taints "budget", and STOPS the scan, so the blob repo is never + /// visited (a visit would probe the healthy public copy and serve 200, which + /// the 503 assertion rules out) and the shed names the budget. Without the + /// budget the acquire would time out at its own 2s, the scan would continue, + /// and the buried blob would serve 200 (the recorded RED). MUTATION (RED): + /// remove the `request_deadline` capture (or make the remaining budget + /// infinite) and this serves 200 again. + #[sqlx::test] + async fn get_by_cid_request_budget_expiry_stops_scan_with_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Seed the blob repo through a LOCAL-ONLY store first, so seeding never + // consults the (deliberately unreachable) Tigris endpoint. + state.repo_store = + crate::git::repo_store::RepoStore::for_testing(repos_dir.clone(), pool.clone()); + let (_, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6f3budget", + "buried", + b"budget expiry proof\n", + ) + .await; + // Swap in a Tigris-backed store over the SAME repos_dir (the seeded bare + // repo stays a fast local hit) and add a NEWER ghost row with no local + // copy: its acquire consults the non-routable endpoint and stalls + // (endpoint-pinned test client, no AWS_* env reads; 10.255.255.1 hangs). + let tigris = crate::git::tigris::TigrisClient::for_testing_with_endpoint( + "test-bucket", + "http://10.255.255.1:9000", + ) + .await; + state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); + state + .db + .upsert_mirror_repo("z6f3budget", "ghost", "/unused-ghost", None, false) + .await + .unwrap(); + let mut cfg = (*state.config).clone(); + cfg.ipfs_request_budget_secs = 1; + cfg.git_acquire_timeout_secs = 2; + state.config = Arc::new(cfg); + + let peer: SocketAddr = "203.0.113.70:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&cid_for_oid(&oid), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted request budget must stop the scan with a retryable 503; \ + scanning on into the later public blob repo would have served 200" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the budget-truncation 503 must carry Retry-After" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("budget"), + "the truncation body must name the budget taint so the operator can \ + map the shed to GITLAWB_IPFS_REQUEST_BUDGET_SECS; got: {body}" + ); + } + + /// F3 clamped walk at expiry: a walk that starts with little budget left runs + /// its git children under `min(git_service_timeout_secs, remaining)`, so the + /// clamp (not any tokio-level abort) is what ends it and a walk can never + /// complete past the budget. Budget 2s, service timeout at its 600s default, + /// fake walk git that sleeps 8s: the walk STARTS (pid file), the walk permit + /// stays held while the blocking walk runs (`available_permits == 0`), the + /// clamped deadline SIGTERM/SIGKILLs the child group at ~2s remaining (the + /// response lands after the ~1s watchdog grace, far before the 8s sleep, and + /// the recorded pid is already dead: a tokio abort would have left it + /// running), the log shows the walk started but never completed, and the + /// request sheds the terminal budget-truncated 503 without ever reaching the + /// OLDER public copy of the same blob (which would have served 200). After + /// the response the permit is free: the spawn_blocking closure genuinely + /// returned. MUTATION (RED): drop the `min` clamp on `walk_timeout` and the + /// walk runs its full 8s sleep (elapsed and log-completion assertions fail). + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_budget_clamps_walk_deadline_and_holds_permit(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let walk_log = tmp.path().join("walks.log"); + let revlist_pid = tmp.path().join("revlist.pid"); + // Fake git for the WALK only: `rev-list` records its pid and a start + // marker, sleeps far past the budget, then records a done marker. Under + // the clamped walk deadline the whole process group is torn down mid + // sleep, so "done" never appears. The 8s sleep also bounds a RED run. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + for-each-ref) : ;;\n\ + rev-parse) echo deadbeef ;;\n\ + rev-list) echo $$ > \"{pid}\"; echo start >> \"{log}\"; sleep 8; echo done >> \"{log}\" ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + pid = revlist_pid.display(), + log = walk_log.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Global walk pool of 1 so the held permit is observable; per-source cap + // permissive so only the global pool matters. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + let mut cfg = (*state.config).clone(); + // The budget is the ONLY thing that can end this walk early: the service + // timeout stays at its generous 600s default. + cfg.ipfs_request_budget_secs = 2; + state.config = Arc::new(cfg); + + // Older row: a plain public copy of the same blob, which must never be + // reached. Newer row: path-scoped, so its blob costs the clamped walk. + let content = b"budget walk clamp proof\n"; + let (_, oid) = + seed_repo_with_blob(&state, tmp.path(), "z6f3clamp", "pubcopy", content).await; + let (walk_id, _) = + seed_repo_with_blob(&state, tmp.path(), "z6f3clamp", "gated", content).await; + state + .db + .set_visibility_rule( + &walk_id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderDDDDDDDDDDDDDDDDDDDDDDDD".to_string()], + "z6f3clamp", + ) + .await + .unwrap(); + + let sem = state.git_ipfs_walk_semaphore.clone(); + let router = ipfs_router(state); + let started = std::time::Instant::now(); + let peer: SocketAddr = "203.0.113.71:5000".parse().unwrap(); + let mut fut = Box::pin(router.oneshot(get_cid(&cid_for_oid(&oid), Some(peer)))); + + // Drive until the fake git's rev-list records its pid: the walk is now in + // the blocking pool and the request future is `.await`ing its join. Stop + // polling the instant the future completes (re-polling would panic). + let mut walk_pid: Option = None; + let mut early = None; + for _ in 0..500 { + let done = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&revlist_pid) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + walk_pid = Some(p); + break; + } + if let Ok(resp) = done { + early = Some(resp.map(|r| r.status())); + break; + } + } + let pid = walk_pid.unwrap_or_else(|| { + panic!( + "the budget-clamped walk must have STARTED (nonzero remaining); early: {early:?}" + ) + }); + // Reap the sleeping child on drop so a RED run leaks no orphan. + struct ReapOnDrop(i32); + impl Drop for ReapOnDrop { + fn drop(&mut self) { + unsafe { + libc::kill(self.0, libc::SIGKILL); + } + } + } + let _cleanup = ReapOnDrop(pid); + + // While the blocking walk runs the permit is HELD: the budget never frees + // a slot whose blocking thread is still burning. + assert_eq!( + sem.available_permits(), + 0, + "the walk permit must stay held while the budget-clamped walk runs" + ); + + let resp = tokio::time::timeout(std::time::Duration::from_secs(20), &mut fut) + .await + .expect("the clamped walk deadline must end the request; it never hung") + .unwrap(); + let elapsed = started.elapsed(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a budget-clamped walk that could not finish leaves no verdict: 503, not 404/200" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("budget"), + "the terminal shed must name the budget taint; got: {body}" + ); + // Deadline-killed at ~remaining, not run to completion: the response + // lands at ~budget + the watchdog's kill/reap slack, well before the 8s + // sleep could have finished. + assert!( + elapsed < std::time::Duration::from_secs(7), + "the clamped git deadline must end the walk at ~remaining; got {elapsed:?}" + ); + // The child group is already dead AT response time: the clamp killed it. + // (A tokio-level abort of the walk future would have answered while the + // blocking thread and its child still ran.) + assert_eq!( + unsafe { libc::kill(pid, 0) }, + -1, + "the walk's git child must be reaped by the clamped deadline before the response" + ); + let log = std::fs::read_to_string(&walk_log).unwrap_or_default(); + assert!( + log.contains("start"), + "the walk must have started (the budget gate passed with remaining > 0)" + ); + assert!( + !log.contains("done"), + "the walk must never complete past the budget; the clamp kills it mid-run" + ); + // The spawn_blocking closure returned and the handler finished: the + // permit is free again (held through the blocking run, no longer). + assert_eq!( + sem.available_permits(), + 1, + "the walk permit must free once the blocking walk genuinely returns" + ); + } + + /// F3 expiry between probe and walk: an object PROBED PRESENT in a + /// path-scoped repo is still not served once the budget is gone; the + /// walk-stage gate taints and stops before any walk starts. The probe is + /// stalled past the 1s budget while still SUCCEEDING via a FIFO at + /// `objects/info/alternates`: real `git cat-file -t` blocks opening it at + /// odb setup until a plain OS thread feeds it an empty alternates list at + /// ~2s, after which the probe finds the genuine loose object and reports + /// "blob". The walk-needing repo thus reaches the walk gate with zero + /// remaining: 503 naming the budget, never a 404 (the walk-gate deny of the + /// fake git's empty allowed-set would have been a verdict) and never a + /// serve, and the fake-git walk log stays EMPTY (no walk ever started). + /// MUTATION (RED): drop the walk-stage budget gate and this decays to the + /// walked 404 with one logged walk. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_budget_expiry_between_probe_and_walk_sheds_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let walk_log = tmp.path().join("walks.log"); + state.git_bin = walk_logging_fake_git(tmp.path(), &walk_log); + let mut cfg = (*state.config).clone(); + cfg.ipfs_request_budget_secs = 1; + state.config = Arc::new(cfg); + + let (repo_id, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6f3probe", + "gated", + b"probe-then-expire proof\n", + ) + .await; + state + .db + .set_visibility_rule( + &repo_id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderEEEEEEEEEEEEEEEEEEEEEEEE".to_string()], + "z6f3probe", + ) + .await + .unwrap(); + + // Stall the REAL-git probe past the budget while still letting it + // succeed: `objects/info/alternates` as a FIFO blocks cat-file's odb + // setup until a writer appears. The feeder thread supplies an EMPTY + // alternates list (open + close, no bytes) after 2s, so the probe then + // resolves the genuine loose object. Non-blocking open with retries so + // a handler that never probes cannot wedge the feeder forever. + let rec = state + .db + .get_repo("z6f3probe", "gated") + .await + .unwrap() + .unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + let fifo = bare.join("objects").join("info").join("alternates"); + let c_path = std::ffi::CString::new(fifo.to_str().unwrap()).unwrap(); + assert_eq!( + unsafe { libc::mkfifo(c_path.as_ptr(), 0o644) }, + 0, + "mkfifo(objects/info/alternates) must succeed" + ); + let feeder = std::thread::spawn(move || { + use std::os::unix::fs::OpenOptionsExt; + std::thread::sleep(std::time::Duration::from_secs(2)); + for _ in 0..300 { + if std::fs::OpenOptions::new() + .write(true) + .custom_flags(libc::O_NONBLOCK) + .open(&fifo) + .is_ok() + { + return; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + } + }); + + let peer: SocketAddr = "203.0.113.72:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&cid_for_oid(&oid), Some(peer))) + .await + .unwrap(); + feeder.join().unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "probed-present with the budget gone must shed the truncation 503: \ + never the walked 404, never a serve" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("budget"), + "the shed must name the budget taint; got: {body}" + ); + let walks = std::fs::read_to_string(&walk_log) + .map(|s| s.lines().count()) + .unwrap_or(0); + assert_eq!( + walks, 0, + "no walk may START once the budget is exhausted, even for a probed-present object" + ); + } + /// Shed at capacity: an exhausted `git_ipfs_walk_semaphore` sheds a `/ipfs/{cid}` /// request with 503 BEFORE any DB/git walk (the acquire is the first thing after CID /// validation), so a lazy DB-free state suffices — exactly like the served-git shed diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 3e0a8e50..d564a554 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -409,6 +409,32 @@ pub struct Config { )] pub ipfs_max_repo_visits: usize, + /// Absolute wall-clock budget for one admitted `GET /ipfs/{cid}` request's + /// acquire+walk lifetime, in seconds. `max_concurrent_ipfs_walks` bounds how + /// MANY requests hold walk slots; this bounds how LONG one admitted request + /// may keep its slot. Without it, each repo iteration draws a fresh + /// `git_acquire_timeout_secs` and each expensive walk a fresh + /// `git_service_timeout_secs`, so one request scanning many repos could hold + /// a scarce walk slot for hours. Every stage (acquire, `cat-file` probe, + /// visibility walk, content read) starts only while budget remains, and the + /// acquire wait and walk deadline are clamped to `min(their own timeout, + /// remaining budget)`; a stage is never started with zero remaining. On + /// exhaustion the scan stops without a verdict and the request sheds a + /// retryable 503 + Retry-After rather than a false 404. Residual overshoot + /// past the budget is bounded by the kill/reap slack of the one in-flight + /// clamped stage (the walk watchdog's SIGTERM grace + SIGKILL settle) plus + /// the `object_type` / `read_object_content` probe subprocesses, which are + /// budget-checked before they start but carry no internal duration clamp. + /// Must be positive. Default: 600s (10 min), matching + /// `git_service_timeout_secs` so a single full-length walk still fits. + #[arg( + long, + env = "GITLAWB_IPFS_REQUEST_BUDGET_SECS", + default_value_t = 600, + value_parser = clap::value_parser!(u64).range(1..) + )] + pub ipfs_request_budget_secs: u64, + /// Per-client-IP rate limit for `GET /ipfs/{cid}`, in requests per hour. The /// route is publicly reachable (`optional_signature`) and each request can drive /// a full-history git walk, so it carries a per-IP flood brake in addition to the @@ -588,6 +614,24 @@ mod tests { ); } + #[test] + fn ipfs_request_budget_secs_defaults_to_600_and_rejects_zero() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).ipfs_request_budget_secs, + 600 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--ipfs-request-budget-secs", "30"]) + .ipfs_request_budget_secs, + 30 + ); + // 0 would expire every /ipfs request at its first stage (unconditional + // 503); clap must reject it. + assert!( + Config::try_parse_from(["gitlawb-node", "--ipfs-request-budget-secs", "0"]).is_err() + ); + } + #[test] fn max_concurrent_reads_per_caller_defaults_and_rejects_out_of_range() { assert_eq!( From daf9195f29192b7dba610238529091e4402e7edd Mon Sep 17 00:00:00 2001 From: t Date: Fri, 17 Jul 2026 18:37:40 -0500 Subject: [PATCH 41/95] fix(node): admit the post-receive git scans to the encrypt pool, defer-not-shed (#174) replication_withheld_set's walk arm, resolve_candidates_for_push's git stages, and fail_closed_full_scan_objects each take a git_encrypt_semaphore permit before their spawn_blocking and carry it inside the closure, so a push burst holds at most pool-many concurrent scans and a started scan finishes with its permit. Deletion-only pushes and repos without path-scoped rules never park. Contention defers the landed push's tail (logged with queue_wait_ms), never errors it; the acquire sites document the accepted disconnect-during-park residual. --- crates/gitlawb-node/src/api/repos.rs | 523 ++++++++++++++++++++++ crates/gitlawb-node/src/git/push_delta.rs | 156 +++++++ crates/gitlawb-node/src/state.rs | 22 +- crates/gitlawb-node/tests/inv22_gates.rs | 33 ++ 4 files changed, 726 insertions(+), 8 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index bb9e743b..f25293b8 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -39,7 +39,15 @@ const ZERO_SHA: &str = "0000000000000000000000000000000000000000"; /// `withheld` is `None`, so an unvetted push neither replicates blobs nor /// announces. Returning both keeps the gate's announce decision a single /// source rather than recomputing it at each call site. +/// +/// The walk arm runs under a `git_encrypt_semaphore` admission permit (#174 F4): +/// by the time the receive-pack tail calls this, the handler's write permit has +/// already been released (receive_pack's AdmissionGuard drops when the git group +/// is reaped), so without the gate a burst of completed pushes accumulates +/// unbounded concurrent full-history walks. `encrypt_sem` is threaded in so the +/// no-walk fast paths (not announceable; no path-scoped rule) never touch it. async fn replication_withheld_set( + encrypt_sem: std::sync::Arc, rules: Option>, owner_did: &str, is_public: bool, @@ -66,7 +74,31 @@ async fn replication_withheld_set( // that off the async worker thread. Some(rules) => { let owner_did = owner_did.to_string(); + // Scan admission (#174 F4): DEFER (await), never shed — dropping the + // walk would skip the vetting and fail the push's replication closed + // for no reason. Accepted residuals, stated honestly: (1) the park + // wait is queue-depth multiplied — post-receive tails are no longer + // admission-bounded once the write permit is released, so N landed + // pushes can queue N walks and the last waits N walk-durations; + // (2) a client-timeout disconnect while parked HERE drops the + // handler future and silently loses this push's replication work — + // the encrypt_inflight coalescing requeue does NOT cover it, because + // this park precedes the `try_begin` spawn gate. + let parked = std::time::Instant::now(); + let permit = encrypt_sem + .acquire_owned() + .await + .expect("git_encrypt_semaphore is never closed"); + tracing::debug!( + repo = %disk_path.display(), + queue_wait_ms = parked.elapsed().as_millis() as u64, + "post-push withheld walk admitted to the scan pool" + ); tokio::task::spawn_blocking(move || { + // The permit lives inside the blocking closure: a started walk + // always completes holding it (a disconnect cannot cancel + // spawn_blocking or leak the permit mid-walk). + let _permit = permit; crate::git::visibility_pack::withheld_blob_oids_bounded( &disk_path, &git_bin, timeout, &rules, is_public, &owner_did, None, ) @@ -102,7 +134,15 @@ async fn replication_withheld_set( /// non-blobs plus allowed blobs. Any error in either walk (or a task panic) /// pins nothing this push, mirroring the degraded-path shape of /// `replication_withheld_set`. +/// +/// Always walks (there is no no-git arm), so the whole blocking scan runs under +/// one `git_encrypt_semaphore` admission permit (#174 F4) — see +/// `replication_withheld_set`'s acquire for the defer rationale and the honest +/// residuals (queue-depth-multiplied park wait; a disconnect while parked loses +/// this push's replication work, uncovered by the coalescing requeue). +#[allow(clippy::too_many_arguments)] async fn fail_closed_full_scan_objects( + encrypt_sem: std::sync::Arc, disk_path: std::path::PathBuf, rules: Vec, is_public: bool, @@ -111,7 +151,20 @@ async fn fail_closed_full_scan_objects( git_bin: String, timeout: std::time::Duration, ) -> Vec { + // Scan admission (#174 F4): DEFER, never shed; the permit moves into the + // closure so a started scan always completes holding it. + let parked = std::time::Instant::now(); + let permit = encrypt_sem + .acquire_owned() + .await + .expect("git_encrypt_semaphore is never closed"); + tracing::debug!( + repo = %disk_path.display(), + queue_wait_ms = parked.elapsed().as_millis() as u64, + "post-push fail-closed full scan admitted to the scan pool" + ); tokio::task::spawn_blocking(move || -> anyhow::Result> { + let _permit = permit; let allowed = crate::git::visibility_pack::replicable_blob_set_bounded( &disk_path, &git_bin, timeout, &rules, is_public, &owner_did, )?; @@ -1385,6 +1438,7 @@ pub async fn git_receive_pack( // never leaks. let rules_opt = state.db.list_visibility_rules(&record.id).await.ok(); let (announce, withheld) = replication_withheld_set( + state.git_encrypt_semaphore.clone(), rules_opt.clone(), &record.owner_did, record.is_public, @@ -1413,6 +1467,7 @@ pub async fn git_receive_pack( .filter(|s| s != ZERO_SHA) .collect(); let pin_set = crate::git::push_delta::resolve_candidates_for_push( + state.git_encrypt_semaphore.clone(), disk_path.clone(), new_tips, old_tips, @@ -1422,6 +1477,7 @@ pub async fn git_receive_pack( .await; if pin_set.full_scan { fail_closed_full_scan_objects( + state.git_encrypt_semaphore.clone(), disk_path.clone(), rules_opt.clone().unwrap_or_default(), record.is_public, @@ -2286,6 +2342,7 @@ mod tests { // Private: no rules at all. let (announce, _) = replication_withheld_set( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), None, OWNER_DID, false, @@ -2298,6 +2355,7 @@ mod tests { // Private: empty rule set, is_public=false → still not listable at root. let (announce, _) = replication_withheld_set( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), Some(vec![]), OWNER_DID, false, @@ -2310,6 +2368,7 @@ mod tests { // Public: empty rule set, is_public=true → listable at root, announces. let (announce, _) = replication_withheld_set( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), Some(vec![]), OWNER_DID, true, @@ -2441,6 +2500,7 @@ mod tests { let result = tokio::time::timeout( Duration::from_secs(10), replication_withheld_set( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), rules, OWNER_DID, true, @@ -4635,6 +4695,469 @@ mod tests { ); } + /// F4 defer proof 1: `replication_withheld_set`'s WALK arm acquires a + /// `git_encrypt_semaphore` permit before its spawn_blocking git walk, deferring + /// (never shedding) when the pool is exhausted — while its no-walk fast paths + /// (no path-scoped rule; not announceable) complete WITHOUT touching the pool. + /// On ungated code the walk runs regardless of a zero-permit pool (RED). + #[cfg(unix)] + #[tokio::test] + async fn replication_walk_defers_when_scan_pool_exhausted() { + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let marker = tmp.path().join("git.ran"); + // Fake git records ANY invocation (the walk's first call is rev-parse), then + // behaves well enough for a successful empty walk: HEAD probe succeeds, + // rev-list lists no commits. + let body = format!( + "#!/bin/sh\necho ran >> \"{}\"\ncase \"$1\" in\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n", + marker.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + let scoped_rules = || Some(vec![vis_rule("/secret/**", &[])]); + + // Zero-permit pool: every gated walk must park forever. + let sem: Arc = Arc::new(Semaphore::new(0)); + + // Fast path A (negative arm): announceable, NO path-scoped rule -> zero git + // work, must complete immediately without acquiring from the empty pool. + let fast = tokio::time::timeout( + Duration::from_millis(500), + replication_withheld_set( + sem.clone(), + Some(vec![]), + OWNER_DID, + true, + tmp.path().to_path_buf(), + git_bin.clone(), + Duration::from_secs(5), + ), + ) + .await + .expect("the no-path-scoped-rule fast path must not park on the scan pool"); + assert_eq!(fast, (true, Some(std::collections::HashSet::new()))); + + // Fast path B (negative arm): not announceable (no rules) -> zero git work, + // must complete immediately without acquiring. + let fast = tokio::time::timeout( + Duration::from_millis(500), + replication_withheld_set( + sem.clone(), + None, + OWNER_DID, + false, + tmp.path().to_path_buf(), + git_bin.clone(), + Duration::from_secs(5), + ), + ) + .await + .expect("the not-announceable fast path must not park on the scan pool"); + assert_eq!(fast, (false, None)); + assert!(!marker.exists(), "the fast paths must spawn no git at all"); + + // Walk arm with the pool exhausted: must DEFER (park), spawning no git. + let blocked = tokio::time::timeout( + Duration::from_millis(500), + replication_withheld_set( + sem.clone(), + scoped_rules(), + OWNER_DID, + true, + tmp.path().to_path_buf(), + git_bin.clone(), + Duration::from_secs(5), + ), + ) + .await; + assert!( + blocked.is_err(), + "the withheld walk must defer (park on admission) when the pool is exhausted" + ); + assert!( + !marker.exists(), + "the withheld walk's git must not spawn while its admission permit is unavailable (F4)" + ); + + // Release admission: the SAME walk now runs (defer, not shed) and succeeds. + sem.add_permits(1); + let ran = replication_withheld_set( + sem, + scoped_rules(), + OWNER_DID, + true, + tmp.path().to_path_buf(), + git_bin, + Duration::from_secs(5), + ) + .await; + assert!( + marker.exists(), + "once admission is available the deferred withheld walk runs its git" + ); + assert_eq!( + ran, + (true, Some(std::collections::HashSet::new())), + "the released walk completes and vets the (empty) withheld set" + ); + } + + /// F4 defer proof 3: `fail_closed_full_scan_objects` ALWAYS walks, so its + /// spawn_blocking is always admission-gated: with the pool exhausted it defers + /// and spawns no git; with a permit the same call runs. Ungated it runs + /// regardless (RED). + #[cfg(unix)] + #[tokio::test] + async fn full_scan_pin_walk_defers_when_scan_pool_exhausted() { + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let marker = tmp.path().join("git.ran"); + let body = format!( + "#!/bin/sh\necho ran >> \"{}\"\ncase \"$1\" in\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n", + marker.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + let candidates = vec!["3333333333333333333333333333333333333333".to_string()]; + + let sem: Arc = Arc::new(Semaphore::new(0)); + let blocked = tokio::time::timeout( + Duration::from_millis(500), + fail_closed_full_scan_objects( + sem.clone(), + tmp.path().to_path_buf(), + vec![vis_rule("/secret/**", &[])], + true, + OWNER_DID.to_string(), + candidates.clone(), + git_bin.clone(), + Duration::from_secs(5), + ), + ) + .await; + assert!( + blocked.is_err(), + "the fail-closed full scan must defer (park on admission) when the pool is exhausted" + ); + assert!( + !marker.exists(), + "the full scan's git must not spawn while its admission permit is unavailable (F4)" + ); + + // Release admission: the SAME scan now runs (defer, not shed). + sem.add_permits(1); + let _objs = fail_closed_full_scan_objects( + sem, + tmp.path().to_path_buf(), + vec![vis_rule("/secret/**", &[])], + true, + OWNER_DID.to_string(), + candidates, + git_bin, + Duration::from_secs(5), + ) + .await; + assert!( + marker.exists(), + "once admission is available the deferred full scan runs its git" + ); + } + + /// Shared fixture for the F4 handler-layer tests: a state whose repo_store and + /// git_bin point at the given tempdir/fake-git, plus a seeded on-disk repo, + /// optionally with a path-scoped rule (so the post-receive walks actually run). + #[cfg(unix)] + async fn f4_state_with_repo( + pool: sqlx::PgPool, + tmp: &std::path::Path, + git_bin: &str, + owner: &str, + name: &str, + path_scoped: bool, + ) -> AppState { + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.git_bin = git_bin.to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo(owner, name, &format!("/unused-{owner}-{name}"), None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + state + .repo_store + .init(&rec.owner_did, &rec.name) + .await + .unwrap(); + if path_scoped { + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkF4ReaderAAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + } + state + } + + /// A pkt-line receive-pack body carrying one branch-create ref update, so the + /// handler's post-receive tail resolves a non-empty new-tip set (the delta + /// scan's git stages run). + fn ref_update_body(new_sha: &str) -> axum::body::Bytes { + let line = format!("{ZERO_SHA} {new_sha} refs/heads/main"); + axum::body::Bytes::from(format!("{:04x}{}0000", line.len() + 4, line)) + } + + /// F4 scenario 2 — push-burst bound at the handler layer: with a scan pool of + /// ONE, two concurrent pushes to two path-scoped repos never have more than one + /// scan's git alive at a time (an atomic mkdir lock in the fake git detects any + /// overlap), and BOTH pushes still succeed 200 — defer, not shed. Two distinct + /// repos on purpose: the per-repo advisory write lock must not be what + /// serializes the scans. + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_burst_scans_serialized_and_both_pushes_succeed(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let lockdir = tmp.path().join("scan.lock"); + let ranfile = tmp.path().join("scan.ran"); + let overlap = tmp.path().join("scan.overlap"); + // receive-pack succeeds instantly; every candidate-scan git op (cat-file / + // rev-list / ls-tree) holds an atomic mkdir lock for 150ms — a second scan + // process alive at the same instant records an overlap. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + receive-pack) cat > /dev/null 2>/dev/null ;;\n\ + rev-parse) echo deadbeef ;;\n\ + cat-file|rev-list|ls-tree)\n\ + if mkdir \"{lock}\" 2>/dev/null; then\n\ + echo 1 >> \"{ran}\"\n\ + sleep 0.15\n\ + rmdir \"{lock}\"\n\ + else\n\ + echo 1 >> \"{over}\"\n\ + fi\n\ + if [ \"$1\" = cat-file ]; then echo commit; fi ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + lock = lockdir.display(), + ran = ranfile.display(), + over = overlap.display(), + ); + let git_bin = write_fake_git(tmp.path(), &body); + + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f4burst1", "b1", true).await; + // Second path-scoped repo on the same state/store. + state + .db + .upsert_mirror_repo("z6f4burst2", "b2", "/unused-z6f4burst2-b2", None, false) + .await + .unwrap(); + let rec2 = state + .db + .get_repo("z6f4burst2", "b2") + .await + .unwrap() + .unwrap(); + state + .repo_store + .init(&rec2.owner_did, &rec2.name) + .await + .unwrap(); + state + .db + .set_visibility_rule( + &rec2.id, + "/secret/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkF4ReaderAAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec2.owner_did, + ) + .await + .unwrap(); + // Scan pool of ONE: at most one post-receive walk may run at a time. + state.git_encrypt_semaphore = Arc::new(Semaphore::new(1)); + + let did = "did:key:z6MkF4BurstPusherAAAAAAAAAAAAAAAAAAAAAAAA"; + let new_sha = "1111111111111111111111111111111111111111"; + let push = |owner: &'static str, name: &'static str, peer: &'static str| { + let state = state.clone(); + tokio::spawn(async move { + git_receive_pack( + State(state), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(peer.parse::().unwrap())), + axum::http::HeaderMap::new(), + ref_update_body(new_sha), + ) + .await + }) + }; + + let (a, b) = ( + push("z6f4burst1", "b1", "203.0.113.71:5000"), + push("z6f4burst2", "b2", "203.0.113.72:5000"), + ); + let a = tokio::time::timeout(std::time::Duration::from_secs(60), a) + .await + .expect("push A must complete — a scan gate must defer, never wedge") + .expect("push A task must not panic"); + let b = tokio::time::timeout(std::time::Duration::from_secs(60), b) + .await + .expect("push B must complete — a scan gate must defer, never wedge") + .expect("push B task must not panic"); + let a = a.expect("push A must succeed"); + let b = b.expect("push B must succeed"); + assert_eq!(a.status(), 200, "push A lands 200 despite scan contention"); + assert_eq!(b.status(), 200, "push B lands 200 despite scan contention"); + + // Let the detached recipients walks drain through the same pool before + // reading the detector files. + tokio::time::sleep(std::time::Duration::from_millis(800)).await; + assert!( + !overlap.exists(), + "with a scan pool of 1, no two scans' git may ever be alive at once \ + (found overlap records: {:?})", + std::fs::read_to_string(&overlap).unwrap_or_default() + ); + let ran = std::fs::read_to_string(&ranfile).unwrap_or_default(); + assert!( + ran.lines().count() >= 6, + "both pushes' scans must actually have run (withheld walk + delta probe + \ + delta rev-list each); got {} runs", + ran.lines().count() + ); + } + + /// F4 scenario 3 — fast-path non-acquisition at the handler layer: a push to a + /// public repo with NO path-scoped rules does zero post-receive git scanning + /// (the withheld short-circuit; a deletion-free flush-only body resolves no new + /// tips), so it must complete 200 even with the scan pool at ZERO permits. + /// A gate that wrongly captured a no-walk path would park this push forever. + /// Note: `resolve_candidates_for_push` spawns git for ANY non-empty new-tip set + /// (the per-tip cat-file probe), so the genuinely git-free negative arm is the + /// no-ref-update body, not a branch-create push. + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_no_scan_fast_path_completes_with_zero_scan_permits(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let marker = tmp.path().join("scan.ran"); + let body = format!( + "#!/bin/sh\ncase \"$1\" in\n receive-pack) cat > /dev/null 2>/dev/null ;;\n cat-file|rev-list|ls-tree) echo 1 >> \"{}\" ;;\n *) : ;;\nesac\nexit 0\n", + marker.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f4fast", "f1", false).await; + state.git_encrypt_semaphore = Arc::new(Semaphore::new(0)); + + let peer: SocketAddr = "203.0.113.73:5000".parse().unwrap(); + let resp = tokio::time::timeout( + std::time::Duration::from_secs(30), + git_receive_pack( + State(state), + Path(("z6f4fast".to_string(), "f1".to_string())), + Extension(crate::auth::AuthenticatedDid( + "did:key:z6MkF4FastPusherAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + )), + crate::rate_limit::PeerAddr(Some(peer)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ), + ) + .await + .expect("a no-scan push must not park on the (empty) scan pool") + .expect("the push must succeed"); + assert_eq!(resp.status(), 200); + assert!( + !marker.exists(), + "the no-walk fast paths must spawn no scan git at all" + ); + } + + /// F4 scenario 4 — landed-push-never-fails: a push whose post-receive walk must + /// park (pool held elsewhere) DEFERS and then returns the receive-pack success + /// once admission frees; contention never converts the landed push into a 5xx. + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_landed_push_defers_on_scan_contention_never_errors(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let body = "#!/bin/sh\ncase \"$1\" in\n receive-pack) cat > /dev/null 2>/dev/null ;;\n rev-parse) echo deadbeef ;;\n cat-file) echo commit ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f4park", "p1", true).await; + let sem = Arc::new(Semaphore::new(1)); + state.git_encrypt_semaphore = sem.clone(); + // The test holds the pool's only permit: the push's withheld walk must park. + let held = sem.clone().acquire_owned().await.unwrap(); + + let peer: SocketAddr = "203.0.113.74:5000".parse().unwrap(); + let mut handle = tokio::spawn(git_receive_pack( + State(state), + Path(("z6f4park".to_string(), "p1".to_string())), + Extension(crate::auth::AuthenticatedDid( + "did:key:z6MkF4ParkPusherAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + )), + crate::rate_limit::PeerAddr(Some(peer)), + axum::http::HeaderMap::new(), + ref_update_body("2222222222222222222222222222222222222222"), + )); + + // Parked, not errored: the handler must NOT complete while the pool is held. + let parked = tokio::time::timeout(std::time::Duration::from_millis(700), &mut handle).await; + assert!( + parked.is_err(), + "the post-receive walk must defer while the scan pool is held; got {parked:?}" + ); + + // Release admission: the SAME push now finishes with the receive-pack 200. + drop(held); + let resp = tokio::time::timeout(std::time::Duration::from_secs(30), handle) + .await + .expect("the deferred push must complete once admission frees") + .expect("the push task must not panic") + .expect("contention must never convert a landed push into an error"); + assert_eq!( + resp.status(), + 200, + "the response is the receive-pack success, not a contention 5xx" + ); + } + // ---- #174 U4 (P2-2): post-push encryption task set bounded by per-repo coalescing ---- // // The residual jatmn found is not the WALK (bounded by `git_encrypt_semaphore`, diff --git a/crates/gitlawb-node/src/git/push_delta.rs b/crates/gitlawb-node/src/git/push_delta.rs index 962c9c65..18f2c4da 100644 --- a/crates/gitlawb-node/src/git/push_delta.rs +++ b/crates/gitlawb-node/src/git/push_delta.rs @@ -282,14 +282,47 @@ pub struct PinCandidateSet { /// it can never leak because the withheld/fail-closed filter still runs on /// whatever set is returned. `full_scan` rides on the returned set so the caller /// knows when the dangling-inclusive filter is required. +/// +/// `scan_sem` is the post-receive scan admission pool (`git_encrypt_semaphore`, +/// #174 F4): both git-spawning stages — the per-tip `cat-file` probe + delta +/// rev-list, and the full-scan fallback — run under ONE permit held for the +/// whole blocking scan. The deletion-only fast path (no new tips) spawns no git +/// and never parks. pub async fn resolve_candidates_for_push( + scan_sem: std::sync::Arc, repo_path: PathBuf, new_tips: Vec, old_tips: Vec, git_bin: String, timeout: Duration, ) -> PinCandidateSet { + // No-git fast path: a deletion-only push resolves to an empty delta without + // spawning any git child, so it must not park on the scan pool. The + // kill-switch disqualifies it — forcing the full scan spawns git. + if new_tips.is_empty() && !force_full_scan() { + tracing::info!(delta = 0usize, repo = %repo_path.display(), "pin candidate set from push delta"); + return PinCandidateSet { + candidates: Vec::new(), + full_scan: false, + }; + } + // Scan admission (#174 F4): DEFER (await), never shed — a dropped scan would + // silently under-pin this push. The permit moves into the blocking closure so + // a started scan always completes holding it. Residuals as documented at + // `replication_withheld_set`'s acquire: park wait is queue-depth multiplied, + // and a client disconnect while parked loses this push's replication work. + let parked = Instant::now(); + let permit = scan_sem + .acquire_owned() + .await + .expect("git_encrypt_semaphore is never closed"); + tracing::debug!( + repo = %repo_path.display(), + queue_wait_ms = parked.elapsed().as_millis() as u64, + "pin-candidate scan admitted to the scan pool" + ); tokio::task::spawn_blocking(move || { + let _permit = permit; // ONE shared deadline for the whole scan, per jatmn ("the same deadline"). let deadline = Instant::now() + timeout; let new_refs: Vec<&str> = new_tips.iter().map(String::as_str).collect(); @@ -547,6 +580,7 @@ mod tests { let c1 = repo.commit_file("a.txt", "one\n"); let c2 = repo.commit_file("b.txt", "two\n"); let set = resolve_candidates_for_push( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), repo.path.clone(), vec![c2.clone()], vec![c1.clone()], @@ -576,6 +610,7 @@ mod tests { .into_iter() .collect(); let set = resolve_candidates_for_push( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), repo.path.clone(), vec![blob], vec![], @@ -588,6 +623,127 @@ mod tests { assert_eq!(got, all, "non-commit tip falls back to full repo scan"); } + /// F4 defer proof 2: `resolve_candidates_for_push`'s git stages (the per-tip + /// cat-file type probe + delta rev-list, and the full-scan fallback) run under a + /// scan-admission permit: with a zero-permit pool the call parks and spawns no + /// git; once a permit is available the SAME call runs (defer, not shed). On + /// ungated code the git runs regardless of the pool (RED). + #[cfg(unix)] + #[tokio::test] + async fn resolve_candidates_defers_when_scan_pool_exhausted() { + use std::os::unix::fs::PermissionsExt; + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + + let dir = tempfile::TempDir::new().unwrap(); + let marker = dir.path().join("git.ran"); + // Fake git records ANY invocation; cat-file reports a commit tip so the + // delta stage proceeds, rev-list yields an empty delta. + let fake = dir.path().join("fakegit"); + std::fs::write( + &fake, + format!( + "#!/bin/sh\necho ran >> \"{}\"\ncase \"$1\" in\n cat-file) echo commit ;;\n *) : ;;\nesac\nexit 0\n", + marker.display() + ), + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + let git_bin = fake.to_str().unwrap().to_string(); + let tip = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef".to_string(); + + let sem: Arc = Arc::new(Semaphore::new(0)); + let blocked = tokio::time::timeout( + Duration::from_millis(500), + resolve_candidates_for_push( + sem.clone(), + dir.path().to_path_buf(), + vec![tip.clone()], + vec![], + git_bin.clone(), + Duration::from_secs(5), + ), + ) + .await; + assert!( + blocked.is_err(), + "the pin-candidate scan must defer (park on admission) when the pool is exhausted" + ); + assert!( + !marker.exists(), + "the scan's git must not spawn while its admission permit is unavailable (F4)" + ); + + // Release admission: the SAME scan now runs (defer, not shed). + sem.add_permits(1); + let set = resolve_candidates_for_push( + sem, + dir.path().to_path_buf(), + vec![tip], + vec![], + git_bin, + Duration::from_secs(5), + ) + .await; + assert!( + marker.exists(), + "once admission is available the deferred scan runs its git" + ); + assert!(!set.full_scan, "commit tip + empty rev-list is a delta"); + assert!(set.candidates.is_empty()); + } + + /// F4 fast-path negative arm: a deletion-only push (no new tips, kill-switch + /// off) computes its empty delta without spawning ANY git child, so it must + /// complete without acquiring from the scan pool — even at zero permits. The + /// per-tip cat-file probe means every push with a non-empty new tip DOES spawn + /// git, so this is the only genuinely git-free stage. + #[cfg(unix)] + #[tokio::test] + async fn resolve_candidates_no_git_fast_path_skips_admission() { + use std::os::unix::fs::PermissionsExt; + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + + let dir = tempfile::TempDir::new().unwrap(); + let marker = dir.path().join("git.ran"); + let fake = dir.path().join("fakegit"); + std::fs::write( + &fake, + format!("#!/bin/sh\necho ran >> \"{}\"\nexit 0\n", marker.display()), + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + + let set = tokio::time::timeout( + Duration::from_millis(500), + resolve_candidates_for_push( + Arc::new(Semaphore::new(0)), + dir.path().to_path_buf(), + vec![], + vec!["deadbeefdeadbeefdeadbeefdeadbeefdeadbeef".to_string()], + fake.to_str().unwrap().to_string(), + Duration::from_secs(5), + ), + ) + .await + .expect("the no-new-tips fast path must not park on the scan pool"); + assert_eq!( + set, + PinCandidateSet { + candidates: Vec::new(), + full_scan: false + } + ); + assert!(!marker.exists(), "the fast path must spawn no git at all"); + } + #[test] fn missing_oid_tip_forces_full_scan() { let repo = Repo::new(); diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index deccf4c7..0badf802 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -110,14 +110,20 @@ pub struct AppState { /// advert pool (each source also capped by `git_push_advert_per_caller` and the /// per-IP push rate limiter), and the reserved POST pool is untouched (#174). pub git_push_advert_semaphore: Arc, - /// Bounds concurrent post-push encrypt-then-pin history walks. Each successful - /// path-scoped push releases its handler write permit and then runs a DETACHED - /// full-history walk (`withheld_blob_recipients_bounded`) to seal withheld blobs; - /// without a cap, N fast pushes spawn N concurrent full-history git walks past - /// `max_concurrent_git_pushes` (which only bounds the in-handler phase). The walk - /// acquires a permit here and DEFERS (blocks) when the pool is full rather than - /// shedding — the work is background and dropping it would lose the recovery copy - /// (#174 P1-e). A pool of its own, not `git_write_semaphore`: a long background + /// Bounds concurrent post-receive git scans. Each successful push releases its + /// handler write permit the moment receive-pack's git group is reaped, then runs + /// up to four scans over the repo: the anonymous withheld walk + /// (`replication_withheld_set`), the pin-candidate scan + /// (`resolve_candidates_for_push`), the fail-closed full scan + /// (`fail_closed_full_scan_objects`), and the DETACHED encrypt-then-pin walk + /// (`withheld_blob_recipients_bounded`). Without a cap, N fast pushes spawn N + /// concurrent full-history git walks past `max_concurrent_git_pushes` (which only + /// bounds the in-handler receive-pack phase) — #174 P1-e closed the detached walk, + /// F4 closed the other three. Each scan acquires ONE permit here per walk and + /// DEFERS (blocks) when the pool is full rather than shedding — dropping the work + /// would lose the recovery copy or silently under-pin the push. No-walk fast + /// paths (not announceable, no path-scoped rule, deletion-only push) never touch + /// the pool. A pool of its own, not `git_write_semaphore`: a long background /// walk must not hold a foreground write slot, and a handler already holding a /// write permit that needed a second would self-deadlock at pool size 1. pub git_encrypt_semaphore: Arc, diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index 7d3fd1c8..bcd0758e 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -87,4 +87,37 @@ fn inv22_concurrency_gates_present_and_not_bypassed() { "U4/P2-2 gate missing: the detached post-push encryption spawn must consult \ encrypt_inflight.try_begin to coalesce per repo (bound the outstanding-task set)" ); + + // F4: every post-receive scan helper acquires a `git_encrypt_semaphore` permit + // BEFORE its spawn_blocking git walk, so a push burst cannot accumulate unbounded + // concurrent scans once the write permit is released. Structural check: within each + // helper the first `acquire_owned` precedes the first `spawn_blocking`. Removing a + // gate pushes the next `acquire_owned` occurrence past the helper's own + // `spawn_blocking` (or off the end of the file), turning the assertion red. + let push_delta = src("git/push_delta.rs"); + for (file_src, file, helper) in [ + (&repos, "api/repos.rs", "fn replication_withheld_set"), + (&repos, "api/repos.rs", "fn fail_closed_full_scan_objects"), + ( + &push_delta, + "git/push_delta.rs", + "fn resolve_candidates_for_push", + ), + ] { + let start = file_src + .find(helper) + .unwrap_or_else(|| panic!("{file}: `{helper}` not found")); + let tail = &file_src[start..]; + let acquire = tail.find("acquire_owned").unwrap_or_else(|| { + panic!("F4 gate missing: {file} `{helper}` no longer acquires a scan permit") + }); + let spawn = tail.find("spawn_blocking").unwrap_or_else(|| { + panic!("{file}: `{helper}` lost its spawn_blocking walk — update this guard") + }); + assert!( + acquire < spawn, + "F4 gate bypassed: {file} `{helper}` must acquire its git_encrypt_semaphore \ + permit BEFORE dispatching the blocking git scan" + ); + } } From 8c95d05661fe5c570ccc60e029297c98fd3b58a9 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 17 Jul 2026 19:17:17 -0500 Subject: [PATCH 42/95] fix(node): requeue coalesced post-push work via an in-lock pending slot and loop-drain (#174) try_begin now merges a losing push's tip pairs into the in-flight key's pending slot in the same critical section as the presence check, bounded at 1024 pairs with a full-scan marker on overflow (force_full_scan threads to the resolver; empty tips never encode the marker). The task loop-drains: finish_or_take_pending either hands back the batch with the key retained, or removes the key and disarms the guard atomically, so a successor task's key survives the guard drop. Drains re-fetch rules fresh and replay the tail pipeline under the U4 helper gates only; the loop holds no task-level pool permit, so a hot repo cannot deadlock the pool at size 1. --- crates/gitlawb-node/src/api/repos.rs | 1035 ++++++++++++++++++--- crates/gitlawb-node/src/git/push_delta.rs | 81 +- crates/gitlawb-node/src/state.rs | 214 ++++- crates/gitlawb-node/tests/inv22_gates.rs | 17 + 4 files changed, 1166 insertions(+), 181 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index f25293b8..2b9cf1d4 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -843,6 +843,244 @@ async fn withheld_recipients_gated( .await } +/// Everything the detached post-push pin/encrypt task needs, cloned once at spawn +/// and shared by the snapshot iteration and every coalesced-drain iteration +/// (#174 F5). +struct EncryptTaskCtx { + ipfs_api: String, + repo_path: std::path::PathBuf, + db: Arc, + repo_id: String, + owner_did: String, + repo_name: String, + irys_url: String, + http_client: Arc, + node_did: String, + node_keypair: Arc, + git_bin: String, + git_timeout: std::time::Duration, + encrypt_sem: Arc, +} + +/// The detached post-push pin/encrypt task (#174 P2-2 + F5): run this push's own +/// pre-resolved snapshot through the pipeline, then loop-drain every push that +/// coalesced against the in-flight key until a finish attempt finds nothing +/// pending (which releases the key in the same critical section). +/// +/// The loop holds NO encrypt-pool permit at the task level: each helper it calls +/// (`replication_withheld_set`, `resolve_candidates_for_push`, +/// `fail_closed_full_scan_objects`, `withheld_recipients_gated`) acquires and +/// releases its own walk permit, so the drain makes progress at pool size 1 — a +/// task-level permit would nest over those same-semaphore acquires and deadlock. +async fn run_encrypt_pin_task( + ctx: EncryptTaskCtx, + guard: crate::state::EncryptInflightGuard, + snapshot_objects: Vec, + snapshot_rules: Option>, + snapshot_is_public: bool, +) { + pin_and_encrypt_objects(&ctx, snapshot_objects, snapshot_rules, snapshot_is_public).await; + let mut guard = guard; + loop { + match guard.finish_or_take_pending() { + crate::state::FinishOutcome::Finished(_) => break, + crate::state::FinishOutcome::Pending(g, pending) => { + guard = g; + if let Some((object_list, rules, is_public)) = + resolve_drain_object_list(&ctx, pending).await + { + pin_and_encrypt_objects(&ctx, object_list, rules, is_public).await; + } + } + } + } +} + +/// Resolve a coalesced-drain iteration's replicable object list. Re-fetches the +/// repo record and visibility rules FRESH — rules tightened between the coalesced +/// push and its drain must be honored, fail closed: a newly-withheld blob is not +/// pinned, and a repo that is no longer announceable (or whose record cannot be +/// re-read) pins nothing at all (`None`). Returns the filtered object list plus +/// the fresh rules/is_public snapshot for the encrypt stage — the same +/// resolution → withheld-filter pipeline the receive-pack tail runs. +async fn resolve_drain_object_list( + ctx: &EncryptTaskCtx, + pending: crate::state::PendingWork, +) -> Option<(Vec, Option>, bool)> { + let record = match ctx.db.get_repo(&ctx.owner_did, &ctx.repo_name).await { + Ok(Some(r)) => r, + Ok(None) => { + tracing::warn!( + repo = %ctx.repo_id, + "coalesced drain: repo record is gone; dropping the pending work" + ); + return None; + } + Err(e) => { + tracing::warn!( + repo = %ctx.repo_id, + err = %e, + "coalesced drain: repo re-fetch failed; pinning nothing (fail closed)" + ); + return None; + } + }; + let rules_opt = ctx.db.list_visibility_rules(&ctx.repo_id).await.ok(); + let (_announce, withheld) = replication_withheld_set( + ctx.encrypt_sem.clone(), + rules_opt.clone(), + &record.owner_did, + record.is_public, + ctx.repo_path.clone(), + ctx.git_bin.clone(), + ctx.git_timeout, + ) + .await; + let withheld_set = match withheld { + Some(w) => w, + None => { + tracing::info!( + repo = %ctx.repo_id, + "coalesced drain: repo is not announceable under current rules; \ + pinning nothing (fail closed)" + ); + return None; + } + }; + let (new_tips, old_tips, force_full_scan) = match pending { + crate::state::PendingWork::Tips(pairs) => { + let new_tips: Vec = pairs + .iter() + .map(|(_, n)| n.clone()) + .filter(|s| s != ZERO_SHA) + .collect(); + let old_tips: Vec = pairs + .into_iter() + .map(|(o, _)| o) + .filter(|s| s != ZERO_SHA) + .collect(); + (new_tips, old_tips, false) + } + // The overflow marker forces the full scan via the explicit flag. It must + // never be encoded as a plain empty-tips call: empty tips resolve to an + // empty delta and would pin nothing (the F5 silent loss again). + crate::state::PendingWork::FullScan => (Vec::new(), Vec::new(), true), + }; + let pin_set = crate::git::push_delta::resolve_candidates_for_push( + ctx.encrypt_sem.clone(), + ctx.repo_path.clone(), + new_tips, + old_tips, + ctx.git_bin.clone(), + ctx.git_timeout, + force_full_scan, + ) + .await; + let object_list = if pin_set.full_scan { + fail_closed_full_scan_objects( + ctx.encrypt_sem.clone(), + ctx.repo_path.clone(), + rules_opt.clone().unwrap_or_default(), + record.is_public, + record.owner_did.clone(), + pin_set.candidates, + ctx.git_bin.clone(), + ctx.git_timeout, + ) + .await + } else { + crate::git::visibility_pack::replicable_objects(pin_set.candidates, &withheld_set) + }; + Some((object_list, rules_opt, record.is_public)) +} + +/// The pin/encrypt pipeline shared by the snapshot iteration and the +/// coalesced-drain iterations: local IPFS pin, then (path-scoped rules only) the +/// admission-gated recipients walk → encrypt-then-pin → Arweave manifest anchor. +async fn pin_and_encrypt_objects( + ctx: &EncryptTaskCtx, + object_list: Vec, + rules: Option>, + is_public: bool, +) { + let pinned = + crate::ipfs_pin::pin_new_objects(&ctx.ipfs_api, &ctx.repo_path, object_list, &ctx.db).await; + if !pinned.is_empty() { + tracing::info!(count = pinned.len(), "pinned git objects to IPFS"); + for (sha, cid) in &pinned { + tracing::info!(sha = %sha, %cid, "pinned"); + } + } + + // Option B1: encrypt-then-pin the withheld blobs so authorized readers can + // recover them when the origin cannot serve them. No path-scoped rule can + // withhold a blob, so withheld_blob_recipients would return an empty map + // after a full per-ref walk; skip it. Mirrors the has_path_scoped_rule gate + // on the other two withheld-walk sites. + if let Some(rules) = rules.filter(|r| visibility_pack::has_path_scoped_rule(r)) { + // Bound the number of concurrent post-push encryption walks (#174 P1-e): + // acquire an admission permit before the full-history walk, deferring + // when the pool is full rather than shedding the recovery pin. + let recip = withheld_recipients_gated( + ctx.encrypt_sem.clone(), + ctx.repo_path.clone(), + ctx.git_bin.clone(), + ctx.git_timeout, + rules, + is_public, + ctx.owner_did.clone(), + ) + .await; + if let Ok(Ok(recipients)) = recip { + let node_seed = ctx.node_keypair.to_seed(); + let delta = crate::encrypted_pin::encrypt_and_pin( + &ctx.ipfs_api, + &ctx.repo_path, + &ctx.db, + &ctx.repo_id, + &node_seed, + &recipients, + ) + .await; + + // Option B3: anchor a per-push manifest of the blobs sealed this + // push to Arweave, so the oid->cid index survives total node loss. + // Best-effort; never fails the push. + if !delta.is_empty() && !ctx.irys_url.is_empty() { + let owner_short = crate::db::normalize_owner_key(&ctx.owner_did); + let repo_slug = format!("{owner_short}/{}", ctx.repo_name); + let ts = chrono::Utc::now().to_rfc3339(); + let manifest = crate::arweave::EncryptedManifest { + repo: &repo_slug, + owner_did: &ctx.owner_did, + node_did: &ctx.node_did, + timestamp: &ts, + blobs: &delta, + }; + match crate::arweave::anchor_encrypted_manifest( + &ctx.http_client, + &ctx.irys_url, + &manifest, + ) + .await + { + Ok(tx) if !tx.is_empty() => tracing::info!( + repo = %repo_slug, + tx_id = %tx, + "anchored encrypted manifest to Arweave" + ), + Ok(_) => {} + Err(e) => tracing::warn!( + repo = %repo_slug, + err = %e, + "encrypted manifest anchor failed" + ), + } + } + } + } +} + /// Map an error from a `smart_http` git service call to the right `AppError`: /// [`smart_http::GitServiceTimeout`] to 504, a malformed client request to 400, /// anything else to a 500 git error. Pure (no logging) so it is unit-testable; @@ -1473,6 +1711,7 @@ pub async fn git_receive_pack( old_tips, state.git_bin.clone(), std::time::Duration::from_secs(state.config.git_service_timeout_secs), + false, ) .await; if pin_set.full_scan { @@ -1497,127 +1736,56 @@ pub async fn git_receive_pack( // Pin new git objects to the local IPFS node (no-op if ipfs_api is empty). // Skipped entirely when the public cannot read the repo (withheld == None). // - // Coalesce per repo (#174 P2-2): this task parks on `git_encrypt_semaphore` - // (which DEFERS when the pool is full rather than dropping the recovery copy). To - // bound the OUTSTANDING parked-task set, only spawn if no encryption task for this - // repo is already in flight; otherwise skip — the pending/next walk over this - // repo's history already covers the newer push's objects, so the recovery copy is - // delayed, never lost. The guard removes the repo key when the task ends (success, - // error, or panic), so a later push is re-admitted (no permanent skip). + // Coalesce-and-requeue per repo (#174 P2-2 + F5): the spawned task's walks park + // on `git_encrypt_semaphore` (which DEFERS when the pool is full rather than + // dropping the recovery copy). To bound the OUTSTANDING task set, at most one + // task per repo is in flight; a push arriving while one is in flight does NOT + // spawn a duplicate — and is NOT dropped either. The in-flight task pins only + // its own pre-spawn object-list snapshot, so this push's (old, new) tip pairs + // are merged into the in-flight key's pending slot in the same critical section + // as the presence check, and the task loop-drains them (fresh rules, fail + // closed) before releasing the key. Without the requeue a coalesced push's pins + // and recovery copies would be silently absent until an unrelated later push + // (the F5 loss). The guard still releases the key on panic (Drop on unwind), so + // a crashed walk never permanently locks the repo out. if withheld.is_some() { - match state.encrypt_inflight.try_begin(&record.id) { - None => { + let tip_pairs: Vec<(String, String)> = ref_updates + .iter() + .map(|u| (u.old_sha.clone(), u.new_sha.clone())) + .collect(); + match state.encrypt_inflight.try_begin(&record.id, tip_pairs) { + crate::state::BeginOutcome::Coalesced => { tracing::debug!( repo = %record.id, - "post-push encryption task already in flight for this repo; coalescing \ - (the pending recovery-copy walk covers this push's objects)" + "post-push encryption task already in flight for this repo; coalesced \ + — this push's tip pairs are queued for that task's drain" ); } - Some(inflight_guard) => { - let object_list_ipfs = object_list.clone(); - let ipfs_api = state.config.ipfs_api.clone(); - let repo_path_clone = disk_path.clone(); - let db_clone = state.db.clone(); - let rules_for_enc = rules_opt.clone(); - let repo_id = record.id.clone(); - let owner_did = record.owner_did.clone(); - let is_public = record.is_public; - let irys_url = state.config.irys_url.clone(); - let http_client = std::sync::Arc::clone(&state.http_client); - let node_did_str = state.node_did.to_string(); - let node_seed = state.node_keypair.to_seed(); - let repo_name = record.name.clone(); - let enc_git_bin = state.git_bin.clone(); - let enc_timeout = - std::time::Duration::from_secs(state.config.git_service_timeout_secs); - let encrypt_sem = state.git_encrypt_semaphore.clone(); - tokio::spawn(async move { - // Held for the whole task; drop (on completion/error/panic) releases the - // repo's coalescing key so the next push for this repo can spawn again. - let _inflight_guard = inflight_guard; - let pinned = crate::ipfs_pin::pin_new_objects( - &ipfs_api, - &repo_path_clone, - object_list_ipfs, - &db_clone, - ) - .await; - if !pinned.is_empty() { - tracing::info!(count = pinned.len(), "pinned git objects to IPFS"); - for (sha, cid) in &pinned { - tracing::info!(sha = %sha, %cid, "pinned"); - } - } - - // Option B1: encrypt-then-pin the withheld blobs so authorized - // readers can recover them when the origin cannot serve them. - // No path-scoped rule can withhold a blob, so withheld_blob_recipients - // would return an empty map after a full per-ref walk; skip it. Mirrors - // the has_path_scoped_rule gate on the other two withheld-walk sites. - if let Some(rules) = - rules_for_enc.filter(|r| visibility_pack::has_path_scoped_rule(r)) - { - // Bound the number of concurrent post-push encryption walks (#174 P1-e): - // acquire an admission permit before the full-history walk, deferring - // when the pool is full rather than shedding the recovery pin. - let recip = withheld_recipients_gated( - encrypt_sem.clone(), - repo_path_clone.clone(), - enc_git_bin.clone(), - enc_timeout, - rules, - is_public, - owner_did.clone(), - ) - .await; - if let Ok(Ok(recipients)) = recip { - let delta = crate::encrypted_pin::encrypt_and_pin( - &ipfs_api, - &repo_path_clone, - &db_clone, - &repo_id, - &node_seed, - &recipients, - ) - .await; - - // Option B3: anchor a per-push manifest of the blobs sealed - // this push to Arweave, so the oid->cid index survives total - // node loss. Best-effort; never fails the push. - if !delta.is_empty() && !irys_url.is_empty() { - let owner_short = crate::db::normalize_owner_key(&owner_did); - let repo_slug = format!("{owner_short}/{repo_name}"); - let ts = chrono::Utc::now().to_rfc3339(); - let manifest = crate::arweave::EncryptedManifest { - repo: &repo_slug, - owner_did: &owner_did, - node_did: &node_did_str, - timestamp: &ts, - blobs: &delta, - }; - match crate::arweave::anchor_encrypted_manifest( - &http_client, - &irys_url, - &manifest, - ) - .await - { - Ok(tx) if !tx.is_empty() => tracing::info!( - repo = %repo_slug, - tx_id = %tx, - "anchored encrypted manifest to Arweave" - ), - Ok(_) => {} - Err(e) => tracing::warn!( - repo = %repo_slug, - err = %e, - "encrypted manifest anchor failed" - ), - } - } - } - } - }); + crate::state::BeginOutcome::Admitted(inflight_guard) => { + let ctx = EncryptTaskCtx { + ipfs_api: state.config.ipfs_api.clone(), + repo_path: disk_path.clone(), + db: state.db.clone(), + repo_id: record.id.clone(), + owner_did: record.owner_did.clone(), + repo_name: record.name.clone(), + irys_url: state.config.irys_url.clone(), + http_client: std::sync::Arc::clone(&state.http_client), + node_did: state.node_did.to_string(), + node_keypair: std::sync::Arc::clone(&state.node_keypair), + git_bin: state.git_bin.clone(), + git_timeout: std::time::Duration::from_secs( + state.config.git_service_timeout_secs, + ), + encrypt_sem: state.git_encrypt_semaphore.clone(), + }; + tokio::spawn(run_encrypt_pin_task( + ctx, + inflight_guard, + object_list.clone(), + rules_opt.clone(), + record.is_public, + )); } } } @@ -5197,9 +5365,9 @@ mod tests { let mut admitted = Vec::new(); let mut coalesced = 0usize; for _ in 0..K { - match inflight.try_begin(repo) { - Some(g) => admitted.push(g), - None => coalesced += 1, + match inflight.try_begin(repo, vec![]) { + crate::state::BeginOutcome::Admitted(g) => admitted.push(g), + crate::state::BeginOutcome::Coalesced => coalesced += 1, } } @@ -5238,12 +5406,15 @@ mod tests { /// one repo in flight cannot starve a second repo's recovery copy. #[test] fn u4_distinct_repos_each_admit_one_encrypt_task() { + use crate::state::BeginOutcome; let inflight = crate::state::EncryptInflight::new(); - let a = inflight.try_begin("owner/repo-a"); - let b = inflight.try_begin("owner/repo-b"); - let c = inflight.try_begin("owner/repo-c"); + let a = inflight.try_begin("owner/repo-a", vec![]); + let b = inflight.try_begin("owner/repo-b", vec![]); + let c = inflight.try_begin("owner/repo-c", vec![]); assert!( - a.is_some() && b.is_some() && c.is_some(), + matches!(&a, BeginOutcome::Admitted(_)) + && matches!(&b, BeginOutcome::Admitted(_)) + && matches!(&c, BeginOutcome::Admitted(_)), "three distinct repos each admit their own encryption task" ); assert_eq!(inflight.len(), 3, "one in-flight entry per distinct repo"); @@ -5258,19 +5429,24 @@ mod tests { /// survives normal completion AND a panic, so no permanent skip / no leaked key. #[test] fn u4_coalesced_repo_is_reprocessed_after_task_ends_not_permanently_skipped() { + use crate::state::{BeginOutcome, FinishOutcome}; let inflight = crate::state::EncryptInflight::new(); let repo = "did:key:z6MkDurableRepoBBBBBBBBBBBBBBBBBBBBBBBBB/repo"; // Push #1 admits and "spawns". A concurrent push #2 (task #1 still in flight) - // coalesces — no duplicate spawn. - let guard1 = inflight.try_begin(repo).expect("first push admits"); + // coalesces — no duplicate spawn; its (empty) tip set is recorded, not lost. + let guard1 = admit(&inflight, repo); assert!( - inflight.try_begin(repo).is_none(), + matches!(inflight.try_begin(repo, vec![]), BeginOutcome::Coalesced), "while task #1 is in flight, push #2 to the same repo coalesces" ); - // Task #1 finishes (encrypt_and_pin ran or errored): guard drops, key released. - drop(guard1); + // Task #1 finishes normally: nothing pending (push #2 carried no tips), so + // the empty-pending check removes the key in its critical section. + assert!( + matches!(guard1.finish_or_take_pending(), FinishOutcome::Finished(_)), + "no pending tips — the task exits and releases the key" + ); assert_eq!( inflight.len(), 0, @@ -5280,18 +5456,21 @@ mod tests { // A LATER push for the SAME repo is admitted again (processed, not skipped // forever). This is what coalesce->shed breaks: shed drops the job and no sweep // re-derives the missing copy, so the recovery copy is permanently lost. - let guard2 = inflight.try_begin(repo).expect( - "a later push for a coalesced repo MUST be re-admitted — durability: the \ - deferred recovery copy is produced eventually, never dropped", - ); + let guard2 = admit(&inflight, repo); + // An errored task (guard dropped without finishing) still releases the key. drop(guard2); assert_eq!(inflight.len(), 0); - // Durability across PANIC: a task that panics mid-walk must still release its key - // (Drop runs on unwind), so one crashed walk never permanently locks a repo out - // of future recovery copies. + // Durability across PANIC: a task that panics mid-walk must still release its + // key (the still-armed guard's Drop runs on unwind), so one crashed walk never + // permanently locks a repo out of future recovery copies. Coalesce real tips + // first: the panic loses them (logged), and the loss must not corrupt the set. let panicked = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - let _g = inflight.try_begin(repo).expect("admit before the panic"); + let _g = admit(&inflight, repo); + assert!(matches!( + inflight.try_begin(repo, vec![("old1".to_string(), "new1".to_string())]), + BeginOutcome::Coalesced + )); assert_eq!(inflight.len(), 1); panic!("simulate the detached encryption task panicking mid-walk"); })); @@ -5302,9 +5481,12 @@ mod tests { "a panicked encryption task still releases its repo key (Drop on unwind) — no \ permanent leak that would block every future recovery copy for the repo" ); + // The next push is re-admitted, and the panicked task's pending tips did NOT + // survive into it (they are lost-and-logged, recovered only by a later push). + let guard3 = admit(&inflight, repo); assert!( - inflight.try_begin(repo).is_some(), - "after a panicked task the repo can still be admitted — durability preserved" + matches!(guard3.finish_or_take_pending(), FinishOutcome::Finished(_)), + "the pre-panic pending tips must not leak into the re-admitted task" ); } @@ -5315,11 +5497,25 @@ mod tests { let inflight = crate::state::EncryptInflight::new(); assert!(inflight.is_empty(), "cold set is empty"); assert!( - inflight.try_begin("owner/first").is_some(), + matches!( + inflight.try_begin("owner/first", vec![]), + crate::state::BeginOutcome::Admitted(_) + ), "the first push on a cold in-flight set must admit (never falsely coalesce)" ); } + /// Unwrap an Admitted outcome (panic on Coalesced) — the u4/u5 suites' shorthand. + fn admit( + inflight: &crate::state::EncryptInflight, + repo: &str, + ) -> crate::state::EncryptInflightGuard { + match inflight.try_begin(repo, vec![]) { + crate::state::BeginOutcome::Admitted(g) => g, + crate::state::BeginOutcome::Coalesced => panic!("expected {repo} to admit"), + } + } + /// Model of the pre-fix / mutated code: no coalescing check, so every push spawns. /// Returns the count of tasks spawned (== the size of the unbounded outstanding set /// the fix prevents), used as the RED comparison in the bound test above. @@ -5327,6 +5523,567 @@ mod tests { (0..pushes).count() } + // ---- #174 U5 (F5): a push that loses try_begin is REQUEUED, never dropped ---- + // + // F5: the in-flight task pins only its own pre-spawn object-list snapshot, so a + // push B arriving while task A is in flight used to be SKIPPED outright (the old + // None arm) — B's pins and recovery copies were silently absent until an + // unrelated later push re-walked the repo. U5 records B's (old, new) tip pairs + // into the in-flight key's pending slot in the SAME critical section as the + // presence check, and A's task loop-drains them before releasing the key. + + /// The F5 lost-update repro. Task A is in flight past its snapshot; push B + /// coalesces carrying its tip pair; when A finishes its snapshot iteration the + /// tracker must hand A exactly B's recorded work with the key retained — and only + /// an empty pending check may remove the key. On pre-U5 code this is RED: the + /// coalesce arm records B's work nowhere and there is no drain surface at all. + #[test] + fn u5_coalesced_push_work_is_drained_by_the_inflight_task() { + use crate::state::{BeginOutcome, FinishOutcome, PendingWork}; + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkF5LostUpdateCCCCCCCCCCCCCCCCCCCCCCCC/repo"; + + // Push A admits; its spawned task is "in flight past its snapshot". + let guard_a = match inflight.try_begin(repo, vec![]) { + BeginOutcome::Admitted(g) => g, + BeginOutcome::Coalesced => panic!("first push must admit"), + }; + + // Push B lands while A is in flight: coalesced, tip pair recorded. + let b_pair = ( + "b0ldb0ldb0ldb0ldb0ldb0ldb0ldb0ldb0ldb0ld".to_string(), + "bnewbnewbnewbnewbnewbnewbnewbnewbnewbnew".to_string(), + ); + match inflight.try_begin(repo, vec![b_pair.clone()]) { + BeginOutcome::Coalesced => {} + BeginOutcome::Admitted(_) => panic!("push B must coalesce while A is in flight"), + } + + // A finishes its snapshot iteration: it must be handed B's work (drained), + // not exit — an exit here is exactly the F5 silent loss. + match guard_a.finish_or_take_pending() { + FinishOutcome::Pending(guard_a, work) => { + assert_eq!( + work, + PendingWork::Tips(vec![b_pair]), + "A drains exactly B's recorded tip pair" + ); + assert_eq!(inflight.len(), 1, "the key is retained while A iterates"); + // Nothing further pending: A now exits and releases the key. + match guard_a.finish_or_take_pending() { + FinishOutcome::Finished(_) => {} + FinishOutcome::Pending(..) => panic!("no second batch was recorded"), + } + assert_eq!( + inflight.len(), + 0, + "an empty pending check at task end releases the key" + ); + } + FinishOutcome::Finished(_) => panic!( + "F5: B's coalesced work vanished — the in-flight task exited without draining it" + ), + } + } + + /// Drain-vs-admit race, both orderings driven deterministically through the lock + /// API (the check+merge and check+remove are each ONE critical section, so a push + /// can only land on one side of A's final pending check — never inside it): + /// before it, the push is merged and A drains it; after it, the key is gone and + /// the push is admitted as a fresh task. Neither ordering loses the work. A + /// check-then-record split (merge moved outside try_begin's critical section) + /// turns ordering 1 RED: the work recorded after A's check is never drained. + #[test] + fn u5_drain_vs_admit_race_loses_no_work_in_either_ordering() { + use crate::state::{BeginOutcome, FinishOutcome, PendingWork}; + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkF5RaceOrderDDDDDDDDDDDDDDDDDDDDDDDDD/repo"; + let pair = ("cold".to_string(), "cnew".to_string()); + + // Ordering 1: push C lands BEFORE A's final pending check → merged in + // try_begin's critical section → A must drain it (key retained). + let guard_a = admit(&inflight, repo); + assert!(matches!( + inflight.try_begin(repo, vec![pair.clone()]), + BeginOutcome::Coalesced + )); + match guard_a.finish_or_take_pending() { + FinishOutcome::Pending(g, work) => { + assert_eq!(work, PendingWork::Tips(vec![pair.clone()])); + assert!(matches!( + g.finish_or_take_pending(), + FinishOutcome::Finished(_) + )); + } + FinishOutcome::Finished(_) => { + panic!("a push merged before the final check must be drained, not lost") + } + } + assert!(inflight.is_empty()); + + // Ordering 2: push C lands AFTER A's final pending check removed the key → + // it must be ADMITTED as a fresh task (its own snapshot covers its work). + let guard_a = admit(&inflight, repo); + assert!(matches!( + guard_a.finish_or_take_pending(), + FinishOutcome::Finished(_) + )); + match inflight.try_begin(repo, vec![pair]) { + BeginOutcome::Admitted(g) => drop(g), + BeginOutcome::Coalesced => panic!( + "a push landing after the key was removed must admit a new task — a \ + coalesce here records work no task will ever drain" + ), + } + } + + /// Exit-vs-successor (the double-remove hazard). A's normal exit removes the key + /// and disarms the guard in ONE critical section, and the disarmed guard is + /// handed back — so its eventual Drop lands in the real remove→drop window. A + /// successor task B admitted inside that window must keep ITS key when A's guard + /// finally drops. With the disarm reverted (Drop removing unconditionally) this + /// is RED: dropping A's guard deletes B's key and the third push falsely admits + /// a second task for the repo. + #[test] + fn u5_disarmed_guard_drop_never_removes_a_successor_key() { + use crate::state::{BeginOutcome, FinishOutcome}; + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkF5DisarmEEEEEEEEEEEEEEEEEEEEEEEEEEEE/repo"; + + // A admits and exits normally; HOLD the disarmed guard to keep the window open. + let guard_a = admit(&inflight, repo); + let disarmed = match guard_a.finish_or_take_pending() { + FinishOutcome::Finished(g) => g, + FinishOutcome::Pending(..) => panic!("nothing was pending"), + }; + assert!(inflight.is_empty(), "A's exit released the key"); + + // Successor B is admitted inside the remove→drop window. + let guard_b = admit(&inflight, repo); + assert_eq!(inflight.len(), 1); + + // A's disarmed guard now drops. B's key must SURVIVE: a third push still + // coalesces against B's in-flight task. + drop(disarmed); + assert_eq!( + inflight.len(), + 1, + "dropping A's disarmed guard must not remove successor B's key" + ); + assert!( + matches!(inflight.try_begin(repo, vec![]), BeginOutcome::Coalesced), + "B's task is still the (only) in-flight task — at-most-one-per-repo holds" + ); + drop(guard_b); + assert!(inflight.is_empty()); + } + + /// Pending overflow: past the 1024-pair bound the slot degrades to the FullScan + /// marker (bounded memory under a hostile push burst); at exactly the bound it + /// stays a Tips batch. The marker is an explicit variant, never an empty tip + /// list — an empty-tips encoding would drain to an empty delta and pin nothing. + #[test] + fn u5_pending_overflow_degrades_to_full_scan_marker() { + use crate::state::{BeginOutcome, FinishOutcome, PendingWork}; + let inflight = crate::state::EncryptInflight::new(); + let pair = |i: usize| (format!("old{i}"), format!("new{i}")); + + // At the bound: exactly 1024 pairs stay a Tips batch. + let repo_at = "owner/at-bound"; + let g = admit(&inflight, repo_at); + assert!(matches!( + inflight.try_begin(repo_at, (0..1024).map(pair).collect()), + BeginOutcome::Coalesced + )); + match g.finish_or_take_pending() { + FinishOutcome::Pending(g, PendingWork::Tips(v)) => { + assert_eq!(v.len(), 1024, "at the bound the pairs are kept verbatim"); + assert!(matches!( + g.finish_or_take_pending(), + FinishOutcome::Finished(_) + )); + } + other => panic!( + "expected a Tips batch at the bound, got {:?}", + match other { + FinishOutcome::Pending(_, w) => Some(w), + FinishOutcome::Finished(_) => None, + } + ), + } + + // Past the bound: the accumulated slot degrades to FullScan and later + // merges are absorbed (still one bounded marker, not a growing list). + let repo_over = "owner/over-bound"; + let g = admit(&inflight, repo_over); + assert!(matches!( + inflight.try_begin(repo_over, (0..1024).map(pair).collect()), + BeginOutcome::Coalesced + )); + assert!(matches!( + inflight.try_begin(repo_over, vec![pair(9999)]), + BeginOutcome::Coalesced + )); + assert!(matches!( + inflight.try_begin(repo_over, vec![pair(10000)]), + BeginOutcome::Coalesced + )); + match g.finish_or_take_pending() { + FinishOutcome::Pending(g, work) => { + assert_eq!( + work, + PendingWork::FullScan, + "overflow degrades to the explicit FullScan marker" + ); + assert!(matches!( + g.finish_or_take_pending(), + FinishOutcome::Finished(_) + )); + } + FinishOutcome::Finished(_) => panic!("the overflowed pending work vanished"), + } + } + + // ---- u5 drain-pipeline fixtures: a real git repo + a DB repo row ---- + + fn u5_git(dir: &std::path::Path, args: &[&str]) -> String { + let out = std::process::Command::new("git") + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + + fn u5_init_repo(dir: &std::path::Path) { + u5_git(dir, &["init", "-q", "-b", "main"]); + u5_git(dir, &["config", "user.email", "t@t"]); + u5_git(dir, &["config", "user.name", "t"]); + } + + /// Commit `name` (parent dirs created) with `body`; returns the commit sha. + fn u5_commit_file(dir: &std::path::Path, name: &str, body: &str) -> String { + let path = dir.join(name); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(&path, body).unwrap(); + u5_git(dir, &["add", name]); + u5_git(dir, &["commit", "-qm", &format!("add {name}")]); + u5_git(dir, &["rev-parse", "HEAD"]) + } + + /// A drain-task context over the test state and an on-disk repo. Empty + /// `ipfs_api`/`irys_url` keep the pin/anchor stages inert (no network). + fn u5_ctx( + state: &AppState, + rec: &crate::db::RepoRecord, + repo_path: std::path::PathBuf, + git_bin: &str, + sem: std::sync::Arc, + ) -> EncryptTaskCtx { + EncryptTaskCtx { + ipfs_api: String::new(), + repo_path, + db: state.db.clone(), + repo_id: rec.id.clone(), + owner_did: rec.owner_did.clone(), + repo_name: rec.name.clone(), + irys_url: String::new(), + http_client: std::sync::Arc::clone(&state.http_client), + node_did: state.node_did.to_string(), + node_keypair: std::sync::Arc::clone(&state.node_keypair), + git_bin: git_bin.to_string(), + git_timeout: std::time::Duration::from_secs(600), + encrypt_sem: sem, + } + } + + /// The drain resolves a coalesced push's tip pair to exactly that push's + /// introduced objects (delta semantics — the F5 observable: push B's pins are + /// recorded by the drain, and pre-existing objects are not re-listed). + #[sqlx::test] + async fn u5_drain_resolves_coalesced_tips_to_their_objects(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + u5_init_repo(tmp.path()); + let c1 = u5_commit_file(tmp.path(), "a.txt", "one\n"); + let c2 = u5_commit_file(tmp.path(), "b.txt", "two\n"); + state + .db + .upsert_mirror_repo("z6u5delta", "d", "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6u5delta", "d").await.unwrap().unwrap(); + let ctx = u5_ctx( + &state, + &rec, + tmp.path().to_path_buf(), + "git", + std::sync::Arc::new(tokio::sync::Semaphore::new(4)), + ); + + // Push B advanced main c1 -> c2 and lost try_begin; its pair was coalesced. + let (list, _rules, is_public) = resolve_drain_object_list( + &ctx, + crate::state::PendingWork::Tips(vec![(c1.clone(), c2.clone())]), + ) + .await + .expect("a public repo drains to a pin list"); + assert!(is_public, "mirror rows are public"); + let got: std::collections::HashSet = list.into_iter().collect(); + let new_blob = u5_git(tmp.path(), &["rev-parse", "HEAD:b.txt"]); + let old_blob = u5_git(tmp.path(), &["rev-parse", &format!("{c1}:a.txt")]); + assert!( + got.contains(&c2) && got.contains(&new_blob), + "B's commit and blob are in the drained pin list (the F5 fix)" + ); + assert!( + !got.contains(&c1) && !got.contains(&old_blob), + "pre-existing objects are not re-listed (delta, not full scan)" + ); + } + + /// The FullScan marker drains through the FLAGGED full-scan path to a NON-EMPTY + /// candidate set. RED arm of the encoding: were the marker a plain empty-tips + /// call, the deletion-only fast path would return an empty delta and the drain + /// would pin nothing (the F5 silent loss resurfacing). + #[sqlx::test] + async fn u5_drain_full_scan_marker_yields_nonempty_candidates(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + u5_init_repo(tmp.path()); + let c1 = u5_commit_file(tmp.path(), "a.txt", "one\n"); + state + .db + .upsert_mirror_repo("z6u5full", "f", "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6u5full", "f").await.unwrap().unwrap(); + let ctx = u5_ctx( + &state, + &rec, + tmp.path().to_path_buf(), + "git", + std::sync::Arc::new(tokio::sync::Semaphore::new(4)), + ); + + let (list, _rules, _pub) = + resolve_drain_object_list(&ctx, crate::state::PendingWork::FullScan) + .await + .expect("a public repo drains to a pin list"); + let got: std::collections::HashSet = list.into_iter().collect(); + assert!( + !got.is_empty(), + "the FullScan drain must enumerate the repo — an empty list means the \ + marker collapsed into the empty-tips fast path" + ); + let blob = u5_git(tmp.path(), &["rev-parse", "HEAD:a.txt"]); + assert!( + got.contains(&c1) && got.contains(&blob), + "the full-scan drain covers the repo's commit and blob" + ); + } + + /// Rules tightened between the coalesced push and its drain are honored, fail + /// closed: the drain re-fetches rules/is_public fresh, so (1) a newly-withheld + /// blob is NOT pinned, and (2) a repo whose root became unreadable to the + /// anonymous public drains to nothing at all. + #[sqlx::test] + async fn u5_drain_honors_rules_tightened_after_the_push(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + u5_init_repo(tmp.path()); + u5_commit_file(tmp.path(), "pub.txt", "public\n"); + let c2 = u5_commit_file(tmp.path(), "secret/hidden.txt", "sealed\n"); + state + .db + .upsert_mirror_repo("z6u5tight", "t", "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6u5tight", "t").await.unwrap().unwrap(); + let ctx = u5_ctx( + &state, + &rec, + tmp.path().to_path_buf(), + "git", + std::sync::Arc::new(tokio::sync::Semaphore::new(4)), + ); + let pending = || crate::state::PendingWork::Tips(vec![(ZERO_SHA.to_string(), c2.clone())]); + + // At push time the repo had no rules. TIGHTEN before the drain: /secret/** + // becomes reader-gated. The drain must re-fetch and withhold the new blob. + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &[READER_DID.to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + let (list, _rules, _pub) = resolve_drain_object_list(&ctx, pending()) + .await + .expect("still announceable at root"); + let got: std::collections::HashSet = list.into_iter().collect(); + let pub_blob = u5_git(tmp.path(), &["rev-parse", "HEAD:pub.txt"]); + let secret_blob = u5_git(tmp.path(), &["rev-parse", "HEAD:secret/hidden.txt"]); + assert!( + got.contains(&pub_blob), + "the still-public blob is pinned by the drain" + ); + assert!( + !got.contains(&secret_blob), + "a blob withheld by a rule added AFTER the push must NOT be pinned by \ + the drain (fresh rules, fail closed)" + ); + + // Tighten further: root becomes reader-gated → not announceable to the + // anonymous public → the drain pins nothing at all. + state + .db + .set_visibility_rule( + &rec.id, + "/", + crate::db::VisibilityMode::A, + &[READER_DID.to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + assert!( + resolve_drain_object_list(&ctx, pending()).await.is_none(), + "a repo no longer announceable under current rules drains to nothing \ + (fail closed)" + ); + } + + /// Hot-repo drain at encrypt-pool size 1: the task loop holds NO task-level + /// permit, so per-iteration helper acquires (withheld walk, candidate scan, + /// recipients walk) each get the pool's single permit in turn and BOTH the + /// snapshot iteration and the coalesced-drain iteration complete. RED if the + /// loop takes a task-level permit: the first helper acquire nests over the + /// same exhausted semaphore and the task parks forever. + #[sqlx::test] + async fn u5_hot_repo_drain_completes_at_pool_size_one(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + u5_init_repo(tmp.path()); + u5_commit_file(tmp.path(), "pub.txt", "public\n"); + let c2 = u5_commit_file(tmp.path(), "secret/hidden.txt", "sealed\n"); + state + .db + .upsert_mirror_repo("z6u5hot", "h", "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6u5hot", "h").await.unwrap().unwrap(); + // A path-scoped rule so every gated walk actually runs (withheld walk on + // the drain, recipients walk on both iterations). READER_DID carries no + // resolvable key, so the encrypt stage plans no seal and stays offline. + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &[READER_DID.to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + let rules = state.db.list_visibility_rules(&rec.id).await.unwrap(); + + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + let ctx = u5_ctx(&state, &rec, tmp.path().to_path_buf(), "git", sem); + + let inflight = crate::state::EncryptInflight::new(); + let guard = admit(&inflight, &rec.id); + assert!(matches!( + inflight.try_begin(&rec.id, vec![(ZERO_SHA.to_string(), c2)]), + crate::state::BeginOutcome::Coalesced + )); + + tokio::time::timeout( + std::time::Duration::from_secs(60), + run_encrypt_pin_task(ctx, guard, Vec::new(), Some(rules), true), + ) + .await + .expect( + "the drain must complete at pool size 1 — a task-level permit would \ + deadlock the helper-internal acquires", + ); + assert!( + inflight.is_empty(), + "the drained task released its repo key on exit" + ); + } + + /// The task LOOP is load-bearing: work coalesced during the snapshot iteration + /// is drained (its candidate scan runs git) before the task exits. RED under + /// the drain-loop revert (task drops its guard after the snapshot without + /// checking pending): the fake git never runs and the marker is absent. + #[cfg(unix)] + #[sqlx::test] + async fn u5_task_drains_coalesced_work_before_exiting(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + let marker = tmp.path().join("git.ran"); + // The drain's candidate scan probes the tip type then walks: report a + // commit tip and an empty rev-list, recording every invocation. + let body = format!( + "#!/bin/sh\necho ran >> \"{}\"\ncase \"$1\" in\n cat-file) echo commit ;;\n *) : ;;\nesac\nexit 0\n", + marker.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + state + .db + .upsert_mirror_repo("z6u5loop", "l", "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6u5loop", "l").await.unwrap().unwrap(); + let ctx = u5_ctx( + &state, + &rec, + tmp.path().to_path_buf(), + &git_bin, + std::sync::Arc::new(tokio::sync::Semaphore::new(4)), + ); + + let inflight = crate::state::EncryptInflight::new(); + let guard = admit(&inflight, &rec.id); + // Push B coalesces mid-flight with a real (created-ref) tip pair. + assert!(matches!( + inflight.try_begin( + &rec.id, + vec![( + ZERO_SHA.to_string(), + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef".to_string(), + )] + ), + crate::state::BeginOutcome::Coalesced + )); + + // Snapshot: empty object list, no rules — the snapshot iteration itself + // spawns no git, so any git invocation below belongs to the DRAIN. + run_encrypt_pin_task(ctx, guard, Vec::new(), None, true).await; + assert!( + marker.exists(), + "the task must drain B's coalesced tips (candidate scan runs git) \ + before exiting — an absent marker is the F5 skip" + ); + assert!( + inflight.is_empty(), + "the empty pending check at task end released the key" + ); + } + /// #174 SC2 (per-source key, U1): the per-caller read sub-cap keys on the /// resolved source IP, NOT the signed DID, so a disposable-DID farm cannot /// multiply its budget. Fill the source IP's single read slot, then drive two diff --git a/crates/gitlawb-node/src/git/push_delta.rs b/crates/gitlawb-node/src/git/push_delta.rs index 18f2c4da..27064357 100644 --- a/crates/gitlawb-node/src/git/push_delta.rs +++ b/crates/gitlawb-node/src/git/push_delta.rs @@ -288,6 +288,13 @@ pub struct PinCandidateSet { /// rev-list, and the full-scan fallback — run under ONE permit held for the /// whole blocking scan. The deletion-only fast path (no new tips) spawns no git /// and never parks. +/// +/// `force_full_scan` forces the full-scan fallback regardless of the tips — the +/// coalesced-drain `PendingWork::FullScan` marker (#174 F5). It composes with +/// the env kill-switch (either forces), and it disqualifies the empty-tips fast +/// path: a forced-scan call with no tips must still enumerate the repo, never +/// silently resolve to an empty delta (which would pin nothing and lose the +/// drained pushes' work — the F5 bug shape). pub async fn resolve_candidates_for_push( scan_sem: std::sync::Arc, repo_path: PathBuf, @@ -295,11 +302,13 @@ pub async fn resolve_candidates_for_push( old_tips: Vec, git_bin: String, timeout: Duration, + force_full_scan: bool, ) -> PinCandidateSet { + let force = force_full_scan || self::force_full_scan(); // No-git fast path: a deletion-only push resolves to an empty delta without - // spawning any git child, so it must not park on the scan pool. The - // kill-switch disqualifies it — forcing the full scan spawns git. - if new_tips.is_empty() && !force_full_scan() { + // spawning any git child, so it must not park on the scan pool. A forced + // full scan (caller flag or kill-switch) disqualifies it — the scan spawns git. + if new_tips.is_empty() && !force { tracing::info!(delta = 0usize, repo = %repo_path.display(), "pin candidate set from push delta"); return PinCandidateSet { candidates: Vec::new(), @@ -327,7 +336,16 @@ pub async fn resolve_candidates_for_push( let deadline = Instant::now() + timeout; let new_refs: Vec<&str> = new_tips.iter().map(String::as_str).collect(); let old_refs: Vec<&str> = old_tips.iter().map(String::as_str).collect(); - match resolve_push_delta(&repo_path, &new_refs, &old_refs, &git_bin, deadline) { + // `force` already folds in the env kill-switch, so the forced arm skips the + // delta machinery outright; the unforced arm goes through the normal + // resolver (whose own env read is false here by construction). + let resolved = if force { + tracing::debug!("full scan forced (coalesced-drain marker or kill-switch)"); + PinCandidates::FullScanRequired + } else { + resolve_push_delta(&repo_path, &new_refs, &old_refs, &git_bin, deadline) + }; + match resolved { PinCandidates::Delta(objs) => { tracing::info!(delta = objs.len(), repo = %repo_path.display(), "pin candidate set from push delta"); PinCandidateSet { candidates: objs, full_scan: false } @@ -586,6 +604,7 @@ mod tests { vec![c1.clone()], "git".to_string(), std::time::Duration::from_secs(600), + false, ) .await; assert!(!set.full_scan, "happy-path delta is not a full scan"); @@ -616,6 +635,7 @@ mod tests { vec![], "git".to_string(), std::time::Duration::from_secs(600), + false, ) .await; assert!(set.full_scan, "non-commit tip is signalled as a full scan"); @@ -665,6 +685,7 @@ mod tests { vec![], git_bin.clone(), Duration::from_secs(5), + false, ), ) .await; @@ -686,6 +707,7 @@ mod tests { vec![], git_bin, Duration::from_secs(5), + false, ) .await; assert!( @@ -730,6 +752,7 @@ mod tests { vec!["deadbeefdeadbeefdeadbeefdeadbeefdeadbeef".to_string()], fake.to_str().unwrap().to_string(), Duration::from_secs(5), + false, ), ) .await @@ -744,6 +767,56 @@ mod tests { assert!(!marker.exists(), "the fast path must spawn no git at all"); } + /// #174 F5: the coalesced-drain FullScan marker is signalled via the explicit + /// `force_full_scan` flag, and a flagged call with NO tips must still enumerate + /// the whole repo (full_scan=true, non-empty candidates). The RED arm of the + /// encoding question: if the marker were encoded as a plain empty-tips call + /// (flag off — the fast path), the drain would resolve to an empty delta and + /// pin nothing, silently losing the coalesced pushes' work. + #[tokio::test] + async fn forced_full_scan_with_no_tips_enumerates_the_repo() { + let repo = Repo::new(); + repo.commit_file("a.txt", "one\n"); + let all: HashSet = list_all_objects(&repo.path, "git", td()) + .unwrap() + .into_iter() + .collect(); + assert!(!all.is_empty(), "fixture repo has objects"); + + let set = resolve_candidates_for_push( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), + repo.path.clone(), + vec![], + vec![], + "git".to_string(), + std::time::Duration::from_secs(600), + true, + ) + .await; + assert!(set.full_scan, "the forced call is signalled as a full scan"); + let got: HashSet = set.candidates.into_iter().collect(); + assert_eq!( + got, all, + "a forced full scan with no tips enumerates the repo — never an empty delta" + ); + + // The discriminator: the SAME empty-tips call without the flag is the + // deletion-only fast path (empty delta, pin nothing). The two must differ, + // or the marker encoding has collapsed into the silent-loss shape. + let unforced = resolve_candidates_for_push( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), + repo.path.clone(), + vec![], + vec![], + "git".to_string(), + std::time::Duration::from_secs(600), + false, + ) + .await; + assert!(!unforced.full_scan); + assert!(unforced.candidates.is_empty()); + } + #[test] fn missing_oid_tip_forces_full_scan() { let repo = Repo::new(); diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 0badf802..4f8ffb0e 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -132,8 +132,9 @@ pub struct AppState { /// detached tasks *spawn and park* on that semaphore's `acquire_owned().await`. /// Before spawning a per-push encryption task, the receive-pack handler consults /// this set: if the repo already has a task in flight it coalesces (skips the - /// duplicate spawn) rather than parking a new unbounded waiter. Coalescing only - /// delays a duplicate walk — it never drops the withheld-blob recovery copy, which + /// duplicate spawn) rather than parking a new unbounded waiter, and its tip pairs + /// are recorded for that task's drain loop (#174 F5). Coalescing only delays the + /// coalesced push's walk — it never drops the withheld-blob recovery copy, which /// `2a54c15` deliberately kept fail-closed (there is no reconciliation sweep to /// re-derive a dropped copy). See [`EncryptInflight`]. pub encrypt_inflight: EncryptInflight, @@ -230,23 +231,80 @@ impl AppState { /// unbounded outstanding set. /// /// This tracks the repo keys with an in-flight encryption task. Before spawning, the -/// handler calls [`try_begin`](Self::try_begin): if a task for the repo is already -/// in-flight it returns `None` and the handler SKIPS spawning a duplicate (coalesce — -/// the newer push's objects are covered by the pending/next walk over the same repo's -/// history). This bounds the outstanding set to <=1 pending task per repo WITHOUT -/// dropping work: coalescing only delays a duplicate walk, it never sheds the recovery -/// copy (there is no reconciliation sweep, so a *dropped* job would be lost forever). +/// handler calls [`try_begin`](Self::try_begin) with the push's (old, new) tip pairs: +/// if no task is in-flight the push is [`Admitted`](BeginOutcome::Admitted) and spawns +/// one; if a task IS in-flight the push [`Coalesces`](BeginOutcome::Coalesced) — no +/// duplicate spawn — and its tip pairs are merged into the in-flight key's pending +/// slot in the SAME critical section as the presence check. The in-flight task pins +/// only its own pre-spawn object-list snapshot, so the merge is what keeps coalescing +/// lossless (#174 F5): the task loop-drains the pending slot via +/// [`EncryptInflightGuard::finish_or_take_pending`] before releasing the key, so a +/// coalesced push's pins and recovery copies are delayed, never dropped (there is no +/// reconciliation sweep, so a *dropped* job would be lost forever). Check-then-record +/// as two lock acquisitions would race the task's final pending check — hence one +/// critical section for both. /// -/// The returned [`EncryptInflightGuard`] is moved into the detached task and removes -/// the repo key on drop — on normal completion, error, OR panic (Drop runs on unwind) -/// — so one crashed walk can never permanently lock a repo out of future recovery -/// copies. +/// The returned [`EncryptInflightGuard`] is moved into the detached task. On normal +/// exit the key is removed (and the guard disarmed) inside `finish_or_take_pending`'s +/// empty-pending critical section; the guard's Drop is the PANIC backstop (Drop runs +/// on unwind), so one crashed walk can never permanently lock a repo out of future +/// recovery copies. #[derive(Clone, Default)] pub struct EncryptInflight { - // std::sync::Mutex: only ever held for O(1) HashSet insert/remove in a sync - // context (right before `tokio::spawn`, and in the guard's Drop) — never across - // an await, so a std Mutex is correct and cheaper than a tokio one. - repos: Arc>>, + // std::sync::Mutex: only ever held for O(1)-ish map ops (insert/remove/merge — + // the merge is an O(pairs) Vec extend bounded by MAX_PENDING_TIP_PAIRS) in a + // sync context, never across an await, so a std Mutex is correct and cheaper + // than a tokio one. Key present == task in flight; the value is the work + // recorded by pushes that coalesced against it. + repos: Arc>>, +} + +/// Cap on the accumulated coalesced tip pairs per repo. Past it the pending slot +/// degrades to [`PendingWork::FullScan`], so a hostile pusher cannot grow the slot +/// without bound while a walk is in flight; the drain then costs one full-repo +/// enumeration instead (the same already-tested fallback the push path uses). +const MAX_PENDING_TIP_PAIRS: usize = 1024; + +/// Work recorded by pushes that coalesced against an in-flight encryption task, +/// drained by that task one batch per loop iteration. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PendingWork { + /// The coalesced pushes' raw (old_sha, new_sha) ref-update pairs, zeros + /// included — the drain strips the create/delete sentinels exactly like the + /// handler tail does. An EMPTY vec is "nothing pending", never a work item. + Tips(Vec<(String, String)>), + /// The pair bound overflowed: drain with a FORCED full-repo scan. This must be + /// signalled explicitly (the `force_full_scan` flag on + /// `resolve_candidates_for_push`), never encoded as an empty tip list — empty + /// tips resolve to an empty delta and would pin nothing (the F5 loss again). + FullScan, +} + +/// Outcome of [`EncryptInflight::try_begin`]. +pub enum BeginOutcome { + /// No task was in flight: the caller spawns one, moving the guard into it. The + /// push's own tip pairs are NOT recorded — the caller's pre-spawn snapshot + /// covers them; the pending slot starts empty. + Admitted(EncryptInflightGuard), + /// A task is in flight; this push's tip pairs were merged into its pending + /// slot (same critical section as the presence check). The in-flight task's + /// drain loop will process them. + Coalesced, +} + +/// Outcome of [`EncryptInflightGuard::finish_or_take_pending`]. +pub enum FinishOutcome { + /// Coalesced work was pending: it is handed back with the still-armed guard + /// (the repo key is retained) and the task must run another drain iteration. + Pending(EncryptInflightGuard, PendingWork), + /// Nothing was pending: the repo key was removed AND the guard disarmed in one + /// critical section, so dropping the returned guard is inert. The task exits. + /// Remove-then-drop as two steps would double-remove: a successor task admitted + /// between them would have ITS key deleted by the late Drop. The disarmed guard + /// is handed back rather than dropped internally so that remove→drop window is + /// real and the disarm is testable; production just lets it fall out of scope + /// (hence the allow). + Finished(#[allow(dead_code)] EncryptInflightGuard), } impl EncryptInflight { @@ -254,19 +312,25 @@ impl EncryptInflight { Self::default() } - /// Try to begin an encryption task for `repo_id`. Returns `Some(guard)` if no task - /// for the repo was in-flight (the caller should spawn), or `None` if one already - /// is (the caller should COALESCE — skip spawning a duplicate). The guard releases - /// the repo key on drop. - pub fn try_begin(&self, repo_id: &str) -> Option { - let mut set = self.repos.lock().expect("encrypt_inflight mutex poisoned"); - if set.insert(repo_id.to_string()) { - Some(EncryptInflightGuard { - repos: Arc::clone(&self.repos), - repo_id: repo_id.to_string(), - }) - } else { - None + /// Begin-or-coalesce an encryption task for `repo_id`, in one critical section. + /// `tip_pairs` is this push's raw (old_sha, new_sha) ref-update list; it is + /// merged into the pending slot only on the [`Coalesced`](BeginOutcome::Coalesced) + /// arm (an admitted caller's own snapshot already covers its pairs). + pub fn try_begin(&self, repo_id: &str, tip_pairs: Vec<(String, String)>) -> BeginOutcome { + let mut map = self.repos.lock().expect("encrypt_inflight mutex poisoned"); + match map.entry(repo_id.to_string()) { + std::collections::hash_map::Entry::Vacant(slot) => { + slot.insert(PendingWork::Tips(Vec::new())); + BeginOutcome::Admitted(EncryptInflightGuard { + repos: Arc::clone(&self.repos), + repo_id: repo_id.to_string(), + armed: true, + }) + } + std::collections::hash_map::Entry::Occupied(mut slot) => { + merge_pending(slot.get_mut(), tip_pairs); + BeginOutcome::Coalesced + } } } @@ -287,21 +351,95 @@ impl EncryptInflight { } } -/// RAII guard removing a repo key from [`EncryptInflight`] when the detached -/// encryption task finishes (drop on completion, error, or panic-unwind). Move-only — -/// there is no reason to clone a guard, and cloning would double-remove. +/// Merge a coalesced push's tip pairs into a repo's pending slot. FullScan absorbs +/// everything; a Tips slot that would exceed [`MAX_PENDING_TIP_PAIRS`] degrades to +/// FullScan rather than growing without bound. +fn merge_pending(slot: &mut PendingWork, pairs: Vec<(String, String)>) { + match slot { + PendingWork::FullScan => {} + PendingWork::Tips(acc) => { + if acc.len().saturating_add(pairs.len()) > MAX_PENDING_TIP_PAIRS { + *slot = PendingWork::FullScan; + } else { + acc.extend(pairs); + } + } + } +} + +/// Guard owned by the detached encryption task for its repo key. Move-only — there +/// is no reason to clone a guard, and cloning would double-remove. Normal exit goes +/// through [`finish_or_take_pending`](Self::finish_or_take_pending); Drop is the +/// panic-path backstop only. pub struct EncryptInflightGuard { - repos: Arc>>, + repos: Arc>>, repo_id: String, + /// True until the normal-exit path removes the key. A disarmed guard's Drop is + /// a no-op: the key slot may already belong to a successor task admitted after + /// our removal, and removing THAT key would break at-most-one-task-per-repo. + armed: bool, +} + +impl EncryptInflightGuard { + /// The task's end-of-iteration step, one critical section: if coalesced work is + /// pending, take it and hand the still-armed guard back (key retained — iterate); + /// if nothing is pending, remove the key and disarm the guard (task exits; the + /// returned guard's Drop is inert). The atomicity is load-bearing both ways: a + /// push landing before this call is merged and therefore drained here; a push + /// landing after it finds the key gone and is admitted as a fresh task. No + /// interleaving can lose the work or admit two tasks for one repo. + pub fn finish_or_take_pending(mut self) -> FinishOutcome { + let mut map = self.repos.lock().expect("encrypt_inflight mutex poisoned"); + match map.get_mut(&self.repo_id) { + Some(PendingWork::Tips(acc)) if acc.is_empty() => { + map.remove(&self.repo_id); + self.armed = false; + drop(map); + FinishOutcome::Finished(self) + } + Some(slot) => { + let work = std::mem::replace(slot, PendingWork::Tips(Vec::new())); + drop(map); + FinishOutcome::Pending(self, work) + } + None => { + // Unreachable while armed (only this method removes a live key), + // but never panic in the release path: treat as finished. + self.armed = false; + drop(map); + FinishOutcome::Finished(self) + } + } + } } impl Drop for EncryptInflightGuard { fn drop(&mut self) { - // A poisoned lock (a prior panic while holding it) still lets us take the inner - // set via into_inner-on-guard; but poisoning here is not expected because the - // only critical sections are the O(1) ops above. Remove best-effort. - if let Ok(mut set) = self.repos.lock() { - set.remove(&self.repo_id); + // Normal exit disarmed us inside finish_or_take_pending's critical section; + // an armed drop means the task ended abnormally (panic-unwind, or a future + // code path that returns without finishing). Release the key so the repo is + // not permanently locked out, and log any pending work this loses — there + // is no sweep, so it stays lost until a later push re-walks the repo. + if !self.armed { + return; + } + // A poisoned lock is not expected (the critical sections above are small + // and panic-free); remove best-effort. + if let Ok(mut map) = self.repos.lock() { + match map.remove(&self.repo_id) { + Some(PendingWork::Tips(acc)) if !acc.is_empty() => tracing::warn!( + repo = %self.repo_id, + lost_tip_pairs = acc.len(), + "encryption task ended abnormally with coalesced pushes pending; \ + their pins/recovery copies are lost until a later push" + ), + Some(PendingWork::FullScan) => tracing::warn!( + repo = %self.repo_id, + "encryption task ended abnormally with a pending full-scan drain; \ + it is lost until a later push" + ), + _ => {} + } } } } diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index bcd0758e..75285f21 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -88,6 +88,23 @@ fn inv22_concurrency_gates_present_and_not_bypassed() { encrypt_inflight.try_begin to coalesce per repo (bound the outstanding-task set)" ); + // F5: coalescing must REQUEUE, not shed. The in-flight task pins only its own + // pre-spawn snapshot, so a coalesced push's tip pairs are recorded on its key and + // the task must loop-drain them (`finish_or_take_pending`) before releasing it. + // Removing the drain reverts to the silent loss: a coalesced push's pins and + // recovery copies are absent until an unrelated later push. Scan only the + // production half of the file — the u5 tests in its `mod tests` also name the + // drain call, and matching them would make this check vacuous. + let repos_production = repos + .split("#[cfg(test)]") + .next() + .expect("split always yields a first chunk"); + assert!( + repos_production.contains("finish_or_take_pending"), + "F5 gate missing: the post-push encryption task must loop-drain coalesced \ + pushes via finish_or_take_pending before releasing its repo key" + ); + // F4: every post-receive scan helper acquires a `git_encrypt_semaphore` permit // BEFORE its spawn_blocking git walk, so a push burst cannot accumulate unbounded // concurrent scans once the write permit is released. Structural check: within each From 48ed0a741f16904e15124033c0618480be3a9932 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 17 Jul 2026 20:16:04 -0500 Subject: [PATCH 43/95] fix(review): close the round's review findings on the #174 fix series object_type now distinguishes repo-level git failures (not-a-repository, error:-prefixed corruption) from genuine object absence, so a corrupt repo taints the /ipfs scan into the retryable 503 instead of vouching for a definitive 404; the drain reads visibility rules by the freshly fetched record id, closing a latent fail-open on id divergence. The four budget gates collapse into one stage-labelled helper, the three scan sites share state::acquire_scan_permit (inv22 tripwire repointed and sever-verified), and the Tigris stall tests use a local accept-and-park listener instead of a non-routable address. --- crates/gitlawb-node/src/api/ipfs.rs | 229 ++++++++++++++++------ crates/gitlawb-node/src/api/repos.rs | 50 ++--- crates/gitlawb-node/src/git/push_delta.rs | 21 +- crates/gitlawb-node/src/git/store.rs | 16 +- crates/gitlawb-node/src/state.rs | 31 +++ crates/gitlawb-node/tests/inv22_gates.rs | 33 +++- 6 files changed, 259 insertions(+), 121 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 28b9cc67..014411e7 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -182,14 +182,35 @@ pub async fn get_by_cid( } } - // Remaining request budget (F3), or None once exhausted. A stage is never - // started with zero remaining; the probe and read subprocesses carry no - // internal duration clamp, so the check before them is their entire bound - // (documented overshoot: at most one unclamped cat-file plus the in-flight - // clamped stage's kill/reap slack). - fn budget_remaining(deadline: std::time::Instant) -> Option { + // Budget gate shared by the four per-stage checks (F3): the remaining + // request budget, or — once exhausted — None, after logging the stage and + // knob and tainting "budget"; the call site only breaks (the scan STOPS, + // leaving this and every later candidate unproven, never a false 404). A + // stage is never started with zero remaining; the probe and read + // subprocesses carry no internal duration clamp, so this pre-start check is + // their entire bound (documented overshoot: at most one unclamped cat-file + // plus the in-flight clamped stage's kill/reap slack). The acquire and walk + // stages clamp their deadlines to the returned remainder. + fn budget_gate( + truncated_by: &mut Vec<&'static str>, + deadline: std::time::Instant, + budget_secs: u64, + repo_name: &str, + stage: &'static str, + ) -> Option { let left = deadline.saturating_duration_since(std::time::Instant::now()); - (!left.is_zero()).then_some(left) + if left.is_zero() { + tracing::warn!( + repo = %repo_name, + stage, + budget_secs, + "/ipfs request budget exhausted before the stage \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); stopping the scan without a verdict" + ); + taint(truncated_by, "budget"); + return None; + } + Some(left) } // Cap on EXPENSIVE walks only (F2): counts the repos that actually require the @@ -216,18 +237,15 @@ pub async fn get_by_cid( continue; } - // Budget gate for the acquire stage (F3): once the request budget is - // exhausted the scan STOPS, leaving this and every later candidate - // unproven (taint, never a false 404). Checked ahead of the visit + // Budget gate for the acquire stage (F3), checked ahead of the visit // bookkeeping so an unstarted acquire is not counted as a visit. - let Some(budget_left) = budget_remaining(request_deadline) else { - tracing::warn!( - repo = %repo.name, - budget_secs = state.config.ipfs_request_budget_secs, - "/ipfs request budget exhausted before the repo acquire \ - (GITLAWB_IPFS_REQUEST_BUDGET_SECS); stopping the scan without a verdict" - ); - taint(&mut truncated_by, "budget"); + let Some(budget_left) = budget_gate( + &mut truncated_by, + request_deadline, + state.config.ipfs_request_budget_secs, + &repo.name, + "repo acquire", + ) else { break; }; @@ -276,13 +294,15 @@ pub async fn get_by_cid( // Budget gate for the probe stage (F3). The probe subprocess has no // internal duration clamp, so this pre-start check is its entire bound. - if budget_remaining(request_deadline).is_none() { - tracing::warn!( - repo = %repo.name, - "/ipfs request budget exhausted before the object-type probe; \ - stopping the scan without a verdict" - ); - taint(&mut truncated_by, "budget"); + if budget_gate( + &mut truncated_by, + request_deadline, + state.config.ipfs_request_budget_secs, + &repo.name, + "object-type probe", + ) + .is_none() + { break; } @@ -311,14 +331,13 @@ pub async fn get_by_cid( // started walk runs its git children under a deadline clamped // to the remainder (the min below), so a walk can never // complete past the budget. - let Some(budget_left) = budget_remaining(request_deadline) else { - tracing::warn!( - repo = %repo.name, - budget_secs = state.config.ipfs_request_budget_secs, - "/ipfs request budget exhausted before the visibility walk \ - (GITLAWB_IPFS_REQUEST_BUDGET_SECS); stopping the scan without a verdict" - ); - taint(&mut truncated_by, "budget"); + let Some(budget_left) = budget_gate( + &mut truncated_by, + request_deadline, + state.config.ipfs_request_budget_secs, + &repo.name, + "visibility walk", + ) else { break; }; // Walk cap (F2), checked at the one site that actually spends a walk: @@ -394,13 +413,15 @@ pub async fn get_by_cid( // serving keeps the terminal arm honest and the stop unconditional; the // retryable 503 tells the caller to come back rather than letting an // over-budget request keep spending. - if budget_remaining(request_deadline).is_none() { - tracing::warn!( - repo = %repo.name, - "/ipfs request budget exhausted before the content read; \ - stopping the scan without a verdict" - ); - taint(&mut truncated_by, "budget"); + if budget_gate( + &mut truncated_by, + request_deadline, + state.config.ipfs_request_budget_secs, + &repo.name, + "content read", + ) + .is_none() + { break; } @@ -600,6 +621,24 @@ mod tests { .to_string() } + /// A local endpoint whose TCP accept succeeds instantly but that never writes + /// an HTTP response, so a Tigris HEAD against it stalls deterministically + /// until the caller's timeout. (A non-routable address hangs only if the + /// network blackholes the SYN — a fast RST would end the stall early.) The + /// accepted sockets are parked in the spawned task, which dies with the + /// test's runtime, so the peer never sees a close mid-test. + async fn silent_tigris_endpoint() -> String { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { + let mut held = Vec::new(); + while let Ok((sock, _)) = listener.accept().await { + held.push(sock); + } + }); + endpoint + } + /// Fake git for the WALK only (`state.git_bin`): empty refs, `rev-parse` /// resolves, and each `rev-list` appends one line to `log` and prints nothing — /// every walked repo yields an EMPTY allowed-set (path-gate deny verdict) and @@ -854,9 +893,9 @@ mod tests { } /// F2 acquire taint: a repo row with NO local copy over a Tigris backend that - /// stalls (non-routable endpoint — the connect just hangs) hits the 1s acquire - /// timeout at the read-acquire site. The skip carries no verdict, so the scan is - /// truncated: retryable 503 + Retry-After, never the old silent-skip 404. + /// stalls (a silent local endpoint — accepted, never answered) hits the 1s + /// acquire timeout at the read-acquire site. The skip carries no verdict, so the + /// scan is truncated: retryable 503 + Retry-After, never the old silent-skip 404. /// MUTATION (RED): drop the taint on the acquire-timeout arm and this decays to /// a 404. #[sqlx::test] @@ -866,12 +905,12 @@ mod tests { std::fs::create_dir_all(&repos_dir).unwrap(); let mut state = crate::test_support::test_state(pool.clone()).await; // Endpoint-pinned test client (no AWS_* env reads — env is racy under a - // parallel test run); 10.255.255.1 is non-routable so the HEAD hangs. - let tigris = crate::git::tigris::TigrisClient::for_testing_with_endpoint( - "test-bucket", - "http://10.255.255.1:9000", - ) - .await; + // parallel test run); the silent local endpoint stalls the HEAD + // deterministically. + let endpoint = silent_tigris_endpoint().await; + let tigris = + crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) + .await; state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; let mut cfg = (*state.config).clone(); @@ -911,9 +950,10 @@ mod tests { /// `object_type` is Err. That is not an absence verdict, so the scan is /// truncated: 503, never 404. A second, real repo probes clean (absent verdict) /// — the one bad row is what taints. NOTE: the probe shells to the real `git` - /// (not `state.git_bin`) and maps a NONZERO cat-file exit to `Ok(None)` (an - /// absent verdict), so a fake git exiting nonzero cannot reach this arm; only a - /// spawn failure is Err, hence the missing-dir recipe. MUTATION (RED): drop the + /// (not `state.git_bin`), and a clean missing/invalid-object nonzero exit is + /// still `Ok(None)` (an absent verdict) — this arm needs a probe that could + /// not RUN, hence the missing-dir spawn failure here; the corrupt-repo test + /// below drives the stderr-discriminated Err. MUTATION (RED): drop the /// taint on the probe-error arm and this decays to a 404. #[sqlx::test] async fn get_by_cid_probe_error_taints_scan_to_503(pool: sqlx::PgPool) { @@ -951,6 +991,76 @@ mod tests { ); } + /// F2 probe taint, corrupt-repo arm: a repo whose git dir EXISTS but is broken + /// (objects/ removed, HEAD garbage) makes the real `cat-file -t` die with the + /// repo-level `fatal: not a git repository` — a probe that could not examine + /// the object store, not an absence verdict, so `object_type` must map it to + /// Err and the scan must shed the probe-tainted 503, never the silent-absence + /// 404. A second, real repo probes clean (absent verdict) — the corrupt row is + /// what taints. MUTATION (RED): map every nonzero cat-file exit back to + /// `Ok(None)` in `object_type` (drop the stderr discrimination) and this + /// decays to a 404. + #[sqlx::test] + async fn get_by_cid_corrupt_repo_dir_probe_error_taints_scan_to_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // Older row: a real repo that probes clean. Newer row: a bare repo whose + // git dir exists on disk but is corrupt at the repo level. + seed_repo_with_blob(&state, tmp.path(), "z6f2corrupt", "real", b"probe clean\n").await; + state + .db + .upsert_mirror_repo("z6f2corrupt", "broken", "/unused-broken", None, false) + .await + .unwrap(); + let rec = state + .db + .get_repo("z6f2corrupt", "broken") + .await + .unwrap() + .unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + std::fs::create_dir_all(&bare).unwrap(); + run_git(&["init", "-q", "--bare", "--object-format=sha256"], &bare); + std::fs::remove_dir_all(bare.join("objects")).unwrap(); + std::fs::write(bare.join("HEAD"), b"junk\n").unwrap(); + + let peer: SocketAddr = "203.0.113.68:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a repo-level cat-file fatal leaves the repo unproven — the scan must \ + shed 503, not report the object absent" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("probe"), + "the shed must name the probe taint; got: {body}" + ); + } + /// F2 read taint: the gate passes (the probe reads the truncated loose object's /// intact "blob 64" header) but the content read fails (`cat-file blob` dies on /// the deflate stream cut mid-content) — the probe just said the object EXISTS @@ -1074,7 +1184,7 @@ mod tests { /// (`ipfs_request_budget_secs`) bounds the whole admitted scan; per-repo /// stages may not each draw a fresh timeout past it. Budget 1s, per-iteration /// acquire timeout 2s; the NEWER row is a Tigris-backed ghost (no local copy, - /// non-routable endpoint) whose acquire stalls, the OLDER row is a plain + /// silent local endpoint) whose acquire stalls, the OLDER row is a plain /// public repo carrying the blob. The ghost's acquire runs clamped to the ~1s /// remainder and times out; at the next repo the budget gate sees zero /// remaining, taints "budget", and STOPS the scan, so the blob repo is never @@ -1105,13 +1215,12 @@ mod tests { .await; // Swap in a Tigris-backed store over the SAME repos_dir (the seeded bare // repo stays a fast local hit) and add a NEWER ghost row with no local - // copy: its acquire consults the non-routable endpoint and stalls - // (endpoint-pinned test client, no AWS_* env reads; 10.255.255.1 hangs). - let tigris = crate::git::tigris::TigrisClient::for_testing_with_endpoint( - "test-bucket", - "http://10.255.255.1:9000", - ) - .await; + // copy: its acquire consults the silent local endpoint and stalls past + // the budget (endpoint-pinned test client, no AWS_* env reads). + let endpoint = silent_tigris_endpoint().await; + let tigris = + crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) + .await; state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); state .db diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 2b9cf1d4..2770ffb9 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -74,30 +74,14 @@ async fn replication_withheld_set( // that off the async worker thread. Some(rules) => { let owner_did = owner_did.to_string(); - // Scan admission (#174 F4): DEFER (await), never shed — dropping the - // walk would skip the vetting and fail the push's replication closed - // for no reason. Accepted residuals, stated honestly: (1) the park - // wait is queue-depth multiplied — post-receive tails are no longer - // admission-bounded once the write permit is released, so N landed - // pushes can queue N walks and the last waits N walk-durations; - // (2) a client-timeout disconnect while parked HERE drops the - // handler future and silently loses this push's replication work — - // the encrypt_inflight coalescing requeue does NOT cover it, because - // this park precedes the `try_begin` spawn gate. - let parked = std::time::Instant::now(); - let permit = encrypt_sem - .acquire_owned() - .await - .expect("git_encrypt_semaphore is never closed"); - tracing::debug!( - repo = %disk_path.display(), - queue_wait_ms = parked.elapsed().as_millis() as u64, - "post-push withheld walk admitted to the scan pool" - ); + // Scan admission (#174 F4): DEFER, never shed — dropping the walk + // would skip the vetting and fail the push's replication closed for + // no reason. Residuals at `acquire_scan_permit`. + let permit = + crate::state::acquire_scan_permit(encrypt_sem, &disk_path, "withheld walk").await; tokio::task::spawn_blocking(move || { // The permit lives inside the blocking closure: a started walk - // always completes holding it (a disconnect cannot cancel - // spawn_blocking or leak the permit mid-walk). + // always completes holding it. let _permit = permit; crate::git::visibility_pack::withheld_blob_oids_bounded( &disk_path, &git_bin, timeout, &rules, is_public, &owner_did, None, @@ -137,9 +121,7 @@ async fn replication_withheld_set( /// /// Always walks (there is no no-git arm), so the whole blocking scan runs under /// one `git_encrypt_semaphore` admission permit (#174 F4) — see -/// `replication_withheld_set`'s acquire for the defer rationale and the honest -/// residuals (queue-depth-multiplied park wait; a disconnect while parked loses -/// this push's replication work, uncovered by the coalescing requeue). +/// `acquire_scan_permit` for the defer rationale and the honest residuals. #[allow(clippy::too_many_arguments)] async fn fail_closed_full_scan_objects( encrypt_sem: std::sync::Arc, @@ -153,16 +135,8 @@ async fn fail_closed_full_scan_objects( ) -> Vec { // Scan admission (#174 F4): DEFER, never shed; the permit moves into the // closure so a started scan always completes holding it. - let parked = std::time::Instant::now(); - let permit = encrypt_sem - .acquire_owned() - .await - .expect("git_encrypt_semaphore is never closed"); - tracing::debug!( - repo = %disk_path.display(), - queue_wait_ms = parked.elapsed().as_millis() as u64, - "post-push fail-closed full scan admitted to the scan pool" - ); + let permit = + crate::state::acquire_scan_permit(encrypt_sem, &disk_path, "fail-closed full scan").await; tokio::task::spawn_blocking(move || -> anyhow::Result> { let _permit = permit; let allowed = crate::git::visibility_pack::replicable_blob_set_bounded( @@ -925,7 +899,11 @@ async fn resolve_drain_object_list( return None; } }; - let rules_opt = ctx.db.list_visibility_rules(&ctx.repo_id).await.ok(); + // record.id, never the spawn-time ctx.repo_id: the record above is re-fetched + // fresh by owner/name, and a delete+re-create between spawn and drain gives + // the row a NEW id — rules read against the stale id come back empty and + // would fail open for the new row. + let rules_opt = ctx.db.list_visibility_rules(&record.id).await.ok(); let (_announce, withheld) = replication_withheld_set( ctx.encrypt_sem.clone(), rules_opt.clone(), diff --git a/crates/gitlawb-node/src/git/push_delta.rs b/crates/gitlawb-node/src/git/push_delta.rs index 27064357..0b569693 100644 --- a/crates/gitlawb-node/src/git/push_delta.rs +++ b/crates/gitlawb-node/src/git/push_delta.rs @@ -315,21 +315,12 @@ pub async fn resolve_candidates_for_push( full_scan: false, }; } - // Scan admission (#174 F4): DEFER (await), never shed — a dropped scan would - // silently under-pin this push. The permit moves into the blocking closure so - // a started scan always completes holding it. Residuals as documented at - // `replication_withheld_set`'s acquire: park wait is queue-depth multiplied, - // and a client disconnect while parked loses this push's replication work. - let parked = Instant::now(); - let permit = scan_sem - .acquire_owned() - .await - .expect("git_encrypt_semaphore is never closed"); - tracing::debug!( - repo = %repo_path.display(), - queue_wait_ms = parked.elapsed().as_millis() as u64, - "pin-candidate scan admitted to the scan pool" - ); + // Scan admission (#174 F4): DEFER, never shed — a dropped scan would + // silently under-pin this push. The permit moves into the blocking closure + // so a started scan always completes holding it. Residuals at + // `acquire_scan_permit`. + let permit = + crate::state::acquire_scan_permit(scan_sem, &repo_path, "pin-candidate scan").await; tokio::task::spawn_blocking(move || { let _permit = permit; // ONE shared deadline for the whole scan, per jatmn ("the same deadline"). diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index 229ee695..5697c229 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -271,7 +271,8 @@ pub struct TreeEntry { /// `/ipfs/` is computed from these same content bytes via /// `gitlawb_core::cid::Cid::from_git_object_bytes`. /// -/// Get just the object type. Returns `None` if the object doesn't exist. +/// Get just the object type. Returns `None` if the object doesn't exist; a +/// probe that could not examine the object store is `Err`, never `None`. pub fn object_type(repo_path: &Path, sha256_hex: &str) -> Result> { let type_output = Command::new("git") .args(["cat-file", "-t", sha256_hex]) @@ -280,6 +281,19 @@ pub fn object_type(repo_path: &Path, sha256_hex: &str) -> Result> .context("failed to run git cat-file -t")?; if !type_output.status.success() { + // A nonzero exit is an ABSENCE verdict only when git could examine the + // object store: missing-object and invalid-oid probes die with a single + // clean `fatal:` line. A broken repo dir (`fatal: not a git repository`) + // or a corrupt object (`error: inflate` / `error: unable to unpack` + // lines before the fatal) proves nothing about absence, so it must + // surface as Err — the /ipfs scan taints on Err rather than treating + // the repo as probed-clean. + let stderr = String::from_utf8_lossy(&type_output.stderr); + if stderr.contains("not a git repository") + || stderr.lines().any(|l| l.starts_with("error:")) + { + bail!("git cat-file -t failed: {}", stderr.trim()); + } return Ok(None); } diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 4f8ffb0e..c4d2b079 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -443,3 +443,34 @@ impl Drop for EncryptInflightGuard { } } } + +/// Admit a post-receive git scan to the shared `git_encrypt_semaphore` pool +/// (#174 F4): DEFER (await), never shed — a dropped scan would lose the push's +/// recovery copy or silently under-pin it. The returned permit must move into +/// the blocking closure so a started scan always completes holding it (a +/// disconnect cannot cancel `spawn_blocking` or leak the permit mid-walk). +/// Accepted residuals, stated once for every caller: (1) the park wait is +/// queue-depth multiplied — post-receive tails are no longer admission-bounded +/// once the write permit is released, so N landed pushes can queue N scans and +/// the last waits N scan-durations; (2) a client-timeout disconnect while +/// parked HERE drops the handler future and silently loses this push's +/// replication work — the `encrypt_inflight` coalescing requeue does NOT cover +/// it, because this park precedes the `try_begin` spawn gate. +pub async fn acquire_scan_permit( + scan_sem: Arc, + repo: &std::path::Path, + stage: &'static str, +) -> tokio::sync::OwnedSemaphorePermit { + let parked = std::time::Instant::now(); + let permit = scan_sem + .acquire_owned() + .await + .expect("git_encrypt_semaphore is never closed"); + tracing::debug!( + repo = %repo.display(), + stage, + queue_wait_ms = parked.elapsed().as_millis() as u64, + "post-receive scan admitted to the scan pool" + ); + permit +} diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index 75285f21..cbfa0d4c 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -105,12 +105,25 @@ fn inv22_concurrency_gates_present_and_not_bypassed() { pushes via finish_or_take_pending before releasing its repo key" ); - // F4: every post-receive scan helper acquires a `git_encrypt_semaphore` permit - // BEFORE its spawn_blocking git walk, so a push burst cannot accumulate unbounded - // concurrent scans once the write permit is released. Structural check: within each - // helper the first `acquire_owned` precedes the first `spawn_blocking`. Removing a - // gate pushes the next `acquire_owned` occurrence past the helper's own - // `spawn_blocking` (or off the end of the file), turning the assertion red. + // F4: every post-receive scan helper admits itself to the shared scan pool via + // `crate::state::acquire_scan_permit` BEFORE its spawn_blocking git walk, so a + // push burst cannot accumulate unbounded concurrent scans once the write permit + // is released. Two halves, both load-bearing: the helper body must actually + // acquire the pool (state.rs sits the helper at the end of the file, so the + // definition tail contains no other `acquire_owned` to match vacuously), and + // within each scan helper the first qualified call precedes the first + // `spawn_blocking`. Severing a call site pushes the next occurrence past the + // helper's own `spawn_blocking` (or off the end of the file), turning the + // assertion red; comments name the helper without the `crate::state::` prefix, + // so this targets the real call sites. + let state_rs = src("state.rs"); + let helper_def = state_rs + .find("fn acquire_scan_permit") + .expect("F4 gate missing: state.rs no longer defines acquire_scan_permit"); + assert!( + state_rs[helper_def..].contains("acquire_owned"), + "F4 gate gutted: acquire_scan_permit must acquire the scan pool via acquire_owned" + ); let push_delta = src("git/push_delta.rs"); for (file_src, file, helper) in [ (&repos, "api/repos.rs", "fn replication_withheld_set"), @@ -125,9 +138,11 @@ fn inv22_concurrency_gates_present_and_not_bypassed() { .find(helper) .unwrap_or_else(|| panic!("{file}: `{helper}` not found")); let tail = &file_src[start..]; - let acquire = tail.find("acquire_owned").unwrap_or_else(|| { - panic!("F4 gate missing: {file} `{helper}` no longer acquires a scan permit") - }); + let acquire = tail + .find("crate::state::acquire_scan_permit(") + .unwrap_or_else(|| { + panic!("F4 gate missing: {file} `{helper}` no longer acquires a scan permit") + }); let spawn = tail.find("spawn_blocking").unwrap_or_else(|| { panic!("{file}: `{helper}` lost its spawn_blocking walk — update this guard") }); From b6d835d61228c01e7055ba6b9c6188e711d3a6e6 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 17 Jul 2026 20:42:48 -0500 Subject: [PATCH 44/95] fix(review): state the probe/read stages' unbounded-hang residual honestly; net the acquire-taint found path The budget-knob docs claimed the overshoot was bounded by one unclamped probe; an unclamped subprocess is not a bound, so the README, .env.example, config help, and module comments now say plainly that a hung git probe holds the walk slot for the hang's duration. New regression: an acquire-tainting ghost row ahead of a healthy public copy must not stop the scan; the later found short-circuit still serves 200. --- .env.example | 12 ++-- README.md | 2 +- crates/gitlawb-node/src/api/ipfs.rs | 102 ++++++++++++++++++++++++++-- crates/gitlawb-node/src/config.rs | 12 ++-- 4 files changed, 111 insertions(+), 17 deletions(-) diff --git a/.env.example b/.env.example index 94685069..8f9bfa2e 100644 --- a/.env.example +++ b/.env.example @@ -180,11 +180,13 @@ GITLAWB_IPFS_MAX_REPOS_WALKED=64 # Default 1024. GITLAWB_IPFS_MAX_REPO_VISITS=1024 # Absolute wall-clock budget for one admitted /ipfs request's acquire+walk -# lifetime (all stages of the whole scan). Bounds how LONG one request may hold -# its walk slot: per-stage timeouts are clamped to the remaining budget and no -# stage starts once it is exhausted (the scan then sheds a retryable 503). -# Residual overshoot: the in-flight clamped stage's kill/reap slack plus one -# unclamped cat-file probe subprocess. Default 600. +# lifetime (all stages of the whole scan). Per-stage clamps bound the acquire +# and walk stages to the remaining budget, and no stage starts once it is +# exhausted (the scan then sheds a retryable 503). The object-type probe and +# content-read cat-file subprocesses are budget-checked before starting but +# have NO duration bound of their own: a hung git probe (corrupt pack, stuck +# filesystem) holds the request's walk slot for the full duration of the hang. +# Default 600. GITLAWB_IPFS_REQUEST_BUDGET_SECS=600 # Max /ipfs/{cid} requests per client IP per hour (route flood brake, distinct # from the concurrency caps above). 0 disables. Default 600. diff --git a/README.md b/README.md index bc338339..cc32bcf7 100644 --- a/README.md +++ b/README.md @@ -349,7 +349,7 @@ Important node settings: | `GITLAWB_IPFS_WALK_PER_SOURCE` | Max concurrent `/ipfs` walks a single source IP may hold. Default 4. | | `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max expensive path-scope visibility walks per `/ipfs/{cid}` request; over-cap repos are skipped and the scan continues, shedding a retryable 503 (not a false 404) if the object is then found nowhere. Default 64. | | `GITLAWB_IPFS_MAX_REPO_VISITS` | Ceiling on repos one `/ipfs/{cid}` request may visit (acquire + probe) past the visibility gate — also the worst-case per-request Tigris fetch count. On exhaustion the scan stops with a retryable 503. Default 1024. | -| `GITLAWB_IPFS_REQUEST_BUDGET_SECS` | Absolute wall-clock budget for one admitted `/ipfs/{cid}` request's acquire+walk lifetime. Per-stage timeouts are clamped to the remaining budget and no stage starts once it is exhausted; the scan then stops with a retryable 503. Overshoot is bounded by the in-flight clamped stage's kill/reap slack plus one unclamped `cat-file` probe. Default 600. | +| `GITLAWB_IPFS_REQUEST_BUDGET_SECS` | Absolute wall-clock budget for one admitted `/ipfs/{cid}` request's acquire+walk lifetime. Per-stage clamps bound the acquire and walk stages to the remaining budget, and no stage starts once it is exhausted; the scan then stops with a retryable 503. The object-type probe and content-read `cat-file` subprocesses are budget-checked before starting but have no duration bound of their own, so a hung git probe (corrupt pack, stuck filesystem) holds the request's walk slot for the full duration of the hang. Default 600. | | `GITLAWB_IPFS_RATE_LIMIT` | Max `/ipfs/{cid}` requests per client IP per hour (route flood brake). 0 disables. Default 600. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 014411e7..41080833 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -64,9 +64,9 @@ use crate::visibility::{visibility_check, Decision}; /// Request budget (F3): one absolute clock (`ipfs_request_budget_secs`) spans /// the whole admitted request. No stage (acquire, probe, walk, content read) /// starts once it is exhausted, and the acquire wait and walk deadline are -/// clamped to the remainder, so an admitted request cannot hold its scarce walk -/// slot past the budget plus the in-flight stage's kill/reap slack and one -/// unclamped probe subprocess. +/// clamped to the remainder. The probe and content-read subprocesses have no +/// duration bound of their own past their pre-start budget check, so a hung +/// git probe holds the request's walk slot for the full duration of the hang. /// /// Scope: this closes the direct unauthenticated scan, including the dangling /// case. A stale-public mirror row still serves withheld content (tracked @@ -188,9 +188,9 @@ pub async fn get_by_cid( // leaving this and every later candidate unproven, never a false 404). A // stage is never started with zero remaining; the probe and read // subprocesses carry no internal duration clamp, so this pre-start check is - // their entire bound (documented overshoot: at most one unclamped cat-file - // plus the in-flight clamped stage's kill/reap slack). The acquire and walk - // stages clamp their deadlines to the returned remainder. + // their entire bound (a hung one holds the request's walk slot for the + // duration of the hang). The acquire and walk stages clamp their deadlines + // to the returned remainder. fn budget_gate( truncated_by: &mut Vec<&'static str>, deadline: std::time::Instant, @@ -944,6 +944,96 @@ mod tests { ); } + /// F2 found-beats-taint on the acquire arm: an acquire timeout taints the + /// scan but must NOT stop it — the loop `continue`s, and a later repo that + /// genuinely carries the object still serves. The NEWER row (visited first + /// under `list_all_repos`' updated_at DESC) is a Tigris-backed ghost whose + /// acquire stalls against the silent endpoint and times out at 1s; the + /// OLDER row is a plain public repo carrying the blob, reached next and + /// served from a cheap probe — found beats taint: 200 with the blob bytes, + /// never the truncation 503. MUTATION (RED): turn the acquire-timeout arm's + /// `continue` into a `break` and the public copy never serves (503). + #[sqlx::test] + async fn get_by_cid_acquire_taint_does_not_block_later_public_copy(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Seed the blob repo through a LOCAL-ONLY store first, so seeding never + // consults the (deliberately unreachable) Tigris endpoint. + state.repo_store = + crate::git::repo_store::RepoStore::for_testing(repos_dir.clone(), pool.clone()); + let content = b"acquire taint continue proof\n"; + let (_, oid) = + seed_repo_with_blob(&state, tmp.path(), "z6f2acqcont", "pubcopy", content).await; + // Swap in a Tigris-backed store over the SAME repos_dir (the seeded bare + // repo stays a fast local hit) and add a NEWER ghost row with no local + // copy: its acquire consults the silent local endpoint and stalls to the + // 1s timeout (endpoint-pinned test client, no AWS_* env reads). + let endpoint = silent_tigris_endpoint().await; + let tigris = + crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) + .await; + state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); + state + .db + .upsert_mirror_repo("z6f2acqcont", "ghost", "/unused-ghost", None, false) + .await + .unwrap(); + let mut cfg = (*state.config).clone(); + cfg.git_acquire_timeout_secs = 1; + state.config = Arc::new(cfg); + + // Ordering precondition: the ghost must be iterated FIRST (updated_at + // DESC — it was upserted after the blob repo), otherwise the pubcopy + // would serve before the taint ever fires and the continue-vs-break + // distinction would go untested. + let order: Vec = state + .db + .list_all_repos() + .await + .unwrap() + .into_iter() + .map(|r| r.name) + .collect(); + let ghost_pos = order.iter().position(|n| n == "ghost").unwrap(); + let pub_pos = order.iter().position(|n| n == "pubcopy").unwrap(); + assert!( + ghost_pos < pub_pos, + "precondition: the stalling ghost must be iterated before the blob repo; got {order:?}" + ); + + let peer: SocketAddr = "203.0.113.73:5000".parse().unwrap(); + let started = std::time::Instant::now(); + let resp = ipfs_router(state) + .oneshot(get_cid(&cid_for_oid(&oid), Some(peer))) + .await + .unwrap(); + // The taint arm demonstrably FIRED on this run: the response can only + // arrive after the ghost's stalled acquire burned its full 1s timeout + // (a cheap skip or a deny verdict would answer near-instantly). + assert!( + started.elapsed() >= std::time::Duration::from_millis(900), + "the ghost's acquire must stall to its timeout before the scan continues; \ + got {:?}", + started.elapsed() + ); + assert_eq!( + resp.status(), + StatusCode::OK, + "an acquire taint must not stop the scan — the later public copy serves" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + assert_eq!( + &body[..], + content.as_slice(), + "the served body must be the blob content from the later public copy" + ); + } + /// F2 probe taint: a repo row whose local dir does not exist (no Tigris) — /// `RepoStore::acquire` returns the path anyway (local passthrough), and the /// `cat-file -t` probe cannot even spawn (missing working dir), so diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index d564a554..87c5d65c 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -420,11 +420,13 @@ pub struct Config { /// acquire wait and walk deadline are clamped to `min(their own timeout, /// remaining budget)`; a stage is never started with zero remaining. On /// exhaustion the scan stops without a verdict and the request sheds a - /// retryable 503 + Retry-After rather than a false 404. Residual overshoot - /// past the budget is bounded by the kill/reap slack of the one in-flight - /// clamped stage (the walk watchdog's SIGTERM grace + SIGKILL settle) plus - /// the `object_type` / `read_object_content` probe subprocesses, which are - /// budget-checked before they start but carry no internal duration clamp. + /// retryable 503 + Retry-After rather than a false 404. The clamps bound + /// only the acquire and walk stages (overshoot there is the walk watchdog's + /// SIGTERM grace + SIGKILL settle); the `object_type` / + /// `read_object_content` probe subprocesses are budget-checked before they + /// start but have NO duration bound of their own, so a hung git probe + /// (corrupt pack, stuck filesystem) holds the request's walk slot for the + /// full duration of the hang. /// Must be positive. Default: 600s (10 min), matching /// `git_service_timeout_secs` so a single full-length walk still fits. #[arg( From 8d79692555758496126370814579b5b9ec0ff8be Mon Sep 17 00:00:00 2001 From: t Date: Sat, 18 Jul 2026 09:51:10 -0500 Subject: [PATCH 45/95] fix(node): make write-acquire cancellation-safe and connection-affine (#174 F1) acquire_write took the pg advisory lock, then awaited Tigris, then built the guard. An outer tokio::time::timeout firing during the Tigris await dropped the future with the lock held and no guard, so no unlock ran and later pushes to the repo spun 60s. release() also unlocked on an arbitrary pooled connection, not the one that locked (session advisory locks are connection-affine), so even the normal release was unreliable. Pin a PoolConnection in RepoWriteGuard, build the guard before the lock query, run the lock loop through it, unlock on that same connection in release(), and add a Drop backstop that spawns a detached affine unlock (Handle::try_current so an off-runtime drop logs instead of panicking). Two distinct-session tests (drop-without-release, affine release) go RED without the fix, GREEN with it. Pinning a connection per active write is a DB-pool DoS unless the pool clears the concurrent-write cap: the shipped default (20) was below max_concurrent_git_pushes (32). Raise the db_max_connections default to 48 and add Config::validate() to reject db_max_connections < max_concurrent_git_pushes + 8 headroom at boot; wire it into startup. --- crates/gitlawb-node/src/config.rs | 76 ++++++++- crates/gitlawb-node/src/git/repo_store.rs | 199 +++++++++++++++++++--- crates/gitlawb-node/src/main.rs | 7 + 3 files changed, 259 insertions(+), 23 deletions(-) diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 87c5d65c..5fb66b55 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -206,11 +206,15 @@ pub struct Config { /// Maximum connections in the PostgreSQL pool. This is a cap, not a floor /// (connections open lazily). Size against the database server's - /// max_connections, remembering admin tooling opens its own pool. + /// max_connections, remembering admin tooling opens its own pool. Each + /// concurrent write pins one pooled connection for its whole duration (the + /// advisory lock in `repo_store::acquire_write` is connection-affine), so this + /// must exceed `max_concurrent_git_pushes` by `DB_POOL_APP_HEADROOM` or slow + /// pushes starve every other DB path — enforced by `Config::validate`. #[arg( long, env = "GITLAWB_DB_MAX_CONNECTIONS", - default_value_t = 20, + default_value_t = 48, value_parser = clap::value_parser!(u32).range(1..) )] pub db_max_connections: u32, @@ -461,6 +465,36 @@ impl Config { } PathBuf::from(&self.key_path) } + + /// DB connections reserved for everything other than held write-locks: auth + /// lookups, visibility-rule reads, the post-receive tail's own DB writes, and + /// admin tooling. A write pins one pooled connection for its whole duration, so + /// the pool must clear the concurrent-write cap by at least this margin. + pub const DB_POOL_APP_HEADROOM: u32 = 8; + + /// Cross-field boot validation. Single-field ranges are enforced by clap; this + /// catches combinations that ship a denial-of-service under otherwise-valid + /// values. Call once at startup and fail fast on `Err`. + pub fn validate(&self) -> Result<(), String> { + // A write pins one pooled connection for its whole duration (the + // connection-affine advisory lock in repo_store::acquire_write), and + // concurrent writes are capped at max_concurrent_git_pushes. If the pool + // does not exceed that cap by DB_POOL_APP_HEADROOM, a burst of slow pushes + // drains every connection and every other DB path 503s. (#174 F1) + let floor = (self.max_concurrent_git_pushes as u64) + (Self::DB_POOL_APP_HEADROOM as u64); + if (self.db_max_connections as u64) < floor { + return Err(format!( + "GITLAWB_DB_MAX_CONNECTIONS ({}) must be at least max_concurrent_git_pushes ({}) \ + + {} headroom = {}: each concurrent write pins one pooled connection for its whole \ + duration, so a smaller pool lets a burst of slow pushes starve every other DB path.", + self.db_max_connections, + self.max_concurrent_git_pushes, + Self::DB_POOL_APP_HEADROOM, + floor + )); + } + Ok(()) + } } #[cfg(test)] @@ -666,4 +700,42 @@ mod tests { 1_048_576 ); } + + /// #174 F1: a connection-affine write lock pins a pooled connection per + /// concurrent write, so the pool must clear `max_concurrent_git_pushes` by + /// `DB_POOL_APP_HEADROOM` or a push burst starves every other DB path. + /// `validate()` must reject an under-sized pool at boot. + #[test] + fn db_pool_must_clear_the_git_push_cap() { + // Shipped defaults validate (48 >= 32 + 8). + Config::parse_from(["gitlawb-node"]) + .validate() + .expect("default config must validate"); + + // An under-sized pool relative to the push cap is rejected (20 < 32 + 8). + let under = Config::parse_from([ + "gitlawb-node", + "--db-max-connections", + "20", + "--max-concurrent-git-pushes", + "32", + ]); + assert!( + under.validate().is_err(), + "db_max_connections 20 below max_concurrent_git_pushes 32 + headroom must be rejected" + ); + + // Exactly at the floor validates (40 == 32 + 8). + let at_floor = Config::parse_from([ + "gitlawb-node", + "--db-max-connections", + "40", + "--max-concurrent-git-pushes", + "32", + ]); + assert!( + at_floor.validate().is_ok(), + "db_max_connections at the floor (pushes + headroom) must validate" + ); + } } diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index a5c367e9..86a1ee8b 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -13,7 +13,8 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use anyhow::{Context, Result}; -use sqlx::PgPool; +use sqlx::pool::PoolConnection; +use sqlx::{PgPool, Postgres}; use tokio::sync::Mutex; use tracing::{debug, info, warn}; @@ -157,13 +158,42 @@ impl RepoStore { let (owner_slug, local_path) = self.local_path(owner_did, repo_name)?; let lock_key = advisory_lock_key(&owner_slug, repo_name); - // Acquire Postgres advisory lock with retry using pg_try_advisory_lock - // to avoid blocking indefinitely on stale locks from crashed connections. + // Pin a dedicated pooled connection and build the guard holding it BEFORE + // issuing the lock query. Session-level pg advisory locks are + // connection-affine (they can only be released on the session that took + // them), so the guard must own the locking connection; and building the + // guard first means any cancellation after the lock is taken — a + // `tokio::time::timeout` firing during the Tigris download below — drops a + // guard that CAN release, closing the leak the outer timeout otherwise + // opened (#174 F1). + let conn = self + .pool + .acquire() + .await + .context("acquiring db connection for the write advisory lock")?; + let mut guard = RepoWriteGuard { + owner_slug: owner_slug.clone(), + repo_name: repo_name.to_string(), + local_path: local_path.clone(), + lock_key, + conn: Some(conn), + locked: false, + released: false, + tigris: self.tigris.clone(), + }; + + // Acquire the advisory lock with retry, through the guard's OWN connection, + // so the matching unlock (in release, or the Drop backstop) runs on the same + // session — pg_advisory_unlock on a different pooled connection is a no-op. let mut acquired = false; for attempt in 0..60 { + let c = guard + .conn + .as_deref_mut() + .expect("write guard holds its connection during acquisition"); let row: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") .bind(lock_key) - .fetch_one(&self.pool) + .fetch_one(&mut *c) .await .context("trying advisory lock")?; if row.0 { @@ -177,9 +207,11 @@ impl RepoStore { if !acquired { anyhow::bail!("could not acquire advisory lock after 60s — possible stale lock for {owner_slug}/{repo_name}"); } + guard.locked = true; - // Always download the latest from Tigris before writing. - // Local disk may be stale if another machine pushed since our last access. + // Always download the latest from Tigris before writing. Local disk may be + // stale if another machine pushed since our last access. The guard already + // owns the lock + its connection, so a cancellation here drops through Drop. if let Some(ref tigris) = self.tigris { if tigris.exists(&owner_slug, repo_name).await.unwrap_or(false) { debug!(repo = %repo_name, "write acquire: downloading latest from tigris"); @@ -197,14 +229,7 @@ impl RepoStore { } } - Ok(RepoWriteGuard { - owner_slug, - repo_name: repo_name.to_string(), - local_path, - lock_key, - pool: self.pool.clone(), - tigris: self.tigris.clone(), - }) + Ok(guard) } /// Initialize a new bare repo on local disk and upload to Tigris. @@ -354,7 +379,16 @@ pub struct RepoWriteGuard { repo_name: String, pub local_path: PathBuf, lock_key: i64, - pool: PgPool, + /// The pooled connection that took the advisory lock. Session-level pg + /// advisory locks are connection-affine, so the guard pins that connection + /// for its whole lifetime and unlocks on it (in `release`, or the `Drop` + /// backstop). `None` only after the connection has been taken to unlock. + conn: Option>, + /// Set once the advisory lock has actually been taken. A guard dropped + /// before the lock is held (or after `release`) performs no unlock. + locked: bool, + /// Set once `release` has run its unlock, making the `Drop` backstop inert. + released: bool, tigris: Option, } @@ -369,7 +403,7 @@ impl RepoWriteGuard { /// half-applied or otherwise inconsistent repo would propagate corruption to /// Tigris (and to every node that later downloads it). The lock is always /// released regardless, to avoid stale locks blocking future writes. - pub async fn release(self, success: bool) { + pub async fn release(mut self, success: bool) { // Upload to Tigris only on success. if success { if let Some(ref tigris) = self.tigris { @@ -384,11 +418,57 @@ impl RepoWriteGuard { warn!(repo = %self.repo_name, "write failed — skipping tigris upload to avoid propagating an inconsistent repo"); } - // Release advisory lock - let _ = sqlx::query("SELECT pg_advisory_unlock($1)") - .bind(self.lock_key) - .execute(&self.pool) - .await; + // Release the advisory lock on the SAME connection that took it (session + // advisory locks are connection-affine). Taking the connection here makes + // the Drop backstop inert. + if self.locked { + if let Some(mut conn) = self.conn.take() { + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(self.lock_key) + .execute(&mut *conn) + .await; + } + } + self.released = true; + } +} + +impl Drop for RepoWriteGuard { + /// Cancellation-safe backstop: if the guard is dropped while still holding the + /// advisory lock (a `tokio::time::timeout` cancelled `acquire_write`, or a + /// handler future was dropped before `release`), unlock on the pinned + /// connection. `Drop` cannot await, so spawn a detached unlock — it runs on the + /// same session (connection-affine). An off-runtime drop falls back to a log; + /// the ~60s stale-lock retry loop in `acquire_write` reclaims it. + fn drop(&mut self) { + if self.released || !self.locked { + return; + } + let Some(mut conn) = self.conn.take() else { + return; + }; + let lock_key = self.lock_key; + let repo_name = self.repo_name.clone(); + match tokio::runtime::Handle::try_current() { + Ok(handle) => { + handle.spawn(async move { + if let Err(e) = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(lock_key) + .execute(&mut *conn) + .await + { + warn!(repo = %repo_name, err = %e, "detached advisory-unlock on write-guard drop failed"); + } + }); + } + Err(_) => { + warn!( + repo = %repo_name, + "RepoWriteGuard dropped off a Tokio runtime; advisory lock not released \ + synchronously — the stale-lock retry loop will reclaim it" + ); + } + } } } @@ -562,4 +642,81 @@ mod tests { ); } } + + // ── advisory-lock cancellation-safety (#174 F1, RED-before/GREEN-after) ── + + /// F1 (P1): dropping a `RepoWriteGuard` WITHOUT calling `release()` — the + /// state a `tokio::time::timeout` cancellation leaves `acquire_write` in when + /// it fires during the Tigris await — must still release the session advisory + /// lock. A checker connection is held OUT of the pool first, so `acquire_write` + /// is forced onto a distinct session; the checker (a different session) then + /// probes the lock, so advisory-lock re-entrancy cannot mask a leak. + /// + /// Load-bearing: RED today (no `Drop` releases the lock → held by + /// `acquire_write`'s session → checker's `pg_try_advisory_lock` returns + /// false). GREEN after the connection-affine `Drop` backstop. + #[sqlx::test] + async fn write_guard_drop_without_release_frees_the_lock(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone()); + let owner = "did:key:z6MkDropBackstopProofAAAAAAAAAAAAAAAAAAAAAA"; + let name = "leaktest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + // Distinct session for the probe: hold it out of the pool BEFORE acquiring, + // so acquire_write cannot use it and a re-entrant probe cannot falsely read free. + let mut checker = pool.acquire().await.expect("checker connection"); + + let guard = store.acquire_write(owner, name).await.expect("acquire"); + // The cancellation shape: drop without release(). + drop(guard); + // Let the detached unlock task run. + tokio::time::sleep(std::time::Duration::from_millis(400)).await; + + let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *checker) + .await + .unwrap(); + assert!( + free, + "advisory lock must be released when the guard is dropped without release()" + ); + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } + + /// F1 (latent non-affine release): `release()` must unlock on the SAME + /// session that locked, so the lock is freed regardless of which pooled + /// connection would service a fresh query. Observed from a distinct session. + #[sqlx::test] + async fn write_guard_release_frees_the_lock_from_a_distinct_session(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone()); + let owner = "did:key:z6MkAffineReleaseProofBBBBBBBBBBBBBBBBBBBB"; + let name = "affinetest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + let mut checker = pool.acquire().await.expect("checker connection"); + let guard = store.acquire_write(owner, name).await.expect("acquire"); + guard.release(false).await; + + let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *checker) + .await + .unwrap(); + assert!( + free, + "release() must free the advisory lock via connection-affine unlock" + ); + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } } diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index ffac65a2..b7bc758e 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -74,6 +74,13 @@ async fn main() -> Result<()> { // bootstrap peers. Operators can opt out via GITLAWB_BOOTSTRAP_DISABLE_SEEDS. bootstrap::merge_seeds(&mut config); + // Fail fast on config combinations that are individually in-range but jointly + // unsafe — notably a DB pool too small for the concurrent-write cap, which + // would let a push burst starve every other DB path (#174 F1). + config + .validate() + .map_err(|e| anyhow::anyhow!("invalid configuration: {e}"))?; + if !config.public_read { warn!( "GITLAWB_PUBLIC_READ=false is reserved; per-repository private-read enforcement is not wired in alpha" From 65d7af2a741a6c8104a1463da02c6fd1d3cdddef Mon Sep 17 00:00:00 2001 From: t Date: Sat, 18 Jul 2026 10:14:27 -0500 Subject: [PATCH 46/95] fix(node): own the post-receive replication tail in a detached task (#174 F2) The post-receive replication tail (withheld/candidate/full-scan resolution, then the encrypt+recovery spawn and the announce spawn) ran inline in the request future and parked on git_encrypt_semaphore before the durable try_begin gate. A client/proxy disconnect while parked dropped the future and silently lost this push's pins, recovery copy, and announcements, with no reconciliation sweep to recover them (state.rs documented this exact residual). Move the whole tail into an independently owned tokio task and return the git response immediately, so a disconnect can no longer drop the parked work. Each push owns its own tail, including its own always-spawned announce, so the announce is never brought under the per-repo encrypt coalescing (which would drop a coalesced push's per-ref announcements). Update state.rs to retire the disconnect-loses-work residual. Test: receive_pack_landed_push_returns_without_parking_on_scan_pool asserts the handler returns 200 while the scan pool is held (RED with the tail inline: the request future parks and the timeout fires; GREEN detached). The burst serialization test is retimed to poll for scan completion since the withheld walk is now detached too. --- crates/gitlawb-node/src/api/repos.rs | 645 ++++++++++++++------------- crates/gitlawb-node/src/state.rs | 14 +- 2 files changed, 352 insertions(+), 307 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 2770ffb9..98e395f6 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1645,314 +1645,336 @@ pub async fn git_receive_pack( } } - // Replication enforcement (Phase 2): decide once per push whether the public - // may read this repo at all and, if so, which blob OIDs must not leave the - // node. `withheld == None` means replicate nothing (private / mode A / - // undetermined): skip every pin so even commit and tree objects (which - // withheld_blob_oids never lists) stay local. `announce` gates the - // network-facing announcements. Fail closed: a private or undetermined repo - // never leaks. - let rules_opt = state.db.list_visibility_rules(&record.id).await.ok(); - let (announce, withheld) = replication_withheld_set( - state.git_encrypt_semaphore.clone(), - rules_opt.clone(), - &record.owner_did, - record.is_public, - disk_path.clone(), - state.git_bin.clone(), - std::time::Duration::from_secs(state.config.git_service_timeout_secs), - ) - .await; - - // Resolve the per-push pin candidate set once, off the async worker, then - // filter to what may actually replicate. Delta path: the reachable-only - // `withheld` set suffices (delta objects are reachable). Full-scan path: the - // candidate set can include dangling blobs the withheld set never classified, - // so fail closed — replicate a blob only if it is reachable AND - // visibility-allowed (#99). Only computed when something will actually - // replicate; every degraded path logs rather than failing silently. - let object_list: Vec = if let Some(withheld_set) = withheld.clone() { - let new_tips: Vec = ref_updates - .iter() - .map(|u| u.new_sha.clone()) - .filter(|s| s != ZERO_SHA) - .collect(); - let old_tips: Vec = ref_updates - .iter() - .map(|u| u.old_sha.clone()) - .filter(|s| s != ZERO_SHA) - .collect(); - let pin_set = crate::git::push_delta::resolve_candidates_for_push( + // #174 F2: move the whole post-receive replication tail into an independently owned + // task. It parks on `git_encrypt_semaphore` (withheld / candidate / full-scan + // resolution), so leaving it in the request future means a client/proxy disconnect + // while parked silently drops this push's pins, recovery copy, and announcements — + // the residual `state.rs` documented, because the durable `try_begin` gate sat after + // the park. The request future now returns the git response immediately and the tail + // runs detached; a disconnect can no longer drop it. Each push owns its own tail, + // including its own always-spawned announce, so per-push announcements are never + // coalesced away (the announce spawn stays out of the per-repo encrypt coalescing). + let did = did.to_string(); + tokio::spawn(async move { + // Replication enforcement (Phase 2): decide once per push whether the public + // may read this repo at all and, if so, which blob OIDs must not leave the + // node. `withheld == None` means replicate nothing (private / mode A / + // undetermined): skip every pin so even commit and tree objects (which + // withheld_blob_oids never lists) stay local. `announce` gates the + // network-facing announcements. Fail closed: a private or undetermined repo + // never leaks. + let rules_opt = state.db.list_visibility_rules(&record.id).await.ok(); + let (announce, withheld) = replication_withheld_set( state.git_encrypt_semaphore.clone(), + rules_opt.clone(), + &record.owner_did, + record.is_public, disk_path.clone(), - new_tips, - old_tips, state.git_bin.clone(), std::time::Duration::from_secs(state.config.git_service_timeout_secs), - false, ) .await; - if pin_set.full_scan { - fail_closed_full_scan_objects( + + // Resolve the per-push pin candidate set once, off the async worker, then + // filter to what may actually replicate. Delta path: the reachable-only + // `withheld` set suffices (delta objects are reachable). Full-scan path: the + // candidate set can include dangling blobs the withheld set never classified, + // so fail closed — replicate a blob only if it is reachable AND + // visibility-allowed (#99). Only computed when something will actually + // replicate; every degraded path logs rather than failing silently. + let object_list: Vec = if let Some(withheld_set) = withheld.clone() { + let new_tips: Vec = ref_updates + .iter() + .map(|u| u.new_sha.clone()) + .filter(|s| s != ZERO_SHA) + .collect(); + let old_tips: Vec = ref_updates + .iter() + .map(|u| u.old_sha.clone()) + .filter(|s| s != ZERO_SHA) + .collect(); + let pin_set = crate::git::push_delta::resolve_candidates_for_push( state.git_encrypt_semaphore.clone(), disk_path.clone(), - rules_opt.clone().unwrap_or_default(), - record.is_public, - record.owner_did.clone(), - pin_set.candidates, + new_tips, + old_tips, state.git_bin.clone(), std::time::Duration::from_secs(state.config.git_service_timeout_secs), + false, ) - .await - } else { - crate::git::visibility_pack::replicable_objects(pin_set.candidates, &withheld_set) - } - } else { - Vec::new() - }; - - // Pin new git objects to the local IPFS node (no-op if ipfs_api is empty). - // Skipped entirely when the public cannot read the repo (withheld == None). - // - // Coalesce-and-requeue per repo (#174 P2-2 + F5): the spawned task's walks park - // on `git_encrypt_semaphore` (which DEFERS when the pool is full rather than - // dropping the recovery copy). To bound the OUTSTANDING task set, at most one - // task per repo is in flight; a push arriving while one is in flight does NOT - // spawn a duplicate — and is NOT dropped either. The in-flight task pins only - // its own pre-spawn object-list snapshot, so this push's (old, new) tip pairs - // are merged into the in-flight key's pending slot in the same critical section - // as the presence check, and the task loop-drains them (fresh rules, fail - // closed) before releasing the key. Without the requeue a coalesced push's pins - // and recovery copies would be silently absent until an unrelated later push - // (the F5 loss). The guard still releases the key on panic (Drop on unwind), so - // a crashed walk never permanently locks the repo out. - if withheld.is_some() { - let tip_pairs: Vec<(String, String)> = ref_updates - .iter() - .map(|u| (u.old_sha.clone(), u.new_sha.clone())) - .collect(); - match state.encrypt_inflight.try_begin(&record.id, tip_pairs) { - crate::state::BeginOutcome::Coalesced => { - tracing::debug!( - repo = %record.id, - "post-push encryption task already in flight for this repo; coalesced \ - — this push's tip pairs are queued for that task's drain" - ); - } - crate::state::BeginOutcome::Admitted(inflight_guard) => { - let ctx = EncryptTaskCtx { - ipfs_api: state.config.ipfs_api.clone(), - repo_path: disk_path.clone(), - db: state.db.clone(), - repo_id: record.id.clone(), - owner_did: record.owner_did.clone(), - repo_name: record.name.clone(), - irys_url: state.config.irys_url.clone(), - http_client: std::sync::Arc::clone(&state.http_client), - node_did: state.node_did.to_string(), - node_keypair: std::sync::Arc::clone(&state.node_keypair), - git_bin: state.git_bin.clone(), - git_timeout: std::time::Duration::from_secs( - state.config.git_service_timeout_secs, - ), - encrypt_sem: state.git_encrypt_semaphore.clone(), - }; - tokio::spawn(run_encrypt_pin_task( - ctx, - inflight_guard, - object_list.clone(), - rules_opt.clone(), + .await; + if pin_set.full_scan { + fail_closed_full_scan_objects( + state.git_encrypt_semaphore.clone(), + disk_path.clone(), + rules_opt.clone().unwrap_or_default(), record.is_public, - )); - } - } - } - - // Pin new git objects to Pinata, then record branch→CID and gossip. - // - // #174 P2-2 scope note: this SECOND detached spawn is deliberately NOT brought - // under the per-repo encryption coalescing above. Two reasons: (1) it does not - // park on `git_encrypt_semaphore` (or any semaphore) — the Pinata `pin_new_objects` - // is a bounded reqwest round-trip, so it does not form the unbounded PARKED-waiter - // set that is the P2-2 residual; it runs to completion under the HTTP client's - // network timeouts. (2) Unlike the idempotent recovery-copy walk, this task does - // PER-PUSH, PER-REF work — branch→CID upserts, gossip publish, GraphQL subscription - // broadcast, Arweave anchoring, and peer notify, each keyed to THIS push's - // ref_updates. Coalescing it against an in-flight task for the same repo would DROP - // a later push's ref-update announcements (a correctness regression), not merely - // delay a duplicate. So it is scoped out with rationale, not brought under the bound. - { - let pinata_jwt = state.config.pinata_jwt.clone(); - let pinata_upload_url = state.config.pinata_upload_url.clone(); - let repo_path_clone = disk_path.clone(); - let db_clone = state.db.clone(); - let http_client = Arc::clone(&state.http_client); - let node_did_str = state.node_did.to_string(); - let repo_slug = format!( - "{}/{}", - crate::db::normalize_owner_key(&record.owner_did), - record.name - ); - let ref_updates_clone = ref_updates - .iter() - .map(|u| (u.ref_name.clone(), u.old_sha.clone(), u.new_sha.clone())) - .collect::>(); - let p2p_handle = state.p2p.clone(); - let pusher_did_clone = did.to_string(); - let db_for_peers = state.db.clone(); - let ref_update_tx = state.ref_update_tx.clone(); - let irys_url = state.config.irys_url.clone(); - let owner_did_for_arweave = record.owner_did.clone(); - let self_public_url = state.config.public_url.clone(); - let node_keypair = Arc::clone(&state.node_keypair); - let object_list_pinata = object_list; - let do_pinata_replication = withheld.is_some(); - tokio::spawn(async move { - let pinned = if do_pinata_replication { - crate::pinata::pin_new_objects( - &http_client, - &pinata_upload_url, - &pinata_jwt, - &repo_path_clone, - object_list_pinata, - &db_clone, + record.owner_did.clone(), + pin_set.candidates, + state.git_bin.clone(), + std::time::Duration::from_secs(state.config.git_service_timeout_secs), ) .await } else { - Vec::new() - }; + crate::git::visibility_pack::replicable_objects(pin_set.candidates, &withheld_set) + } + } else { + Vec::new() + }; - if !pinned.is_empty() { - tracing::info!(count = pinned.len(), "pinned git objects to Pinata"); + // Pin new git objects to the local IPFS node (no-op if ipfs_api is empty). + // Skipped entirely when the public cannot read the repo (withheld == None). + // + // Coalesce-and-requeue per repo (#174 P2-2 + F5): the spawned task's walks park + // on `git_encrypt_semaphore` (which DEFERS when the pool is full rather than + // dropping the recovery copy). To bound the OUTSTANDING task set, at most one + // task per repo is in flight; a push arriving while one is in flight does NOT + // spawn a duplicate — and is NOT dropped either. The in-flight task pins only + // its own pre-spawn object-list snapshot, so this push's (old, new) tip pairs + // are merged into the in-flight key's pending slot in the same critical section + // as the presence check, and the task loop-drains them (fresh rules, fail + // closed) before releasing the key. Without the requeue a coalesced push's pins + // and recovery copies would be silently absent until an unrelated later push + // (the F5 loss). The guard still releases the key on panic (Drop on unwind), so + // a crashed walk never permanently locks the repo out. + if withheld.is_some() { + let tip_pairs: Vec<(String, String)> = ref_updates + .iter() + .map(|u| (u.old_sha.clone(), u.new_sha.clone())) + .collect(); + match state.encrypt_inflight.try_begin(&record.id, tip_pairs) { + crate::state::BeginOutcome::Coalesced => { + tracing::debug!( + repo = %record.id, + "post-push encryption task already in flight for this repo; coalesced \ + — this push's tip pairs are queued for that task's drain" + ); + } + crate::state::BeginOutcome::Admitted(inflight_guard) => { + let ctx = EncryptTaskCtx { + ipfs_api: state.config.ipfs_api.clone(), + repo_path: disk_path.clone(), + db: state.db.clone(), + repo_id: record.id.clone(), + owner_did: record.owner_did.clone(), + repo_name: record.name.clone(), + irys_url: state.config.irys_url.clone(), + http_client: std::sync::Arc::clone(&state.http_client), + node_did: state.node_did.to_string(), + node_keypair: std::sync::Arc::clone(&state.node_keypair), + git_bin: state.git_bin.clone(), + git_timeout: std::time::Duration::from_secs( + state.config.git_service_timeout_secs, + ), + encrypt_sem: state.git_encrypt_semaphore.clone(), + }; + tokio::spawn(run_encrypt_pin_task( + ctx, + inflight_guard, + object_list.clone(), + rules_opt.clone(), + record.is_public, + )); + } } + } + + // Pin new git objects to Pinata, then record branch→CID and gossip. + // + // #174 P2-2 scope note: this SECOND detached spawn is deliberately NOT brought + // under the per-repo encryption coalescing above. Two reasons: (1) it does not + // park on `git_encrypt_semaphore` (or any semaphore) — the Pinata `pin_new_objects` + // is a bounded reqwest round-trip, so it does not form the unbounded PARKED-waiter + // set that is the P2-2 residual; it runs to completion under the HTTP client's + // network timeouts. (2) Unlike the idempotent recovery-copy walk, this task does + // PER-PUSH, PER-REF work — branch→CID upserts, gossip publish, GraphQL subscription + // broadcast, Arweave anchoring, and peer notify, each keyed to THIS push's + // ref_updates. Coalescing it against an in-flight task for the same repo would DROP + // a later push's ref-update announcements (a correctness regression), not merely + // delay a duplicate. So it is scoped out with rationale, not brought under the bound. + { + let pinata_jwt = state.config.pinata_jwt.clone(); + let pinata_upload_url = state.config.pinata_upload_url.clone(); + let repo_path_clone = disk_path.clone(); + let db_clone = state.db.clone(); + let http_client = Arc::clone(&state.http_client); + let node_did_str = state.node_did.to_string(); + let repo_slug = format!( + "{}/{}", + crate::db::normalize_owner_key(&record.owner_did), + record.name + ); + let ref_updates_clone = ref_updates + .iter() + .map(|u| (u.ref_name.clone(), u.old_sha.clone(), u.new_sha.clone())) + .collect::>(); + let p2p_handle = state.p2p.clone(); + let pusher_did_clone = did.to_string(); + let db_for_peers = state.db.clone(); + let ref_update_tx = state.ref_update_tx.clone(); + let irys_url = state.config.irys_url.clone(); + let owner_did_for_arweave = record.owner_did.clone(); + let self_public_url = state.config.public_url.clone(); + let node_keypair = Arc::clone(&state.node_keypair); + let object_list_pinata = object_list; + let do_pinata_replication = withheld.is_some(); + tokio::spawn(async move { + let pinned = if do_pinata_replication { + crate::pinata::pin_new_objects( + &http_client, + &pinata_upload_url, + &pinata_jwt, + &repo_path_clone, + object_list_pinata, + &db_clone, + ) + .await + } else { + Vec::new() + }; - // Build sha→cid map from pinned objects - let cid_map: std::collections::HashMap = pinned.into_iter().collect(); + if !pinned.is_empty() { + tracing::info!(count = pinned.len(), "pinned git objects to Pinata"); + } - // Record branch→CID for each ref update and publish gossip - for (ref_name, old_sha, new_sha) in &ref_updates_clone { - let cid = cid_map.get(new_sha).map(|s| s.as_str()); + // Build sha→cid map from pinned objects + let cid_map: std::collections::HashMap = + pinned.into_iter().collect(); - if let Some(cid_str) = cid { - let _ = db_clone - .upsert_branch_cid(&repo_slug, ref_name, new_sha, cid_str, &node_did_str) - .await; + // Record branch→CID for each ref update and publish gossip + for (ref_name, old_sha, new_sha) in &ref_updates_clone { + let cid = cid_map.get(new_sha).map(|s| s.as_str()); + + if let Some(cid_str) = cid { + let _ = db_clone + .upsert_branch_cid( + &repo_slug, + ref_name, + new_sha, + cid_str, + &node_did_str, + ) + .await; + } + + if announce { + if let Some(p2p) = &p2p_handle { + p2p.publish_ref_update(crate::p2p::RefUpdateEvent { + node_did: node_did_str.clone(), + pusher_did: pusher_did_clone.clone(), + repo: repo_slug.clone(), + ref_name: ref_name.clone(), + old_sha: old_sha.clone(), + new_sha: new_sha.clone(), + timestamp: chrono::Utc::now().to_rfc3339(), + cert_id: None, + cid: cid.map(|s| s.to_string()), + }) + .await; + } + } } + // Broadcast ref update to GraphQL subscription listeners — one per ref. + // Gated on `announce`: /graphql/ws is unauthenticated (mounted after + // the optional_signature layer), and the subscription resolver has no + // caller to gate against, so only publicly-readable ref updates may + // reach anonymous subscribers. Mirrors the gossip (above) and Arweave + // (below) sends, which are already `announce`-gated. Without this a + // private-repo push would leak live ref metadata over the socket — + // the subscription analog of #112/#114. + let now_ts = chrono::Utc::now().to_rfc3339(); if announce { - if let Some(p2p) = &p2p_handle { - p2p.publish_ref_update(crate::p2p::RefUpdateEvent { - node_did: node_did_str.clone(), - pusher_did: pusher_did_clone.clone(), + for (ref_name, old_sha, new_sha) in &ref_updates_clone { + let _ = ref_update_tx.send(crate::state::RefUpdateBroadcast { repo: repo_slug.clone(), ref_name: ref_name.clone(), old_sha: old_sha.clone(), new_sha: new_sha.clone(), - timestamp: chrono::Utc::now().to_rfc3339(), - cert_id: None, - cid: cid.map(|s| s.to_string()), - }) - .await; + pusher_did: pusher_did_clone.clone(), + node_did: node_did_str.clone(), + timestamp: now_ts.clone(), + }); } } - } - - // Broadcast ref update to GraphQL subscription listeners — one per ref. - // Gated on `announce`: /graphql/ws is unauthenticated (mounted after - // the optional_signature layer), and the subscription resolver has no - // caller to gate against, so only publicly-readable ref updates may - // reach anonymous subscribers. Mirrors the gossip (above) and Arweave - // (below) sends, which are already `announce`-gated. Without this a - // private-repo push would leak live ref metadata over the socket — - // the subscription analog of #112/#114. - let now_ts = chrono::Utc::now().to_rfc3339(); - if announce { - for (ref_name, old_sha, new_sha) in &ref_updates_clone { - let _ = ref_update_tx.send(crate::state::RefUpdateBroadcast { - repo: repo_slug.clone(), - ref_name: ref_name.clone(), - old_sha: old_sha.clone(), - new_sha: new_sha.clone(), - pusher_did: pusher_did_clone.clone(), - node_did: node_did_str.clone(), - timestamp: now_ts.clone(), - }); - } - } - // Arweave permanent anchoring — fire for each ref update. - // Suppressed for repos the public cannot read (public permanent ledger). - if announce && !irys_url.is_empty() { - for (ref_name, old_sha, new_sha) in &ref_updates_clone { - let cid = cid_map.get(new_sha).cloned(); - let anchor = crate::arweave::RefAnchor { - repo: repo_slug.clone(), - owner_did: owner_did_for_arweave.clone(), - ref_name: ref_name.clone(), - old_sha: old_sha.clone(), - new_sha: new_sha.clone(), - cid: cid.clone(), - timestamp: now_ts.clone(), - node_did: node_did_str.clone(), - }; - match crate::arweave::anchor_ref_update(&http_client, &irys_url, &anchor).await - { - Ok(tx_id) if !tx_id.is_empty() => { - let arweave_url = crate::arweave::arweave_url(&tx_id); - let _ = db_clone - .record_arweave_anchor(&crate::db::RecordAnchorInput { - repo: &repo_slug, - owner_did: &owner_did_for_arweave, - ref_name, - old_sha, - new_sha, - cid: cid.as_deref(), - irys_tx_id: &tx_id, - arweave_url: &arweave_url, - node_did: &node_did_str, - }) - .await; + // Arweave permanent anchoring — fire for each ref update. + // Suppressed for repos the public cannot read (public permanent ledger). + if announce && !irys_url.is_empty() { + for (ref_name, old_sha, new_sha) in &ref_updates_clone { + let cid = cid_map.get(new_sha).cloned(); + let anchor = crate::arweave::RefAnchor { + repo: repo_slug.clone(), + owner_did: owner_did_for_arweave.clone(), + ref_name: ref_name.clone(), + old_sha: old_sha.clone(), + new_sha: new_sha.clone(), + cid: cid.clone(), + timestamp: now_ts.clone(), + node_did: node_did_str.clone(), + }; + match crate::arweave::anchor_ref_update(&http_client, &irys_url, &anchor) + .await + { + Ok(tx_id) if !tx_id.is_empty() => { + let arweave_url = crate::arweave::arweave_url(&tx_id); + let _ = db_clone + .record_arweave_anchor(&crate::db::RecordAnchorInput { + repo: &repo_slug, + owner_did: &owner_did_for_arweave, + ref_name, + old_sha, + new_sha, + cid: cid.as_deref(), + irys_tx_id: &tx_id, + arweave_url: &arweave_url, + node_did: &node_did_str, + }) + .await; + } + Ok(_) => {} + Err(e) => { + tracing::warn!(repo=%repo_slug, err=%e, "Arweave anchor failed") + } } - Ok(_) => {} - Err(e) => tracing::warn!(repo=%repo_slug, err=%e, "Arweave anchor failed"), } } - } - // HTTP peer notification — notify all known peers to pull from us. - // This is the reliable fallback when Gossipsub p2p is not yet connected. - // Suppressed for repos the public cannot read. Runs last so a slow or - // unreachable peer cannot delay the local GraphQL broadcast or Arweave - // anchoring above; this is the lowest-priority best-effort step. - if announce { - if let Ok(peers) = db_for_peers.list_peers().await { - for peer in peers { - if peer.http_url.is_empty() { - continue; - } - let peer_url = peer.http_url.trim_end_matches('/'); - if let Some(self_url) = self_public_url.as_deref() { - if peer_url == self_url.trim_end_matches('/') { + // HTTP peer notification — notify all known peers to pull from us. + // This is the reliable fallback when Gossipsub p2p is not yet connected. + // Suppressed for repos the public cannot read. Runs last so a slow or + // unreachable peer cannot delay the local GraphQL broadcast or Arweave + // anchoring above; this is the lowest-priority best-effort step. + if announce { + if let Ok(peers) = db_for_peers.list_peers().await { + for peer in peers { + if peer.http_url.is_empty() { continue; } + let peer_url = peer.http_url.trim_end_matches('/'); + if let Some(self_url) = self_public_url.as_deref() { + if peer_url == self_url.trim_end_matches('/') { + continue; + } + } + let notify_url = format!("{peer_url}{SYNC_NOTIFY_PATH}"); + notify_peer_of_refs( + &http_client, + node_keypair.as_ref(), + &peer.did, + ¬ify_url, + &repo_slug, + &ref_updates_clone, + &node_did_str, + &pusher_did_clone, + ) + .await; } - let notify_url = format!("{peer_url}{SYNC_NOTIFY_PATH}"); - notify_peer_of_refs( - &http_client, - node_keypair.as_ref(), - &peer.did, - ¬ify_url, - &repo_slug, - &ref_updates_clone, - &node_did_str, - &pusher_did_clone, - ) - .await; } } - } - }); - } + }); + } + }); Ok(result) } @@ -5179,9 +5201,23 @@ mod tests { assert_eq!(a.status(), 200, "push A lands 200 despite scan contention"); assert_eq!(b.status(), 200, "push B lands 200 despite scan contention"); - // Let the detached recipients walks drain through the same pool before - // reading the detector files. - tokio::time::sleep(std::time::Duration::from_millis(800)).await; + // Wait for both pushes' detached scan tails to drain through the pool of 1 + // before reading the detector files. The WHOLE tail (withheld walk included) now + // runs detached (#174 F2), so poll until every expected scan has run rather than a + // fixed sleep, which is load-sensitive under a parallel test run. + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); + loop { + let ran = std::fs::read_to_string(&ranfile) + .unwrap_or_default() + .lines() + .count(); + if ran >= 6 || std::time::Instant::now() >= deadline { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + // Small settle so the last scan's rmdir has landed before the overlap check. + tokio::time::sleep(std::time::Duration::from_millis(100)).await; assert!( !overlap.exists(), "with a scan pool of 1, no two scans' git may ever be alive at once \ @@ -5252,9 +5288,20 @@ mod tests { /// F4 scenario 4 — landed-push-never-fails: a push whose post-receive walk must /// park (pool held elsewhere) DEFERS and then returns the receive-pack success /// once admission frees; contention never converts the landed push into a 5xx. + /// #174 F2 (RED-before/GREEN-after): the post-receive replication tail parks on + /// `git_encrypt_semaphore` (withheld/candidate/full-scan resolution). Leaving it in the + /// request future means a client/proxy disconnect while parked silently loses this + /// push's pins, recovery copy, and announcements (state.rs documented this residual). + /// The fix moves the whole tail into an independently owned task, so the handler + /// returns its receive-pack 200 WITHOUT waiting on the scan pool and a disconnect can + /// no longer drop the work. + /// + /// Load-bearing: with the tail inline (pre-fix) the handler parks while the pool is + /// held and does NOT return within the bound (RED — the timeout fires). With the + /// detached tail it returns 200 promptly (GREEN). #[cfg(unix)] #[sqlx::test] - async fn receive_pack_landed_push_defers_on_scan_contention_never_errors(pool: sqlx::PgPool) { + async fn receive_pack_landed_push_returns_without_parking_on_scan_pool(pool: sqlx::PgPool) { use axum::extract::{Path, State}; use axum::Extension; use std::net::SocketAddr; @@ -5268,40 +5315,38 @@ mod tests { f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f4park", "p1", true).await; let sem = Arc::new(Semaphore::new(1)); state.git_encrypt_semaphore = sem.clone(); - // The test holds the pool's only permit: the push's withheld walk must park. + // Hold the pool's only permit: the post-receive scan would park if it ran in the + // request future. let held = sem.clone().acquire_owned().await.unwrap(); let peer: SocketAddr = "203.0.113.74:5000".parse().unwrap(); - let mut handle = tokio::spawn(git_receive_pack( - State(state), - Path(("z6f4park".to_string(), "p1".to_string())), - Extension(crate::auth::AuthenticatedDid( - "did:key:z6MkF4ParkPusherAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), - )), - crate::rate_limit::PeerAddr(Some(peer)), - axum::http::HeaderMap::new(), - ref_update_body("2222222222222222222222222222222222222222"), - )); - - // Parked, not errored: the handler must NOT complete while the pool is held. - let parked = tokio::time::timeout(std::time::Duration::from_millis(700), &mut handle).await; - assert!( - parked.is_err(), - "the post-receive walk must defer while the scan pool is held; got {parked:?}" - ); - - // Release admission: the SAME push now finishes with the receive-pack 200. - drop(held); - let resp = tokio::time::timeout(std::time::Duration::from_secs(30), handle) - .await - .expect("the deferred push must complete once admission frees") - .expect("the push task must not panic") - .expect("contention must never convert a landed push into an error"); + // The handler must return its receive-pack 200 WITHOUT waiting on the held scan + // pool — the tail is owned by a detached task. Pre-fix this times out. + let resp = tokio::time::timeout( + std::time::Duration::from_secs(5), + git_receive_pack( + State(state), + Path(("z6f4park".to_string(), "p1".to_string())), + Extension(crate::auth::AuthenticatedDid( + "did:key:z6MkF4ParkPusherAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + )), + crate::rate_limit::PeerAddr(Some(peer)), + axum::http::HeaderMap::new(), + ref_update_body("2222222222222222222222222222222222222222"), + ), + ) + .await + .expect("the handler must return without parking on the held scan pool") + .expect("contention must never convert a landed push into an error"); assert_eq!( resp.status(), 200, - "the response is the receive-pack success, not a contention 5xx" + "the response is the receive-pack success, returned before the detached tail runs" ); + + // The detached tail is still owned: release admission and let it drain cleanly. + drop(held); + tokio::time::sleep(std::time::Duration::from_millis(300)).await; } // ---- #174 U4 (P2-2): post-push encryption task set bounded by per-repo coalescing ---- diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index c4d2b079..8396a369 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -449,13 +449,13 @@ impl Drop for EncryptInflightGuard { /// recovery copy or silently under-pin it. The returned permit must move into /// the blocking closure so a started scan always completes holding it (a /// disconnect cannot cancel `spawn_blocking` or leak the permit mid-walk). -/// Accepted residuals, stated once for every caller: (1) the park wait is -/// queue-depth multiplied — post-receive tails are no longer admission-bounded -/// once the write permit is released, so N landed pushes can queue N scans and -/// the last waits N scan-durations; (2) a client-timeout disconnect while -/// parked HERE drops the handler future and silently loses this push's -/// replication work — the `encrypt_inflight` coalescing requeue does NOT cover -/// it, because this park precedes the `try_begin` spawn gate. +/// Accepted residual, stated once for every caller: the park wait is queue-depth +/// multiplied — post-receive tails are no longer admission-bounded once the write +/// permit is released, so N landed pushes can queue N scans and the last waits N +/// scan-durations. A client-timeout disconnect no longer loses the work (#174 F2): +/// the whole post-receive replication tail runs in an independently owned task, so +/// dropping the request future cannot drop this parked scan — the park no longer +/// precedes any durable-record gate in a cancellable future. pub async fn acquire_scan_permit( scan_sem: Arc, repo: &std::path::Path, From 759c9b6aa387f15cc0f2f88ed3dfb1f0d13a8cc8 Mon Sep 17 00:00:00 2001 From: t Date: Sat, 18 Jul 2026 10:36:16 -0500 Subject: [PATCH 47/95] fix(node): bound the /ipfs cat-file probe and content read (#174 F3) The GET /ipfs/{cid} object-type probe and content read ran bare synchronous Command::output() on the async worker, budget-checked only before they started. A hung or corrupt object store therefore pinned a Tokio runtime worker and both held IPFS admission permits indefinitely, so enough requests exhausted the route despite GITLAWB_IPFS_REQUEST_BUDGET_SECS. The walk beside them already used spawn_blocking + the reaped bounded runner; these two stages were left bare. Extract run_bounded_git_raw (returns ExitStatus/stdout/stderr so callers can classify exit codes; run_bounded_git stays a thin bail-on-nonzero wrapper for the walk callers) and add store::object_type_bounded / read_object_content_bounded on it, preserving object_type's Ok(None)-on-absence vs Err classification so the serve path's 404-vs-503 semantics are unchanged. Run both /ipfs stages in spawn_blocking under a deadline clamped to min(git timeout, remaining budget), with the IPFS permits held across the awaited spawn_blocking. Test: get_by_cid_hung_probe_is_reaped_and_sheds_503 hangs git on a FIFO alternates with no feeder; the reaped probe sheds a 503 in bounded time (GREEN), while a neutralized deadline blocks the handler forever (RED). The existing probe-error / corrupt-repo tests still assert the absence-vs-error classification at the handler. --- crates/gitlawb-node/src/api/ipfs.rs | 158 +++++++++++------- crates/gitlawb-node/src/git/store.rs | 56 +++++++ .../gitlawb-node/src/git/visibility_pack.rs | 39 ++++- 3 files changed, 185 insertions(+), 68 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 41080833..43e418ae 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -292,32 +292,52 @@ pub async fn get_by_cid( } }; - // Budget gate for the probe stage (F3). The probe subprocess has no - // internal duration clamp, so this pre-start check is its entire bound. - if budget_gate( + // Budget gate for the probe stage (F3): a probe is never STARTED with zero + // remaining, and a started probe now runs its git child under a deadline + // clamped to the remainder (below), so it can never complete past the budget. + let Some(probe_budget) = budget_gate( &mut truncated_by, request_deadline, state.config.ipfs_request_budget_secs, &repo.name, "object-type probe", - ) - .is_none() - { + ) else { break; - } + }; // Check whether the object exists in this repo before any expensive // reachability walk. This prevents random-CID spray from triggering // full-history git walks on repos that don't carry the object. Absent - // (`Ok(None)`) is a VERDICT; a probe that could not run is not. - let obj_type = match store::object_type(&repo_path, &sha256_hex) { - Ok(Some(t)) => t, - Ok(None) => continue, - Err(e) => { + // (`Ok(None)`) is a VERDICT; a probe that could not run is not. The + // `git cat-file -t` shells out, so run it OFF the async worker under the + // reaped bounded runner (#174 F3) — a hung/corrupt object store cannot pin a + // runtime worker or the held IPFS permits past the deadline. + let probe_deadline = std::time::Instant::now() + + std::cmp::min( + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + probe_budget, + ); + // The probe shells to the real `git` (as `object_type` historically did), + // independent of `state.git_bin` (which tests point at a fake walk git). + let probe_path = repo_path.clone(); + let probe_sha = sha256_hex.clone(); + let obj_type = match tokio::task::spawn_blocking(move || { + store::object_type_bounded("git", &probe_path, &probe_sha, probe_deadline) + }) + .await + { + Ok(Ok(Some(t))) => t, + Ok(Ok(None)) => continue, + Ok(Err(e)) => { tracing::warn!(repo = %repo.name, err = %e, "object-type probe failed during /ipfs scan; skipping repo without a verdict"); taint(&mut truncated_by, "probe"); continue; } + Err(join_err) => { + tracing::warn!(repo = %repo.name, err = %join_err, "object-type probe task panicked during /ipfs scan; skipping repo without a verdict"); + taint(&mut truncated_by, "probe"); + continue; + } }; // Per-blob gating only applies when a path-scoped rule exists (KTD4). @@ -413,28 +433,52 @@ pub async fn get_by_cid( // serving keeps the terminal arm honest and the stop unconditional; the // retryable 503 tells the caller to come back rather than letting an // over-budget request keep spending. - if budget_gate( + let Some(read_budget) = budget_gate( &mut truncated_by, request_deadline, state.config.ipfs_request_budget_secs, &repo.name, "content read", - ) - .is_none() - { + ) else { break; - } + }; // Now that we've passed the gate, read the content. A failed read after a // passed gate is not an absence verdict — the probe just said the object - // exists here — so the skip taints the scan. - let content = match store::read_object_content(&repo_path, &sha256_hex, &obj_type) { - Ok(c) => c, - Err(e) => { + // exists here — so the skip taints the scan. Like the probe, the read shells + // out to `git cat-file `, so run it OFF the async worker under the reaped + // bounded runner clamped to the remaining budget (#174 F3). + let read_deadline = std::time::Instant::now() + + std::cmp::min( + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + read_budget, + ); + // Real `git`, as the read historically used, independent of `state.git_bin`. + let read_path = repo_path.clone(); + let read_sha = sha256_hex.clone(); + let read_type = obj_type.clone(); + let content = match tokio::task::spawn_blocking(move || { + store::read_object_content_bounded( + "git", + &read_path, + &read_sha, + &read_type, + read_deadline, + ) + }) + .await + { + Ok(Ok(c)) => c, + Ok(Err(e)) => { tracing::warn!(repo = %repo.name, err = %e, "object content read failed during /ipfs scan; skipping repo without a verdict"); taint(&mut truncated_by, "read"); continue; } + Err(join_err) => { + tracing::warn!(repo = %repo.name, err = %join_err, "object content read task panicked during /ipfs scan; skipping repo without a verdict"); + taint(&mut truncated_by, "read"); + continue; + } }; // 3. Return the content with IPFS-style headers @@ -1539,22 +1583,22 @@ mod tests { ); } - /// F3 expiry between probe and walk: an object PROBED PRESENT in a - /// path-scoped repo is still not served once the budget is gone; the - /// walk-stage gate taints and stops before any walk starts. The probe is - /// stalled past the 1s budget while still SUCCEEDING via a FIFO at - /// `objects/info/alternates`: real `git cat-file -t` blocks opening it at - /// odb setup until a plain OS thread feeds it an empty alternates list at - /// ~2s, after which the probe finds the genuine loose object and reports - /// "blob". The walk-needing repo thus reaches the walk gate with zero - /// remaining: 503 naming the budget, never a 404 (the walk-gate deny of the - /// fake git's empty allowed-set would have been a verdict) and never a - /// serve, and the fake-git walk log stays EMPTY (no walk ever started). - /// MUTATION (RED): drop the walk-stage budget gate and this decays to the - /// walked 404 with one logged walk. + /// #174 F3 hung-probe reap (RED-before/GREEN-after): the `git cat-file -t` + /// probe runs OFF the async worker under the reaped bounded runner, so a hung or + /// corrupt object store cannot pin a runtime worker or the held IPFS permits. + /// `objects/info/alternates` is a FIFO with no writer, so real `git cat-file -t` + /// blocks at odb setup forever. With the probe bounded to + /// `min(git_service_timeout, remaining budget)` (~1s here), the watchdog tears the + /// git process group down at the deadline and the probe returns Err — a taint, not + /// a verdict — so the scan sheds a retryable 503 naming the probe, no walk ever + /// starts, and the whole request returns in bounded time. + /// + /// Load-bearing: with the probe on the bare async worker (pre-fix) this FIFO blocks + /// the handler forever (no feeder frees it) and the request hangs — the wrapping + /// timeout fires (RED). With the reaped bounded probe it returns 503 promptly. #[cfg(unix)] #[sqlx::test] - async fn get_by_cid_budget_expiry_between_probe_and_walk_sheds_503(pool: sqlx::PgPool) { + async fn get_by_cid_hung_probe_is_reaped_and_sheds_503(pool: sqlx::PgPool) { let tmp = tempfile::TempDir::new().unwrap(); let mut state = crate::test_support::test_state(pool.clone()).await; let repos_dir = tmp.path().join("repos"); @@ -1587,12 +1631,10 @@ mod tests { .await .unwrap(); - // Stall the REAL-git probe past the budget while still letting it - // succeed: `objects/info/alternates` as a FIFO blocks cat-file's odb - // setup until a writer appears. The feeder thread supplies an EMPTY - // alternates list (open + close, no bytes) after 2s, so the probe then - // resolves the genuine loose object. Non-blocking open with retries so - // a handler that never probes cannot wedge the feeder forever. + // Hang the REAL-git probe indefinitely: `objects/info/alternates` as a FIFO + // with no writer blocks `git cat-file -t` at odb setup forever. There is no + // feeder — the reaped bounded runner must tear the git process group down at + // the deadline; a bare unbounded probe would block the handler here. let rec = state .db .get_repo("z6f3probe", "gated") @@ -1611,28 +1653,16 @@ mod tests { 0, "mkfifo(objects/info/alternates) must succeed" ); - let feeder = std::thread::spawn(move || { - use std::os::unix::fs::OpenOptionsExt; - std::thread::sleep(std::time::Duration::from_secs(2)); - for _ in 0..300 { - if std::fs::OpenOptions::new() - .write(true) - .custom_flags(libc::O_NONBLOCK) - .open(&fifo) - .is_ok() - { - return; - } - std::thread::sleep(std::time::Duration::from_millis(50)); - } - }); - let peer: SocketAddr = "203.0.113.72:5000".parse().unwrap(); - let resp = ipfs_router(state) - .oneshot(get_cid(&cid_for_oid(&oid), Some(peer))) - .await - .unwrap(); - feeder.join().unwrap(); + // The request must return in bounded time: the reaped probe sheds a 503; a + // bare unbounded probe would block on the FIFO forever (no feeder frees it). + let resp = tokio::time::timeout( + std::time::Duration::from_secs(15), + ipfs_router(state).oneshot(get_cid(&cid_for_oid(&oid), Some(peer))), + ) + .await + .expect("the hung probe must be reaped, not block the handler") + .unwrap(); assert_eq!( resp.status(), StatusCode::SERVICE_UNAVAILABLE, @@ -1651,8 +1681,8 @@ mod tests { .unwrap(); let body = String::from_utf8_lossy(&body); assert!( - body.contains("budget"), - "the shed must name the budget taint; got: {body}" + body.contains("probe"), + "the shed must name the reaped-probe taint; got: {body}" ); let walks = std::fs::read_to_string(&walk_log) .map(|s| s.lines().count()) diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index 5697c229..d5a0d1a1 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -320,6 +320,62 @@ pub fn read_object_content(repo_path: &Path, sha256_hex: &str, obj_type: &str) - Ok(content_output.stdout) } +/// Bounded, reaped variant of [`object_type`] for the async `/ipfs` serve path +/// (#174 F3): runs `git cat-file -t` off the caller's runtime through the +/// process-group + watchdog reaper, so a hung or corrupt object store cannot pin a +/// runtime worker or an IPFS admission permit past `deadline`. Exit classification +/// is identical to [`object_type`] — a missing object is `Ok(None)`, a store-access +/// failure is `Err` — so the serve path's 404-vs-503 semantics are unchanged. +pub fn object_type_bounded( + git_bin: &str, + repo_path: &Path, + sha256_hex: &str, + deadline: std::time::Instant, +) -> Result> { + let (status, stdout, stderr) = crate::git::visibility_pack::run_bounded_git_raw( + git_bin, + &["cat-file", "-t", sha256_hex], + repo_path, + &[], + deadline, + )?; + if !status.success() { + // Same classification as `object_type` (a nonzero exit is an ABSENCE verdict + // only when git could examine the object store); U5 refines the absence arm + // with an out-of-band readability check. + let stderr = String::from_utf8_lossy(&stderr); + if stderr.contains("not a git repository") + || stderr.lines().any(|l| l.starts_with("error:")) + { + bail!("git cat-file -t failed: {}", stderr.trim()); + } + return Ok(None); + } + Ok(Some(String::from_utf8_lossy(&stdout).trim().to_string())) +} + +/// Bounded, reaped variant of [`read_object_content`] for the async `/ipfs` serve +/// path (#174 F3). Same teardown guarantees as [`object_type_bounded`]. +pub fn read_object_content_bounded( + git_bin: &str, + repo_path: &Path, + sha256_hex: &str, + obj_type: &str, + deadline: std::time::Instant, +) -> Result> { + let (status, stdout, stderr) = crate::git::visibility_pack::run_bounded_git_raw( + git_bin, + &["cat-file", obj_type, sha256_hex], + repo_path, + &[], + deadline, + )?; + if !status.success() { + bail!("git cat-file failed: {}", String::from_utf8_lossy(&stderr)); + } + Ok(stdout) +} + /// Read a git object by its SHA-256 hex object ID. /// /// Returns `(object_type, content_bytes)` where `content_bytes` is the raw diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index 9f07020d..e69c4ceb 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -67,13 +67,13 @@ fn child_terminated_without_reaping(pid: i32) -> bool { } #[cfg(unix)] -pub(crate) fn run_bounded_git( +pub(crate) fn run_bounded_git_raw( git_bin: &str, args: &[&str], repo_path: &Path, stdin_bytes: &[u8], deadline: Instant, -) -> Result> { +) -> Result<(std::process::ExitStatus, Vec, Vec)> { use std::io::{Read, Write}; use std::os::unix::process::CommandExt; use std::sync::mpsc::RecvTimeoutError; @@ -191,6 +191,23 @@ pub(crate) fn run_bounded_git( if killed && !status.success() { return Err(crate::git::smart_http::GitServiceTimeout.into()); } + Ok((status, out, err)) +} + +/// Bounded git returning only stdout, `bail!`ing on any nonzero exit. The thin +/// wrapper the walk callers use. Probes that must distinguish exit classes — +/// `git cat-file` absence vs an object-store access failure — call +/// [`run_bounded_git_raw`] and classify the status/stderr themselves. +#[cfg(unix)] +pub(crate) fn run_bounded_git( + git_bin: &str, + args: &[&str], + repo_path: &Path, + stdin_bytes: &[u8], + deadline: Instant, +) -> Result> { + let label = args.first().copied().unwrap_or("git"); + let (status, out, err) = run_bounded_git_raw(git_bin, args, repo_path, stdin_bytes, deadline)?; if !status.success() { anyhow::bail!("git {label} failed: {}", String::from_utf8_lossy(&err)); } @@ -210,13 +227,13 @@ pub(crate) fn run_bounded_git( /// the Unix version's signature and result semantics so every caller compiles on all /// targets (#174). #[cfg(not(unix))] -pub(crate) fn run_bounded_git( +pub(crate) fn run_bounded_git_raw( git_bin: &str, args: &[&str], repo_path: &Path, stdin_bytes: &[u8], deadline: Instant, -) -> Result> { +) -> Result<(std::process::ExitStatus, Vec, Vec)> { use std::io::{Read, Write}; use std::sync::mpsc::RecvTimeoutError; @@ -283,6 +300,20 @@ pub(crate) fn run_bounded_git( if killed && !status.success() { return Err(crate::git::smart_http::GitServiceTimeout.into()); } + Ok((status, out, err)) +} + +/// Non-Unix thin wrapper matching the Unix [`run_bounded_git`] semantics. +#[cfg(not(unix))] +pub(crate) fn run_bounded_git( + git_bin: &str, + args: &[&str], + repo_path: &Path, + stdin_bytes: &[u8], + deadline: Instant, +) -> Result> { + let label = args.first().copied().unwrap_or("git"); + let (status, out, err) = run_bounded_git_raw(git_bin, args, repo_path, stdin_bytes, deadline)?; if !status.success() { anyhow::bail!("git {label} failed: {}", String::from_utf8_lossy(&err)); } From 6dc2b2e912587b14d1b80c8db87426231d29be53 Mon Sep 17 00:00:00 2001 From: t Date: Sat, 18 Jul 2026 10:49:18 -0500 Subject: [PATCH 48/95] fix(node): share one whole-scan deadline across both full-scan phases (#174 F4) fail_closed_full_scan_objects ran phase 1 (replicable_blob_set_bounded) then phase 2 (all_blob_oids) with a FRESH Instant::now() + timeout for phase 2, under one held scan permit. A large-but-successful phase 1 plus a full phase 2 held the permit ~2x the configured budget instead of the one whole-scan budget. Compute one deadline before phase 1 and share it: phase 2 takes the shared Instant directly, phase 1 takes the remaining-until-deadline. Total occupancy is now ~1x. The completeness cost is honest: a repo whose phase 1 nears the budget under-pins this push (fail closed) rather than over-holding, so size the budget so both phases normally fit. Test: full_scan_shares_one_deadline_across_both_phases drives a slow-but- successful phase 1 that eats the budget; the shared deadline reaps phase 2 and the scan fails closed (empty). RED before: phase 2's fresh budget completed and kept the non-blob candidate. --- crates/gitlawb-node/src/api/repos.rs | 64 +++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 2 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 98e395f6..ede6c5e1 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -139,10 +139,25 @@ async fn fail_closed_full_scan_objects( crate::state::acquire_scan_permit(encrypt_sem, &disk_path, "fail-closed full scan").await; tokio::task::spawn_blocking(move || -> anyhow::Result> { let _permit = permit; + // One whole-scan deadline shared across both phases (#174 F4). A fresh + // `Instant::now() + timeout` for phase 2 let a large-but-successful phase 1 plus + // a full phase 2 hold the scan permit ~2x the configured budget. Sharing the + // deadline caps total occupancy at ~1x: phase 1 runs against the remaining + // budget, and if it consumes the budget phase 2 gets what is left and fails + // closed (pins nothing) rather than over-holding — the safe direction. The cost + // is honest: a genuinely large repo whose phase 1 nears the budget under-pins + // this push rather than the previous silent ~2x hold; size the budget so both + // phases normally fit. + let deadline = std::time::Instant::now() + timeout; let allowed = crate::git::visibility_pack::replicable_blob_set_bounded( - &disk_path, &git_bin, timeout, &rules, is_public, &owner_did, + &disk_path, + &git_bin, + deadline.saturating_duration_since(std::time::Instant::now()), + &rules, + is_public, + &owner_did, )?; - let all_blobs = crate::git::push_delta::all_blob_oids(&disk_path, &git_bin, std::time::Instant::now() + timeout)?; + let all_blobs = crate::git::push_delta::all_blob_oids(&disk_path, &git_bin, deadline)?; Ok(crate::git::visibility_pack::replicable_objects_fail_closed( candidates, &allowed, &all_blobs, )) @@ -5036,6 +5051,51 @@ mod tests { ); } + /// #174 F4 (RED-before/GREEN-after): the two full-scan phases share ONE whole-scan + /// deadline. Phase 1 (`replicable_blob_set_bounded`) succeeds but consumes almost + /// the whole budget; phase 2 (`all_blob_oids`) then gets only the remainder. With a + /// shared deadline phase 2 is reaped and the scan fails closed (pins nothing) — the + /// safe direction. With a FRESH `Instant::now() + timeout` for phase 2 (pre-fix) it + /// gets a full second budget, completes with an empty blob set, and the non-blob + /// candidate is kept — so the result is NON-empty (RED) and the permit is held ~2x. + #[cfg(unix)] + #[tokio::test] + async fn full_scan_shares_one_deadline_across_both_phases() { + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + // Phase 1 (ls-tree) sleeps 1.5s and succeeds (empty tree); phase 2 + // (cat-file --batch-all-objects) sleeps 1.5s. With a 2s whole-scan budget the + // shared deadline leaves phase 2 only ~0.5s, so it is reaped; a fresh 2s budget + // would let it finish. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-parse) echo deadbeef ;;\n rev-list) echo aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ;;\n ls-tree) sleep 1.5 ;;\n cat-file) case \"$*\" in *--batch-all-objects*) sleep 1.5 ;; *) : ;; esac ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + // A candidate that is NOT a blob (never appears in all_blob_oids): kept by + // replicable_objects_fail_closed only if phase 2 actually ran to completion. + let candidates = vec!["cccccccccccccccccccccccccccccccccccccccc".to_string()]; + + let sem: Arc = Arc::new(Semaphore::new(1)); + let objs = fail_closed_full_scan_objects( + sem, + tmp.path().to_path_buf(), + vec![vis_rule("/secret/**", &[])], + true, + OWNER_DID.to_string(), + candidates, + git_bin, + Duration::from_secs(2), + ) + .await; + assert!( + objs.is_empty(), + "a large-but-successful phase 1 must leave phase 2 only the SHARED whole-scan \ + remainder, so it reaps and the scan fails closed; got {objs:?} (a fresh phase-2 \ + budget completed the scan and kept the candidate — the ~2x-budget bug)" + ); + } + /// Shared fixture for the F4 handler-layer tests: a state whose repo_store and /// git_bin point at the given tempdir/fake-git, plus a seeded on-disk repo, /// optionally with a path-scoped rule (so the post-receive walks actually run). From d39fc8de9cf5a3f15c42d23675fc5741f0412302 Mon Sep 17 00:00:00 2001 From: t Date: Sat, 18 Jul 2026 10:57:16 -0500 Subject: [PATCH 49/95] fix(node): disambiguate absence from an unreadable object store in the /ipfs probe (#174 F5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit object_type_bounded treated any bare `fatal:` cat-file exit as absence. A packed object whose pack/idx is transiently unreadable (permissions, or a mid-repack race) emits `could not get object info` — byte-identical to a genuine miss — so a present object could return Ok(None) and the /ipfs scan a definitive 404 instead of the intended retryable 503. The stderr cannot separate the two (the CID is pre-validated, so that string is also the only legitimate absence path), so the disambiguation must be out of band. On the collided fatal, probe object_store_readable (objects/ listable, every pack/idx openable). Unreadable -> Err (taint -> 503). Readable -> re-probe once; still absent -> Ok(None), now present -> the type. This narrows but does not close the concurrent-repack TOCTOU (the check samples a different instant), stated honestly rather than claimed deterministic. Test: object_type_bounded_unreadable_pack_is_error_not_absence packs a real repo, chmod 000 the pack, and asserts Err (root-guarded). RED with the readability check disabled: the present blob returned Ok(None). --- crates/gitlawb-node/src/git/store.rs | 199 +++++++++++++++++++++++++-- 1 file changed, 188 insertions(+), 11 deletions(-) diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index d5a0d1a1..f9d65e9d 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -339,19 +339,86 @@ pub fn object_type_bounded( &[], deadline, )?; - if !status.success() { - // Same classification as `object_type` (a nonzero exit is an ABSENCE verdict - // only when git could examine the object store); U5 refines the absence arm - // with an out-of-band readability check. - let stderr = String::from_utf8_lossy(&stderr); - if stderr.contains("not a git repository") - || stderr.lines().any(|l| l.starts_with("error:")) - { - bail!("git cat-file -t failed: {}", stderr.trim()); + if status.success() { + return Ok(Some(String::from_utf8_lossy(&stdout).trim().to_string())); + } + // A nonzero exit whose stderr proves git could not EXAMINE the object store — + // "not a git repository", or any `error:` line (corrupt loose object, bad idx) — + // is not an absence verdict, so surface it as Err (#173/#174: never a false 404). + let stderr = String::from_utf8_lossy(&stderr); + if stderr.contains("not a git repository") || stderr.lines().any(|l| l.starts_with("error:")) { + bail!("git cat-file -t failed: {}", stderr.trim()); + } + // Any other bare `fatal:` ("could not get object info") is the absence-vs- + // unreadable-pack COLLISION (#174 F5): a genuinely missing object and a packed + // object whose pack/idx is unreadable (permissions, or a mid-repack race) emit the + // identical string. Disambiguate OUT OF BAND: if the object store is not readable, + // this is not an absence verdict -> Err (taint -> retryable 503). + if !object_store_readable(repo_path) { + bail!( + "git cat-file -t inconclusive: object store not readable at {} (not an absence verdict)", + repo_path.display() + ); + } + // Store readable: re-probe once. An object still absent on a confirmed-readable + // store is very likely truly absent (Ok(None)); if a mid-repack race resolved and + // the re-probe now finds it, return the type. This narrows, but cannot fully close, + // the concurrent-repack window (the readability check samples a different instant). + let (status2, stdout2, stderr2) = crate::git::visibility_pack::run_bounded_git_raw( + git_bin, + &["cat-file", "-t", sha256_hex], + repo_path, + &[], + deadline, + )?; + if status2.success() { + return Ok(Some(String::from_utf8_lossy(&stdout2).trim().to_string())); + } + let stderr2 = String::from_utf8_lossy(&stderr2); + if stderr2.contains("not a git repository") || stderr2.lines().any(|l| l.starts_with("error:")) + { + bail!("git cat-file -t failed: {}", stderr2.trim()); + } + Ok(None) +} + +/// Best-effort check that a repo's object store is readable, used to disambiguate a +/// genuine missing-object `git cat-file` fatal from an unreadable or racing pack +/// (both emit "could not get object info"). Returns false on any unreadable +/// `objects/` dir or any pack/idx that cannot be opened (EACCES / EIO), so the +/// caller surfaces an error rather than a false absence. Cheap — a couple of readdir +/// plus open probes. It narrows, but does not close, the concurrent-repack TOCTOU: it +/// samples a different instant than the failing cat-file. +fn object_store_readable(repo_path: &Path) -> bool { + let objects = repo_path.join("objects"); + // The objects dir itself must be listable; drain the iterator so a mid-listing + // EACCES/EIO surfaces, not just the initial open. + let Ok(entries) = std::fs::read_dir(&objects) else { + return false; + }; + for entry in entries { + if entry.is_err() { + return false; } - return Ok(None); } - Ok(Some(String::from_utf8_lossy(&stdout).trim().to_string())) + // Every pack file and its index must be openable for read. A loose-only store + // (no pack dir) is fine — the objects readdir above already proved reachability. + if let Ok(pack_entries) = std::fs::read_dir(objects.join("pack")) { + for entry in pack_entries { + let Ok(entry) = entry else { + return false; + }; + let path = entry.path(); + if matches!( + path.extension().and_then(|s| s.to_str()), + Some("pack") | Some("idx") + ) && std::fs::File::open(&path).is_err() + { + return false; + } + } + } + true } /// Bounded, reaped variant of [`read_object_content`] for the async `/ipfs` serve @@ -571,4 +638,114 @@ mod tests { "unchanged file must not appear: {names:?}" ); } + + /// #174 F5 (RED-before/GREEN-after): a packed object whose pack/idx is unreadable + /// makes `git cat-file -t` emit "could not get object info" — byte-identical to a + /// genuine miss. `object_type_bounded` must report absence ONLY when the object + /// store is confirmed readable; an unreadable store is Err (-> retryable 503), + /// never Ok(None) (-> a wrong 404 for a present object). + #[cfg(unix)] + #[test] + fn object_type_bounded_unreadable_pack_is_error_not_absence() { + use std::os::unix::fs::PermissionsExt; + let td = tempfile::TempDir::new().unwrap(); + let work = td.path().join("work"); + let bare = td.path().join("bare.git"); + std::fs::create_dir_all(&work).unwrap(); + let g = |args: &[&str], dir: &Path| { + assert!( + Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + g(&["init", "-q", "--object-format=sha256", "."], &work); + g(&["config", "user.email", "t@t"], &work); + g(&["config", "user.name", "t"], &work); + std::fs::write(work.join("file.txt"), b"packed f5 content\n").unwrap(); + g(&["add", "file.txt"], &work); + g(&["commit", "-qm", "c1"], &work); + let blob = String::from_utf8( + Command::new("git") + .args(["rev-parse", "HEAD:file.txt"]) + .current_dir(&work) + .output() + .unwrap() + .stdout, + ) + .unwrap() + .trim() + .to_string(); + g( + &[ + "clone", + "-q", + "--bare", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + td.path(), + ); + g(&["gc", "-q"], &bare); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + + // Readable store: the packed blob probes present; a genuine miss is Ok(None). + assert_eq!( + super::object_type_bounded("git", &bare, &blob, deadline) + .unwrap() + .as_deref(), + Some("blob"), + "a packed blob on a readable store must probe present" + ); + assert!( + super::object_type_bounded("git", &bare, &"0".repeat(64), deadline) + .unwrap() + .is_none(), + "a genuinely-absent object on a readable store must be Ok(None)" + ); + + // Make the pack unreadable: cat-file -t now emits the collided fatal for the + // PRESENT blob. It must surface as Err, not a false Ok(None). + let pack_dir = bare.join("objects").join("pack"); + let set_pack_mode = |mode: u32| { + for e in std::fs::read_dir(&pack_dir).unwrap() { + let p = e.unwrap().path(); + if matches!( + p.extension().and_then(|s| s.to_str()), + Some("pack") | Some("idx") + ) { + let mut perms = std::fs::metadata(&p).unwrap().permissions(); + perms.set_mode(mode); + std::fs::set_permissions(&p, perms).unwrap(); + } + } + }; + set_pack_mode(0o000); + // Root bypasses file permissions, so the chmod won't block reads there; only + // assert the error path when the pack is genuinely unreadable to this process. + let a_pack = std::fs::read_dir(&pack_dir) + .unwrap() + .filter_map(|e| e.ok()) + .map(|e| e.path()) + .find(|p| p.extension().and_then(|s| s.to_str()) == Some("pack")); + let genuinely_unreadable = a_pack + .as_ref() + .map(|p| std::fs::File::open(p).is_err()) + .unwrap_or(false); + let res = super::object_type_bounded("git", &bare, &blob, deadline); + set_pack_mode(0o644); // restore so TempDir cleanup succeeds + + if genuinely_unreadable { + assert!( + res.is_err(), + "an unreadable pack must surface as Err (-> retryable 503), not Ok(None) \ + (-> a wrong 404 for a present object); got {res:?}" + ); + } + } } From 448d38458defead064141573626410a3d68ff9bd Mon Sep 17 00:00:00 2001 From: t Date: Sat, 18 Jul 2026 11:13:32 -0500 Subject: [PATCH 50/95] fix(node): bound concurrent pin-loop memory; document the per-repo task cap (#174 F6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EncryptInflight bounds the outstanding post-push encryption-task set to one per repo, but its state.rs doc implied a global cap it does not deliver, and each pin loop (ipfs_pin + pinata pin_new_objects) holds a full per-push object-id list while walking it — so N distinct authenticated first-pushes could hold N MB-scale lists at once, unbounded. Add a global pin_semaphore (GITLAWB_MAX_CONCURRENT_PIN_TASKS, default 8) and gate BOTH pin loops through it: they DEFER when the pool is full, never drop a pin. Tighten the state.rs comment to state the bound is PER-REPO, with the cross-repo residual (throttled by auth + rate limits, memory bounded by the pin permit) called out honestly. Also correct .env.example GITLAWB_DB_MAX_CONNECTIONS (20 -> 48): with the #174 F1 boot validation, 20 is below max_concurrent_git_pushes(32)+headroom(8) and would fail startup — closing the INV-24 doc gap from the F1 change. Tests: pin_new_objects_gated_defers_when_pin_pool_exhausted (RED without the permit acquire); max_concurrent_pin_tasks range/default. --- .env.example | 14 ++++- crates/gitlawb-node/src/api/repos.rs | 83 ++++++++++++++++++++++++- crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/config.rs | 38 +++++++++++ crates/gitlawb-node/src/main.rs | 3 + crates/gitlawb-node/src/state.rs | 29 ++++++--- crates/gitlawb-node/src/test_support.rs | 1 + 7 files changed, 155 insertions(+), 14 deletions(-) diff --git a/.env.example b/.env.example index 8f9bfa2e..9913b41b 100644 --- a/.env.example +++ b/.env.example @@ -24,8 +24,11 @@ DATABASE_URL=postgresql://gitlawb:changeme@localhost:5432/gitlawb # ── Database pool & startup resilience ──────────────────────────────────── # Maximum connections in the PostgreSQL pool. A cap, not a floor — # connections open lazily. Size against the DB server's max_connections, -# remembering admin tooling opens its own pool. -GITLAWB_DB_MAX_CONNECTIONS=20 +# remembering admin tooling opens its own pool. Each concurrent write pins one +# connection for its whole duration (the connection-affine advisory lock), so the +# node REJECTS at boot any value below GITLAWB_MAX_CONCURRENT_GIT_PUSHES + 8 +# headroom — keep this comfortably above that (default 48 for pushes 32). +GITLAWB_DB_MAX_CONNECTIONS=48 # Seconds a request waits for a pool connection before failing with 503. GITLAWB_DB_ACQUIRE_TIMEOUT_SECS=5 # Upper bound on each startup connect+migrate attempt, in seconds. Keep it @@ -141,6 +144,13 @@ GITLAWB_MAX_CONCURRENT_GIT_OPS=128 # Retry-After. Default 32. GITLAWB_MAX_CONCURRENT_GIT_PUSHES=32 +# Max concurrent post-push pin loops (IPFS + Pinata pin_new_objects) across all +# repos. Each loop holds a full per-push object-id list while pinning, so this +# bounds that MB-scale memory even though the per-repo encrypt-task set already +# caps the task COUNT. A loop DEFERS (waits) when the pool is full, never drops a +# pin. Default 8. +GITLAWB_MAX_CONCURRENT_PIN_TASKS=8 + # Max concurrent read ops (upload-pack + the upload-pack info/refs advertisement) # a single caller may hold, so one caller cannot monopolize the read pool. Keyed # on the resolved SOURCE IP, never the DID: a signature does not move a caller off diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index ede6c5e1..6e6d5f3a 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -849,6 +849,7 @@ struct EncryptTaskCtx { git_bin: String, git_timeout: std::time::Duration, encrypt_sem: Arc, + pin_sem: Arc, } /// The detached post-push pin/encrypt task (#174 P2-2 + F5): run this push's own @@ -990,14 +991,40 @@ async fn resolve_drain_object_list( /// The pin/encrypt pipeline shared by the snapshot iteration and the /// coalesced-drain iterations: local IPFS pin, then (path-scoped rules only) the /// admission-gated recipients walk → encrypt-then-pin → Arweave manifest anchor. +/// Pin `object_list` to the local IPFS node under the global pin-admission permit +/// (#174 F6). `EncryptInflight` bounds the pin-task COUNT to one per repo, but each +/// pin loop holds a full per-push object-id list while walking it; this permit bounds +/// how many such MB-scale lists are held at once across all repos. DEFERS (waits) when +/// the pool is full — never drops, since a dropped pin loses the replication copy. +async fn pin_new_objects_gated( + pin_sem: &Arc, + ipfs_api: &str, + repo_path: &std::path::Path, + object_list: Vec, + db: &Arc, +) -> Vec<(String, String)> { + let _permit = pin_sem + .clone() + .acquire_owned() + .await + .expect("pin_semaphore is never closed"); + crate::ipfs_pin::pin_new_objects(ipfs_api, repo_path, object_list, db).await +} + async fn pin_and_encrypt_objects( ctx: &EncryptTaskCtx, object_list: Vec, rules: Option>, is_public: bool, ) { - let pinned = - crate::ipfs_pin::pin_new_objects(&ctx.ipfs_api, &ctx.repo_path, object_list, &ctx.db).await; + let pinned = pin_new_objects_gated( + &ctx.pin_sem, + &ctx.ipfs_api, + &ctx.repo_path, + object_list, + &ctx.db, + ) + .await; if !pinned.is_empty() { tracing::info!(count = pinned.len(), "pinned git objects to IPFS"); for (sha, cid) in &pinned { @@ -1782,6 +1809,7 @@ pub async fn git_receive_pack( state.config.git_service_timeout_secs, ), encrypt_sem: state.git_encrypt_semaphore.clone(), + pin_sem: state.pin_semaphore.clone(), }; tokio::spawn(run_encrypt_pin_task( ctx, @@ -1833,8 +1861,16 @@ pub async fn git_receive_pack( let node_keypair = Arc::clone(&state.node_keypair); let object_list_pinata = object_list; let do_pinata_replication = withheld.is_some(); + // Same global pin-admission bound as the IPFS loop (#174 F6): the Pinata pin + // loop also holds this push's full object-id list while pinning it, so it must + // share the cap. It DEFERS on a full pool rather than dropping the pin. + let pin_sem_pinata = state.pin_semaphore.clone(); tokio::spawn(async move { let pinned = if do_pinata_replication { + let _pin_permit = pin_sem_pinata + .acquire_owned() + .await + .expect("pin_semaphore is never closed"); crate::pinata::pin_new_objects( &http_client, &pinata_upload_url, @@ -5096,6 +5132,48 @@ mod tests { ); } + /// #174 F6 (RED-before/GREEN-after): a post-push pin loop holds this push's full + /// object-id list while walking it, so concurrent pin loops across many repos must + /// be bounded by a global permit, not just the per-repo task count. `pin_new_objects_gated` + /// DEFERS (waits) when the pin pool is exhausted rather than running unbounded. + /// + /// Load-bearing: without the permit acquire the pin loop runs immediately even with + /// the pool held (RED — the deferral assertion fails). With it, it parks. + #[sqlx::test] + async fn pin_new_objects_gated_defers_when_pin_pool_exhausted(pool: sqlx::PgPool) { + use std::sync::Arc; + use tokio::sync::Semaphore; + + let state = crate::test_support::test_state(pool).await; + let db = state.db.clone(); + let tmp = tempfile::TempDir::new().unwrap(); + let pin_sem = Arc::new(Semaphore::new(1)); + // Hold the only pin permit. + let held = pin_sem.clone().acquire_owned().await.unwrap(); + + // Empty ipfs_api makes the pin itself a no-op, but the loop must still DEFER on + // the exhausted pin pool rather than run. + let blocked = tokio::time::timeout( + std::time::Duration::from_millis(500), + pin_new_objects_gated(&pin_sem, "", tmp.path(), vec![], &db), + ) + .await; + assert!( + blocked.is_err(), + "a pin loop must defer while the pin pool is exhausted (#174 F6)" + ); + + // Release admission: the SAME call now completes. + drop(held); + let out = tokio::time::timeout( + std::time::Duration::from_secs(5), + pin_new_objects_gated(&pin_sem, "", tmp.path(), vec![], &db), + ) + .await + .expect("the pin loop completes once admission frees"); + assert!(out.is_empty(), "an empty ipfs_api pins nothing"); + } + /// Shared fixture for the F4 handler-layer tests: a state whose repo_store and /// git_bin point at the given tempdir/fake-git, plus a seeded on-disk repo, /// optionally with a path-scoped rule (so the post-receive walks actually run). @@ -5884,6 +5962,7 @@ mod tests { git_bin: git_bin.to_string(), git_timeout: std::time::Duration::from_secs(600), encrypt_sem: sem, + pin_sem: std::sync::Arc::new(tokio::sync::Semaphore::new(64)), } } diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 912108a0..c1fa6624 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -524,6 +524,7 @@ mod tests { git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + pin_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), encrypt_inflight: crate::state::EncryptInflight::new(), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 5fb66b55..d70be5ea 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -319,6 +319,24 @@ pub struct Config { )] pub max_concurrent_git_pushes: usize, + /// Max concurrent post-push pin loops (`ipfs_pin` and `pinata` + /// `pin_new_objects`) across all repos. `EncryptInflight` bounds the outstanding + /// pin-task COUNT to one per repo, but each pin loop holds a full per-push + /// object-id list (up to `git_max_pack_bytes` worth of OIDs) while it walks it, + /// so N distinct repos could hold N such lists at once. This caps how many run + /// concurrently, bounding that MB-scale memory regardless of how many repos an + /// authenticated actor pushes to (#174 F6). Beyond it a pin loop DEFERS (waits), + /// never drops — a dropped pin would lose the object's replication copy. + /// + /// Default: 8. Must be between 1 and 1_048_576. + #[arg( + long, + env = "GITLAWB_MAX_CONCURRENT_PIN_TASKS", + default_value_t = 8, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub max_concurrent_pin_tasks: usize, + /// Maximum concurrent read operations (`upload-pack` and the upload-pack /// `info/refs` advertisement) a single caller may hold at once, so one caller /// cannot monopolize the `max_concurrent_git_ops` read pool (#174). Callers are @@ -518,6 +536,26 @@ mod tests { ); } + #[test] + fn max_concurrent_pin_tasks_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).max_concurrent_pin_tasks, + 8 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-pin-tasks", "2"]) + .max_concurrent_pin_tasks, + 2 + ); + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-pin-tasks", "0"]).is_err() + ); + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-pin-tasks", "1048577"]) + .is_err() + ); + } + #[test] fn max_concurrent_git_ops_defaults_and_rejects_out_of_range() { assert_eq!( diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index b7bc758e..4c4c1da8 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -402,6 +402,9 @@ async fn main() -> Result<()> { git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new( config.max_concurrent_git_pushes, )), + // Bounds concurrent post-push pin loops (their MB-scale object-id lists) across + // all repos (#174 F6), independent of the per-repo encrypt-task coalescing below. + pin_semaphore: Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_pin_tasks)), // Coalesces the DETACHED post-push encryption tasks per repo so a rapid pusher // cannot grow the outstanding parked-waiter set past one task per repo (#174 // P2-2). No knob: it is a natural cap (one entry per distinct repo), not a diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 8396a369..77799bd0 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -127,16 +127,25 @@ pub struct AppState { /// walk must not hold a foreground write slot, and a handler already holding a /// write permit that needed a second would self-deadlock at pool size 1. pub git_encrypt_semaphore: Arc, - /// Bounds the OUTSTANDING post-push encryption-task set by per-repo coalescing - /// (#174 P2-2). `git_encrypt_semaphore` caps *active* walks; this caps how many - /// detached tasks *spawn and park* on that semaphore's `acquire_owned().await`. - /// Before spawning a per-push encryption task, the receive-pack handler consults - /// this set: if the repo already has a task in flight it coalesces (skips the - /// duplicate spawn) rather than parking a new unbounded waiter, and its tip pairs - /// are recorded for that task's drain loop (#174 F5). Coalescing only delays the - /// coalesced push's walk — it never drops the withheld-blob recovery copy, which - /// `2a54c15` deliberately kept fail-closed (there is no reconciliation sweep to - /// re-derive a dropped copy). See [`EncryptInflight`]. + /// Bounds concurrent post-push pin loops (`ipfs_pin` / `pinata` `pin_new_objects`) + /// across all repos (#174 F6). `encrypt_inflight` caps the pin-task COUNT to one + /// per repo, but each pin loop holds a full per-push object-id list while walking + /// it, so N distinct repos could hold N such MB-scale lists at once. This caps how + /// many run concurrently; a loop DEFERS (waits) when the pool is full, never drops. + pub pin_semaphore: Arc, + /// Bounds the outstanding post-push encryption-task set to at most one PER REPO by + /// coalescing (#174 P2-2). This is NOT a global cap: N distinct repos still admit N + /// tasks; the cross-repo residual (an authenticated actor pushing to many repos + /// leaves many parked tasks) is throttled by auth plus the per-IP/per-DID rate + /// limits, and its real cost — the MB-scale per-push object-id list each pin loop + /// holds — is bounded by `pin_semaphore`, not this. `git_encrypt_semaphore` caps + /// *active* walks; this caps duplicate SPAWNS per repo. Before spawning a per-push + /// encryption task, the receive-pack handler consults this set: if the repo already + /// has a task in flight it coalesces (skips the duplicate spawn) rather than parking + /// a new waiter, and its tip pairs are recorded for that task's drain loop (#174 F5). + /// Coalescing only delays the coalesced push's walk — it never drops the withheld-blob + /// recovery copy, which `2a54c15` deliberately kept fail-closed (there is no + /// reconciliation sweep to re-derive a dropped copy). See [`EncryptInflight`]. pub encrypt_inflight: EncryptInflight, /// Per-caller concurrency sub-cap on the read pool: each caller (keyed on the /// resolved source IP, #174 U1) may hold at most `max_concurrent_reads_per_caller` diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index df095b44..cd58f19e 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -87,6 +87,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + pin_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), encrypt_inflight: crate::state::EncryptInflight::new(), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( From 93cdf308baad7b31043e4f8c96e96ab2e24f4d44 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 09:14:01 -0500 Subject: [PATCH 51/95] fix(node): make RepoWriteGuard::release cancellation-safe (#174 F4) Await pg_advisory_unlock while the pooled connection is still owned by self; take it and set released only after the await resolves. A cancel mid-unlock now leaves the Drop backstop armed (conn still Some), so the session-level advisory lock is no longer leaked onto a pooled connection. Adds a cancel-mid-unlock regression test (via a test-only pre-unlock gate) and an append-only inv22 gate row asserting the take/released orderings. --- crates/gitlawb-node/src/git/repo_store.rs | 151 +++++++++++++++++++++- crates/gitlawb-node/tests/inv22_gates.rs | 47 +++++++ 2 files changed, 195 insertions(+), 3 deletions(-) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 86a1ee8b..d88594e7 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -180,6 +180,8 @@ impl RepoStore { locked: false, released: false, tigris: self.tigris.clone(), + #[cfg(test)] + test_pre_unlock_gate: None, }; // Acquire the advisory lock with retry, through the guard's OWN connection, @@ -390,6 +392,13 @@ pub struct RepoWriteGuard { /// Set once `release` has run its unlock, making the `Drop` backstop inert. released: bool, tigris: Option, + /// Test-only seam: when set, `release` parks on this gate at the exact point + /// it is about to await `pg_advisory_unlock` (connection still owned, not yet + /// released). Dropping the `release` future while it is parked reproduces a + /// mid-unlock cancellation, so a test can assert the `Drop` backstop still + /// frees the session lock. Never set outside tests. + #[cfg(test)] + test_pre_unlock_gate: Option>, } impl RepoWriteGuard { @@ -419,10 +428,22 @@ impl RepoWriteGuard { } // Release the advisory lock on the SAME connection that took it (session - // advisory locks are connection-affine). Taking the connection here makes - // the Drop backstop inert. + // advisory locks are connection-affine). Unlock through the connection + // while it is STILL owned by `self` — do not `take()` it first. If this + // future is cancelled during the unlock await, `self` is dropped with + // `conn == Some(..)` and `released == false`, so the `Drop` backstop still + // runs the detached unlock. `released` is set only AFTER the await + // resolves, so a cancellation cannot make the backstop inert (#174 F4). if self.locked { - if let Some(mut conn) = self.conn.take() { + #[cfg(test)] + let pre_unlock_gate = self.test_pre_unlock_gate.clone(); + if let Some(conn) = self.conn.as_deref_mut() { + // Test-only: park right before the unlock await so a test can drop + // this future mid-unlock (connection owned, not yet released). + #[cfg(test)] + if let Some(gate) = pre_unlock_gate { + gate.notified().await; + } let _ = sqlx::query("SELECT pg_advisory_unlock($1)") .bind(self.lock_key) .execute(&mut *conn) @@ -719,4 +740,128 @@ mod tests { .execute(&mut *checker) .await; } + + // ── cancellation-safe unlock (#174 F4, RED-before/GREEN-after) ────────── + + /// F4 (P1): a cancellation DURING the unlock await must still free the session + /// advisory lock. `release` unlocks through the connection while `self` still + /// owns it, so if the future is dropped mid-unlock, `Drop` sees `conn == Some` + /// + `locked && !released` and runs its detached-unlock backstop. A test-only + /// gate parks `release` at the exact pre-unlock point; dropping the future + /// there reproduces the cancellation. + /// + /// Load-bearing: RED on the original ordering (`self.conn.take()` before the + /// await → at cancellation `self.conn == None` → `Drop` skips → the local + /// connection returns to the pool with the session lock still held → the + /// checker's `pg_try_advisory_lock` returns false). GREEN after the reorder. + #[sqlx::test] + async fn write_guard_release_cancelled_mid_unlock_frees_the_lock(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone()); + let owner = "did:key:z6MkCancelMidUnlockProofCCCCCCCCCCCCCCCCCC"; + let name = "canceltest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + // Distinct session for the probe, held out of the pool before acquiring. + let mut checker = pool.acquire().await.expect("checker connection"); + + let mut guard = store.acquire_write(owner, name).await.expect("acquire"); + // Arm the pre-unlock gate; it is never notified, so `release` parks on it + // with the connection still owned and `released` still false. + let gate = Arc::new(tokio::sync::Notify::new()); + guard.test_pre_unlock_gate = Some(gate); + + // Box the future so we can drop it ourselves (tokio::pin! keeps it alive to + // end of scope, which would defer the guard's Drop past the assertions). + let mut fut = Box::pin(guard.release(false)); + let parked = + tokio::time::timeout(std::time::Duration::from_millis(300), fut.as_mut()).await; + assert!( + parked.is_err(), + "release should park on the pre-unlock gate, not complete" + ); + // Cancel mid-unlock: dropping the boxed future runs RepoWriteGuard::drop. + drop(fut); + + // Let the Drop backstop's detached unlock task run. + tokio::time::sleep(std::time::Duration::from_millis(400)).await; + + let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *checker) + .await + .unwrap(); + assert!( + free, + "advisory lock must be freed when release is cancelled mid-unlock — the \ + connection must stay owned by the guard so Drop's backstop can run" + ); + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } + + /// F4: the ordinary success path still frees the lock, and a second write + /// acquire on the same repo returns promptly (no stale-lock retry loop). + #[sqlx::test] + async fn write_guard_release_true_frees_lock_and_second_acquire_succeeds(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone()); + let owner = "did:key:z6MkReleaseTrueProofDDDDDDDDDDDDDDDDDDDDDD"; + let name = "reltruetest"; + + let guard = store + .acquire_write(owner, name) + .await + .expect("first acquire"); + guard.release(true).await; + + let again = tokio::time::timeout( + std::time::Duration::from_secs(2), + store.acquire_write(owner, name), + ) + .await + .expect("second acquire_write must not hit the ~60s stale-lock retry loop") + .expect("second acquire"); + again.release(true).await; + } + + /// F4: releasing a guard that never took the lock (`locked == false`, the state + /// acquire_write leaves after a failed acquisition) must not unlock or panic. + #[sqlx::test] + async fn write_guard_release_when_not_locked_does_not_unlock_or_panic(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let owner = "did:key:z6MkNotLockedProofEEEEEEEEEEEEEEEEEEEEEE"; + let name = "notlockedtest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + let guard = RepoWriteGuard { + owner_slug: slug, + repo_name: name.to_string(), + local_path: dir.path().to_path_buf(), + lock_key: key, + conn: Some(pool.acquire().await.expect("conn")), + locked: false, + released: false, + tigris: None, + test_pre_unlock_gate: None, + }; + // Must complete without panic and issue no unlock. + guard.release(false).await; + + let mut checker = pool.acquire().await.expect("checker"); + let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *checker) + .await + .unwrap(); + assert!(free, "release on an unlocked guard must not touch the key"); + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } } diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index cbfa0d4c..d21ec194 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -153,3 +153,50 @@ fn inv22_concurrency_gates_present_and_not_bypassed() { ); } } + +/// F4 (repo_store advisory-unlock cancellation safety): `RepoWriteGuard::release` +/// must await `pg_advisory_unlock` while `self` still owns the pooled connection, +/// and must not mark itself `released` until that await resolves. Either shape, +/// reintroduced, re-opens the mid-unlock cancellation leak: taking the connection +/// early leaves `Drop` with `conn == None`, and setting `released = true` early +/// leaves the `Drop` backstop inert — both strand the session lock on cancellation. +/// +/// Scoped to the `release` fn body: the `Drop` impl legitimately takes the +/// connection and unlocks, so a whole-file scan would match it and read as a false +/// pass. Reverting either ordering turns this red (proven load-bearing). +#[test] +fn f4_release_keeps_conn_owned_until_unlock_resolves() { + let repo_store = src("git/repo_store.rs"); + + let rel_start = repo_store + .find("pub async fn release(mut self") + .expect("F4 gate: repo_store.rs no longer defines RepoWriteGuard::release"); + let rel_end = repo_store[rel_start..] + .find("impl Drop for RepoWriteGuard") + .map(|off| rel_start + off) + .expect("F4 gate: release fn / Drop impl markers moved — update this guard"); + let release_body = &repo_store[rel_start..rel_end]; + + let unlock = release_body + .find("pg_advisory_unlock") + .expect("F4 gate: release must still issue pg_advisory_unlock"); + let before_unlock = &release_body[..unlock]; + + // (a) the connection must still be owned by `self` at the unlock await. + assert!( + !before_unlock.contains("self.conn.take()"), + "F4 regression: RepoWriteGuard::release takes self.conn BEFORE awaiting \ + pg_advisory_unlock. A cancellation during the unlock await then strands the \ + session advisory lock (Drop sees conn == None and skips its backstop). \ + Unlock through the still-owned connection instead." + ); + // (b) `released` must not be set before the unlock await — the other + // reintroduction shape a single-reorder check on (a) alone is blind to. + assert!( + !before_unlock.contains("released = true"), + "F4 regression: RepoWriteGuard::release sets `released = true` BEFORE awaiting \ + pg_advisory_unlock. A cancellation during the await then leaves the Drop \ + backstop inert (it early-returns on released). Set released only AFTER the \ + unlock await resolves." + ); +} From e0e207e6455dd3b51aa85085611e6b06c2c3f770 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 09:36:09 -0500 Subject: [PATCH 52/95] fix(node): classify /ipfs object probe with a version-stable signal (#174 F5) Probe existence via `git cat-file --batch-check` (structured ` missing` on exit 0) instead of matching English `cat-file -t` prose, so a git wording change cannot flip a miss into an error or vice-versa. A bad-config/corrupt repo on a readable store is now a terminal, non-retryable 500 (not a false 404, and not a retryable 503 that would fan out cat-file retries); a transient unreadable store stays 503. Error bodies are opaque (no raw git stderr). Layers onto the existing object_store_readable + re-probe disambiguation. --- crates/gitlawb-node/src/api/ipfs.rs | 126 ++++++++- crates/gitlawb-node/src/git/store.rs | 369 +++++++++++++++++++++++---- 2 files changed, 445 insertions(+), 50 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 43e418ae..1910a14b 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -61,6 +61,13 @@ use crate::visibility::{visibility_check, Decision}; /// the truncation sources — existing content is never misreported absent /// because of unrelated repos or transient faults. /// +/// Deterministic fault (F5/U4): a candidate repo that is persistently broken (a +/// corrupt repo, a bad `.git/config`) also yields no absence verdict, but a retry +/// cannot fix it, so a scan that found nothing sheds a TERMINAL, non-retryable 500 +/// (opaque body) rather than the retryable 503 — checked first so a deterministic +/// fault is never downgraded, and gated on nothing-served so a healthy repo that +/// carries the object still serves. +/// /// Request budget (F3): one absolute clock (`ipfs_request_budget_secs`) spans /// the whole admitted request. No stage (acquire, probe, walk, content read) /// starts once it is exhausted, and the acquire wait and walk deadline are @@ -181,6 +188,13 @@ pub async fn get_by_cid( truncated_by.push(source); } } + // A DETERMINISTIC probe fault (a corrupt repo / bad `.git/config`; #174 F5/U4) is + // separate from a transient taint: a retry cannot fix it, so a scan that found + // nothing must NOT shed the retryable 503 (which would invite a conformant client + // to retry-storm a fresh `git cat-file` per attempt against the broken repo). It + // sheds a terminal, non-retryable 500 instead — but only if nothing served, so one + // corrupt repo never masks a healthy repo that carries the object. + let mut deterministic_fault = false; // Budget gate shared by the four per-stage checks (F3): the remaining // request budget, or — once exhausted — None, after logging the stage and @@ -328,11 +342,22 @@ pub async fn get_by_cid( { Ok(Ok(Some(t))) => t, Ok(Ok(None)) => continue, - Ok(Err(e)) => { - tracing::warn!(repo = %repo.name, err = %e, "object-type probe failed during /ipfs scan; skipping repo without a verdict"); + // Transient probe fault (unreadable/mid-repack store): unproven, retryable. + Ok(Err(store::ProbeError::Transient(e))) => { + tracing::warn!(repo = %repo.name, err = %e, "object-type probe hit a transient store fault during /ipfs scan; skipping repo without a verdict"); taint(&mut truncated_by, "probe"); continue; } + // Deterministic probe fault (corrupt repo / bad config): a retry cannot fix + // it, so it does NOT taint (which would shed a retryable 503). It records a + // terminal condition that the terminal arm renders as a non-retryable 500 + // only if nothing served. The raw git detail stays in the log; the client + // body is opaque. + Ok(Err(store::ProbeError::Deterministic(e))) => { + tracing::warn!(repo = %repo.name, err = %e, "object-type probe hit a deterministic fault (corrupt repo/config) during /ipfs scan; skipping repo without a verdict"); + deterministic_fault = true; + continue; + } Err(join_err) => { tracing::warn!(repo = %repo.name, err = %join_err, "object-type probe task panicked during /ipfs scan; skipping repo without a verdict"); taint(&mut truncated_by, "probe"); @@ -500,6 +525,21 @@ pub async fn get_by_cid( return Ok((StatusCode::OK, headers, content).into_response()); } + // Deterministic fault (F5/U4): a candidate repo is persistently broken (corrupt + // repo / bad `.git/config`), so the object is not proven absent AND a retry cannot + // change that. Shed a TERMINAL, non-retryable 500 rather than the retryable 503 + // below — a 503 would let a conformant client retry-storm a fresh `git cat-file` + // per attempt against the broken repo. This is checked before the transient 503 so + // a deterministic fault is never downgraded to a retryable status. The body is + // opaque (a generic message via `AppError::Git` -> 500, no Retry-After): the raw + // git stderr — which leaks filesystem paths / config — was logged at the probe, and + // never reaches the client. + if deterministic_fault { + return Err(AppError::Git( + "ipfs object probe could not complete: a candidate repository is corrupt".into(), + )); + } + // Truncated scan (F2): at least one candidate repo yielded no verdict, so the // object is not proven absent. A 404 here would misreport existing content, so // shed retryable instead — Overloaded is the single 503 + Retry-After site in @@ -1195,6 +1235,88 @@ mod tests { ); } + /// #174 F5/U4 (RED-before/GREEN-after): a candidate repo with a corrupt + /// `.git/config` makes `git cat-file` die with `fatal: bad config line N` while + /// `objects/` stays readable. That is a DETERMINISTIC fault, not an absence, and a + /// retry cannot fix it — so the scan must shed a TERMINAL, non-retryable 500, never + /// the old false 404 (`Ok(None)` fell through) and never the retryable 503 (which + /// would invite a conformant client to retry-storm a fresh `cat-file` per attempt). + /// A second, healthy repo probes clean (absent verdict); the corrupt row is what + /// forces the 500. The body must be OPAQUE — no raw git stderr, no filesystem path. + /// MUTATION (RED): route the deterministic fault back to `Ok(None)` in + /// `object_type_bounded` and this decays to a 404; classify it Transient and it + /// decays to a retryable 503. + #[sqlx::test] + async fn get_by_cid_bad_config_repo_is_terminal_500_not_404_or_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // A healthy repo that probes clean (would give a definitive 404 on its own) plus + // a repo whose bare git dir has a corrupt config (objects/ intact). + seed_repo_with_blob(&state, tmp.path(), "z6f5clean", "real", b"probe clean\n").await; + state + .db + .upsert_mirror_repo("z6f5badcfg", "broken", "/unused-badcfg", None, false) + .await + .unwrap(); + let rec = state + .db + .get_repo("z6f5badcfg", "broken") + .await + .unwrap() + .unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + std::fs::create_dir_all(&bare).unwrap(); + run_git(&["init", "-q", "--bare", "--object-format=sha256"], &bare); + // Corrupt the config; leave objects/ readable (the readable-store + git-fails + // combination is exactly what makes this deterministic, not transient). + { + use std::io::Write; + let mut cfg = std::fs::OpenOptions::new() + .append(true) + .open(bare.join("config")) + .unwrap(); + cfg.write_all(b"\n[broken section\nnot a valid = = = line\n") + .unwrap(); + } + + let peer: SocketAddr = "203.0.113.69:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::INTERNAL_SERVER_ERROR, + "a bad-config (deterministic) repo fault must shed a terminal 500, never a \ + 404 (false absence) or a retryable 503" + ); + assert!( + resp.headers().get("retry-after").is_none(), + "a terminal 500 must NOT advertise a retry (that is the whole point vs 503)" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + !body.contains("bad config") + && !body.contains(tmp.path().to_str().unwrap()) + && !body.contains(".git") + && !body.contains("fatal"), + "the 500 body must be opaque — no raw git stderr / config text / filesystem \ + path; got: {body}" + ); + } + /// F2 read taint: the gate passes (the probe reads the truncated loose object's /// intact "blob 64" header) but the content read fails (`cat-file blob` dies on /// the deflate stream cut mid-content) — the probe just said the object EXISTS diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index f9d65e9d..a18ab472 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -320,66 +320,164 @@ pub fn read_object_content(repo_path: &Path, sha256_hex: &str, obj_type: &str) - Ok(content_output.stdout) } -/// Bounded, reaped variant of [`object_type`] for the async `/ipfs` serve path -/// (#174 F3): runs `git cat-file -t` off the caller's runtime through the -/// process-group + watchdog reaper, so a hung or corrupt object store cannot pin a -/// runtime worker or an IPFS admission permit past `deadline`. Exit classification -/// is identical to [`object_type`] — a missing object is `Ok(None)`, a store-access -/// failure is `Err` — so the serve path's 404-vs-503 semantics are unchanged. -pub fn object_type_bounded( +/// Why an `/ipfs` existence probe could not return an absence verdict (#174 F5/U4). +/// The caller (`api::ipfs`) maps the variant to an HTTP status, and the split exists +/// only for that mapping: a `Transient` fault is retryable (503), a `Deterministic` +/// fault is terminal (500). +/// +/// The discriminator is object-store readability, NOT any English `git` wording, so a +/// future `git` message change cannot silently reclassify a fault (KTD-4): if the +/// store cannot be read (an unreadable or mid-repack pack, a removed `objects/` dir, +/// a permissions fault) the fault may clear on its own -> `Transient`; if the store IS +/// readable yet `git` still fails (a corrupt repo, a bad `.git/config`) a retry cannot +/// fix it -> `Deterministic`, so a conformant client is told not to retry-storm a +/// fresh `git cat-file` per attempt against a persistently broken repo. +#[derive(Debug)] +pub enum ProbeError { + /// Retryable (-> 503): the object store could not be read right now. + Transient(anyhow::Error), + /// Terminal (-> 500): a persistent, deterministic fault a retry cannot fix. + Deterministic(anyhow::Error), +} + +impl std::fmt::Display for ProbeError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ProbeError::Transient(e) => write!(f, "transient probe fault: {e}"), + ProbeError::Deterministic(e) => write!(f, "deterministic probe fault: {e}"), + } + } +} + +impl std::error::Error for ProbeError {} + +/// Structured outcome of one `git cat-file --batch-check` existence probe. +enum BatchProbe { + /// Present: git printed ` ` on exit 0. + Present(String), + /// A structured, CLEAN absence: git printed ` missing` on exit 0 with no + /// `error:` diagnostics on stderr. This is the ONLY signal that can become an + /// `Ok(None)` (404) absence verdict — and even then only after the store-readable + /// disambiguation below, since an unreadable pack ALSO prints a clean `missing`. + Missing, + /// git could not honestly examine the object store: a hard exit (bad config, not a + /// git repo) OR an exit-0 `missing` accompanied by `error:` diagnostics (a corrupt + /// loose object still prints `missing` on stdout but complains on stderr). Never an + /// absence verdict. Carries the raw git detail for the server log only. + Fault(anyhow::Error), +} + +/// One bounded, reaped `git cat-file --batch-check` probe (the oid is fed on stdin, so +/// no prose is matched to decide presence — the `missing` token and the exit code are +/// the structured signal). See [`object_type_bounded`] for the surrounding teardown +/// guarantees. +fn batch_check_probe( git_bin: &str, repo_path: &Path, sha256_hex: &str, deadline: std::time::Instant, -) -> Result> { +) -> std::result::Result { + let stdin = format!("{sha256_hex}\n"); let (status, stdout, stderr) = crate::git::visibility_pack::run_bounded_git_raw( git_bin, - &["cat-file", "-t", sha256_hex], + &["cat-file", "--batch-check"], repo_path, - &[], + stdin.as_bytes(), deadline, - )?; - if status.success() { - return Ok(Some(String::from_utf8_lossy(&stdout).trim().to_string())); - } - // A nonzero exit whose stderr proves git could not EXAMINE the object store — - // "not a git repository", or any `error:` line (corrupt loose object, bad idx) — - // is not an absence verdict, so surface it as Err (#173/#174: never a false 404). + ) + // A spawn/timeout failure of the reaped child is not deterministic — retry it. + .map_err(ProbeError::Transient)?; + let stderr = String::from_utf8_lossy(&stderr); - if stderr.contains("not a git repository") || stderr.lines().any(|l| l.starts_with("error:")) { - bail!("git cat-file -t failed: {}", stderr.trim()); - } - // Any other bare `fatal:` ("could not get object info") is the absence-vs- - // unreadable-pack COLLISION (#174 F5): a genuinely missing object and a packed - // object whose pack/idx is unreadable (permissions, or a mid-repack race) emit the - // identical string. Disambiguate OUT OF BAND: if the object store is not readable, - // this is not an absence verdict -> Err (taint -> retryable 503). - if !object_store_readable(repo_path) { - bail!( - "git cat-file -t inconclusive: object store not readable at {} (not an absence verdict)", - repo_path.display() - ); + // A corrupt object makes `--batch-check` print `missing` on stdout (exit 0) yet + // emit `error:` lines on stderr; those diagnostics disqualify a clean-absence read + // regardless of the exit code, so they are checked before anything else. + let has_error_diag = stderr.lines().any(|l| l.starts_with("error:")); + + if status.success() && !has_error_diag { + let line = String::from_utf8_lossy(&stdout); + let line = line.trim(); + // ` missing` is the structured absence token; ` ` is a + // hit. Anything else on a "success" exit is unexpected and not an absence. + if line + .rsplit(' ') + .next() + .is_some_and(|last| last == "missing") + { + return Ok(BatchProbe::Missing); + } + let mut parts = line.split_whitespace(); + if let (Some(_oid), Some(ty), Some(_size)) = (parts.next(), parts.next(), parts.next()) { + return Ok(BatchProbe::Present(ty.to_string())); + } + return Ok(BatchProbe::Fault(anyhow::anyhow!( + "unexpected git cat-file --batch-check output: {line:?}" + ))); } - // Store readable: re-probe once. An object still absent on a confirmed-readable - // store is very likely truly absent (Ok(None)); if a mid-repack race resolved and - // the re-probe now finds it, return the type. This narrows, but cannot fully close, - // the concurrent-repack window (the readability check samples a different instant). - let (status2, stdout2, stderr2) = crate::git::visibility_pack::run_bounded_git_raw( - git_bin, - &["cat-file", "-t", sha256_hex], - repo_path, - &[], - deadline, - )?; - if status2.success() { - return Ok(Some(String::from_utf8_lossy(&stdout2).trim().to_string())); + + Ok(BatchProbe::Fault(anyhow::anyhow!( + "git cat-file --batch-check failed (exit {:?}): {}", + status.code(), + stderr.trim() + ))) +} + +/// Bounded, reaped variant of [`object_type`] for the async `/ipfs` serve path +/// (#174 F3/F5): runs `git cat-file --batch-check` off the caller's runtime through the +/// process-group + watchdog reaper, so a hung or corrupt object store cannot pin a +/// runtime worker or an IPFS admission permit past `deadline`. +/// +/// Absence is keyed on `--batch-check`'s STRUCTURED ` missing` token on exit 0, +/// never on any English `fatal:` wording (KTD-4): a genuinely-absent object is the only +/// `Ok(None)` (404) path. A probe that could not honestly examine the store is a +/// [`ProbeError`], split by object-store readability into `Transient` (retryable 503) +/// and `Deterministic` (terminal 500) so the serve path can shed the right status. +pub fn object_type_bounded( + git_bin: &str, + repo_path: &Path, + sha256_hex: &str, + deadline: std::time::Instant, +) -> std::result::Result, ProbeError> { + match batch_check_probe(git_bin, repo_path, sha256_hex, deadline)? { + BatchProbe::Present(ty) => Ok(Some(ty)), + BatchProbe::Fault(detail) => Err(classify_store_fault(repo_path, detail)), + BatchProbe::Missing => { + // A clean `missing` is the absence-vs-unreadable-pack COLLISION (#174 F5): + // a genuinely missing object AND a packed object whose pack/idx is + // unreadable (permissions, or a mid-repack race) both print an identical + // clean `missing`. Disambiguate OUT OF BAND on store readability — an + // unreadable store is not an absence verdict (taint -> retryable 503). + if !object_store_readable(repo_path) { + return Err(ProbeError::Transient(anyhow::anyhow!( + "git cat-file inconclusive: object store not readable at {} (not an absence verdict)", + repo_path.display() + ))); + } + // Store readable: re-probe once. Still `missing` on a confirmed-readable + // store is very likely truly absent (Ok(None)); a mid-repack race that + // resolved returns the type. This narrows, but cannot fully close, the + // concurrent-repack window (the readability check samples a different + // instant than the failing probe). + match batch_check_probe(git_bin, repo_path, sha256_hex, deadline)? { + BatchProbe::Present(ty) => Ok(Some(ty)), + BatchProbe::Fault(detail) => Err(classify_store_fault(repo_path, detail)), + BatchProbe::Missing => Ok(None), + } + } } - let stderr2 = String::from_utf8_lossy(&stderr2); - if stderr2.contains("not a git repository") || stderr2.lines().any(|l| l.starts_with("error:")) - { - bail!("git cat-file -t failed: {}", stderr2.trim()); +} + +/// Classify a probe fault by object-store readability (#174 F5/U4). An unreadable store +/// may be a transient permissions/mid-repack condition (retryable 503); a readable +/// store on which git still fails is a persistent, deterministic fault — a corrupt repo +/// or a bad `.git/config` — that a retry cannot fix (terminal 500). The `detail` is +/// carried for the server log; the client-facing body is opaque (set by the caller). +fn classify_store_fault(repo_path: &Path, detail: anyhow::Error) -> ProbeError { + if object_store_readable(repo_path) { + ProbeError::Deterministic(detail) + } else { + ProbeError::Transient(detail) } - Ok(None) } /// Best-effort check that a repo's object store is readable, used to disambiguate a @@ -748,4 +846,179 @@ mod tests { ); } } + + /// Shared setup: a bare sha256 repo carrying one committed blob. Returns the repo + /// path and the blob's oid. + #[cfg(unix)] + fn bare_repo_with_blob(td: &std::path::Path) -> (std::path::PathBuf, String) { + let work = td.join("work"); + let bare = td.join("bare.git"); + std::fs::create_dir_all(&work).unwrap(); + let g = |args: &[&str], dir: &Path| { + assert!( + Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + g(&["init", "-q", "--object-format=sha256", "."], &work); + g(&["config", "user.email", "t@t"], &work); + g(&["config", "user.name", "t"], &work); + std::fs::write(work.join("file.txt"), b"f5 u4 content\n").unwrap(); + g(&["add", "file.txt"], &work); + g(&["commit", "-qm", "c1"], &work); + let blob = String::from_utf8( + Command::new("git") + .args(["rev-parse", "HEAD:file.txt"]) + .current_dir(&work) + .output() + .unwrap() + .stdout, + ) + .unwrap() + .trim() + .to_string(); + g( + &[ + "clone", + "-q", + "--bare", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + td, + ); + (bare, blob) + } + + /// #174 F5/U4 (RED-before/GREEN-after, the CORE regression guard): a repo with a + /// corrupt `.git/config` makes `git cat-file` die with `fatal: bad config line N` + /// (exit 128, NO `error:` line) while `objects/` stays fully readable. The old + /// `-t` path let that fall through to `Ok(None)` — a false 404 for content that + /// may well exist. `object_type_bounded` must instead classify it as a + /// DETERMINISTIC fault (a retry cannot fix it) so the serve path renders a + /// terminal 500, never a 404 and never a retryable 503. + /// + /// LOAD-BEARING: revert the classification (route `BatchProbe::Fault` on a readable + /// store back to `Ok(None)`, or drop the `has_error_diag`/exit checks so a hard + /// `fatal:` is read as `missing`) and this goes RED — the probe reports the corrupt + /// repo as an absent object. + #[cfg(unix)] + #[test] + fn object_type_bounded_bad_config_is_deterministic_not_absence() { + let td = tempfile::TempDir::new().unwrap(); + let (bare, blob) = bare_repo_with_blob(td.path()); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + + // Baseline on the healthy repo: present blob probes present, genuine miss is + // the clean Ok(None) absence verdict. + assert_eq!( + super::object_type_bounded("git", &bare, &blob, deadline) + .unwrap() + .as_deref(), + Some("blob"), + "a present blob on a healthy readable store must probe present" + ); + assert!( + super::object_type_bounded("git", &bare, &"0".repeat(64), deadline) + .unwrap() + .is_none(), + "a genuinely-absent object on a healthy readable store must be Ok(None) (404)" + ); + + // Corrupt the config; objects/ is untouched (and stays readable). + { + use std::io::Write; + let mut cfg = std::fs::OpenOptions::new() + .append(true) + .open(bare.join("config")) + .unwrap(); + cfg.write_all(b"\n[broken section\nnot a valid = = = line\n") + .unwrap(); + } + assert!( + super::object_store_readable(&bare), + "config corruption must leave objects/ readable (that is the whole point: \ + a readable store + a git failure == deterministic, not transient)" + ); + + // Probing the PRESENT blob under the bad config must be a DETERMINISTIC fault, + // never Ok(None) (the old false 404) and never a Transient (retryable 503). + let res = super::object_type_bounded("git", &bare, &blob, deadline); + assert!( + matches!(res, Err(super::ProbeError::Deterministic(_))), + "a bad-config fatal on a readable store must be a terminal Deterministic \ + fault (-> 500), never Ok(None) (-> false 404) or Transient (-> 503); got {res:?}" + ); + // And a genuinely-absent oid under the bad config is ALSO not an absence verdict. + let res_absent = super::object_type_bounded("git", &bare, &"0".repeat(64), deadline); + assert!( + matches!(res_absent, Err(super::ProbeError::Deterministic(_))), + "even a would-be-absent oid must not read as Ok(None) once the config is \ + corrupt; got {res_absent:?}" + ); + } + + /// #174 F5/U4: a corrupt LOOSE object makes `git cat-file --batch-check` print + /// ` missing` on stdout (exit 0) yet emit `error:` diagnostics on stderr. The + /// clean-`missing` absence path must NOT fire here — the `error:` line disqualifies + /// a clean-absence read — so the probe surfaces a fault, not a false Ok(None) 404. + /// The object store is readable (a corrupt object file still opens), so this is a + /// Deterministic fault. LOAD-BEARING: drop the `has_error_diag` guard and a corrupt + /// object reads as `missing` -> Ok(None) -> false 404 (RED). + #[cfg(unix)] + #[test] + fn object_type_bounded_corrupt_loose_object_is_fault_not_absence() { + use std::os::unix::fs::PermissionsExt; + let td = tempfile::TempDir::new().unwrap(); + let work = td.path().join("loose"); + std::fs::create_dir_all(&work).unwrap(); + let g = |args: &[&str]| { + assert!( + Command::new("git") + .args(args) + .current_dir(&work) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + g(&["init", "-q", "--object-format=sha256", "."]); + g(&["config", "user.email", "t@t"]); + g(&["config", "user.name", "t"]); + std::fs::write(work.join("f.txt"), b"loose object content\n").unwrap(); + g(&["add", "f.txt"]); + g(&["commit", "-qm", "c1"]); + let blob = String::from_utf8( + Command::new("git") + .args(["rev-parse", "HEAD:f.txt"]) + .current_dir(&work) + .output() + .unwrap() + .stdout, + ) + .unwrap() + .trim() + .to_string(); + + // Overwrite the loose object file with non-zlib garbage (it is 0o444 by default). + let obj = work.join(".git/objects").join(&blob[0..2]).join(&blob[2..]); + let mut perms = std::fs::metadata(&obj).unwrap().permissions(); + perms.set_mode(0o644); + std::fs::set_permissions(&obj, perms).unwrap(); + std::fs::write(&obj, b"garbage not a zlib stream").unwrap(); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + let res = super::object_type_bounded("git", &work, &blob, deadline); + assert!( + res.is_err(), + "a corrupt loose object (error: on stderr, `missing` on stdout) must be Err, \ + never a false Ok(None) 404; got {res:?}" + ); + } } From 9180467973c5b25b3e2a20f401c7bd532d91abf1 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 10:00:40 -0500 Subject: [PATCH 53/95] fix(node): clamp the initial /ipfs metadata queries to the request budget (#174 F6) list_all_repos and list_visibility_rules_for_repos ran as bare awaits while holding the scarce IPFS walk permits, so a query blocked in Postgres pinned the slots past GITLAWB_IPFS_REQUEST_BUDGET_SECS and shed later requests as 503s. Wrap both in the remaining request-deadline budget; on timeout the RAII permits drop and the handler sheds a retryable budget 503. Fails closed: a timeout on the visibility-rules query denies the request rather than serving an unfiltered listing. Appends an inv22 gate row asserting both queries stay deadline-wrapped. --- crates/gitlawb-node/src/api/ipfs.rs | 263 ++++++++++++++++++++++- crates/gitlawb-node/tests/inv22_gates.rs | 41 ++++ 2 files changed, 293 insertions(+), 11 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 1910a14b..7f977a81 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -148,22 +148,65 @@ pub async fn get_by_cid( None => None, }; - // 2. Search all repos for an object with this SHA-256 - let repos = state - .db - .list_all_repos() - .await - .map_err(AppError::Internal)?; + // 2. Search all repos for an object with this SHA-256. + // + // F6/KTD-5: both initial metadata queries run while the scarce walk permits + // acquired above are ALREADY held (RAII, for the whole request) and BEFORE the + // per-repo loop's first budget gate. With no pool statement_timeout, a query + // blocked in Postgres would otherwise pin those walk slots for the entire stall + // — past the request budget — capacity-503'ing later requests. Clamp BOTH to the + // remaining request budget. On timeout return the same retryable budget 503 the + // later stages shed (the "budget" source); returning here drops the RAII permits, + // freeing the slot. list_visibility_rules_for_repos is the access-control query, + // so its timeout returns BEFORE the loop — the scan can NEVER run with an empty + // rule map and serve an unfiltered listing that exposes private repos (FAIL CLOSED). + let repos = match tokio::time::timeout( + request_deadline.saturating_duration_since(std::time::Instant::now()), + state.db.list_all_repos(), + ) + .await + { + Ok(Ok(repos)) => repos, + Ok(Err(e)) => return Err(AppError::Internal(e)), + Err(_elapsed) => { + tracing::warn!( + budget_secs = state.config.ipfs_request_budget_secs, + "/ipfs list_all_repos exceeded the request budget \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); shedding a retryable 503 and freeing the walk permit" + ); + return Err(AppError::Overloaded(format!( + "ipfs scan incomplete (budget) for CID {cid_str}; retry shortly" + ))); + } + }; // Fetch every repo's visibility rules in one query rather than one per row // (the gate runs each row against its OWN rules — KTD2a). A row absent from // the map has no rules. let repo_ids: Vec = repos.iter().map(|r| r.id.clone()).collect(); - let rules_by_repo = state - .db - .list_visibility_rules_for_repos(&repo_ids) - .await - .map_err(AppError::Internal)?; + let rules_by_repo = match tokio::time::timeout( + request_deadline.saturating_duration_since(std::time::Instant::now()), + state.db.list_visibility_rules_for_repos(&repo_ids), + ) + .await + { + Ok(Ok(rules)) => rules, + Ok(Err(e)) => return Err(AppError::Internal(e)), + // FAIL CLOSED (security-critical): a timeout on the access-control query must + // DENY. Returning here — before the loop — means the scan can never fall + // through and apply an empty rule map, which would serve an unfiltered listing + // exposing private repos. It sheds the same retryable budget 503 as above. + Err(_elapsed) => { + tracing::warn!( + budget_secs = state.config.ipfs_request_budget_secs, + "/ipfs list_visibility_rules_for_repos exceeded the request budget \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); denying (fail closed) and freeing the walk permit" + ); + return Err(AppError::Overloaded(format!( + "ipfs scan incomplete (budget) for CID {cid_str}; retry shortly" + ))); + } + }; // Request-scoped memo of the per-repo allowed-blob set (KTD1, #126). The // caller is constant for one request, so `repo.id` alone is a safe, @@ -2416,4 +2459,202 @@ mod tests { "a different IP must not be braked by another IP's exhausted bucket" ); } + + /// F6/KTD-5: the two initial metadata queries (`list_all_repos`, + /// `list_visibility_rules_for_repos`) run AFTER the scarce walk permits are + /// acquired (held RAII for the whole request) but BEFORE the per-repo loop's + /// first budget gate. Pre-fix they were bare awaits with no deadline, so a query + /// blocked in Postgres pinned the walk slot for the whole stall, past the request + /// budget. Here we hold an ACCESS EXCLUSIVE lock on `repos` so `list_all_repos` + /// blocks; with the budget clamp the request sheds a retryable budget 503 within + /// ~budget and FREES the walk permit, and a follow-up (lock released) is served. + /// + /// Load-bearing: pre-fix the bare await blocks on the lock until the 10s wrapping + /// timeout fires (RED — "never returned within budget"). After the fix it returns + /// the 503 at ~1s and the permit is free again. MUTATION (RED): drop the + /// `tokio::time::timeout` around `list_all_repos` and this hangs past the wrap. + #[sqlx::test] + async fn get_by_cid_stalled_metadata_query_frees_walk_permit(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Global walk pool of 1 so the held/freed permit is directly observable; + // per-source cap permissive so only the global pool matters. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + let mut cfg = (*state.config).clone(); + cfg.ipfs_request_budget_secs = 1; + state.config = Arc::new(cfg); + + let sem = state.git_ipfs_walk_semaphore.clone(); + let router = ipfs_router(state); + + // Hold an ACCESS EXCLUSIVE lock on `repos` on a dedicated pooled connection: + // `list_all_repos`' SELECT needs ACCESS SHARE, which conflicts, so it blocks + // at lock acquisition regardless of row count. + let mut lock_conn = pool.acquire().await.unwrap(); + sqlx::raw_sql("BEGIN; LOCK TABLE repos IN ACCESS EXCLUSIVE MODE;") + .execute(&mut *lock_conn) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.80:5000".parse().unwrap(); + let started = std::time::Instant::now(); + let resp = tokio::time::timeout( + std::time::Duration::from_secs(10), + router.clone().oneshot(get_cid(&valid_cid(), Some(peer))), + ) + .await + .expect("the budget clamp must return within budget; a bare await hangs on the lock") + .unwrap(); + let elapsed = started.elapsed(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a metadata query blocked past the request budget must shed a retryable 503" + ); + assert!( + elapsed < std::time::Duration::from_secs(3), + "the clamp must end the request at ~budget (1s); got {elapsed:?} \ + (pre-fix the bare await blocks on the lock for the whole stall)" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("budget"), + "the shed must name the budget taint so it maps to \ + GITLAWB_IPFS_REQUEST_BUDGET_SECS; got: {body}" + ); + // The scarce walk permit was RAII-dropped on the early return, not pinned for + // the stall: the slot is free again the instant the request returns. + assert_eq!( + sem.available_permits(), + 1, + "the walk permit must be freed on the budget-shed path, not held for the stall" + ); + + // Release the lock; a follow-up request is now SERVED (404 — empty DB), never + // capacity-503'd, proving the slot was not left pinned. + sqlx::raw_sql("ROLLBACK") + .execute(&mut *lock_conn) + .await + .unwrap(); + drop(lock_conn); + let resp2 = router + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp2.status(), + StatusCode::NOT_FOUND, + "with the permit freed and the lock released, a follow-up is served (404), \ + not capacity-503'd" + ); + } + + /// F6/KTD-5 FAIL CLOSED (security-critical): `list_visibility_rules_for_repos` is + /// the access-control query. If its timeout let the handler fall through with an + /// empty rule map, the loop would apply no visibility rules and serve an unfiltered + /// listing — exposing a public repo's path-restricted blob. Here a PUBLIC repo + /// carries the blob under a path-scoped rule that denies anon; `visibility_rules` + /// is locked ACCESS EXCLUSIVE so the rule query blocks. The fix returns the budget + /// 503 BEFORE the loop, so the handler NEVER serves (never 200). + /// + /// Load-bearing: pre-fix the bare await blocks on the lock until the 10s wrap fires + /// (RED). After the fix it sheds the 503 at ~1s. The `assert_ne!(200)` is the + /// fail-closed guard: a naive fix that `unwrap_or_default()`s the rules on timeout + /// and falls through would serve the blob 200 and trip it. + #[sqlx::test] + async fn get_by_cid_visibility_rule_timeout_fails_closed(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool.clone()); + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + + // A PUBLIC repo (upsert_mirror_repo sets is_public=true) carrying the blob, + // with a path-scoped rule restricting src/** to a reader that is NOT the anon + // caller. Rules applied => the blob is denied; rules skipped (fall-through) => + // the public repo serves it (the exposure this guard forbids). + let (repo_id, oid) = + seed_repo_with_blob(&state, tmp.path(), "z6f3failclosed", "gated", b"private\n").await; + state + .db + .set_visibility_rule( + &repo_id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderDDDDDDDDDDDDDDDDDDDDDDDD".to_string()], + "z6f3failclosed", + ) + .await + .unwrap(); + + let mut cfg = (*state.config).clone(); + cfg.ipfs_request_budget_secs = 1; + state.config = Arc::new(cfg); + let sem = state.git_ipfs_walk_semaphore.clone(); + let router = ipfs_router(state); + + // Lock `visibility_rules` ACCESS EXCLUSIVE: list_all_repos (on `repos`) still + // succeeds, but list_visibility_rules_for_repos blocks on the rule query. + let mut lock_conn = pool.acquire().await.unwrap(); + sqlx::raw_sql("BEGIN; LOCK TABLE visibility_rules IN ACCESS EXCLUSIVE MODE;") + .execute(&mut *lock_conn) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.81:5000".parse().unwrap(); + let started = std::time::Instant::now(); + let resp = tokio::time::timeout( + std::time::Duration::from_secs(10), + router.oneshot(get_cid(&cid_for_oid(&oid), Some(peer))), + ) + .await + .expect("the budget clamp must return within budget; a bare await hangs on the lock") + .unwrap(); + let elapsed = started.elapsed(); + + let status = resp.status(); + // Fail closed: the handler must NEVER emit the listing with no rules applied. + assert_ne!( + status, + StatusCode::OK, + "a visibility-rule query timeout must DENY, never serve the path-restricted \ + blob from the public repo (that would expose private content)" + ); + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "a visibility-rule query blocked past the budget must shed the retryable budget 503" + ); + assert!( + elapsed < std::time::Duration::from_secs(3), + "the clamp must end the request at ~budget (1s); got {elapsed:?}" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("budget"), + "the fail-closed shed must name the budget taint; got: {body}" + ); + assert_eq!( + sem.available_permits(), + 1, + "the walk permit must be freed on the fail-closed budget-shed path" + ); + + sqlx::raw_sql("ROLLBACK") + .execute(&mut *lock_conn) + .await + .unwrap(); + drop(lock_conn); + } } diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index d21ec194..756806c0 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -200,3 +200,44 @@ fn f4_release_keeps_conn_owned_until_unlock_resolves() { unlock await resolves." ); } + +/// F6/KTD-5 (initial IPFS metadata queries deadline-wrapped): `get_by_cid` acquires +/// the scarce walk permits (RAII, held for the whole request) BEFORE its two initial +/// metadata queries, and the per-repo loop's first budget gate runs only later. So +/// both `list_all_repos` and `list_visibility_rules_for_repos` must be clamped to the +/// remaining request budget — otherwise a query blocked in Postgres pins the walk slot +/// for the whole stall, past the budget. This scans the PRODUCTION half of `api/ipfs.rs` +/// (the `mod tests` half names the same calls in its own harness and would make the +/// check vacuous) and asserts each query call is immediately preceded by a +/// `tokio::time::timeout(` wrapper. Removing either wrapper turns this red (proven +/// load-bearing). +#[test] +fn f6_ipfs_metadata_queries_are_deadline_wrapped() { + let ipfs = src("api/ipfs.rs"); + let production = ipfs + .split("#[cfg(test)]") + .next() + .expect("split always yields a first chunk"); + + for call in [".list_all_repos()", ".list_visibility_rules_for_repos("] { + assert_eq!( + production.matches(call).count(), + 1, + "F6 guard stale: `{call}` must appear exactly once in the production half \ + of api/ipfs.rs (the deadline-wrapped handler call) — update this guard" + ); + let idx = production + .find(call) + .unwrap_or_else(|| panic!("F6 gate: api/ipfs.rs no longer calls `{call}`")); + // The wrapper opens a few lines above the call (match tokio::time::timeout( ... + // remaining budget ..., )); a 240-char lookback covers it without + // reaching the previous statement. + let window = &production[idx.saturating_sub(240)..idx]; + assert!( + window.contains("tokio::time::timeout("), + "F6 gate missing: `{call}` must be wrapped in tokio::time::timeout(...) \ + clamped to the remaining request budget. An unwrapped await pins the held \ + walk permit for the whole DB stall, past GITLAWB_IPFS_REQUEST_BUDGET_SECS." + ); + } +} From a2be8bab136396c382d8810d77355290a49c16bb Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 10:19:50 -0500 Subject: [PATCH 54/95] fix(node): bound post-receive Pinata replication memory (#174 F2) The detached Pinata task moved each push's full object-id list into the closure and parked it on pin_semaphore.acquire_owned(); a tokio semaphore bounds active holders, not the waiter queue, so a slow Pinata backend let N queued pushes each retain an MB-scale OID list (unbounded memory/tasks). Capture only the small ref-update tuples and re-derive the object set via the existing reaped/deadline-bounded git helpers AFTER the pin permit is acquired, so retained memory is O(ref tuples), not O(object lists). Per-ref effects are unchanged (still one spawn per push, no coalescing or shedding, so no dropped announcements). Adds an inv22 gate row forbidding a retained object list across the acquire, plus re-derivation-equivalence and reaping tests. --- crates/gitlawb-node/src/api/repos.rs | 276 +++++++++++++++++++++-- crates/gitlawb-node/tests/inv22_gates.rs | 65 ++++++ 2 files changed, 327 insertions(+), 14 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 6e6d5f3a..0f12e720 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -988,6 +988,96 @@ async fn resolve_drain_object_list( Some((object_list, rules_opt, record.is_public)) } +/// Re-derive the Pinata replication object set for a push from its ref-update +/// tuples (#174 F2 / KTD-3). +/// +/// The detached Pinata task used to MOVE the push's full pre-resolved object list +/// into its closure and hold it across the `pin_semaphore` await. Under a slow +/// Pinata backend every later push then parked a fresh task each still retaining +/// an MB-scale OID list, so outstanding memory grew O(pushes x object-list) — +/// unbounded. The task now captures only the small `(ref, old, new)` tuples and +/// calls this once a pin slot frees, re-deriving the SAME OID set via +/// `git rev-list` (the delta scan) filtered by the current withheld set. Retained +/// memory is O(ref tuples); the object list is materialized only inside the +/// pin-bounded section, so at most `pin_semaphore` permits' worth exist at once. +/// +/// Coalescing / shedding were rejected because the task's per-ref work is +/// non-idempotent (branch->CID upsert, gossip, GraphQL broadcast, Arweave anchor, +/// peer-notify); dropping a later push's task drops its announcements. Only the +/// retained object list is dropped here, not the task, so every push's effects +/// still fire exactly once. +/// +/// Fresh by design, exactly like `resolve_drain_object_list`: the withheld set +/// and candidate set are recomputed from the current rules, so a rule tightened +/// since the push is honored and fails closed (a newly-withheld blob is not +/// pinned; a no-longer-announceable repo pins nothing). Every git child runs +/// through the same INV-22 bounded, process-group-reaped helpers the sibling +/// post-receive scans use (`replication_withheld_set`, +/// `resolve_candidates_for_push`, `fail_closed_full_scan_objects`). +#[allow(clippy::too_many_arguments)] +async fn pinata_object_list_for_refs( + encrypt_sem: Arc, + disk_path: std::path::PathBuf, + ref_updates: &[(String, String, String)], + rules_opt: Option>, + is_public: bool, + owner_did: String, + git_bin: String, + timeout: std::time::Duration, +) -> Vec { + let (_announce, withheld) = replication_withheld_set( + encrypt_sem.clone(), + rules_opt.clone(), + &owner_did, + is_public, + disk_path.clone(), + git_bin.clone(), + timeout, + ) + .await; + // Not announceable, or the withheld walk failed: replicate nothing (fail + // closed) — mirrors the receive-pack tail's `withheld == None` handling. + let withheld_set = match withheld { + Some(w) => w, + None => return Vec::new(), + }; + let new_tips: Vec = ref_updates + .iter() + .map(|(_, _, new)| new.clone()) + .filter(|s| s != ZERO_SHA) + .collect(); + let old_tips: Vec = ref_updates + .iter() + .map(|(_, old, _)| old.clone()) + .filter(|s| s != ZERO_SHA) + .collect(); + let pin_set = crate::git::push_delta::resolve_candidates_for_push( + encrypt_sem.clone(), + disk_path.clone(), + new_tips, + old_tips, + git_bin.clone(), + timeout, + false, + ) + .await; + if pin_set.full_scan { + fail_closed_full_scan_objects( + encrypt_sem, + disk_path, + rules_opt.unwrap_or_default(), + is_public, + owner_did, + pin_set.candidates, + git_bin, + timeout, + ) + .await + } else { + crate::git::visibility_pack::replicable_objects(pin_set.candidates, &withheld_set) + } +} + /// The pin/encrypt pipeline shared by the snapshot iteration and the /// coalesced-drain iterations: local IPFS pin, then (path-scoped rules only) the /// admission-gated recipients walk → encrypt-then-pin → Arweave manifest anchor. @@ -1825,16 +1915,20 @@ pub async fn git_receive_pack( // Pin new git objects to Pinata, then record branch→CID and gossip. // // #174 P2-2 scope note: this SECOND detached spawn is deliberately NOT brought - // under the per-repo encryption coalescing above. Two reasons: (1) it does not - // park on `git_encrypt_semaphore` (or any semaphore) — the Pinata `pin_new_objects` - // is a bounded reqwest round-trip, so it does not form the unbounded PARKED-waiter - // set that is the P2-2 residual; it runs to completion under the HTTP client's - // network timeouts. (2) Unlike the idempotent recovery-copy walk, this task does - // PER-PUSH, PER-REF work — branch→CID upserts, gossip publish, GraphQL subscription - // broadcast, Arweave anchoring, and peer notify, each keyed to THIS push's - // ref_updates. Coalescing it against an in-flight task for the same repo would DROP - // a later push's ref-update announcements (a correctness regression), not merely - // delay a duplicate. So it is scoped out with rationale, not brought under the bound. + // under the per-repo encryption coalescing above, because unlike the idempotent + // recovery-copy walk it does PER-PUSH, PER-REF work — branch→CID upserts, gossip + // publish, GraphQL subscription broadcast, Arweave anchoring, and peer notify, each + // keyed to THIS push's ref_updates. Coalescing (or shedding) it against an in-flight + // task for the same repo would DROP a later push's ref-update announcements (a + // correctness regression), not merely delay a duplicate. So the task stays one per + // push and every push's effects fire exactly once. + // + // #174 F2 / KTD-3: {bounded memory, no dropped effects, no handler latency} are + // jointly unsatisfiable by coalesce/shed/block, so instead of retaining the full + // object list we bound the thing that actually accumulates. The task captures only + // the small ref tuples and RE-DERIVES the object set inside the worker once a pin + // slot frees (see `pinata_object_list_for_refs`); the MB-scale OID list is never + // held by a parked task. { let pinata_jwt = state.config.pinata_jwt.clone(); let pinata_upload_url = state.config.pinata_upload_url.clone(); @@ -1859,11 +1953,22 @@ pub async fn git_receive_pack( let owner_did_for_arweave = record.owner_did.clone(); let self_public_url = state.config.public_url.clone(); let node_keypair = Arc::clone(&state.node_keypair); - let object_list_pinata = object_list; let do_pinata_replication = withheld.is_some(); + // #174 F2 / KTD-3: capture only the small inputs the re-derivation needs; the + // MB-scale object list is NOT moved in. `pinata_object_list_for_refs` recomputes + // it from these once a pin slot frees. rules/owner/is_public drive the fresh + // fail-closed withheld filter; encrypt_sem + git_bin + timeout keep the re-derive + // git children under the same INV-22 bounded, group-reaped scan admission. + let pinata_rules_opt = rules_opt.clone(); + let pinata_owner_did = record.owner_did.clone(); + let pinata_is_public = record.is_public; + let pinata_git_bin = state.git_bin.clone(); + let pinata_git_timeout = + std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let pinata_encrypt_sem = state.git_encrypt_semaphore.clone(); // Same global pin-admission bound as the IPFS loop (#174 F6): the Pinata pin - // loop also holds this push's full object-id list while pinning it, so it must - // share the cap. It DEFERS on a full pool rather than dropping the pin. + // loop holds a re-derived object-id list while pinning it, so it shares the cap. + // It DEFERS on a full pool rather than dropping the pin. let pin_sem_pinata = state.pin_semaphore.clone(); tokio::spawn(async move { let pinned = if do_pinata_replication { @@ -1871,12 +1976,28 @@ pub async fn git_receive_pack( .acquire_owned() .await .expect("pin_semaphore is never closed"); + // Re-derive the object set now that a pin slot is free (#174 F2 / + // KTD-3). A parked task retained only `ref_updates_clone` (O(ref + // tuples)), never this list, so a slow Pinata backend cannot grow + // outstanding memory O(pushes x object-list). Fresh + fail-closed; + // each git child is INV-22 bounded and process-group reaped. + let object_list = pinata_object_list_for_refs( + pinata_encrypt_sem, + repo_path_clone.clone(), + &ref_updates_clone, + pinata_rules_opt, + pinata_is_public, + pinata_owner_did, + pinata_git_bin, + pinata_git_timeout, + ) + .await; crate::pinata::pin_new_objects( &http_client, &pinata_upload_url, &pinata_jwt, &repo_path_clone, - object_list_pinata, + object_list, &db_clone, ) .await @@ -6127,6 +6248,133 @@ mod tests { ); } + /// #174 F2 / KTD-3 re-derivation equivalence: the object set the Pinata worker + /// re-derives from ONLY the ref tuples (`pinata_object_list_for_refs`, run once a + /// pin slot frees) must equal exactly what the old retained `object_list` would + /// have pinned — the inline-resolved delta, filtered by the withheld set. If the + /// two differ, the memory fix changed what gets pinned; they must not. + #[tokio::test] + async fn f2_pinata_rederivation_equals_retained_object_list() { + let tmp = tempfile::TempDir::new().unwrap(); + u5_init_repo(tmp.path()); + let c1 = u5_commit_file(tmp.path(), "a.txt", "one\n"); + let c2 = u5_commit_file(tmp.path(), "b.txt", "two\n"); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(4)); + let timeout = std::time::Duration::from_secs(600); + + // What the OLD retained-list task WOULD have pinned: the inline pipeline the + // receive-pack tail ran before moving `object_list` into the closure — the + // delta for main c1 -> c2, filtered by the (empty) withheld set. + let candidates = crate::git::push_delta::resolve_candidates_for_push( + sem.clone(), + tmp.path().to_path_buf(), + vec![c2.clone()], + vec![c1.clone()], + "git".to_string(), + timeout, + false, + ) + .await; + assert!( + !candidates.full_scan, + "the c1 -> c2 push is a delta, not a full scan" + ); + let retained: std::collections::HashSet = + crate::git::visibility_pack::replicable_objects( + candidates.candidates, + &std::collections::HashSet::new(), + ) + .into_iter() + .collect(); + + // What the worker re-derives from only the (ref, old, new) tuples. Empty rules + // + is_public => announceable, withheld = {} (the common Pinata case). + let ref_updates = vec![("refs/heads/main".to_string(), c1.clone(), c2.clone())]; + let rederived: std::collections::HashSet = pinata_object_list_for_refs( + sem.clone(), + tmp.path().to_path_buf(), + &ref_updates, + Some(Vec::new()), + true, + "z6MkPinataOwnerAAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + "git".to_string(), + timeout, + ) + .await + .into_iter() + .collect(); + + let new_blob = u5_git(tmp.path(), &["rev-parse", "HEAD:b.txt"]); + assert!( + retained.contains(&c2) && retained.contains(&new_blob), + "the push introduced the new commit and blob" + ); + assert_eq!( + rederived, retained, + "the worker's git rev-list re-derivation must yield exactly the object set \ + the retained list would have pinned — the memory fix must not change what pins" + ); + } + + /// #174 F2 / KTD-3 reaped + deadline-bounded: the worker's re-derivation git children + /// run through the same INV-22 bounded, process-group-reaped helpers the sibling scans + /// use. On a git that hangs on both `rev-list` and `--batch-all-objects`, + /// `pinata_object_list_for_refs` must RETURN within the watchdog budget (the group is + /// SIGKILLed + reaped at the deadline), not block. A bare `Command::output()` here + /// would hang past the ceiling (RED). + #[cfg(unix)] + #[tokio::test] + async fn f2_pinata_rederivation_is_deadline_bounded_and_reaped() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::TempDir::new().unwrap(); + // Empty rules => replication_withheld_set short-circuits (no git). The tip + // peel (`cat-file -t`) reports a commit so the delta stage proceeds; rev-list + // and the full-scan `cat-file --batch-all-objects` both hang (bounded 30s so a + // broken test cannot leak a permanent orphan). + let fake = dir.path().join("fakegit"); + std::fs::write( + &fake, + "#!/bin/sh\ncase \"$1\" in\n \ + cat-file) case \"$*\" in *--batch-all-objects*) i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;; *) echo commit ;; esac ;;\n \ + rev-list) i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;;\n \ + *) : ;;\nesac\nexit 0\n", + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + let git_bin = fake.to_str().unwrap().to_string(); + + let ref_updates = vec![( + "refs/heads/main".to_string(), + ZERO_SHA.to_string(), + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef".to_string(), + )]; + let got = tokio::time::timeout( + std::time::Duration::from_secs(10), + pinata_object_list_for_refs( + std::sync::Arc::new(tokio::sync::Semaphore::new(4)), + dir.path().to_path_buf(), + &ref_updates, + Some(Vec::new()), + true, + "z6MkPinataOwnerAAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + git_bin, + std::time::Duration::from_millis(400), + ), + ) + .await + .expect( + "pinata_object_list_for_refs must return within the watchdog budget — a hang \ + means the re-derivation git is not deadline-bounded / group-reaped (RED)", + ); + assert!( + got.is_empty(), + "a hung git yields nothing this push (the reconciliation sweep backstops)" + ); + } + /// Hot-repo drain at encrypt-pool size 1: the task loop holds NO task-level /// permit, so per-iteration helper acquires (withheld walk, candidate scan, /// recipients walk) each get the pool's single permit in turn and BOTH the diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index 756806c0..2962dc4b 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -241,3 +241,68 @@ fn f6_ipfs_metadata_queries_are_deadline_wrapped() { ); } } + +/// #174 F2 / KTD-3: the detached post-receive Pinata replication task must enqueue +/// only the push's `(ref, old, new)` tuples and RE-DERIVE its object set inside the +/// worker once a pin slot frees — it must NOT move the full per-push object list into +/// the closure and hold it across the `pin_semaphore` acquire. Retaining the list makes +/// every parked task (under a slow Pinata backend) hold an MB-scale OID list, so +/// outstanding memory grows O(pushes x object-list). Coalescing/shedding the task is +/// forbidden (its per-ref effects are non-idempotent), so the fix bounds the retained +/// data, not the task count. +/// +/// Two load-bearing checks, both against the PRODUCTION half of `api/repos.rs` (the +/// `mod tests` half names the same identifiers in its own harness and would make the +/// scan vacuous): (a) the closure-local `object_list_pinata` binding — the retain form — +/// must be gone; reintroducing `let object_list_pinata = object_list;` turns this red. +/// (b) the re-derivation (`pinata_object_list_for_refs`) must appear AFTER the Pinata +/// pin permit is acquired, so the object list is materialized only inside the pin-bounded +/// section and a parked task holds O(ref tuples). +#[test] +fn f2_pinata_enqueues_refs_not_retained_object_lists() { + let repos = src("api/repos.rs"); + let production = repos + .split("#[cfg(test)]") + .next() + .expect("split always yields a first chunk"); + + // (a) the retained-list form must be gone from production. + assert!( + !production.contains("object_list_pinata"), + "F2/KTD-3 regression: the Pinata task retains a full per-push object list \ + (`object_list_pinata`) across the pin-permit acquire. Enqueue only the ref \ + tuples and re-derive the object set inside the worker (pinata_object_list_for_refs)." + ); + + // (b) the re-derivation runs AFTER the pin permit is acquired. Anchor on the + // Pinata pin-admission clone so the window is the Pinata task, not the sibling + // IPFS/encrypt spawn (which shares `pin_semaphore` but never re-derives). + let anchor = production + .find("let pin_sem_pinata") + .expect("F2 gate stale: the Pinata pin-admission clone (pin_sem_pinata) moved"); + let tail = &production[anchor..]; + let acquire = tail + .find(".acquire_owned()") + .expect("F2 gate: the Pinata task no longer acquires a pin permit (acquire_owned)"); + let rederive = tail.find("pinata_object_list_for_refs(").expect( + "F2 gate missing: the Pinata task must re-derive its object set via \ + pinata_object_list_for_refs; it can no longer move a pre-resolved list into the closure", + ); + assert!( + acquire < rederive, + "F2 gate bypassed: the Pinata object-set re-derivation must run AFTER the pin \ + permit is acquired, so a parked task never holds the MB-scale object list" + ); + + // The re-derivation is driven by the push's ref tuples (the enqueued unit), not a + // retained object list — tie "enqueue ref_updates" to the call explicitly. + assert!( + tail[rederive..].starts_with("pinata_object_list_for_refs(") + && tail[rederive..] + .get(..400) + .map(|w| w.contains("ref_updates_clone")) + .unwrap_or(false), + "F2 gate: pinata_object_list_for_refs must re-derive from the push's ref tuples \ + (ref_updates_clone), the small unit the parked task retains" + ); +} From a333ddc1fc7b68830ad6d2d7a55967f64256b0b0 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 10:58:37 -0500 Subject: [PATCH 55/95] fix(node): serialize a disconnected push's repo until its group is reaped (#174 F3) On client disconnect, RepoWriteGuard::Drop released the per-repo advisory lock immediately (it holds no child/pgid), while KillGroupOnDrop's detached reaper was still tearing down the git process group for ~4s. A second same-node push could then acquire the repo and race the first's still-writing objects/ dir and Tigris upload. Add an in-process per-repo write lease (block-and-wait, keyed like the advisory lock) that SUPPLEMENTS the retained cluster-wide PG lock. The lease rides into the disconnect reaper via a clone on the write-path AdmissionGuard (None on all read paths) and a second clone held across guard.release() for the clean-path Tigris upload, so it frees only after the group is reaped. A bounded-wait steal reclaims a leaked lease (block-and-wait has no coalesce degradation). The lease is always taken before the PG lock and released after, so the two serializers cannot invert. Deterministic fake-git race test + inv22 gate row. --- crates/gitlawb-node/src/api/repos.rs | 262 ++++++++++++++++++++- crates/gitlawb-node/src/auth/mod.rs | 1 + crates/gitlawb-node/src/git/smart_http.rs | 20 ++ crates/gitlawb-node/src/main.rs | 5 + crates/gitlawb-node/src/state.rs | 274 ++++++++++++++++++++++ crates/gitlawb-node/src/test_support.rs | 1 + crates/gitlawb-node/tests/inv22_gates.rs | 50 ++++ 7 files changed, 612 insertions(+), 1 deletion(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 0f12e720..9774148e 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1626,6 +1626,25 @@ pub async fn git_receive_pack( } } + // Per-repo in-process write lease (#174 U2/F3): SUPPLEMENTS the cluster-wide pg + // advisory lock. Acquire it BEFORE acquire_write (one consistent order everywhere, + // so the two serializers can never invert into a self-hang) so a second SAME-NODE + // push to this repo blocks here rather than racing a disconnected first push's git + // group while its detached reaper is still tearing it down over the shared local + // objects/ dir. Taking it before the pg lock also means a blocked second writer pins + // no pooled pg connection while it waits. The lease rides the write-path + // AdmissionGuard into the reaper (clone (a)) and spans the clean-path Tigris upload + // in guard.release (clone (b)); it frees only when the LAST clone drops. steal_after + // is set well above any legitimate hold (a full receive-pack under + // git_service_timeout + the ~4s reaper cap + the Tigris upload), so the bounded-wait + // reclaim only ever fires for a genuinely leaked lease, never a merely-slow push. + let lease_steal_after = + std::time::Duration::from_secs(state.config.git_service_timeout_secs * 2 + 60); + let lease = state + .repo_write_leases + .acquire(&record.id, lease_steal_after) + .await; + tracing::debug!(repo = %name, "acquiring write lock"); // Bound the write acquire under `git_acquire_timeout_secs`. acquire_write's // advisory-lock loop already caps at ~60s, but its per-iteration @@ -1661,7 +1680,13 @@ pub async fn git_receive_pack( // instant a disconnect drops this future while the detached reaper runs (#174 P1-a). // The handler keeps no copy. This is independent of the write-lock `guard.release` // below: admission tracks the git process lifetime, the write lock tracks the repo. - let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + // Clone (a) of the write lease rides this AdmissionGuard: on a client disconnect the + // guard moves into KillGroupOnDrop's detached reaper, so the lease frees only after + // the receive-pack group is reaped — NOT at the disconnect instant (which is exactly + // when RepoWriteGuard::Drop frees the pg lock). Tying the lease to RepoWriteGuard + // instead would drop it at disconnect and reopen the F3 race. + let admission = + smart_http::AdmissionGuard::new(_permit, _caller_permit).with_lease(lease.clone()); let receive_result = smart_http::receive_pack( &state.git_bin, &disk_path, @@ -1675,6 +1700,12 @@ pub async fn git_receive_pack( // from blocking subsequent pushes. Only upload to Tigris when the push // succeeded; uploading a half-applied repo would propagate corruption. guard.release(receive_result.is_ok()).await; + // Clean path: clone (a) already dropped inside run_git_service when the receive-pack + // group was reaped; clone (b) held here spanned the success-only Tigris upload that + // ran inside release() above. Drop it now so a second same-repo push proceeds the + // moment this write is durable, rather than at end of the (longer) handler tail. On + // the disconnect path this line is never reached — clone (a) rides the reaper (F3). + drop(lease); let result = receive_result.map_err(|e| { let app = git_service_app_error(&e); @@ -6665,4 +6696,233 @@ mod tests { "repo creation must be IP-throttled before signature verification" ); } + + // ── #174 U2 / F3: second same-repo push serialized until a disconnected first ── + // push's git process GROUP is reaped (RepoWriteLease riding the disconnect reaper). + + /// `kill(pid, 0)` liveness probe (same-uid here, so EPERM never applies). + #[cfg(unix)] + fn f3_alive(pid: i32) -> bool { + unsafe { libc::kill(pid, 0) == 0 } + } + + /// F3 (P1, RED-before/GREEN-after): on a client disconnect DURING receive-pack the + /// disconnected push's git group is torn down by KillGroupOnDrop's detached reaper + /// (~4s TERM/grace/KILL/reap), while RepoWriteGuard::Drop releases the pg advisory + /// lock at the disconnect INSTANT. Without the in-process write lease, a second + /// same-node push then acquires the freed pg lock and mutates the shared local repo + /// WHILE the first group is still writing — a torn snapshot. The lease is held by the + /// write-path AdmissionGuard, which rides that reaper, so the second push must not run + /// its receive-pack (mutate the repo) until the first group is reaped. + /// + /// The fake git labels the pushes by receive-pack arrival order (atomic mkdir): the + /// first (push A) forks a SIGTERM-IGNORING descendant, records its pid, then hangs + /// (so A can be dropped mid-transfer and its group survives the SIGTERM grace); the + /// second (push B, a DIFFERENT source) records that its receive-pack ran — i.e. that + /// B mutated the repo. The load-bearing invariant is strictly ordered, not + /// time-windowed: B's marker must NEVER appear while A's descendant is still alive. + /// + /// Load-bearing: pre-fix (no lease) A's disconnect frees the pg lock, B's + /// acquire_write succeeds within its ~1s retry, and B's receive-pack runs (marker + /// appears) WHILE A's descendant is still alive — RED. With the lease the reaper + /// holds it until the group is ESRCH-gone, so B's marker appears only AFTER — GREEN. + #[cfg(unix)] + #[sqlx::test] + async fn f3_second_push_serialized_until_disconnected_group_reaped(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let tmp = tempfile::TempDir::new().unwrap(); + let seq_a = tmp.path().join("seq_a"); // first receive-pack wins this mkdir = push A + let descfile = tmp.path().join("desc.pid"); // A's SIGTERM-ignoring descendant pid + let b_ran = tmp.path().join("b.ran"); // set when B's receive-pack runs (B mutates) + // receive-pack: first invocation (A) forks a TERM-ignoring descendant (bounded + // loop so a RED run leaks no permanent orphan), records its pid, and hangs in + // `wait`; second (B) records that it ran. rev-parse feeds any tail probe. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + receive-pack)\n\ + cat >/dev/null 2>/dev/null\n\ + if mkdir \"{seq}\" 2>/dev/null; then\n\ + sh -c 'trap \"\" TERM; echo $$ > \"{desc}\"; i=0; while [ $i -lt 60 ]; do sleep 0.1; i=$((i+1)); done' &\n\ + wait\n\ + else\n\ + echo 1 > \"{bran}\"\n\ + fi ;;\n\ + rev-parse) echo deadbeef ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + seq = seq_a.display(), + desc = descfile.display(), + bran = b_ran.display(), + ); + let git_bin = write_fake_git(tmp.path(), &body); + // One repo; A and B push to it (same record.id -> same lease key). Non-path-scoped + // + flush-only body -> no post-receive scans to muddy the observation. + let state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f3repo", "r1", false).await; + let did = "did:key:z6MkF3PusherAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + + // Push A: drive its handler future in slices until it reaches receive-pack and the + // fake records its descendant pid (A now holds the lease and is hung). + let mut fut_a = Box::pin(git_receive_pack( + State(state.clone()), + Path(("z6f3repo".to_string(), "r1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some("203.0.113.81:5000".parse::().unwrap())), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + )); + let mut desc: Option = None; + for _ in 0..1000 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut_a).await; + if let Some(p) = std::fs::read_to_string(&descfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + desc = Some(p); + break; + } + } + let desc = desc.expect("push A must reach receive-pack and record its descendant pid"); + assert!( + f3_alive(desc), + "A's descendant must be alive before the disconnect" + ); + + // Push B: a DIFFERENT source, same repo. It blocks on the lease A holds. + let state_b = state.clone(); + let handle_b = tokio::spawn(async move { + git_receive_pack( + State(state_b), + Path(("z6f3repo".to_string(), "r1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some( + "203.0.113.82:5000".parse::().unwrap(), + )), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await + }); + // Give B time to reach and block on the lease acquire. + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + assert!( + !b_ran.exists(), + "B must not have mutated the repo while A legitimately holds the lease" + ); + + // Client disconnect on A: drop its future. RepoWriteGuard::Drop frees the pg lock + // immediately; the write-path AdmissionGuard (carrying the lease's clone (a)) + // rides KillGroupOnDrop's detached reaper, which now tears down A's group. + drop(fut_a); + + // Load-bearing ordering invariant: while A's descendant is still alive (group not + // yet reaped), B must NOT have run its receive-pack. Poll until the descendant is + // gone; every step it is alive, B's marker must be absent. Pre-fix, B's marker + // appears here (RED); with the lease it can only appear after the reap (GREEN). + let mut reaped = false; + for _ in 0..800 { + if !f3_alive(desc) { + reaped = true; + break; + } + assert!( + !b_ran.exists(), + "F3 RED: push B mutated the repo while push A's disconnected git group \ + was still alive (descendant pid {desc}) — the second writer must be \ + serialized until the first group is reaped" + ); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + // Safety net so a RED run never leaks the orphan. + unsafe { + libc::kill(desc, libc::SIGKILL); + } + assert!( + reaped, + "A's disconnected group must be reaped within the teardown cap" + ); + + // GREEN tail: once the group is reaped the lease frees and B proceeds — its + // receive-pack runs (marker appears) and it returns 200. + let mut b_mutated = false; + for _ in 0..1000 { + if b_ran.exists() { + b_mutated = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert!( + b_mutated, + "push B must proceed and mutate the repo once A's group is reaped" + ); + let resp = tokio::time::timeout(std::time::Duration::from_secs(30), handle_b) + .await + .expect("push B must complete once the lease frees") + .expect("push B task must not panic") + .expect("push B must succeed"); + assert_eq!(resp.status(), 200, "push B lands 200 after serialization"); + } + + /// F3 clean-path no-regression: a clean push (no disconnect) releases the lease after + /// the receive-pack group is reaped and the (success-only) Tigris upload in + /// guard.release runs, so the per-repo lease entry is GC'd and a second same-repo + /// push proceeds immediately. A lease that failed to free on the clean path would + /// wedge every subsequent push to the repo. + #[cfg(unix)] + #[sqlx::test] + async fn f3_clean_push_frees_lease_and_second_push_proceeds(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let tmp = tempfile::TempDir::new().unwrap(); + // Clean receive-pack: drain stdin, exit 0. No hang, no descendant. + let body = "#!/bin/sh\ncase \"$1\" in\n receive-pack) cat >/dev/null 2>/dev/null ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + let state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f3clean", "c1", false).await; + let did = "did:key:z6MkF3CleanPusherAAAAAAAAAAAAAAAAAAAAAAAA"; + + let push = |st: AppState, peer: &'static str| async move { + tokio::time::timeout( + std::time::Duration::from_secs(30), + git_receive_pack( + State(st), + Path(("z6f3clean".to_string(), "c1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(peer.parse::().unwrap())), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ), + ) + .await + .expect("a clean push must not wedge on the lease") + .expect("the push must succeed") + }; + + let a = push(state.clone(), "203.0.113.91:5000").await; + assert_eq!(a.status(), 200, "clean push A lands 200"); + // The clean push freed its lease (both clones dropped) -> the entry GC'd. + assert!( + state.repo_write_leases.is_empty(), + "a clean push must free the per-repo lease (Drop-frees-key) so it never wedges" + ); + + let b = push(state.clone(), "203.0.113.92:5000").await; + assert_eq!( + b.status(), + 200, + "a second same-repo push proceeds after a clean first" + ); + assert!( + state.repo_write_leases.is_empty(), + "the lease entry must be freed again after the second clean push" + ); + } } diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index c1fa6624..431ac983 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -526,6 +526,7 @@ mod tests { git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), pin_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), encrypt_inflight: crate::state::EncryptInflight::new(), + repo_write_leases: crate::state::RepoWriteLeases::new(), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index f8b46ce8..9ac742ec 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -33,17 +33,37 @@ pub struct AdmissionGuard { // 'static` so the guard can move into the detached reaper task. _global: Option>, _caller: Option>, + // Per-repo write lease (#174 U2/F3), `Some` ONLY on the receive-pack write path + // (via [`with_lease`](Self::with_lease)); `None` on every read path and every + // non-receive-pack write path. It rides this guard into `KillGroupOnDrop`'s detached + // reaper, so on a client disconnect the lease frees only after the disconnected + // push's git group is reaped — serializing a second same-node push against the + // still-writing group. A `RepoWriteLease` clone is `Send + 'static` and holds no pg + // connection, so it travels with the reaper cleanly. A stray `Some` on a READ path + // would wrongly serialize upload-pack against pushes, hence the None-everywhere-else + // discipline. + _lease: Option, } impl AdmissionGuard { /// Take ownership of the global permit and an optional per-caller permit. Both are /// erased to `Box` — the guard's only job is to hold them until it drops. + /// No lease is attached here; the receive-pack write path adds one via + /// [`with_lease`](Self::with_lease), so every other call site is lease-free. pub fn new(global: impl Send + 'static, caller: Option) -> Self { Self { _global: Some(Box::new(global)), _caller: caller.map(|c| Box::new(c) as Box), + _lease: None, } } + + /// Attach the per-repo write lease (#174 U2/F3). Called ONLY on the receive-pack + /// write path, so the lease rides the disconnect reaper; read paths never call this. + pub fn with_lease(mut self, lease: crate::state::RepoWriteLease) -> Self { + self._lease = Some(lease); + self + } } /// Handle `GET /:owner/:repo/info/refs?service=git-upload-pack` diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 4c4c1da8..d6c0df46 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -410,6 +410,11 @@ async fn main() -> Result<()> { // P2-2). No knob: it is a natural cap (one entry per distinct repo), not a // sized pool. encrypt_inflight: crate::state::EncryptInflight::new(), + // Per-repo in-process write-lease serializer (#174 U2/F3): supplements the pg + // advisory lock so a disconnected push's still-reaping git group can't be raced + // by a second same-node push. Natural cap (one entry per contended repo, freed + // when unreferenced), no sized knob. + repo_write_leases: crate::state::RepoWriteLeases::new(), git_read_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( config.max_concurrent_reads_per_caller, ), diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 77799bd0..213b8b35 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -147,6 +147,20 @@ pub struct AppState { /// recovery copy, which `2a54c15` deliberately kept fail-closed (there is no /// reconciliation sweep to re-derive a dropped copy). See [`EncryptInflight`]. pub encrypt_inflight: EncryptInflight, + /// Per-repo in-process write serializer that SUPPLEMENTS the cluster-wide pg + /// advisory lock on the receive-pack path (#174 U2/F3). On a client disconnect + /// mid-`receive-pack`, `RepoWriteGuard::Drop` releases the pg advisory lock at the + /// disconnect instant, but the disconnected push's git process GROUP is still + /// being torn down by `KillGroupOnDrop`'s detached reaper (~4s TERM/grace/KILL/reap) + /// over the shared LOCAL objects/ dir — so a second SAME-NODE push could acquire + /// the repo and race the still-writing group into a torn snapshot. This lease is + /// held by the write-path `AdmissionGuard`, which rides that reaper, so a second + /// same-repo push blocks until the first group is reaped. It is per-NODE (the + /// corruption is same-node: shared local objects/ + in-process reaper, and the + /// disconnect path uploads nothing to Tigris), so it needs no cross-node counterpart + /// and does NOT replace the pg lock (which stays the genuine cluster-wide serializer). + /// See [`RepoWriteLeases`]. + pub repo_write_leases: RepoWriteLeases, /// Per-caller concurrency sub-cap on the read pool: each caller (keyed on the /// resolved source IP, #174 U1) may hold at most `max_concurrent_reads_per_caller` /// in-flight read ops, so one caller cannot monopolize `git_read_semaphore` @@ -453,6 +467,182 @@ impl Drop for EncryptInflightGuard { } } +/// Per-repo in-process write-lease serializer (#174 U2/F3). Keyed by the repo's DB +/// id (1:1 with the pg advisory lock's owner/name key), each entry is a one-permit +/// semaphore: the receive-pack handler takes it BEFORE `acquire_write` (see the acquire +/// order note on [`acquire`](Self::acquire)) and a second same-repo push BLOCKS on it — +/// block-and-wait, NOT coalesce. It mirrors [`EncryptInflight`]'s keyed-map + guard + +/// Drop-frees-key STRUCTURE; the semantics differ (block-and-wait, so there is no +/// lossy-coalesce degradation to fall back on). +#[derive(Clone, Default)] +pub struct RepoWriteLeases { + // std::sync::Mutex: held only for O(1) map ops (get-or-create + refcount) in a sync + // context, never across an await — the semaphore wait happens OUTSIDE this lock. + repos: Arc>>, +} + +/// A per-repo lease entry: the one-permit semaphore plus a refcount of the handlers +/// currently referencing it (holding or waiting). While `refs > 0` every acquirer +/// shares the SAME semaphore, so mutual exclusion holds; the entry is removed only when +/// `refs` hits 0 (no one references it), so a fresh entry can never split serialization. +struct LeaseSlot { + sem: Arc, + refs: usize, +} + +impl RepoWriteLeases { + pub fn new() -> Self { + Self::default() + } + + /// Acquire the per-repo write lease, blocking until it is free (a second same-repo + /// writer waits). `steal_after` bounds that wait: past it the acquirer STEALS + /// (proceeds permit-less) rather than block forever. + /// + /// Why a bounded steal: block-and-wait has no degradation of its own (unlike the + /// coalescing [`EncryptInflight`], whose lost key merely delays a best-effort copy), + /// and unlike the pg advisory lock (60s stale reclaim) an in-process waiter has no + /// reclaim — so a leaked/never-run Drop (runtime teardown without unwind, task abort, + /// `mem::forget`) would otherwise wedge the repo permanently. A stealer takes NO + /// permit and touches NO count, so a merely-slow holder that later drops can never + /// leave the semaphore over-counted; the caller must therefore set `steal_after` + /// safely ABOVE any legitimate hold (a full receive-pack under + /// `git_service_timeout_secs` + the ~4s reaper cap + the Tigris upload). + /// + /// Acquire order (one consistent order everywhere, so no inversion self-hang): the + /// lease is taken BEFORE the pg advisory lock (`acquire_write`) and released AFTER + /// it. Nothing anywhere takes the pg lock before this lease, so the two serializers + /// can never deadlock; taking the lease first also means a blocked second writer + /// pins no pooled pg connection while it waits. + pub async fn acquire(&self, repo_id: &str, steal_after: std::time::Duration) -> RepoWriteLease { + // Take the entry refcount BEFORE the await, so the entry cannot be GC'd out from + // under a waiter (a fresh entry for a new acquirer would split serialization). + let sem = { + let mut map = self.repos.lock().expect("repo_write_leases mutex poisoned"); + let slot = map.entry(repo_id.to_string()).or_insert_with(|| LeaseSlot { + sem: Arc::new(tokio::sync::Semaphore::new(1)), + refs: 0, + }); + slot.refs += 1; + Arc::clone(&slot.sem) + }; + // Cancellation-safe refcount: hold a reservation across the (cancellable) wait so + // that if this acquire future is DROPPED mid-wait — a client disconnect while a + // second same-repo push is blocked here — the reservation's Drop still decrements + // the ref it just took, rather than stranding it (which would defeat the + // Drop-frees-key GC). On success the reservation is `forget`-transferred into the + // returned guard, which then owns the single decrement. + let reservation = RefReservation { + repos: Arc::clone(&self.repos), + repo_id: repo_id.to_string(), + }; + let permit = match tokio::time::timeout(steal_after, Arc::clone(&sem).acquire_owned()).await + { + Ok(Ok(p)) => Some(p), + // The semaphore is never closed; treat the (unreachable) closed case as a + // steal so acquire always makes forward progress. + Ok(Err(_closed)) => None, + Err(_elapsed) => { + tracing::warn!( + repo = %repo_id, + steal_after_secs = steal_after.as_secs(), + "repo write-lease wait exceeded the steal bound; presuming a leaked \ + lease and proceeding permit-less (in-process serializer reclaim)" + ); + None + } + }; + // Transfer the ref from the reservation to the guard: forget the reservation (so + // it does NOT decrement) and let the guard own the single decrement on its Drop. + std::mem::forget(reservation); + RepoWriteLease(Arc::new(LeaseGuardInner { + repos: Arc::clone(&self.repos), + repo_id: repo_id.to_string(), + _permit: permit, + })) + } + + /// Number of repos with a live lease entry. Test/metrics observability. + #[allow(dead_code)] + pub fn len(&self) -> usize { + self.repos + .lock() + .expect("repo_write_leases mutex poisoned") + .len() + } + + #[allow(dead_code)] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +/// Shared-ownership handle to a held repo write lease (#174 U2/F3). `Clone` hands a +/// second holder a handle to the SAME inner guard; the lease (permit + map refcount) +/// frees only when the LAST clone drops. The receive-pack handler makes two: +/// * clone (a) rides the write-path [`AdmissionGuard`] into `KillGroupOnDrop`'s +/// detached reaper, so on a client disconnect it drops only AFTER the git group is +/// reaped (this is the F3 fix — a lease tied to `RepoWriteGuard` would instead drop +/// at the disconnect instant, reopening the race); +/// * clone (b) is held by the handler across `guard.release()`, so on the clean path +/// it spans the success-only Tigris upload that runs inside `release`, AFTER +/// `receive_pack` has already dropped clone (a) inside `run_git_service`. +/// +/// `Send + 'static` with NO pg connection (just an `Arc`), so it can ride the reaper. +#[derive(Clone)] +pub struct RepoWriteLease(#[allow(dead_code)] Arc); + +struct LeaseGuardInner { + repos: Arc>>, + repo_id: String, + // `None` only on the steal path (the bounded wait elapsed). Dropping `None` releases + // no permit, so a stealer never corrupts the semaphore's permit count. + _permit: Option, +} + +impl Drop for LeaseGuardInner { + fn drop(&mut self) { + // Runs exactly ONCE per handler acquisition — when the last `RepoWriteLease` + // clone drops (the Arc strong count hits 0) — so the refcount decrements once, + // however many clones existed. `_permit` drops after this body, releasing the + // semaphore permit so a waiting acquirer proceeds. + release_lease_ref(&self.repos, &self.repo_id); + } +} + +/// Holds the entry refcount across the cancellable wait inside +/// [`RepoWriteLeases::acquire`]. If that acquire future is dropped mid-wait, this Drop +/// decrements the ref it took; on success `acquire` `forget`s it and hands the ref to +/// the returned [`LeaseGuardInner`], so the ref is decremented exactly once either way. +struct RefReservation { + repos: Arc>>, + repo_id: String, +} + +impl Drop for RefReservation { + fn drop(&mut self) { + release_lease_ref(&self.repos, &self.repo_id); + } +} + +/// Decrement a lease entry's refcount and remove it once no handler references it, so +/// the map cannot grow without bound (Drop-frees-key, like `EncryptInflight`). Safe +/// under block-and-wait: while `refs > 0` every acquirer shares the SAME semaphore, and +/// a fresh entry is created only after `refs` hits 0, when no one references the old one. +fn release_lease_ref( + repos: &Arc>>, + repo_id: &str, +) { + if let Ok(mut map) = repos.lock() { + if let Some(slot) = map.get_mut(repo_id) { + slot.refs = slot.refs.saturating_sub(1); + if slot.refs == 0 { + map.remove(repo_id); + } + } + } +} + /// Admit a post-receive git scan to the shared `git_encrypt_semaphore` pool /// (#174 F4): DEFER (await), never shed — a dropped scan would lose the push's /// recovery copy or silently under-pin it. The returned permit must move into @@ -483,3 +673,87 @@ pub async fn acquire_scan_permit( ); permit } + +#[cfg(test)] +mod repo_write_lease_tests { + use super::RepoWriteLeases; + use std::time::Duration; + + /// #174 U2/F3 lease mechanics: block-and-wait serialization on the same repo, + /// no serialization across distinct repos, Drop-frees-key GC, and the bounded-wait + /// steal reclaim so a leaked (never-run Drop) holder cannot wedge the repo forever. + #[tokio::test] + async fn serializes_same_repo_frees_key_and_steals_on_leak() { + let leases = RepoWriteLeases::new(); + let big = Duration::from_secs(3600); + + // Block-and-wait: a second same-repo acquire waits while the first is held. + let a = leases.acquire("repo1", big).await; + let blocked = + tokio::time::timeout(Duration::from_millis(200), leases.acquire("repo1", big)).await; + assert!( + blocked.is_err(), + "a second same-repo acquire must block while the first lease is held" + ); + // ... and proceeds once the first frees. + drop(a); + let b = tokio::time::timeout(Duration::from_millis(500), leases.acquire("repo1", big)) + .await + .expect("the second acquire must proceed once the first lease frees"); + drop(b); + + // Drop-frees-key: with no holders the entry is removed (bounded map growth). + assert!( + leases.is_empty(), + "the lease entry must be removed once no handler references it" + ); + + // Distinct repos never serialize against each other. + let x = leases.acquire("repoX", big).await; + let _y = tokio::time::timeout(Duration::from_millis(200), leases.acquire("repoY", big)) + .await + .expect("distinct repos must not serialize"); + drop(x); + drop(_y); + + // Steal-on-timeout reclaim: a leaked holder (never-run Drop, simulated by + // mem::forget) must NOT wedge the repo — the bounded wait proceeds permit-less. + let leaked = leases.acquire("repoZ", big).await; + std::mem::forget(leaked); + let stolen = tokio::time::timeout( + Duration::from_secs(2), + leases.acquire("repoZ", Duration::from_millis(150)), + ) + .await + .expect("a leaked lease must be reclaimed by the bounded-wait steal, not hang forever"); + drop(stolen); + } + + /// Cancellation safety: dropping an acquire future while it is BLOCKED waiting for + /// the lease (a client disconnect on a second same-repo push) must not strand the + /// entry refcount — after the holder frees and the waiter is cancelled, the key GCs. + #[tokio::test] + async fn cancelled_waiter_does_not_strand_the_refcount() { + let leases = RepoWriteLeases::new(); + let big = Duration::from_secs(3600); + + let a = leases.acquire("repoC", big).await; + // A waiter blocks, then is cancelled (its acquire future dropped) mid-wait. + let cancelled = + tokio::time::timeout(Duration::from_millis(150), leases.acquire("repoC", big)).await; + assert!( + cancelled.is_err(), + "the waiter must be blocked, then cancelled" + ); + + // Release the holder. If the cancelled waiter had stranded its ref, the entry + // would never GC; assert it does once the holder frees. + drop(a); + // Let any pending Drop bookkeeping settle. + tokio::task::yield_now().await; + assert!( + leases.is_empty(), + "a cancelled waiter must not strand the entry refcount (key must GC)" + ); + } +} diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index cd58f19e..b88ae463 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -89,6 +89,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), pin_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), encrypt_inflight: crate::state::EncryptInflight::new(), + repo_write_leases: crate::state::RepoWriteLeases::new(), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( 8, diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index 2962dc4b..0ce54088 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -306,3 +306,53 @@ fn f2_pinata_enqueues_refs_not_retained_object_lists() { (ref_updates_clone), the small unit the parked task retains" ); } + +/// #174 U2 / F3 (second same-repo writer serialized until a disconnected first push's +/// git group is reaped): `git_receive_pack` must take the per-repo in-process write +/// lease and CARRY it on the write-path `AdmissionGuard` (via `.with_lease`), so the +/// lease rides `KillGroupOnDrop`'s detached reaper and frees only after the group is +/// reaped — supplementing the pg advisory lock, which `RepoWriteGuard::Drop` releases at +/// the disconnect instant. Tying the lease to `RepoWriteGuard` instead (or dropping the +/// `.with_lease` carry) reopens the race; the behavioral RED/GREEN test +/// (`f3_second_push_serialized_until_disconnected_group_reaped`) is the real bar, this +/// is the completeness tripwire. Scanned against the PRODUCTION half of `api/repos.rs` +/// (the `mod tests` half names these identifiers in its own harness). +#[test] +fn f3_second_writer_leased_until_reap() { + let repos = src("api/repos.rs"); + let smart_http = src("git/smart_http.rs"); + let repos_production = repos + .split("#[cfg(test)]") + .next() + .expect("split always yields a first chunk"); + + // The lease is acquired, then carried by the AdmissionGuard (.with_lease), before + // receive_pack runs the write. Severing any of the three turns this red. + let lease_acquire = repos_production.find("repo_write_leases").expect( + "F3 gate missing: git_receive_pack no longer takes the per-repo write lease \ + (state.repo_write_leases)", + ); + let with_lease = repos_production.find(".with_lease(").expect( + "F3 gate missing: the write-path AdmissionGuard no longer carries the lease \ + (.with_lease). The lease must ride the disconnect reaper via the AdmissionGuard, \ + NOT RepoWriteGuard (which drops at the disconnect instant, reopening F3).", + ); + let receive = repos_production + .find("smart_http::receive_pack(") + .expect("F3 gate stale: git_receive_pack no longer calls smart_http::receive_pack"); + assert!( + lease_acquire < with_lease && with_lease < receive, + "F3 gate bypassed: the write lease must be acquired, then carried by the \ + AdmissionGuard (.with_lease), BEFORE receive_pack runs the write" + ); + + // The AdmissionGuard must actually hold the lease (Option) via a + // with_lease setter, so it travels into the detached reaper. Removing the field or + // method turns this red. + assert!( + smart_http.contains("_lease: Option") + && smart_http.contains("pub fn with_lease("), + "F3 gate missing: AdmissionGuard must hold an Option set via \ + with_lease, so the lease rides the guard into KillGroupOnDrop's reaper" + ); +} From f02c5e1ebd6c365b2d29bdb1890e1398e8011338 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 11:54:09 -0500 Subject: [PATCH 56/95] fix(node): take the write permits after the per-repo lease, not before (#174 F3 review) Code review found a DoS the F3 lease introduced: git_receive_pack acquired the global and per-source write permits before block-waiting on the per-repo lease, so a second same-repo push that parked on the lease pinned a scarce global write-pool slot (1 of 32) for up to steal_after while sending zero bytes. A few hostile sources stacking same-repo pushes could hold every slot on idle lease-waiters and shed 503 on every push to every other repo node-wide. Move both permit acquisitions to after the lease acquire so a lease-blocked waiter pins no write-pool slot, and keep a cheap non-holding availability peek before the DB lookup so a push flood on a saturated pool still sheds 503 without touching Postgres (the authoritative held permit is taken after the lease). Adds a blocked-waiter-holds-no-permit regression test. --- crates/gitlawb-node/src/api/repos.rs | 192 ++++++++++++++++++++++++--- 1 file changed, 171 insertions(+), 21 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 9774148e..836e5625 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1536,27 +1536,16 @@ pub async fn git_receive_pack( body: Bytes, ) -> Result { let name = smart_http_repo_name(&repo)?; - // Per-source write sub-cap (#174 P1-d): before the global write permit so one - // source IP cannot occupy the whole write pool via many slow authenticated pushes - // and 503 every other source. Owner enforcement defaults off, so any valid did:key - // is accepted (auth != authz), and the 600/hour push limiter bounds arrival RATE, - // not in-flight concurrency — so without this a single host minting disposable DIDs - // saturates the pool. Keyed on the resolved source IP, NEVER the signed DID (a DID - // farm defeats a DID key); no resolvable key -> global write pool only. - let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); - let _caller_permit = acquire_read_caller_permit( - &state.git_write_per_caller, - caller_key.as_deref(), - name, - "receive-pack", - )?; - // Shed with a 503 before spawning git when the concurrency cap is saturated. - // Pushes draw from the dedicated WRITE pool, separate from reads, so a flood of - // anonymous reads cannot shed an authenticated push (#174). Taken after the - // per-source cap above so one source cannot occupy global slots it would be - // sub-cap-denied for; still before the Tigris acquire_write, bounding concurrent - // fresh acquires (INV-10); held for the whole op. - let _permit = git_permit(&state.git_write_semaphore)?; + // Fast-path shed BEFORE the DB lookup when the write pool is saturated, so a push + // flood on a full pool does not hit Postgres per request. Best-effort (racy) and + // NON-holding: the authoritative, held permit is taken after the per-repo lease below + // (so a lease-blocked waiter pins no write slot — #174 F3 review). This peek only + // restores the cheap pre-DB shed the permit reorder would otherwise have lost. + if state.git_write_semaphore.available_permits() == 0 { + return Err(AppError::Overloaded( + "git service at capacity, retry shortly".into(), + )); + } tracing::info!(owner = %owner, repo = %name, "receive-pack request"); let record = state .db @@ -1645,6 +1634,31 @@ pub async fn git_receive_pack( .acquire(&record.id, lease_steal_after) .await; + // Admission permits are taken HERE, AFTER the per-repo lease and BEFORE acquire_write. + // Ordering is the fix (#174 P2 DoS): the lease is a block-and-wait serializer, so a + // second same-repo push can park on `acquire` above for up to steal_after. Taking the + // scarce write permits only once we own the lease means a lease-blocked waiter pins NO + // write-pool slot while it waits. Otherwise a few hostile sources could stack same-repo + // pushes, hold every global slot on zero-byte lease-waiters, and shed 503 on every push + // to every OTHER repo node-wide. Still before acquire_write, so the git op stays + // admission-gated (INV-10) and a saturated pool sheds 503 before spawning git. + // + // Per-source sub-cap first (#174 P1-d): one source IP cannot occupy the whole write + // pool via many slow pushes. Owner enforcement defaults off, so any valid did:key is + // accepted (auth != authz) and the push rate limiter bounds arrival RATE, not in-flight + // concurrency. Keyed on the resolved source IP, NEVER the signed DID (a DID farm defeats + // a DID key); no resolvable key -> global write pool only. Then the global write permit: + // pushes draw from the dedicated WRITE pool, separate from reads, and it is held for the + // whole op (moved into the AdmissionGuard below). + let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); + let _caller_permit = acquire_read_caller_permit( + &state.git_write_per_caller, + caller_key.as_deref(), + name, + "receive-pack", + )?; + let _permit = git_permit(&state.git_write_semaphore)?; + tracing::debug!(repo = %name, "acquiring write lock"); // Bound the write acquire under `git_acquire_timeout_secs`. acquire_write's // advisory-lock loop already caps at ~60s, but its per-iteration @@ -6925,4 +6939,140 @@ mod tests { "the lease entry must be freed again after the second clean push" ); } + + /// F3 DoS (P2, RED-before/GREEN-after): a second same-repo push that BLOCKS on the + /// per-repo write lease must hold NO global write permit while it waits. The lease + /// is a block-and-wait serializer, so a lease-blocked waiter can sit for up to + /// steal_after (~a full git_service_timeout window). If it grabs a scarce global + /// write-pool slot BEFORE blocking, a handful of hostile sources can stack same-repo + /// pushes, pin every write slot on lease-waiters sending zero bytes, and shed 503 on + /// every push to every OTHER repo node-wide. The fix acquires the lease BEFORE the + /// two write permits, so a blocked waiter pins no slot. + /// + /// Load-bearing invariant: with the write pool sized to 2, push A holds the lease and + /// is in-flight in receive-pack (1 permit held), and same-repo push B is blocked on + /// the lease, `git_write_semaphore.available_permits()` must stay 1 (only A holds). + /// Pre-fix B takes its permit BEFORE blocking on the lease, draining the pool to 0 + /// (RED). With the reorder B blocks before any permit, so the pool stays at 1 (GREEN). + #[cfg(unix)] + #[sqlx::test] + async fn f3_lease_blocked_waiter_holds_no_write_permit(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let seq_a = tmp.path().join("seq_a"); // first receive-pack wins this mkdir = push A + let a_inpack = tmp.path().join("a.inpack"); // set when A reaches receive-pack (holds lease+permit) + let b_ran = tmp.path().join("b.ran"); // set when B's receive-pack runs (B got past the lease) + // receive-pack: first invocation (A) marks that it reached the pack and hangs in a + // bounded loop (so a RED run leaks no permanent orphan); second (B) marks it ran. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + receive-pack)\n\ + cat >/dev/null 2>/dev/null\n\ + if mkdir \"{seq}\" 2>/dev/null; then\n\ + echo 1 > \"{ainp}\"\n\ + i=0; while [ $i -lt 100 ]; do sleep 0.1; i=$((i+1)); done\n\ + else\n\ + echo 1 > \"{bran}\"\n\ + fi ;;\n\ + rev-parse) echo deadbeef ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + seq = seq_a.display(), + ainp = a_inpack.display(), + bran = b_ran.display(), + ); + let git_bin = write_fake_git(tmp.path(), &body); + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f3dos", "d1", false).await; + // Size the write pool to 2 so one in-flight holder (A) leaves exactly one slot + // free, and a pool-holding waiter (B, pre-fix) would drain it to zero. Sizing to + // 1 would 503 B on the pool before it could block on the lease, hiding the bug. + state.git_write_semaphore = Arc::new(Semaphore::new(2)); + let did = "did:key:z6MkF3DosPusherAAAAAAAAAAAAAAAAAAAAAAAAAA"; + + // Push A: drive its handler future in slices until it reaches receive-pack (it now + // holds the lease and one write permit and is hung). available_permits() drops to 1. + let mut fut_a = Box::pin(git_receive_pack( + State(state.clone()), + Path(("z6f3dos".to_string(), "d1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some("203.0.113.71:5000".parse::().unwrap())), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + )); + for _ in 0..1000 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut_a).await; + if a_inpack.exists() { + break; + } + } + assert!( + a_inpack.exists(), + "push A must reach receive-pack and hold the lease" + ); + assert_eq!( + state.git_write_semaphore.available_permits(), + 1, + "with the pool sized to 2, the single in-flight holder (A) leaves one slot free" + ); + + // Push B: a DIFFERENT source, same repo. It blocks on the lease A holds. + let state_b = state.clone(); + let handle_b = tokio::spawn(async move { + git_receive_pack( + State(state_b), + Path(("z6f3dos".to_string(), "d1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some( + "203.0.113.72:5000".parse::().unwrap(), + )), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await + }); + + // Load-bearing check: while B is a lease-blocked waiter the pool must stay at 1 + // (only A holds a permit). Poll the invariant across a full window; pre-fix B + // grabs the last slot within ms and the pool falls to 0 (RED), post-fix it never + // does (GREEN). A stable state, not a one-shot race: B stays blocked on the lease + // (steal_after is far larger than this window) so once it settles the pool holds. + for _ in 0..100 { + assert_eq!( + state.git_write_semaphore.available_permits(), + 1, + "F3 DoS RED: a lease-blocked same-repo waiter took a global write permit \ + while sending zero bytes, draining the pool — a blocked waiter must pin \ + no write-pool slot" + ); + assert!( + !b_ran.exists(), + "B must not have run receive-pack while A holds the lease" + ); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + // Non-vacuous: B is genuinely parked on the lease, not returned early. + assert!( + !handle_b.is_finished(), + "push B must still be blocked on the lease at this point" + ); + + // Client disconnect on A: drop its future. The write-path AdmissionGuard rides the + // reaper, freeing the lease once A's group is reaped; B then proceeds. + drop(fut_a); + let resp = tokio::time::timeout(std::time::Duration::from_secs(30), handle_b) + .await + .expect("push B must complete once A's group is reaped and the lease frees") + .expect("push B task must not panic") + .expect("push B must succeed"); + assert_eq!(resp.status(), 200, "push B lands 200 after serialization"); + assert!(b_ran.exists(), "push B ran its receive-pack once unblocked"); + } } From e2be6f3019a3fe2b6d941f0d94722fb10843ef2d Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 11:54:09 -0500 Subject: [PATCH 57/95] fix(node): don't 500 an /ipfs miss when a transient taint co-occurs (#174 F5 review) The deterministic-fault terminal 500 was checked before the transient truncated_by 503, unconditionally. When one repo is corrupt (deterministic) while a DIFFERENT repo is transiently skipped in the same scan, the requested CID may live in the transient repo -- a retryable 503 is correct, but the handler returned a terminal 500 and a conformant client won't retry, hiding retrievable content until the unrelated corrupt repo is repaired. Gate the 500 on truncated_by.is_empty(): it fires only when a deterministic fault is the SOLE reason nothing served; a co-occurring transient taint falls through to the retryable 503. A pure deterministic fault still terminally 500s an absent lookup by design. Adds a co-occurrence regression test. --- crates/gitlawb-node/src/api/ipfs.rs | 126 ++++++++++++++++++++++++++-- 1 file changed, 120 insertions(+), 6 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 7f977a81..5e7ae05a 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -572,12 +572,19 @@ pub async fn get_by_cid( // repo / bad `.git/config`), so the object is not proven absent AND a retry cannot // change that. Shed a TERMINAL, non-retryable 500 rather than the retryable 503 // below — a 503 would let a conformant client retry-storm a fresh `git cat-file` - // per attempt against the broken repo. This is checked before the transient 503 so - // a deterministic fault is never downgraded to a retryable status. The body is - // opaque (a generic message via `AppError::Git` -> 500, no Retry-After): the raw - // git stderr — which leaks filesystem paths / config — was logged at the probe, and - // never reaches the client. - if deterministic_fault { + // per attempt against the broken repo. The body is opaque (a generic message via + // `AppError::Git` -> 500, no Retry-After): the raw git stderr — which leaks + // filesystem paths / config — was logged at the probe, and never reaches the client. + // + // Gate on `truncated_by.is_empty()`: the terminal 500 fires only when a + // deterministic fault is the SOLE reason nothing served. When a TRANSIENT taint + // co-occurs (a DIFFERENT repo was skipped by budget / acquire / walk-cap / probe / + // visit-ceiling), the requested object may live in that transiently-skipped repo, + // so fall through to the retryable 503 below — a retry can re-probe it and serve + // the content. Reporting a terminal 500 there would wrongly tell a conformant + // client not to retry, leaving reachable content unreachable until the unrelated + // broken repo is repaired. + if deterministic_fault && truncated_by.is_empty() { return Err(AppError::Git( "ipfs object probe could not complete: a candidate repository is corrupt".into(), )); @@ -1360,6 +1367,113 @@ mod tests { ); } + /// #174 F5 co-occurrence (RED-before/GREEN-after): a deterministic fault on ONE + /// repo and a TRANSIENT taint on a DIFFERENT repo occur in the same scan, and the + /// requested CID is served by neither. The transiently-skipped repo could hold the + /// object, so a retry can surface it — the correct shed is the RETRYABLE 503, not + /// the terminal 500. Two broken repos drive it, both local so the outcome is + /// deterministic: a bad-`config` repo whose `objects/` stays readable is a + /// DETERMINISTIC probe fault (`deterministic_fault = true`), while a repo whose + /// `objects/` dir is removed is a TRANSIENT probe fault (taints "probe"). A third + /// healthy repo probes clean (absent verdict) so nothing serves. Before the fix the + /// terminal `if deterministic_fault` arm fired first and shed 500 unconditionally, + /// hiding the transiently-skipped repo behind a non-retryable status. MUTATION + /// (RED): drop the `&& truncated_by.is_empty()` gate and this shes 500 again. + #[sqlx::test] + async fn get_by_cid_deterministic_fault_with_cooccurring_transient_taint_is_503_not_500( + pool: sqlx::PgPool, + ) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // Healthy repo that probes clean (definitive absence on its own). + seed_repo_with_blob(&state, tmp.path(), "z6f5coclean", "real", b"probe clean\n").await; + + // Bad-config repo: objects/ readable, config corrupt -> DETERMINISTIC fault. + state + .db + .upsert_mirror_repo("z6f5cobadcfg", "broken", "/unused-badcfg", None, false) + .await + .unwrap(); + let rec = state + .db + .get_repo("z6f5cobadcfg", "broken") + .await + .unwrap() + .unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + std::fs::create_dir_all(&bare).unwrap(); + run_git(&["init", "-q", "--bare", "--object-format=sha256"], &bare); + { + use std::io::Write; + let mut cfg = std::fs::OpenOptions::new() + .append(true) + .open(bare.join("config")) + .unwrap(); + cfg.write_all(b"\n[broken section\nnot a valid = = = line\n") + .unwrap(); + } + + // Corrupt-dir repo: objects/ removed -> TRANSIENT probe fault (taints "probe"), + // a DIFFERENT repo than the deterministic one above. + state + .db + .upsert_mirror_repo("z6f5cocorrupt", "broken", "/unused-corrupt", None, false) + .await + .unwrap(); + let rec2 = state + .db + .get_repo("z6f5cocorrupt", "broken") + .await + .unwrap() + .unwrap(); + let bare2 = state + .repo_store + .acquire(&rec2.owner_did, &rec2.name) + .await + .unwrap(); + std::fs::create_dir_all(&bare2).unwrap(); + run_git(&["init", "-q", "--bare", "--object-format=sha256"], &bare2); + std::fs::remove_dir_all(bare2.join("objects")).unwrap(); + std::fs::write(bare2.join("HEAD"), b"junk\n").unwrap(); + + let peer: SocketAddr = "203.0.113.71:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a deterministic fault co-occurring with a transient taint on a DIFFERENT \ + repo must shed the retryable 503 (a retry can surface the object in the \ + transiently-skipped repo), never the terminal 500" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the co-occurrence 503 must carry Retry-After" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("probe"), + "the shed must name the transient probe taint; got: {body}" + ); + } + /// F2 read taint: the gate passes (the probe reads the truncated loose object's /// intact "blob 64" header) but the content read fails (`cat-file blob` dies on /// the deflate stream cut mid-content) — the probe just said the object EXISTS From e6f1fec861bb840e6883c92db1315cccce3aadb1 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 12:02:46 -0500 Subject: [PATCH 58/95] docs(node): correct three lease/replication comments from code review (#174) - steal_after: the reclaim is NOT a guarantee that only a leaked lease is reclaimed. A waiter's timeout starts at acquire(), not at the FIFO head, so a same-repo backlog whose cumulative wait exceeds steal_after can steal while an earlier waiter still writes; correctness rests on the retained pg advisory lock (which serializes the stealer at acquire_write), not on the bound. - pinata_object_list_for_refs: recomputes from the tail-start rules snapshot, not a fresh read at pin-worker time, so a rule tightened AFTER tail-start is not reflected (matches the old retained-list behavior; reconciliation sweep is the backstop). Removed the overstated 'rule tightened since the push is honored'. - RepoWriteGuard::Drop: note that on runtime shutdown the detached unlock task may be dropped before it polls, bounded by pool teardown / connection close. --- crates/gitlawb-node/src/api/repos.rs | 22 +++++++++++++++------- crates/gitlawb-node/src/git/repo_store.rs | 5 ++++- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 836e5625..f258a4bc 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1007,10 +1007,13 @@ async fn resolve_drain_object_list( /// retained object list is dropped here, not the task, so every push's effects /// still fire exactly once. /// -/// Fresh by design, exactly like `resolve_drain_object_list`: the withheld set -/// and candidate set are recomputed from the current rules, so a rule tightened -/// since the push is honored and fails closed (a newly-withheld blob is not -/// pinned; a no-longer-announceable repo pins nothing). Every git child runs +/// Exactly like `resolve_drain_object_list`: the withheld and candidate sets are +/// recomputed from the rules snapshot captured at post-receive tail start (NOT a +/// fresh read at pin-worker time), so a rule tightened up to that point is honored +/// and the filter always fails closed (a newly-withheld blob is not pinned; a +/// no-longer-announceable repo pins nothing). A tightening AFTER tail-start — before +/// a slow re-derivation runs — is not reflected, matching the old retained-list +/// behavior; the reconciliation sweep is the backstop. Every git child runs /// through the same INV-22 bounded, process-group-reaped helpers the sibling /// post-receive scans use (`replication_withheld_set`, /// `resolve_candidates_for_push`, `fail_closed_full_scan_objects`). @@ -1624,9 +1627,14 @@ pub async fn git_receive_pack( // no pooled pg connection while it waits. The lease rides the write-path // AdmissionGuard into the reaper (clone (a)) and spans the clean-path Tigris upload // in guard.release (clone (b)); it frees only when the LAST clone drops. steal_after - // is set well above any legitimate hold (a full receive-pack under - // git_service_timeout + the ~4s reaper cap + the Tigris upload), so the bounded-wait - // reclaim only ever fires for a genuinely leaked lease, never a merely-slow push. + // is sized above ONE legitimate hold (a full receive-pack under git_service_timeout + + // the ~4s reaper cap + the Tigris upload). It is NOT a guarantee that only a leaked + // lease is reclaimed: a waiter's timeout starts at acquire(), not at the head of the + // FIFO queue, so a same-repo backlog whose CUMULATIVE wait exceeds steal_after can + // steal while an earlier waiter is still writing. Correctness does not rest on the + // bound — on the non-disconnect path the retained pg advisory lock still serializes + // the stealer at acquire_write (a spurious 503, not a race); the only corruption-capable + // overlap is the ~4s disconnect/reap window, which the reaper-carried clone (a) covers. let lease_steal_after = std::time::Duration::from_secs(state.config.git_service_timeout_secs * 2 + 60); let lease = state diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index d88594e7..bd27a186 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -460,7 +460,10 @@ impl Drop for RepoWriteGuard { /// handler future was dropped before `release`), unlock on the pinned /// connection. `Drop` cannot await, so spawn a detached unlock — it runs on the /// same session (connection-affine). An off-runtime drop falls back to a log; - /// the ~60s stale-lock retry loop in `acquire_write` reclaims it. + /// the ~60s stale-lock retry loop in `acquire_write` reclaims it. On runtime + /// SHUTDOWN the spawned unlock task may be dropped before it polls, so the unlock + /// may not run — but shutdown tears down the pool, and closing the connection + /// releases the session-level advisory lock server-side, so this too is bounded. fn drop(&mut self) { if self.released || !self.locked { return; From 34a92c78c787fafdf5b2c608c2d9e893240b6c93 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 27 Jul 2026 22:45:48 -0500 Subject: [PATCH 59/95] fix(node): take the per-source write permit before parking on the repo lease A push buffers its whole pack before the handler runs, then parks on the per-repo write lease for up to git_service_timeout_secs * 2 + 60, which is 1260s at defaults. Both admission permits were taken after that park, so nothing accounted for waiters: any number of requests could sit on one contended repo, each holding a fully buffered body, invisible to both semaphores. The per-source sub-cap now runs before the park. It already exists, is sized at a quarter of the push pool, and is keyed on the resolved source IP, so parked bodies are bounded at four per source without a new knob or a new cap to tune. The global permit deliberately stays below the lease. Moving it up is what f02c5e1 fixed: a lease-blocked waiter holding a global slot sheds pushes to every other repo node-wide. A per-source key does not have that shape, since exhausting it denies only the source doing the exhausting, so the two permits belong on opposite sides of the park and the comment there now says why. The load-bearing test is not that a capped source sheds, which any cap satisfies, including the global one this rejects. It is that a different source still parks and is served while the first is at its cap. Keying the permit on a constant leaves the shed test green and turns only that one red. Moving the global permit above the lease turns the f02c5e1 guard red. Bounded per source, not globally. A distributed set of sources can still each hold their four, and the unbounded-total term is the pre-auth buffering in #263 rather than the lease. --- crates/gitlawb-node/src/api/repos.rs | 449 +++++++++++++++++++++++++-- 1 file changed, 428 insertions(+), 21 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index f258a4bc..768496c2 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1541,8 +1541,9 @@ pub async fn git_receive_pack( let name = smart_http_repo_name(&repo)?; // Fast-path shed BEFORE the DB lookup when the write pool is saturated, so a push // flood on a full pool does not hit Postgres per request. Best-effort (racy) and - // NON-holding: the authoritative, held permit is taken after the per-repo lease below - // (so a lease-blocked waiter pins no write slot — #174 F3 review). This peek only + // NON-holding: the authoritative, held GLOBAL write permit is taken after the per-repo + // lease below (so a lease-blocked waiter pins no global write slot, #174 F3 review; + // the per-SOURCE sub-cap is taken above the lease, see there). This peek only // restores the cheap pre-DB shed the permit reorder would otherwise have lost. if state.git_write_semaphore.available_permits() == 0 { return Err(AppError::Overloaded( @@ -1637,27 +1638,31 @@ pub async fn git_receive_pack( // overlap is the ~4s disconnect/reap window, which the reaper-carried clone (a) covers. let lease_steal_after = std::time::Duration::from_secs(state.config.git_service_timeout_secs * 2 + 60); - let lease = state - .repo_write_leases - .acquire(&record.id, lease_steal_after) - .await; - // Admission permits are taken HERE, AFTER the per-repo lease and BEFORE acquire_write. - // Ordering is the fix (#174 P2 DoS): the lease is a block-and-wait serializer, so a - // second same-repo push can park on `acquire` above for up to steal_after. Taking the - // scarce write permits only once we own the lease means a lease-blocked waiter pins NO - // write-pool slot while it waits. Otherwise a few hostile sources could stack same-repo - // pushes, hold every global slot on zero-byte lease-waiters, and shed 503 on every push - // to every OTHER repo node-wide. Still before acquire_write, so the git op stays - // admission-gated (INV-10) and a saturated pool sheds 503 before spawning git. + // The two write admission permits are DELIBERATELY split around the lease park. // - // Per-source sub-cap first (#174 P1-d): one source IP cannot occupy the whole write - // pool via many slow pushes. Owner enforcement defaults off, so any valid did:key is - // accepted (auth != authz) and the push rate limiter bounds arrival RATE, not in-flight - // concurrency. Keyed on the resolved source IP, NEVER the signed DID (a DID farm defeats - // a DID key); no resolvable key -> global write pool only. Then the global write permit: - // pushes draw from the dedicated WRITE pool, separate from reads, and it is held for the - // whole op (moved into the AdmissionGuard below). + // Per-source sub-cap BEFORE the lease (#174 P1-d, and F1 review). One source IP + // cannot occupy the whole write pool via many slow pushes: owner enforcement defaults + // off, so any valid did:key is accepted (auth != authz) and the push rate limiter + // bounds arrival RATE, not in-flight concurrency. It is taken here, above the park, + // because `body: Bytes` means axum has ALREADY buffered the entire pack (up to + // max_pack_bytes) before this function runs, and `acquire` below is a block-and-wait + // serializer whose waiter set is unbounded (state.rs) and whose wait runs to + // steal_after (git_service_timeout_secs * 2 + 60 = 1260s at defaults). With both + // permits below the lease nothing accounted for parked waiters, so one source could + // stack unbounded buffered bodies on a contended repo. The sub-cap is + // (max_concurrent_git_pushes / 8).max(1) = 4 at defaults, so it bounds parked bodies + // at 4 per source. Keyed on the resolved source IP, NEVER the signed DID (a DID farm + // defeats a DID key); no resolvable key -> global write pool only. + // + // The GLOBAL write permit deliberately stays BELOW the lease (#174 P2 DoS, f02c5e1). + // That is a scarce node-wide resource: a lease-blocked waiter holding one lets a few + // hostile sources stack same-repo pushes, pin every global slot on zero-byte + // lease-waiters, and shed 503 on every push to every OTHER repo node-wide. A + // per-source key is not global, so exhausting it denies only the exhausting source. + // Self-inflicted denial is the property that makes the sub-cap safe above the park and + // the global permit unsafe there. Both still land before acquire_write, so the git op + // stays admission-gated (INV-10) and a saturated pool sheds 503 before spawning git. let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); let _caller_permit = acquire_read_caller_permit( &state.git_write_per_caller, @@ -1665,6 +1670,12 @@ pub async fn git_receive_pack( name, "receive-pack", )?; + + let lease = state + .repo_write_leases + .acquire(&record.id, lease_steal_after) + .await; + let _permit = git_permit(&state.git_write_semaphore)?; tracing::debug!(repo = %name, "acquiring write lock"); @@ -7083,4 +7094,400 @@ mod tests { assert_eq!(resp.status(), 200, "push B lands 200 after serialization"); assert!(b_ran.exists(), "push B ran its receive-pack once unblocked"); } + + /// Fake git for the U1 lease-park tests. The FIRST receive-pack invocation marks + /// that it reached the pack and hangs in a BOUNDED loop (so a RED run leaks no + /// permanent orphan); every later invocation marks that it got past the lease. + /// Returns `(git_bin, a_inpack marker, later_ran marker)`. + #[cfg(unix)] + fn f1_hanging_first_git( + tmp: &std::path::Path, + ) -> (String, std::path::PathBuf, std::path::PathBuf) { + let seq_a = tmp.join("f1_seq_a"); // first receive-pack wins this mkdir = push A + let a_inpack = tmp.join("f1_a.inpack"); + let later_ran = tmp.join("f1_later.ran"); + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + receive-pack)\n\ + cat >/dev/null 2>/dev/null\n\ + if mkdir \"{seq}\" 2>/dev/null; then\n\ + echo 1 > \"{ainp}\"\n\ + i=0; while [ $i -lt 100 ]; do sleep 0.1; i=$((i+1)); done\n\ + else\n\ + echo 1 > \"{later}\"\n\ + fi ;;\n\ + rev-parse) echo deadbeef ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + seq = seq_a.display(), + ainp = a_inpack.display(), + later = later_ran.display(), + ); + (write_fake_git(tmp, &body), a_inpack, later_ran) + } + + /// #174 F1 (U1, RED-before/GREEN-after): a source already at its per-source WRITE + /// sub-cap must SHED on a contended repo rather than park on the per-repo write + /// lease. `git_receive_pack` takes `body: Bytes`, so axum has already buffered the + /// whole pack (up to `max_pack_bytes`, 2 GB by default) before the handler runs, + /// and the lease park runs to `lease_steal_after` = `git_service_timeout_secs * 2 + /// + 60` = 1260s at defaults. The waiter set in `RepoWriteLeases::acquire` is + /// unbounded, so with BOTH admission permits taken after the lease nothing accounts + /// for parked bodies. Taking the per-caller write permit BEFORE the lease bounds + /// them at `(max_concurrent_git_pushes / 8).max(1)` = 4 per source. + /// + /// Load-bearing: push A (source X) holds the lease and X's only per-caller slot. + /// Pre-fix push B (also X) parks on the lease and does not return inside the test + /// window (RED). Post-fix it sheds on the per-caller cap immediately (GREEN). The + /// elapsed assertion is the point: an error returned after `steal_after` would be + /// no fix at all. + #[cfg(unix)] + #[sqlx::test] + async fn f1_capped_source_sheds_instead_of_parking_on_a_contended_lease(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let tmp = tempfile::TempDir::new().unwrap(); + let (git_bin, a_inpack, _later_ran) = f1_hanging_first_git(tmp.path()); + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f1cap", "c1", false).await; + // Per-source write cap of 1 so push A alone exhausts source X's budget. The + // global write pool stays generous (64), so nothing here sheds on it. + state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + let did = "did:key:z6MkF1CapPusherAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let src_x: SocketAddr = "203.0.113.61:5000".parse().unwrap(); + + // Push A: drive its handler future in slices until it reaches receive-pack. It + // now holds the per-repo lease AND source X's single per-caller write slot. + let mut fut_a = Box::pin(git_receive_pack( + State(state.clone()), + Path(("z6f1cap".to_string(), "c1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(src_x)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + )); + for _ in 0..1000 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut_a).await; + if a_inpack.exists() { + break; + } + } + assert!( + a_inpack.exists(), + "push A must reach receive-pack and hold the lease plus source X's write slot" + ); + + // Push B: SAME source, same repo. It must be turned away, not parked. + let started = std::time::Instant::now(); + let b = tokio::time::timeout( + std::time::Duration::from_secs(10), + git_receive_pack( + State(state.clone()), + Path(("z6f1cap".to_string(), "c1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(src_x)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ), + ) + .await; + let elapsed = started.elapsed(); + let b = b.expect( + "F1 RED: a source already at its per-source write cap PARKED on the contended \ + repo lease holding a fully buffered pack body. The per-caller write permit \ + must be taken BEFORE the lease so the request sheds instead of waiting out \ + steal_after (1260s at defaults)", + ); + assert!( + matches!(b, Err(AppError::Overloaded(_))), + "the capped source's second push must shed with Overloaded/503; got {b:?}" + ); + assert!( + elapsed < std::time::Duration::from_secs(5), + "the shed must return promptly, not after the lease park; took {elapsed:?}" + ); + + drop(fut_a); + } + + /// #174 F1 (U1) THE DISCRIMINATOR: while source X sits at its per-source write cap + /// on a contended repo, a push from a DIFFERENT source Y must still park on the + /// lease and be served once the lease frees. This is what distinguishes moving the + /// PER-CALLER permit above the lease from the rejected alternative of moving the + /// GLOBAL write permit (or any node-wide waiter cap) above it: exhausting a + /// per-source key denies only the exhausting source, so the denial is + /// self-inflicted. Key the permit on a constant instead of `read_caller_key`'s + /// resolved source IP and Y sheds too, which is exactly the node-wide shed + /// `f02c5e1` was written to prevent. + /// + /// Load-bearing on both halves: X's second push sheds promptly (RED pre-fix, where + /// it parks) AND Y's push stays parked rather than shedding (RED under the + /// constant-key mutation). + #[cfg(unix)] + #[sqlx::test] + async fn f1_capped_source_shed_leaves_a_different_source_parked_and_served(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let tmp = tempfile::TempDir::new().unwrap(); + let (git_bin, a_inpack, later_ran) = f1_hanging_first_git(tmp.path()); + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f1disc", "d1", false).await; + state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + let did = "did:key:z6MkF1DiscPusherAAAAAAAAAAAAAAAAAAAAAAAAA"; + let src_x: SocketAddr = "203.0.113.63:5000".parse().unwrap(); + let src_y: SocketAddr = "203.0.113.64:5000".parse().unwrap(); + + // Push A (source X) reaches receive-pack: holds the lease and X's only slot. + let mut fut_a = Box::pin(git_receive_pack( + State(state.clone()), + Path(("z6f1disc".to_string(), "d1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(src_x)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + )); + for _ in 0..1000 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut_a).await; + if a_inpack.exists() { + break; + } + } + assert!(a_inpack.exists(), "push A must reach receive-pack"); + + // Push C from source Y: it has its own budget, so it takes Y's permit and parks + // on the lease A holds. + let state_c = state.clone(); + let handle_c = tokio::spawn(async move { + git_receive_pack( + State(state_c), + Path(("z6f1disc".to_string(), "d1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(src_y)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await + }); + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + assert!( + !handle_c.is_finished(), + "F1 discriminator RED: a push from a DIFFERENT source was turned away while \ + source X was at its cap. The write sub-cap must be keyed on the resolved \ + source IP, so exhausting it denies only the exhausting source" + ); + + // Push B from source X: shed, promptly, while C stays parked. + let started = std::time::Instant::now(); + let b = tokio::time::timeout( + std::time::Duration::from_secs(10), + git_receive_pack( + State(state.clone()), + Path(("z6f1disc".to_string(), "d1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(src_x)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ), + ) + .await + .expect("the capped source must shed rather than park on the contended lease"); + assert!( + matches!(b, Err(AppError::Overloaded(_))), + "source X's second push must shed with Overloaded/503; got {b:?}" + ); + assert!( + started.elapsed() < std::time::Duration::from_secs(5), + "source X's shed must be prompt" + ); + assert!( + !handle_c.is_finished(), + "source Y must still be parked on the lease, unaffected by X hitting its cap" + ); + assert!( + !later_ran.exists(), + "no second receive-pack may run while A holds the lease" + ); + + // Free the lease: A disconnects. Source Y's parked push is served. + drop(fut_a); + let resp = tokio::time::timeout(std::time::Duration::from_secs(30), handle_c) + .await + .expect("push C must complete once A's group is reaped and the lease frees") + .expect("push C task must not panic") + .expect("push C must succeed"); + assert_eq!( + resp.status(), + 200, + "the unaffected source's parked push lands 200 once the lease frees" + ); + assert!( + later_ran.exists(), + "push C ran its receive-pack once the lease freed" + ); + } + + /// #174 F1 (U1) key shape: the write sub-cap is keyed on the SOURCE, not the repo. + /// A source at its cap is shed on EVERY repo (otherwise one source could hold four + /// buffered bodies per repo, and the bound would be `repos x cap x max_pack_bytes`), + /// while a different source is shed on none. Uncontended leases here: this pins the + /// key's shape, not the acquisition order. + #[sqlx::test] + async fn f1_write_cap_key_is_per_source_not_per_repo(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + let mut state = crate::test_support::test_state(pool).await; + state.git_write_semaphore = Arc::new(Semaphore::new(4)); + state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + for name in ["k1", "k2"] { + state + .db + .upsert_mirror_repo( + "z6f1key", + name, + &format!("/tmp/f1-key-{name}-nonexistent"), + None, + false, + ) + .await + .unwrap(); + } + + let did = "did:key:z6MkF1KeyPusherAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let capped: SocketAddr = "203.0.113.65:5000".parse().unwrap(); + let other: SocketAddr = "203.0.113.66:5000".parse().unwrap(); + let _slot = state + .git_write_per_caller + .try_acquire(&capped.ip().to_string()) + .expect("pin the capped source at its single write slot"); + + let push = |peer: SocketAddr, repo: &'static str| { + let st = state.clone(); + async move { + git_receive_pack( + State(st), + Path(("z6f1key".to_string(), repo.to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(peer)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await + } + }; + + for repo in ["k1", "k2"] { + let r = push(capped, repo).await; + assert!( + matches!(r, Err(AppError::Overloaded(_))), + "the capped source must shed on repo {repo} too: the sub-cap is per \ + source, not per repo; got {r:?}" + ); + } + let r = push(other, "k2").await; + assert!( + !matches!(r, Err(AppError::Overloaded(_))), + "a different source must not be shed on any repo while the capped source \ + holds its slot; got {r:?}" + ); + } + + /// #174 F1 (U1) fallback: a caller with no resolvable source key (no trusted + /// header, no peer address) takes no per-caller permit and is bounded by the global + /// write pool only, exactly as before the move. `acquire_read_caller_permit` + /// returns `Ok(None)` for a `None` key, so it must never 503 here. + #[sqlx::test] + async fn f1_write_cap_is_inert_without_a_resolvable_source_key(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::sync::Arc; + use tokio::sync::Semaphore; + + let mut state = crate::test_support::test_state(pool).await; + state.git_write_semaphore = Arc::new(Semaphore::new(4)); + state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 1); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6f1none", "n1", "/tmp/f1-none-nonexistent", None, false) + .await + .unwrap(); + // Saturate the limiter's only key slot as well, so a request that DID resolve a + // key would be shed. The keyless request must still pass. + let _slot = state + .git_write_per_caller + .try_acquire("203.0.113.67") + .expect("occupy the limiter's single key slot"); + + let r = git_receive_pack( + State(state.clone()), + Path(("z6f1none".to_string(), "n1".to_string())), + Extension(crate::auth::AuthenticatedDid( + "did:key:z6MkF1NoKeyPusherAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + )), + crate::rate_limit::PeerAddr(None), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await; + assert!( + !matches!(r, Err(AppError::Overloaded(_))), + "a caller with no resolvable source key must fall back to the global write \ + pool only, never shed on the per-caller cap; got {r:?}" + ); + } + + /// #174 F1 (U1) no-regression: moving the per-caller permit above the lease must + /// not leak it. Two SEQUENTIAL clean pushes from the SAME source with the sub-cap + /// set to 1 must both land 200; if the first push's permit outlived its request the + /// second would 503, breaking every repeat pusher. + #[cfg(unix)] + #[sqlx::test] + async fn f1_sequential_pushes_from_one_source_release_the_write_permit(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let tmp = tempfile::TempDir::new().unwrap(); + let body = "#!/bin/sh\ncase \"$1\" in\n receive-pack) cat >/dev/null 2>/dev/null ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f1seq", "s1", false).await; + state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + let did = "did:key:z6MkF1SeqPusherAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let src: SocketAddr = "203.0.113.68:5000".parse().unwrap(); + + for attempt in 1..=2 { + let resp = tokio::time::timeout( + std::time::Duration::from_secs(30), + git_receive_pack( + State(state.clone()), + Path(("z6f1seq".to_string(), "s1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(src)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ), + ) + .await + .expect("a clean push must not wedge") + .unwrap_or_else(|e| panic!("push {attempt} from the same source must succeed: {e:?}")); + assert_eq!(resp.status(), 200, "clean push {attempt} lands 200"); + } + assert_eq!( + state.git_write_per_caller.tracked_keys(), + 0, + "every per-source write permit must be released when its push completes" + ); + } } From 92a7e58b238efc80516218c34c3d70963e0e4cac Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Mon, 27 Jul 2026 23:02:15 -0500 Subject: [PATCH 60/95] fix(node): end the session when the advisory unlock errors, instead of pooling a locked connection `release` discarded the unlock's result and set `released` regardless, so an unlock that errored while its session stayed alive (a statement timeout, an admin cancel) returned a connection to the pool still holding the repo's advisory lock, with the Drop backstop already disarmed by that flag. The error is now captured. On failure the connection is taken and closed, which ends the session and makes Postgres drop the lock server-side, and only then is the guard marked released, because the lock went with the session rather than with the unlock. `close()` rather than `detach()` since we are already in an async fn and can wait for the teardown deterministically. Disposal is the only mechanism on this path. It cannot coexist with an armed Drop backstop: disposal needs the connection taken, and Drop returns early when there is none, while leaving the connection in place so Drop can use it just puts the locked connection back in the pool when that spawned task ends. The Drop doc now says so rather than claiming to cover a case it cannot. The cancellation path is untouched. It was already correct, since `released` is set after the await, and its test still passes unmodified. The new tests assert against `pg_locks` from a connection held out of the pool before the acquire, never by calling `acquire_write` a second time. That shortcut would pass whether or not the fix is present, because the pool can hand back the same connection and session advisory locks are re-entrant. The unlock error is injected by leaving the pinned connection in an aborted transaction, which errors the next statement while keeping the session and its lock alive, and the test asserts that precondition before releasing. Not covered: that the failure path still skips the Tigris upload. The test store builds with no Tigris client, so the upload site is unobservable here without a new seam, and that block is untouched by this change. --- crates/gitlawb-node/src/git/repo_store.rs | 204 +++++++++++++++++++++- 1 file changed, 197 insertions(+), 7 deletions(-) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index bd27a186..2bc15e0a 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -384,7 +384,9 @@ pub struct RepoWriteGuard { /// The pooled connection that took the advisory lock. Session-level pg /// advisory locks are connection-affine, so the guard pins that connection /// for its whole lifetime and unlocks on it (in `release`, or the `Drop` - /// backstop). `None` only after the connection has been taken to unlock. + /// backstop). `None` only after the connection has been taken, either to run + /// the detached unlock in `Drop` or to be closed when `release`'s unlock + /// errored (#174 F3b). conn: Option>, /// Set once the advisory lock has actually been taken. A guard dropped /// before the lock is held (or after `release`) performs no unlock. @@ -437,17 +439,49 @@ impl RepoWriteGuard { if self.locked { #[cfg(test)] let pre_unlock_gate = self.test_pre_unlock_gate.clone(); - if let Some(conn) = self.conn.as_deref_mut() { + let unlock = if let Some(conn) = self.conn.as_deref_mut() { // Test-only: park right before the unlock await so a test can drop // this future mid-unlock (connection owned, not yet released). #[cfg(test)] if let Some(gate) = pre_unlock_gate { gate.notified().await; } - let _ = sqlx::query("SELECT pg_advisory_unlock($1)") - .bind(self.lock_key) - .execute(&mut *conn) - .await; + Some( + sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(self.lock_key) + .execute(&mut *conn) + .await, + ) + } else { + None + }; + // An unlock that ERRORS is a different failure from a cancellation: the + // await resolved, so `Drop` is about to be made inert by `released` + // below, but the session is still alive and still holds the lock + // (statement timeout, admin cancel, aborted transaction). Returning that + // `PoolConnection` to the pool would hand the next caller a connection + // holding a lock nobody tracks (#174 F3b). Connection disposal is the + // single mechanism here, and it is why we do not instead try to keep the + // `Drop` backstop armed: disposal needs `conn.take()`, and `Drop` + // early-returns on `conn == None`. Ending the session is what frees the + // lock, so `released = true` still holds. + if let Some(Err(e)) = unlock { + warn!(repo = %self.repo_name, err = %e, + "advisory unlock failed, closing the connection so the session ends and postgres drops the lock"); + if let Some(conn) = self.conn.take() { + // `close()` over `detach()`: both consume the `PoolConnection` by + // value in sqlx 0.8.6, but we are in an async fn, so `close()` + // sends Terminate and waits for the socket to go down before + // `release` returns. `detach()` would only end the session + // whenever the returned `PgConnection` is dropped and its + // background close completes. If this future is cancelled during + // `close()`, the connection is dropped mid-close, which still + // tears the session down. + if let Err(e) = conn.close().await { + warn!(repo = %self.repo_name, err = %e, + "closing the write-lock connection failed, the session teardown still frees the lock server-side"); + } + } } } self.released = true; @@ -458,7 +492,10 @@ impl Drop for RepoWriteGuard { /// Cancellation-safe backstop: if the guard is dropped while still holding the /// advisory lock (a `tokio::time::timeout` cancelled `acquire_write`, or a /// handler future was dropped before `release`), unlock on the pinned - /// connection. `Drop` cannot await, so spawn a detached unlock — it runs on the + /// connection. This is NOT the backstop for an unlock that ran and returned an + /// error: that case is closed inside `release` by disposing of the connection, + /// because `Drop` early-returns on `conn == None` and the two mechanisms cannot + /// both apply (#174 F3b). `Drop` cannot await, so spawn a detached unlock — it runs on the /// same session (connection-affine). An off-runtime drop falls back to a log; /// the ~60s stale-lock retry loop in `acquire_write` reclaims it. On runtime /// SHUTDOWN the spawned unlock task may be dropped before it polls, so the unlock @@ -831,6 +868,159 @@ mod tests { again.release(true).await; } + // ── unlock error disposes the connection (#174 F3b, RED-before/GREEN-after) ─ + + /// Put the guard's pinned connection into a failed-transaction state, so the + /// next statement on it errors while the SESSION stays alive and keeps holding + /// the session-level advisory lock (those survive a transaction abort; only + /// `pg_advisory_xact_lock` would not). This is the smallest injection that + /// reproduces F3b's shape: `pg_advisory_unlock` returning `Err` on a live, + /// still-locking session. No production seam is needed because the tests live + /// in this module and can reach `conn` directly. + async fn poison_guard_connection(guard: &mut RepoWriteGuard) { + let conn = guard + .conn + .as_deref_mut() + .expect("guard holds its connection before release"); + sqlx::query("BEGIN") + .execute(&mut *conn) + .await + .expect("open a transaction on the guard connection"); + let poisoned = sqlx::query("SELECT 1 / 0").execute(&mut *conn).await; + assert!( + poisoned.is_err(), + "the poison statement must fail so the transaction is aborted" + ); + } + + /// F3b (P1): when `pg_advisory_unlock` ERRORS while the session is still alive + /// (statement timeout, admin cancel, aborted transaction), the lock must not + /// survive `release`. The old code discarded the error with `let _ =` and set + /// `released = true` anyway, so `Drop` early-returned and the `PoolConnection` + /// went back to the pool still holding the session lock. + /// + /// Observed from a SEPARATE connection held out of the pool before acquiring: + /// session advisory locks are re-entrant and counted, so probing from the + /// holding session (or via a fresh `acquire_write` that may be handed the same + /// connection) would report free whether or not the fix is present. + /// + /// Load-bearing: RED before the fix (lock still held → `pg_try_advisory_lock` + /// returns false), GREEN after (the errored connection is closed, so the + /// session ends and Postgres drops the lock). + #[sqlx::test] + async fn write_guard_release_with_failing_unlock_frees_the_lock(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone()); + let owner = "did:key:z6MkUnlockErrorProofFFFFFFFFFFFFFFFFFFFFFF"; + let name = "unlockerrtest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + // Distinct session for the probe, held out of the pool before acquiring. + let mut checker = pool.acquire().await.expect("checker connection"); + + let mut guard = store.acquire_write(owner, name).await.expect("acquire"); + poison_guard_connection(&mut guard).await; + + // Sanity: the poisoned session is still alive and still holds the lock, so + // the assertion below measures the release path and not a dead session. + let (free_before,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *checker) + .await + .unwrap(); + assert!( + !free_before, + "the poisoned session must still hold the lock before release" + ); + + guard.release(false).await; + // Give a background close a moment to reach the server. + tokio::time::sleep(std::time::Duration::from_millis(400)).await; + + let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *checker) + .await + .unwrap(); + assert!( + free, + "an errored pg_advisory_unlock must not leave the lock held: release \ + must dispose of the connection so the session ends" + ); + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } + + /// F3b: the connection that saw the unlock error must leave the pool entirely, + /// rather than being handed to the next caller while still holding the lock. + /// `PgPool::size()` counts the connections the pool owns, so closing the + /// errored one is observable as a drop in that count. + #[sqlx::test] + async fn write_guard_release_with_failing_unlock_does_not_return_the_connection( + pool: sqlx::PgPool, + ) { + let dir = tempfile::TempDir::new().unwrap(); + let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone()); + let owner = "did:key:z6MkUnlockErrorPoolProofGGGGGGGGGGGGGGGGGG"; + let name = "unlockerrpooltest"; + + let mut guard = store.acquire_write(owner, name).await.expect("acquire"); + poison_guard_connection(&mut guard).await; + let size_before = pool.size(); + assert!(size_before > 0, "the pool owns the guard's connection"); + + guard.release(false).await; + tokio::time::sleep(std::time::Duration::from_millis(400)).await; + + assert_eq!( + pool.size(), + size_before - 1, + "the connection that saw the unlock error must be closed, not returned \ + to the pool still holding the session lock" + ); + } + + /// F3b regression guard on the success path: a normal unlock keeps the + /// connection in the pool and marks the guard released, so the disposal branch + /// is confined to the error case. + #[sqlx::test] + async fn write_guard_release_success_keeps_the_connection_and_frees_the_lock( + pool: sqlx::PgPool, + ) { + let dir = tempfile::TempDir::new().unwrap(); + let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone()); + let owner = "did:key:z6MkUnlockOkProofHHHHHHHHHHHHHHHHHHHHHHHH"; + let name = "unlockoktest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + let mut checker = pool.acquire().await.expect("checker connection"); + let guard = store.acquire_write(owner, name).await.expect("acquire"); + let size_before = pool.size(); + + guard.release(false).await; + tokio::time::sleep(std::time::Duration::from_millis(400)).await; + + assert_eq!( + pool.size(), + size_before, + "a successful unlock must leave the connection in the pool" + ); + let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *checker) + .await + .unwrap(); + assert!(free, "the success path must still free the lock"); + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } + /// F4: releasing a guard that never took the lock (`locked == false`, the state /// acquire_write leaves after a failed acquisition) must not unlock or panic. #[sqlx::test] From 316bdc0cbb827849271a5290af04397f9e158f02 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 28 Jul 2026 00:05:04 -0500 Subject: [PATCH 61/95] fix(node): coalesce a repeat push before it parks on the encrypt pool, not after The post-push tail resolved the withheld set and materialized the object list before it ever consulted the per-repo coalescer, so rapid pushes to one repo each parked on the encrypt semaphore and repeated the scan before discovering they should have coalesced. The gate now runs first, keyed on the cheap listable-at-root predicate that `replication_withheld_set` already computes from rules alone before it touches the semaphore. Three things made this less mechanical than it looks, all of them ways a naive move would have shipped a regression. The withheld set had a second consumer. It also gated the Pinata pin and the branch-to-CID upsert, so a coalesced push that skipped the walk would have pinned nothing and recorded no CID, while a test asserting merely that the announce spawn ran would have stayed green. That gate moves to the announce predicate too. Coalescing must not return from the tail. The announce spawn sits below the coalescer in the same task, and returning early would drop gossip, the GraphQL broadcast, Arweave anchoring and peer notify, which are per-push and non-idempotent. A flag skips the encrypt section and falls through. The publish paths are gated on announce and iterate the ref updates, not the object list, so handing the tail the cheap predicate would have made a repo whose walk is failing start gossiping and anchoring, losing the fail-closed property. `pinata_object_list_for_refs` already recomputes the real set inside the pin permit, so it now returns that verdict alongside the list and the tail uses it. No additional walk is introduced. Moving the gate earlier also opens a window where a push can coalesce while the admitted one is still walking. If that walk then fails, the admitted task now still spawns the drain with an empty snapshot so those tips are consumed, rather than dropping the guard and discarding them. Note a released guard and a dropped guard are indistinguishable by the in-flight map, since Drop clears the key either way, so that case is asserted on drained work rather than on the map being empty. Each of the four is pinned by a mutation: walking anyway, returning early, leaving the Pinata gate on the withheld set, and dropping instead of draining each turn exactly one test red. --- crates/gitlawb-node/src/api/repos.rs | 1291 +++++++++++++++++++------- crates/gitlawb-node/src/state.rs | 13 + 2 files changed, 983 insertions(+), 321 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 768496c2..667baf9a 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1027,8 +1027,8 @@ async fn pinata_object_list_for_refs( owner_did: String, git_bin: String, timeout: std::time::Duration, -) -> Vec { - let (_announce, withheld) = replication_withheld_set( +) -> (bool, Vec) { + let (announce, withheld) = replication_withheld_set( encrypt_sem.clone(), rules_opt.clone(), &owner_did, @@ -1039,10 +1039,14 @@ async fn pinata_object_list_for_refs( ) .await; // Not announceable, or the withheld walk failed: replicate nothing (fail - // closed) — mirrors the receive-pack tail's `withheld == None` handling. + // closed), mirroring the receive-pack tail's `withheld == None` handling. The + // announce decision is returned with the list because this recomputation is the + // tail's only walk once coalescing runs ahead of it (#174 F2a): a repo whose + // walk is failing must still suppress gossip, the GraphQL broadcast, Arweave and + // peer-notify, and `announce` is false on exactly those arms. let withheld_set = match withheld { Some(w) => w, - None => return Vec::new(), + None => return (announce, Vec::new()), }; let new_tips: Vec = ref_updates .iter() @@ -1064,7 +1068,7 @@ async fn pinata_object_list_for_refs( false, ) .await; - if pin_set.full_scan { + let object_list = if pin_set.full_scan { fail_closed_full_scan_objects( encrypt_sem, disk_path, @@ -1078,7 +1082,8 @@ async fn pinata_object_list_for_refs( .await } else { crate::git::visibility_pack::replicable_objects(pin_set.candidates, &withheld_set) - } + }; + (announce, object_list) } /// The pin/encrypt pipeline shared by the snapshot iteration and the @@ -1851,16 +1856,82 @@ pub async fn git_receive_pack( // including its own always-spawned announce, so per-push announcements are never // coalesced away (the announce spawn stays out of the per-repo encrypt coalescing). let did = did.to_string(); - tokio::spawn(async move { - // Replication enforcement (Phase 2): decide once per push whether the public - // may read this repo at all and, if so, which blob OIDs must not leave the - // node. `withheld == None` means replicate nothing (private / mode A / - // undetermined): skip every pin so even commit and tree objects (which - // withheld_blob_oids never lists) stay local. `announce` gates the - // network-facing announcements. Fail closed: a private or undetermined repo - // never leaks. - let rules_opt = state.db.list_visibility_rules(&record.id).await.ok(); - let (announce, withheld) = replication_withheld_set( + tokio::spawn(post_receive_replication_tail( + state, + record, + ref_updates, + disk_path, + did, + )); + + Ok(result) +} + +/// The detached post-receive replication tail (#174 F2): everything a landed push +/// still owes after its git response has been returned: the replication decision, +/// the per-repo-coalesced pin/encrypt task, and this push's own Pinata + announce +/// task. Split out of `git_receive_pack` so the ordering the coalescing gate depends +/// on is directly testable; the handler spawns it and returns. +async fn post_receive_replication_tail( + state: AppState, + record: RepoRecord, + ref_updates: Vec, + disk_path: std::path::PathBuf, + did: String, +) { + // Replication enforcement (Phase 2): decide once per push whether the public + // may read this repo at all and, if so, which blob OIDs must not leave the + // node. `withheld == None` means this push pins nothing (private / mode A / + // undetermined, or a walk that failed): skip every pin so even commit and tree + // objects (which withheld_blob_oids never lists) stay local. Fail closed: a + // private or undetermined repo never leaks. The announce decision that gates + // the network-facing sends is taken separately, below. + let rules_opt = state.db.list_visibility_rules(&record.id).await.ok(); + + // #174 F2a: take the per-repo coalescing key BEFORE the walk, not after it. + // `replication_withheld_set` decides announceability from the rules snapshot + // alone and returns `(false, None)` before it touches the scan pool or spawns + // any git, so the same predicate can be evaluated here and used to gate + // `try_begin`. With the gate below the walk, rapid pushes to one repo each + // parked on `git_encrypt_semaphore` and re-ran the walk plus the object-list + // materialization before finding out they were going to coalesce; now a push + // that will coalesce does none of that. Not announceable is the same as + // before: nothing replicates, so no key is taken and no walk runs. + let announce_at_root = match &rules_opt { + Some(rules) => { + crate::visibility::listable_at_root(rules, record.is_public, &record.owner_did, None) + } + None => false, + }; + let mut coalesced = false; + let mut inflight = None; + if announce_at_root { + let tip_pairs: Vec<(String, String)> = ref_updates + .iter() + .map(|u| (u.old_sha.clone(), u.new_sha.clone())) + .collect(); + match state.encrypt_inflight.try_begin(&record.id, tip_pairs) { + crate::state::BeginOutcome::Coalesced => { + coalesced = true; + tracing::debug!( + repo = %record.id, + "post-push encryption task already in flight for this repo; coalesced \ + — this push's tip pairs are queued for that task's drain" + ); + } + crate::state::BeginOutcome::Admitted(guard) => inflight = Some(guard), + } + } + + // The walk feeds this push's own pin/encrypt snapshot, so it is skipped both + // when nothing may replicate and when this push coalesced (the in-flight + // task drains its tips). The walk's own announce decision is deliberately + // not kept here: it does not exist on the coalesced path, and the Pinata / + // announce tail below re-derives it (see `do_pinata_replication`). + let withheld = if coalesced || !announce_at_root { + None + } else { + replication_withheld_set( state.git_encrypt_semaphore.clone(), rules_opt.clone(), &record.owner_did, @@ -1869,193 +1940,202 @@ pub async fn git_receive_pack( state.git_bin.clone(), std::time::Duration::from_secs(state.config.git_service_timeout_secs), ) - .await; + .await + .1 + }; - // Resolve the per-push pin candidate set once, off the async worker, then - // filter to what may actually replicate. Delta path: the reachable-only - // `withheld` set suffices (delta objects are reachable). Full-scan path: the - // candidate set can include dangling blobs the withheld set never classified, - // so fail closed — replicate a blob only if it is reachable AND - // visibility-allowed (#99). Only computed when something will actually - // replicate; every degraded path logs rather than failing silently. - let object_list: Vec = if let Some(withheld_set) = withheld.clone() { - let new_tips: Vec = ref_updates - .iter() - .map(|u| u.new_sha.clone()) - .filter(|s| s != ZERO_SHA) - .collect(); - let old_tips: Vec = ref_updates - .iter() - .map(|u| u.old_sha.clone()) - .filter(|s| s != ZERO_SHA) - .collect(); - let pin_set = crate::git::push_delta::resolve_candidates_for_push( + // Resolve the per-push pin candidate set once, off the async worker, then + // filter to what may actually replicate. Delta path: the reachable-only + // `withheld` set suffices (delta objects are reachable). Full-scan path: the + // candidate set can include dangling blobs the withheld set never classified, + // so fail closed — replicate a blob only if it is reachable AND + // visibility-allowed (#99). Only computed when something will actually + // replicate; every degraded path logs rather than failing silently. + let object_list: Vec = if let Some(withheld_set) = withheld { + let new_tips: Vec = ref_updates + .iter() + .map(|u| u.new_sha.clone()) + .filter(|s| s != ZERO_SHA) + .collect(); + let old_tips: Vec = ref_updates + .iter() + .map(|u| u.old_sha.clone()) + .filter(|s| s != ZERO_SHA) + .collect(); + let pin_set = crate::git::push_delta::resolve_candidates_for_push( + state.git_encrypt_semaphore.clone(), + disk_path.clone(), + new_tips, + old_tips, + state.git_bin.clone(), + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + false, + ) + .await; + if pin_set.full_scan { + fail_closed_full_scan_objects( state.git_encrypt_semaphore.clone(), disk_path.clone(), - new_tips, - old_tips, + rules_opt.clone().unwrap_or_default(), + record.is_public, + record.owner_did.clone(), + pin_set.candidates, state.git_bin.clone(), std::time::Duration::from_secs(state.config.git_service_timeout_secs), - false, ) - .await; - if pin_set.full_scan { - fail_closed_full_scan_objects( - state.git_encrypt_semaphore.clone(), - disk_path.clone(), - rules_opt.clone().unwrap_or_default(), - record.is_public, - record.owner_did.clone(), - pin_set.candidates, - state.git_bin.clone(), - std::time::Duration::from_secs(state.config.git_service_timeout_secs), - ) - .await - } else { - crate::git::visibility_pack::replicable_objects(pin_set.candidates, &withheld_set) - } + .await } else { - Vec::new() - }; - - // Pin new git objects to the local IPFS node (no-op if ipfs_api is empty). - // Skipped entirely when the public cannot read the repo (withheld == None). - // - // Coalesce-and-requeue per repo (#174 P2-2 + F5): the spawned task's walks park - // on `git_encrypt_semaphore` (which DEFERS when the pool is full rather than - // dropping the recovery copy). To bound the OUTSTANDING task set, at most one - // task per repo is in flight; a push arriving while one is in flight does NOT - // spawn a duplicate — and is NOT dropped either. The in-flight task pins only - // its own pre-spawn object-list snapshot, so this push's (old, new) tip pairs - // are merged into the in-flight key's pending slot in the same critical section - // as the presence check, and the task loop-drains them (fresh rules, fail - // closed) before releasing the key. Without the requeue a coalesced push's pins - // and recovery copies would be silently absent until an unrelated later push - // (the F5 loss). The guard still releases the key on panic (Drop on unwind), so - // a crashed walk never permanently locks the repo out. - if withheld.is_some() { - let tip_pairs: Vec<(String, String)> = ref_updates - .iter() - .map(|u| (u.old_sha.clone(), u.new_sha.clone())) - .collect(); - match state.encrypt_inflight.try_begin(&record.id, tip_pairs) { - crate::state::BeginOutcome::Coalesced => { - tracing::debug!( - repo = %record.id, - "post-push encryption task already in flight for this repo; coalesced \ - — this push's tip pairs are queued for that task's drain" - ); - } - crate::state::BeginOutcome::Admitted(inflight_guard) => { - let ctx = EncryptTaskCtx { - ipfs_api: state.config.ipfs_api.clone(), - repo_path: disk_path.clone(), - db: state.db.clone(), - repo_id: record.id.clone(), - owner_did: record.owner_did.clone(), - repo_name: record.name.clone(), - irys_url: state.config.irys_url.clone(), - http_client: std::sync::Arc::clone(&state.http_client), - node_did: state.node_did.to_string(), - node_keypair: std::sync::Arc::clone(&state.node_keypair), - git_bin: state.git_bin.clone(), - git_timeout: std::time::Duration::from_secs( - state.config.git_service_timeout_secs, - ), - encrypt_sem: state.git_encrypt_semaphore.clone(), - pin_sem: state.pin_semaphore.clone(), - }; - tokio::spawn(run_encrypt_pin_task( - ctx, - inflight_guard, - object_list.clone(), - rules_opt.clone(), - record.is_public, - )); - } - } + crate::git::visibility_pack::replicable_objects(pin_set.candidates, &withheld_set) } + } else { + Vec::new() + }; - // Pin new git objects to Pinata, then record branch→CID and gossip. - // - // #174 P2-2 scope note: this SECOND detached spawn is deliberately NOT brought - // under the per-repo encryption coalescing above, because unlike the idempotent - // recovery-copy walk it does PER-PUSH, PER-REF work — branch→CID upserts, gossip - // publish, GraphQL subscription broadcast, Arweave anchoring, and peer notify, each - // keyed to THIS push's ref_updates. Coalescing (or shedding) it against an in-flight - // task for the same repo would DROP a later push's ref-update announcements (a - // correctness regression), not merely delay a duplicate. So the task stays one per - // push and every push's effects fire exactly once. - // - // #174 F2 / KTD-3: {bounded memory, no dropped effects, no handler latency} are - // jointly unsatisfiable by coalesce/shed/block, so instead of retaining the full - // object list we bound the thing that actually accumulates. The task captures only - // the small ref tuples and RE-DERIVES the object set inside the worker once a pin - // slot frees (see `pinata_object_list_for_refs`); the MB-scale OID list is never - // held by a parked task. - { - let pinata_jwt = state.config.pinata_jwt.clone(); - let pinata_upload_url = state.config.pinata_upload_url.clone(); - let repo_path_clone = disk_path.clone(); - let db_clone = state.db.clone(); - let http_client = Arc::clone(&state.http_client); - let node_did_str = state.node_did.to_string(); - let repo_slug = format!( - "{}/{}", - crate::db::normalize_owner_key(&record.owner_did), - record.name - ); - let ref_updates_clone = ref_updates - .iter() - .map(|u| (u.ref_name.clone(), u.old_sha.clone(), u.new_sha.clone())) - .collect::>(); - let p2p_handle = state.p2p.clone(); - let pusher_did_clone = did.to_string(); - let db_for_peers = state.db.clone(); - let ref_update_tx = state.ref_update_tx.clone(); - let irys_url = state.config.irys_url.clone(); - let owner_did_for_arweave = record.owner_did.clone(); - let self_public_url = state.config.public_url.clone(); - let node_keypair = Arc::clone(&state.node_keypair); - let do_pinata_replication = withheld.is_some(); - // #174 F2 / KTD-3: capture only the small inputs the re-derivation needs; the - // MB-scale object list is NOT moved in. `pinata_object_list_for_refs` recomputes - // it from these once a pin slot frees. rules/owner/is_public drive the fresh - // fail-closed withheld filter; encrypt_sem + git_bin + timeout keep the re-derive - // git children under the same INV-22 bounded, group-reaped scan admission. - let pinata_rules_opt = rules_opt.clone(); - let pinata_owner_did = record.owner_did.clone(); - let pinata_is_public = record.is_public; - let pinata_git_bin = state.git_bin.clone(); - let pinata_git_timeout = - std::time::Duration::from_secs(state.config.git_service_timeout_secs); - let pinata_encrypt_sem = state.git_encrypt_semaphore.clone(); - // Same global pin-admission bound as the IPFS loop (#174 F6): the Pinata pin - // loop holds a re-derived object-id list while pinning it, so it shares the cap. - // It DEFERS on a full pool rather than dropping the pin. - let pin_sem_pinata = state.pin_semaphore.clone(); - tokio::spawn(async move { - let pinned = if do_pinata_replication { - let _pin_permit = pin_sem_pinata - .acquire_owned() - .await - .expect("pin_semaphore is never closed"); - // Re-derive the object set now that a pin slot is free (#174 F2 / - // KTD-3). A parked task retained only `ref_updates_clone` (O(ref - // tuples)), never this list, so a slow Pinata backend cannot grow - // outstanding memory O(pushes x object-list). Fresh + fail-closed; - // each git child is INV-22 bounded and process-group reaped. - let object_list = pinata_object_list_for_refs( - pinata_encrypt_sem, - repo_path_clone.clone(), - &ref_updates_clone, - pinata_rules_opt, - pinata_is_public, - pinata_owner_did, - pinata_git_bin, - pinata_git_timeout, - ) - .await; + // Pin new git objects to the local IPFS node (no-op if ipfs_api is empty). + // Skipped entirely when the public cannot read the repo (no key was taken). + // + // Coalesce-and-requeue per repo (#174 P2-2 + F5): the spawned task's walks park + // on `git_encrypt_semaphore` (which DEFERS when the pool is full rather than + // dropping the recovery copy). To bound the OUTSTANDING task set, at most one + // task per repo is in flight; a push arriving while one is in flight does NOT + // spawn a duplicate — and is NOT dropped either. The in-flight task pins only + // its own pre-spawn object-list snapshot, so this push's (old, new) tip pairs + // are merged into the in-flight key's pending slot in the same critical section + // as the presence check, and the task loop-drains them (fresh rules, fail + // closed) before releasing the key. Without the requeue a coalesced push's pins + // and recovery copies would be silently absent until an unrelated later push + // (the F5 loss). The guard still releases the key on panic (Drop on unwind), so + // a crashed walk never permanently locks the repo out. + // + // #174 F2a: the key was taken above, so this is only the spawn. An admitted + // push ALWAYS spawns, including when its own walk failed and `object_list` is + // therefore empty: pushes can have coalesced into the pending slot while that + // walk ran, and the task's drain loop is what consumes them. Releasing or + // dropping the guard instead would discard that work with a warn (the F5 loss + // class again), and the two are indistinguishable from outside (Drop removes + // the key whenever the guard is still armed). + if let Some(inflight_guard) = inflight { + let ctx = EncryptTaskCtx { + ipfs_api: state.config.ipfs_api.clone(), + repo_path: disk_path.clone(), + db: state.db.clone(), + repo_id: record.id.clone(), + owner_did: record.owner_did.clone(), + repo_name: record.name.clone(), + irys_url: state.config.irys_url.clone(), + http_client: std::sync::Arc::clone(&state.http_client), + node_did: state.node_did.to_string(), + node_keypair: std::sync::Arc::clone(&state.node_keypair), + git_bin: state.git_bin.clone(), + git_timeout: std::time::Duration::from_secs(state.config.git_service_timeout_secs), + encrypt_sem: state.git_encrypt_semaphore.clone(), + pin_sem: state.pin_semaphore.clone(), + }; + tokio::spawn(run_encrypt_pin_task( + ctx, + inflight_guard, + object_list, + rules_opt.clone(), + record.is_public, + )); + } + + // Pin new git objects to Pinata, then record branch→CID and gossip. + // + // #174 P2-2 scope note: this SECOND detached spawn is deliberately NOT brought + // under the per-repo encryption coalescing above, because unlike the idempotent + // recovery-copy walk it does PER-PUSH, PER-REF work — branch→CID upserts, gossip + // publish, GraphQL subscription broadcast, Arweave anchoring, and peer notify, each + // keyed to THIS push's ref_updates. Coalescing (or shedding) it against an in-flight + // task for the same repo would DROP a later push's ref-update announcements (a + // correctness regression), not merely delay a duplicate. So the task stays one per + // push and every push's effects fire exactly once. + // + // #174 F2 / KTD-3: {bounded memory, no dropped effects, no handler latency} are + // jointly unsatisfiable by coalesce/shed/block, so instead of retaining the full + // object list we bound the thing that actually accumulates. The task captures only + // the small ref tuples and RE-DERIVES the object set inside the worker once a pin + // slot frees (see `pinata_object_list_for_refs`); the MB-scale OID list is never + // held by a parked task. + { + let pinata_jwt = state.config.pinata_jwt.clone(); + let pinata_upload_url = state.config.pinata_upload_url.clone(); + let repo_path_clone = disk_path.clone(); + let db_clone = state.db.clone(); + let http_client = Arc::clone(&state.http_client); + let node_did_str = state.node_did.to_string(); + let repo_slug = format!( + "{}/{}", + crate::db::normalize_owner_key(&record.owner_did), + record.name + ); + let ref_updates_clone = ref_updates + .iter() + .map(|u| (u.ref_name.clone(), u.old_sha.clone(), u.new_sha.clone())) + .collect::>(); + let p2p_handle = state.p2p.clone(); + let pusher_did_clone = did.to_string(); + let db_for_peers = state.db.clone(); + let ref_update_tx = state.ref_update_tx.clone(); + let irys_url = state.config.irys_url.clone(); + let owner_did_for_arweave = record.owner_did.clone(); + let self_public_url = state.config.public_url.clone(); + let node_keypair = Arc::clone(&state.node_keypair); + // #174 F2a: gated on the cheap announce predicate, not on `withheld`. + // `withheld` is None for a push that coalesced (it never walked), and + // this task's work is per-push and non-idempotent, so keying it on the + // walk result would silently stop pinning and stop recording a branch to + // CID mapping for every coalesced push. The fail-closed source for this + // path is now `pinata_object_list_for_refs`'s own recomputation of + // `replication_withheld_set` inside the pin permit: it returns an empty + // list AND announce=false when the walk fails or the repo may not + // replicate, so neither blobs nor announcements escape an unvetted push. + let do_pinata_replication = announce_at_root; + // #174 F2 / KTD-3: capture only the small inputs the re-derivation needs; the + // MB-scale object list is NOT moved in. `pinata_object_list_for_refs` recomputes + // it from these once a pin slot frees. rules/owner/is_public drive the fresh + // fail-closed withheld filter; encrypt_sem + git_bin + timeout keep the re-derive + // git children under the same INV-22 bounded, group-reaped scan admission. + let pinata_rules_opt = rules_opt.clone(); + let pinata_owner_did = record.owner_did.clone(); + let pinata_is_public = record.is_public; + let pinata_git_bin = state.git_bin.clone(); + let pinata_git_timeout = + std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let pinata_encrypt_sem = state.git_encrypt_semaphore.clone(); + // Same global pin-admission bound as the IPFS loop (#174 F6): the Pinata pin + // loop holds a re-derived object-id list while pinning it, so it shares the cap. + // It DEFERS on a full pool rather than dropping the pin. + let pin_sem_pinata = state.pin_semaphore.clone(); + tokio::spawn(async move { + // `announce` comes back from the re-derivation below rather than from + // the tail's own walk (#174 F2a): a coalesced push has no walk of its + // own, and this is the recomputation that fails closed for it. When + // the repo is not announceable at root there is no re-derivation and + // no announcement either, which is the same answer the walk gave. + let (announce, pinned) = if do_pinata_replication { + let _pin_permit = pin_sem_pinata + .acquire_owned() + .await + .expect("pin_semaphore is never closed"); + // Re-derive the object set now that a pin slot is free (#174 F2 / + // KTD-3). A parked task retained only `ref_updates_clone` (O(ref + // tuples)), never this list, so a slow Pinata backend cannot grow + // outstanding memory O(pushes x object-list). Fresh + fail-closed; + // each git child is INV-22 bounded and process-group reaped. + let (announce, object_list) = pinata_object_list_for_refs( + pinata_encrypt_sem, + repo_path_clone.clone(), + &ref_updates_clone, + pinata_rules_opt, + pinata_is_public, + pinata_owner_did, + pinata_git_bin, + pinata_git_timeout, + ) + .await; + ( + announce, crate::pinata::pin_new_objects( &http_client, &pinata_upload_url, @@ -2064,155 +2144,145 @@ pub async fn git_receive_pack( object_list, &db_clone, ) - .await - } else { - Vec::new() - }; + .await, + ) + } else { + (false, Vec::new()) + }; - if !pinned.is_empty() { - tracing::info!(count = pinned.len(), "pinned git objects to Pinata"); - } + if !pinned.is_empty() { + tracing::info!(count = pinned.len(), "pinned git objects to Pinata"); + } - // Build sha→cid map from pinned objects - let cid_map: std::collections::HashMap = - pinned.into_iter().collect(); + // Build sha→cid map from pinned objects + let cid_map: std::collections::HashMap = pinned.into_iter().collect(); - // Record branch→CID for each ref update and publish gossip - for (ref_name, old_sha, new_sha) in &ref_updates_clone { - let cid = cid_map.get(new_sha).map(|s| s.as_str()); - - if let Some(cid_str) = cid { - let _ = db_clone - .upsert_branch_cid( - &repo_slug, - ref_name, - new_sha, - cid_str, - &node_did_str, - ) - .await; - } + // Record branch→CID for each ref update and publish gossip + for (ref_name, old_sha, new_sha) in &ref_updates_clone { + let cid = cid_map.get(new_sha).map(|s| s.as_str()); - if announce { - if let Some(p2p) = &p2p_handle { - p2p.publish_ref_update(crate::p2p::RefUpdateEvent { - node_did: node_did_str.clone(), - pusher_did: pusher_did_clone.clone(), - repo: repo_slug.clone(), - ref_name: ref_name.clone(), - old_sha: old_sha.clone(), - new_sha: new_sha.clone(), - timestamp: chrono::Utc::now().to_rfc3339(), - cert_id: None, - cid: cid.map(|s| s.to_string()), - }) - .await; - } - } + if let Some(cid_str) = cid { + let _ = db_clone + .upsert_branch_cid(&repo_slug, ref_name, new_sha, cid_str, &node_did_str) + .await; } - // Broadcast ref update to GraphQL subscription listeners — one per ref. - // Gated on `announce`: /graphql/ws is unauthenticated (mounted after - // the optional_signature layer), and the subscription resolver has no - // caller to gate against, so only publicly-readable ref updates may - // reach anonymous subscribers. Mirrors the gossip (above) and Arweave - // (below) sends, which are already `announce`-gated. Without this a - // private-repo push would leak live ref metadata over the socket — - // the subscription analog of #112/#114. - let now_ts = chrono::Utc::now().to_rfc3339(); if announce { - for (ref_name, old_sha, new_sha) in &ref_updates_clone { - let _ = ref_update_tx.send(crate::state::RefUpdateBroadcast { + if let Some(p2p) = &p2p_handle { + p2p.publish_ref_update(crate::p2p::RefUpdateEvent { + node_did: node_did_str.clone(), + pusher_did: pusher_did_clone.clone(), repo: repo_slug.clone(), ref_name: ref_name.clone(), old_sha: old_sha.clone(), new_sha: new_sha.clone(), - pusher_did: pusher_did_clone.clone(), - node_did: node_did_str.clone(), - timestamp: now_ts.clone(), - }); + timestamp: chrono::Utc::now().to_rfc3339(), + cert_id: None, + cid: cid.map(|s| s.to_string()), + }) + .await; } } + } - // Arweave permanent anchoring — fire for each ref update. - // Suppressed for repos the public cannot read (public permanent ledger). - if announce && !irys_url.is_empty() { - for (ref_name, old_sha, new_sha) in &ref_updates_clone { - let cid = cid_map.get(new_sha).cloned(); - let anchor = crate::arweave::RefAnchor { - repo: repo_slug.clone(), - owner_did: owner_did_for_arweave.clone(), - ref_name: ref_name.clone(), - old_sha: old_sha.clone(), - new_sha: new_sha.clone(), - cid: cid.clone(), - timestamp: now_ts.clone(), - node_did: node_did_str.clone(), - }; - match crate::arweave::anchor_ref_update(&http_client, &irys_url, &anchor) - .await - { - Ok(tx_id) if !tx_id.is_empty() => { - let arweave_url = crate::arweave::arweave_url(&tx_id); - let _ = db_clone - .record_arweave_anchor(&crate::db::RecordAnchorInput { - repo: &repo_slug, - owner_did: &owner_did_for_arweave, - ref_name, - old_sha, - new_sha, - cid: cid.as_deref(), - irys_tx_id: &tx_id, - arweave_url: &arweave_url, - node_did: &node_did_str, - }) - .await; - } - Ok(_) => {} - Err(e) => { - tracing::warn!(repo=%repo_slug, err=%e, "Arweave anchor failed") - } + // Broadcast ref update to GraphQL subscription listeners — one per ref. + // Gated on `announce`: /graphql/ws is unauthenticated (mounted after + // the optional_signature layer), and the subscription resolver has no + // caller to gate against, so only publicly-readable ref updates may + // reach anonymous subscribers. Mirrors the gossip (above) and Arweave + // (below) sends, which are already `announce`-gated. Without this a + // private-repo push would leak live ref metadata over the socket — + // the subscription analog of #112/#114. + let now_ts = chrono::Utc::now().to_rfc3339(); + if announce { + for (ref_name, old_sha, new_sha) in &ref_updates_clone { + let _ = ref_update_tx.send(crate::state::RefUpdateBroadcast { + repo: repo_slug.clone(), + ref_name: ref_name.clone(), + old_sha: old_sha.clone(), + new_sha: new_sha.clone(), + pusher_did: pusher_did_clone.clone(), + node_did: node_did_str.clone(), + timestamp: now_ts.clone(), + }); + } + } + + // Arweave permanent anchoring — fire for each ref update. + // Suppressed for repos the public cannot read (public permanent ledger). + if announce && !irys_url.is_empty() { + for (ref_name, old_sha, new_sha) in &ref_updates_clone { + let cid = cid_map.get(new_sha).cloned(); + let anchor = crate::arweave::RefAnchor { + repo: repo_slug.clone(), + owner_did: owner_did_for_arweave.clone(), + ref_name: ref_name.clone(), + old_sha: old_sha.clone(), + new_sha: new_sha.clone(), + cid: cid.clone(), + timestamp: now_ts.clone(), + node_did: node_did_str.clone(), + }; + match crate::arweave::anchor_ref_update(&http_client, &irys_url, &anchor).await + { + Ok(tx_id) if !tx_id.is_empty() => { + let arweave_url = crate::arweave::arweave_url(&tx_id); + let _ = db_clone + .record_arweave_anchor(&crate::db::RecordAnchorInput { + repo: &repo_slug, + owner_did: &owner_did_for_arweave, + ref_name, + old_sha, + new_sha, + cid: cid.as_deref(), + irys_tx_id: &tx_id, + arweave_url: &arweave_url, + node_did: &node_did_str, + }) + .await; + } + Ok(_) => {} + Err(e) => { + tracing::warn!(repo=%repo_slug, err=%e, "Arweave anchor failed") } } } + } - // HTTP peer notification — notify all known peers to pull from us. - // This is the reliable fallback when Gossipsub p2p is not yet connected. - // Suppressed for repos the public cannot read. Runs last so a slow or - // unreachable peer cannot delay the local GraphQL broadcast or Arweave - // anchoring above; this is the lowest-priority best-effort step. - if announce { - if let Ok(peers) = db_for_peers.list_peers().await { - for peer in peers { - if peer.http_url.is_empty() { + // HTTP peer notification — notify all known peers to pull from us. + // This is the reliable fallback when Gossipsub p2p is not yet connected. + // Suppressed for repos the public cannot read. Runs last so a slow or + // unreachable peer cannot delay the local GraphQL broadcast or Arweave + // anchoring above; this is the lowest-priority best-effort step. + if announce { + if let Ok(peers) = db_for_peers.list_peers().await { + for peer in peers { + if peer.http_url.is_empty() { + continue; + } + let peer_url = peer.http_url.trim_end_matches('/'); + if let Some(self_url) = self_public_url.as_deref() { + if peer_url == self_url.trim_end_matches('/') { continue; } - let peer_url = peer.http_url.trim_end_matches('/'); - if let Some(self_url) = self_public_url.as_deref() { - if peer_url == self_url.trim_end_matches('/') { - continue; - } - } - let notify_url = format!("{peer_url}{SYNC_NOTIFY_PATH}"); - notify_peer_of_refs( - &http_client, - node_keypair.as_ref(), - &peer.did, - ¬ify_url, - &repo_slug, - &ref_updates_clone, - &node_did_str, - &pusher_did_clone, - ) - .await; } + let notify_url = format!("{peer_url}{SYNC_NOTIFY_PATH}"); + notify_peer_of_refs( + &http_client, + node_keypair.as_ref(), + &peer.did, + ¬ify_url, + &repo_slug, + &ref_updates_clone, + &node_did_str, + &pusher_did_clone, + ) + .await; } } - }); - } - }); - - Ok(result) + } + }); + } } /// GET /api/v1/repos/{owner}/{repo}/refs @@ -6365,6 +6435,7 @@ mod tests { timeout, ) .await + .1 .into_iter() .collect(); @@ -6434,7 +6505,7 @@ mod tests { means the re-derivation git is not deadline-bounded / group-reaped (RED)", ); assert!( - got.is_empty(), + got.1.is_empty(), "a hung git yields nothing this push (the reconciliation sweep backstops)" ); } @@ -7490,4 +7561,582 @@ mod tests { "every per-source write permit must be released when its push completes" ); } + // ---- #174 F2a: the coalescing gate runs BEFORE the withheld walk ---- + // + // These drive `post_receive_replication_tail` directly (the handler's detached + // tail, extracted so the ordering the gate depends on is observable) over a REAL + // git repo, with a logging git shim in front of the real binary. The shim's log + // is the seam: `ls-tree` lines are withheld-walk children, and a line naming a + // tip sha attributes a scan to the push that pushed it. + + /// A git shim that appends its argv to `log`, then delegates to the real git, so + /// the walks stay real while every child is observable. + #[cfg(unix)] + fn f2a_logging_git(dir: &std::path::Path, log: &std::path::Path) -> String { + write_fake_git( + dir, + &format!( + "#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"{}\"\nexec git \"$@\"\n", + log.display() + ), + ) + } + + fn f2a_log(log: &std::path::Path) -> String { + std::fs::read_to_string(log).unwrap_or_default() + } + + /// Withheld-walk children run so far. `ls-tree` is the walk's signature child + /// (`blob_paths` lists every reachable commit's tree); the delta scan and the + /// full-scan fallback use `rev-list` / `cat-file` instead. + fn f2a_walks(log: &std::path::Path) -> usize { + f2a_log(log) + .lines() + .filter(|l| l.starts_with("ls-tree")) + .count() + } + + /// A state whose git is the shim, plus a repo row (optionally path-scoped, so + /// the withheld walk actually runs rather than taking the no-rule shortcut). + /// The repo's on-disk path is passed to the tail directly, so no repo_store or + /// receive-pack plumbing is involved. + async fn f2a_state( + pool: sqlx::PgPool, + git_bin: &str, + owner: &str, + name: &str, + path_scoped: bool, + ) -> (AppState, crate::db::RepoRecord) { + let mut state = crate::test_support::test_state(pool).await; + state.git_bin = git_bin.to_string(); + state + .db + .upsert_mirror_repo(owner, name, "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + if path_scoped { + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkF2aReaderAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + } + (state, rec) + } + + fn f2a_update(ref_name: &str, new_sha: &str) -> Vec { + vec![RefUpdate { + old_sha: ZERO_SHA.to_string(), + new_sha: new_sha.to_string(), + ref_name: ref_name.to_string(), + }] + } + + const F2A_PUSHER: &str = "did:key:z6MkF2aPusherAAAAAAAAAAAAAAAAAAAAAAAAAA"; + + /// Scenario 1 (the finding). A second rapid push to the same repo coalesces + /// WITHOUT running the withheld walk. Asserted on the walk's git children, not + /// on the `Coalesced` outcome: with `try_begin` below the walk (the pre-fix + /// order) the second push still parks on the scan pool and re-walks, which is + /// exactly the accumulation jatmn found. + /// + /// The pin pool's only permit is held for the whole test, so both pushes' pin + /// tasks park before doing any git of their own: every `ls-tree` in the log is a + /// tail walk. + #[cfg(unix)] + #[sqlx::test] + async fn f2a_coalesced_push_does_not_run_the_withheld_walk(pool: sqlx::PgPool) { + let repo = tempfile::TempDir::new().unwrap(); + let bin = tempfile::TempDir::new().unwrap(); + u5_init_repo(repo.path()); + let c1 = u5_commit_file(repo.path(), "a.txt", "one\n"); + let c2 = u5_commit_file(repo.path(), "secret/s.txt", "two\n"); + let log = bin.path().join("git.log"); + let git_bin = f2a_logging_git(bin.path(), &log); + let (mut state, rec) = f2a_state(pool, &git_bin, "z6f2acoal", "c1", true).await; + state.pin_semaphore = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + let _held = state.pin_semaphore.clone().acquire_owned().await.unwrap(); + + post_receive_replication_tail( + state.clone(), + rec.clone(), + f2a_update("refs/heads/main", &c2), + repo.path().to_path_buf(), + F2A_PUSHER.to_string(), + ) + .await; + let after_first = f2a_walks(&log); + assert!( + after_first >= 1, + "the admitted push must have run the withheld walk; log:\n{}", + f2a_log(&log) + ); + assert_eq!( + state.encrypt_inflight.len(), + 1, + "the admitted push's task holds the repo key while it is parked on the pin pool" + ); + + post_receive_replication_tail( + state.clone(), + rec.clone(), + f2a_update("refs/heads/second", &c1), + repo.path().to_path_buf(), + F2A_PUSHER.to_string(), + ) + .await; + + assert_eq!( + f2a_walks(&log), + after_first, + "a push that coalesces must not run the withheld walk at all; log:\n{}", + f2a_log(&log) + ); + assert_eq!( + state.encrypt_inflight.pending_for(&rec.id), + Some(crate::state::PendingWork::Tips(vec![( + ZERO_SHA.to_string(), + c1.clone() + )])), + "the coalesced push's tip pairs are queued for the in-flight task's drain" + ); + } + /// Poll `cond` until it holds, with a bound so a regression fails the test + /// rather than hanging the suite. + async fn f2a_wait_for(mut cond: impl FnMut() -> bool, what: &str) { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); + while !cond() { + assert!( + std::time::Instant::now() < deadline, + "timed out waiting for {what}" + ); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + } + + /// A `rev-list --objects` line names the tips a DELTA scan was asked to resolve, + /// so it attributes that scan to one push's tips. The withheld walk's own + /// `rev-list --all` / `ls-tree` lines never carry a tip as an argument this way. + fn f2a_delta_scanned(log: &std::path::Path, tip: &str) -> bool { + f2a_log(log) + .lines() + .any(|l| l.starts_with("rev-list --objects") && l.contains(tip)) + } + + /// Scenario 2. The tip pairs a coalesced push queues are consumed by the + /// in-flight task's drain, which is what makes coalescing lossless. Asserted on + /// the drained WORK (the delta scan the drain runs for those tips), not on the + /// key going empty: an armed guard's Drop empties the key too, so `is_empty` + /// cannot tell a drain from a discard. + /// + /// The coalesced tips are injected through `try_begin` directly rather than by a + /// second tail. A second tail would spawn its own Pinata worker, which re-derives + /// from the SAME tips, and the two scans are indistinguishable in the git log; the + /// tail-to-`try_begin` half is covered by scenario 1's pending-slot assertion. + #[cfg(unix)] + #[sqlx::test] + async fn f2a_coalesced_tips_are_drained_by_the_inflight_task(pool: sqlx::PgPool) { + let repo = tempfile::TempDir::new().unwrap(); + let bin = tempfile::TempDir::new().unwrap(); + u5_init_repo(repo.path()); + u5_commit_file(repo.path(), "a.txt", "one\n"); + let c2 = u5_commit_file(repo.path(), "secret/s.txt", "two\n"); + let log = bin.path().join("git.log"); + let git_bin = f2a_logging_git(bin.path(), &log); + let (mut state, rec) = f2a_state(pool, &git_bin, "z6f2adrain", "d1", true).await; + state.pin_semaphore = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + let held = state.pin_semaphore.clone().acquire_owned().await.unwrap(); + + // Push A is admitted; its task then parks on the held pin pool, key retained. + post_receive_replication_tail( + state.clone(), + rec.clone(), + f2a_update("refs/heads/main", &c2), + repo.path().to_path_buf(), + F2A_PUSHER.to_string(), + ) + .await; + + // A later push lands while A's task is in flight: it coalesces, and its tip + // advances main to a commit no other push in this test names. + let c3 = u5_commit_file(repo.path(), "b.txt", "three\n"); + assert!( + matches!( + state + .encrypt_inflight + .try_begin(&rec.id, vec![(c2.clone(), c3.clone())]), + crate::state::BeginOutcome::Coalesced + ), + "a push arriving while a task is in flight must coalesce" + ); + assert!( + !f2a_delta_scanned(&log, &c3), + "nothing has scanned the coalesced tip yet" + ); + + drop(held); + f2a_wait_for( + || f2a_delta_scanned(&log, &c3), + "the in-flight task's drain to resolve the coalesced push's tips", + ) + .await; + } + + /// Scenario 3 (trap 4). A push coalesces WHILE the admitted push is walking, the + /// admitted walk then FAILS, and the coalesced work is still drained. Moving + /// `try_begin` above the walk opens this window, so the failed-walk arm must + /// still spawn the task (with an empty snapshot) rather than let the guard go: + /// dropping it discards the pending tips with a warn. + /// + /// The git shim fails the FIRST `rev-list --all` (the withheld walk's commit + /// enumeration) after signalling that the walk has started and waiting for the + /// test to inject the coalescing push, then behaves normally, so the drain that + /// follows is a real one. + #[cfg(unix)] + #[sqlx::test] + async fn f2a_walk_failure_still_drains_the_coalesced_work(pool: sqlx::PgPool) { + let repo = tempfile::TempDir::new().unwrap(); + let bin = tempfile::TempDir::new().unwrap(); + u5_init_repo(repo.path()); + u5_commit_file(repo.path(), "a.txt", "one\n"); + let c2 = u5_commit_file(repo.path(), "secret/s.txt", "two\n"); + let c3 = u5_commit_file(repo.path(), "b.txt", "three\n"); + let log = bin.path().join("git.log"); + let started = bin.path().join("walk.started"); + let go = bin.path().join("walk.go"); + let once = bin.path().join("walk.once"); + let git_bin = write_fake_git( + bin.path(), + &format!( + "#!/bin/sh\n\ + printf '%s\\n' \"$*\" >> \"{log}\"\n\ + case \"$*\" in\n\ + 'rev-list --all'*)\n\ + if [ ! -f \"{once}\" ]; then\n\ + : > \"{once}\"\n\ + : > \"{started}\"\n\ + while [ ! -f \"{go}\" ]; do sleep 0.05; done\n\ + exit 1\n\ + fi ;;\n\ + esac\n\ + exec git \"$@\"\n", + log = log.display(), + once = once.display(), + started = started.display(), + go = go.display(), + ), + ); + let (state, rec) = f2a_state(pool, &git_bin, "z6f2afail", "f1", true).await; + + let tail = tokio::spawn(post_receive_replication_tail( + state.clone(), + rec.clone(), + f2a_update("refs/heads/main", &c2), + repo.path().to_path_buf(), + F2A_PUSHER.to_string(), + )); + f2a_wait_for(|| started.exists(), "the admitted push's walk to start").await; + + // Mid-walk arrival: the key is already taken, so this push coalesces into the + // slot the walking task owns. (With the gate back below the walk it would be + // ADMITTED here instead, and this assertion is what catches that.) + assert!( + matches!( + state + .encrypt_inflight + .try_begin(&rec.id, vec![(c2.clone(), c3.clone())]), + crate::state::BeginOutcome::Coalesced + ), + "a push arriving mid-walk must coalesce, not start a second task" + ); + std::fs::write(&go, b"").unwrap(); + tail.await.unwrap(); + + f2a_wait_for( + || f2a_delta_scanned(&log, &c3), + "the failed walk's task to drain the coalesced push's tips", + ) + .await; + } + + /// Mount a Pinata upload endpoint that assigns every object the same CID, and + /// point the state at it. Returns the server (kept alive by the caller) and CID. + async fn f2a_pinata(state: &mut AppState) -> (mockito::ServerGuard, String) { + let cid = "bafyf2acoalescedmapping".to_string(); + let mut server = mockito::Server::new_async().await; + server + .mock("POST", "/") + .with_status(200) + .with_body(format!(r#"{{"data":{{"cid":"{cid}"}}}}"#)) + .expect_at_least(1) + .create_async() + .await; + let mut cfg = (*state.config).clone(); + cfg.pinata_jwt = "f2a-test-jwt".to_string(); + cfg.pinata_upload_url = server.url(); + state.config = std::sync::Arc::new(cfg); + (server, cid) + } + + /// Poll the branch to CID table until the push's mapping lands (the Pinata + /// worker is detached), bounded so a regression fails rather than hangs. + async fn f2a_wait_for_branch_cid( + db: &crate::db::Db, + slug: &str, + what: &str, + ) -> Vec { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); + loop { + let rows = db.list_branch_cids(slug).await.unwrap(); + if !rows.is_empty() { + return rows; + } + assert!( + std::time::Instant::now() < deadline, + "timed out waiting for {what}" + ); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + } + + fn f2a_slug(rec: &crate::db::RepoRecord) -> String { + format!( + "{}/{}", + crate::db::normalize_owner_key(&rec.owner_did), + rec.name + ) + } + + /// Scenario 4 (traps 1 and 2). A coalesced push still does its own per-push work: + /// it records a branch to CID mapping and broadcasts its own ref update. Both + /// would be lost by returning early on `Coalesced`, and the mapping alone would be + /// lost by leaving the Pinata gate on `withheld.is_some()` (a coalesced push never + /// walks, so its `withheld` is None) while a test that only checked "the spawn + /// ran" stayed green. + #[cfg(unix)] + #[sqlx::test] + async fn f2a_coalesced_push_still_pins_and_announces(pool: sqlx::PgPool) { + let repo = tempfile::TempDir::new().unwrap(); + let bin = tempfile::TempDir::new().unwrap(); + u5_init_repo(repo.path()); + let c1 = u5_commit_file(repo.path(), "a.txt", "one\n"); + let c2 = u5_commit_file(repo.path(), "secret/s.txt", "two\n"); + let log = bin.path().join("git.log"); + let git_bin = f2a_logging_git(bin.path(), &log); + let (mut state, rec) = f2a_state(pool, &git_bin, "z6f2apin", "p1", true).await; + let (_server, cid) = f2a_pinata(&mut state).await; + let mut updates = state.ref_update_tx.subscribe(); + + // A task for this repo is already in flight, so the push below coalesces. + let _inflight = match state.encrypt_inflight.try_begin(&rec.id, Vec::new()) { + crate::state::BeginOutcome::Admitted(g) => g, + crate::state::BeginOutcome::Coalesced => panic!("the first begin must admit"), + }; + + post_receive_replication_tail( + state.clone(), + rec.clone(), + vec![RefUpdate { + old_sha: c1.clone(), + new_sha: c2.clone(), + ref_name: "refs/heads/main".to_string(), + }], + repo.path().to_path_buf(), + F2A_PUSHER.to_string(), + ) + .await; + + let slug = f2a_slug(&rec); + let mapped = f2a_wait_for_branch_cid( + &state.db, + &slug, + "the coalesced push's branch to CID mapping", + ) + .await; + assert_eq!( + mapped.len(), + 1, + "one mapping, for the ref this push advanced" + ); + assert_eq!(mapped[0].ref_name, "refs/heads/main"); + assert_eq!(mapped[0].sha, c2, "mapped to the tip this push landed"); + assert_eq!(mapped[0].cid, cid); + + let broadcast = updates + .try_recv() + .expect("a coalesced push still fires its own announce"); + assert_eq!(broadcast.new_sha, c2); + assert_eq!(broadcast.ref_name, "refs/heads/main"); + } + + /// Scenario 5 (trap 3, fail-closed). On a repo whose withheld walk is failing, a + /// coalesced push must not publish. Before the gate moved, every push on such a + /// repo got `announce = false` from its own walk; a coalesced push has no walk, so + /// the announce decision now comes from the Pinata worker's recomputation, which + /// fails closed the same way. Asserted on the broadcast channel: nothing is sent. + #[cfg(unix)] + #[sqlx::test] + async fn f2a_coalesced_push_on_a_failing_walk_does_not_publish(pool: sqlx::PgPool) { + let repo = tempfile::TempDir::new().unwrap(); + let bin = tempfile::TempDir::new().unwrap(); + u5_init_repo(repo.path()); + let c1 = u5_commit_file(repo.path(), "a.txt", "one\n"); + let c2 = u5_commit_file(repo.path(), "secret/s.txt", "two\n"); + let log = bin.path().join("git.log"); + // Every withheld walk fails: the repo cannot be vetted, so it must neither + // replicate nor announce. + let git_bin = write_fake_git( + bin.path(), + &format!( + "#!/bin/sh\n\ + printf '%s\\n' \"$*\" >> \"{log}\"\n\ + case \"$*\" in 'rev-list --all'*) exit 1 ;; esac\n\ + exec git \"$@\"\n", + log = log.display(), + ), + ); + let (mut state, rec) = f2a_state(pool, &git_bin, "z6f2aclosed", "x1", true).await; + let (_server, _cid) = f2a_pinata(&mut state).await; + let mut updates = state.ref_update_tx.subscribe(); + + let _inflight = match state.encrypt_inflight.try_begin(&rec.id, Vec::new()) { + crate::state::BeginOutcome::Admitted(g) => g, + crate::state::BeginOutcome::Coalesced => panic!("the first begin must admit"), + }; + + post_receive_replication_tail( + state.clone(), + rec.clone(), + vec![RefUpdate { + old_sha: c1.clone(), + new_sha: c2.clone(), + ref_name: "refs/heads/main".to_string(), + }], + repo.path().to_path_buf(), + F2A_PUSHER.to_string(), + ) + .await; + + // The worker has reached its recomputation (and failed it) once the walk's + // commit enumeration shows up in the log; give the rest of the task a settle. + f2a_wait_for( + || { + f2a_log(&log) + .lines() + .any(|l| l.starts_with("rev-list --all")) + }, + "the Pinata worker's fail-closed recomputation", + ) + .await; + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + + assert!( + matches!( + updates.try_recv(), + Err(tokio::sync::broadcast::error::TryRecvError::Empty) + ), + "a push whose replication could not be vetted must not broadcast" + ); + assert!( + state + .db + .list_branch_cids(&f2a_slug(&rec)) + .await + .unwrap() + .is_empty(), + "and it must pin nothing, so it maps no CID" + ); + } + + /// Scenario 6. A repo the anonymous public cannot read at root takes no key, runs + /// no walk, and spawns no git at all: the cheap predicate answers before anything + /// is acquired, exactly as `replication_withheld_set`'s own early return did. + #[cfg(unix)] + #[sqlx::test] + async fn f2a_private_repo_takes_no_key_and_runs_no_git(pool: sqlx::PgPool) { + let repo = tempfile::TempDir::new().unwrap(); + let bin = tempfile::TempDir::new().unwrap(); + u5_init_repo(repo.path()); + let c1 = u5_commit_file(repo.path(), "a.txt", "one\n"); + let log = bin.path().join("git.log"); + let git_bin = f2a_logging_git(bin.path(), &log); + let (state, mut rec) = f2a_state(pool, &git_bin, "z6f2apriv", "v1", false).await; + rec.is_public = false; + + post_receive_replication_tail( + state.clone(), + rec.clone(), + f2a_update("refs/heads/main", &c1), + repo.path().to_path_buf(), + F2A_PUSHER.to_string(), + ) + .await; + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + + assert!( + state.encrypt_inflight.is_empty(), + "a repo that replicates nothing must not take the coalescing key" + ); + assert_eq!( + f2a_log(&log), + "", + "and it must spawn no git: no walk, no candidate scan, no re-derivation" + ); + } + + /// Scenario 7. The first push is unaffected: it is admitted, it runs the walk, and + /// it still does the full per-push work (pin, mapping, announce). + #[cfg(unix)] + #[sqlx::test] + async fn f2a_first_push_is_admitted_and_does_the_full_work(pool: sqlx::PgPool) { + let repo = tempfile::TempDir::new().unwrap(); + let bin = tempfile::TempDir::new().unwrap(); + u5_init_repo(repo.path()); + let c1 = u5_commit_file(repo.path(), "a.txt", "one\n"); + let c2 = u5_commit_file(repo.path(), "secret/s.txt", "two\n"); + let log = bin.path().join("git.log"); + let git_bin = f2a_logging_git(bin.path(), &log); + let (mut state, rec) = f2a_state(pool, &git_bin, "z6f2afirst", "f1", true).await; + let (_server, cid) = f2a_pinata(&mut state).await; + let mut updates = state.ref_update_tx.subscribe(); + + post_receive_replication_tail( + state.clone(), + rec.clone(), + vec![RefUpdate { + old_sha: c1.clone(), + new_sha: c2.clone(), + ref_name: "refs/heads/main".to_string(), + }], + repo.path().to_path_buf(), + F2A_PUSHER.to_string(), + ) + .await; + + assert!( + f2a_walks(&log) >= 1, + "the admitted push runs the withheld walk itself; log:\n{}", + f2a_log(&log) + ); + let slug = f2a_slug(&rec); + let mapped = f2a_wait_for_branch_cid( + &state.db, + &slug, + "the admitted push's branch to CID mapping", + ) + .await; + assert_eq!(mapped[0].sha, c2); + assert_eq!(mapped[0].cid, cid); + let broadcast = updates + .try_recv() + .expect("the admitted push fires its announce"); + assert_eq!(broadcast.new_sha, c2); + } } diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 213b8b35..654399c7 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -372,6 +372,19 @@ impl EncryptInflight { pub fn is_empty(&self) -> bool { self.len() == 0 } + + /// The work currently queued against a repo's in-flight task, or `None` when + /// no task holds the key. Test-only observability: it lets a test assert that + /// a coalesced push's tip pairs were actually recorded, rather than that the + /// outcome merely was `Coalesced`. + #[cfg(test)] + pub fn pending_for(&self, repo_id: &str) -> Option { + self.repos + .lock() + .expect("encrypt_inflight mutex poisoned") + .get(repo_id) + .cloned() + } } /// Merge a coalesced push's tip pairs into a repo's pending slot. FullScan absorbs From b71201ef406c57ef8e05de7045528b31d7fd0f61 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 28 Jul 2026 02:40:37 -0500 Subject: [PATCH 62/95] test(node): observe the write-cap shed as state, not as a wall-clock deadline The two F1 tests asserted "the capped source shed rather than parked" by racing a ten second timeout, and drove the holding push in ten millisecond slices. Under `cargo test --workspace` that is a race against the scheduler rather than a test: a reviewer saw two of five runs go red, and under load the pair stretched from 1.8s to 19.2s while the fake git holding the lease only lived for 10s, past which every assertion in both tests is meaningless. Shed versus park is a question of which path ran, so both are now read as state. A shed is `Err(Overloaded)`. A park is a second reference on the lease entry, exposed by a test-only `refs_for`; `acquire` takes that reference synchronously before the cancellable wait, so a waiter is visible the instant it reaches the lease, and it stays visible because the steal bound is 1260s. The tests spawn their pushes and poll until one of the two mutually exclusive states appears, so a starved machine iterates more rather than failing. Every wall-clock assertion is gone. The worst of them was a sleep followed by `!is_finished()`: a runtime that had not yet polled that task satisfied it vacuously, which means the constant-key mutation could have gone green under load, and the discriminator is the whole reason that test exists. All three mutations still go red, now in seconds rather than by timeout expiry: the per-caller acquire below the lease, the global permit above it, and the per-caller key replaced with a constant. Also fixes a third flake found on the way: the acquire-deadline test's follow-up push inherited the deliberately short two second acquire timeout, which has to cover a real advisory-lock round trip, so its `Overloaded` was indistinguishable from the drained-pool `Overloaded` the assertion was actually testing for. The sqlx `databases_pkey` collisions seen alongside this are machine residue, not a leak from these tests: sqlx derives the test database name from a hash of the test path and does cleanup-then-insert without a lock, so any two concurrent test binaries running the same path collide. The orphan count here held at 53 across every run. --- crates/gitlawb-node/src/api/repos.rs | 252 ++++++++++++++++++--------- crates/gitlawb-node/src/state.rs | 15 ++ 2 files changed, 185 insertions(+), 82 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 7d8bbe2a..62516148 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -5090,6 +5090,14 @@ mod tests { .execute(&mut *lock_conn) .await .expect("release the advisory lock"); + // The follow-up asserts an ADMISSION property (the freed write slot is usable), + // so it must not inherit the 2s acquire deadline the shed above is testing. That + // budget covers a real advisory-lock round trip, and on a loaded machine it + // expires and returns the same `Overloaded` this assertion reads as a drained + // pool: a correct run going red for a reason the test is not about. + let mut followup_cfg = (*state.config).clone(); + followup_cfg.git_acquire_timeout_secs = 120; + state.config = std::sync::Arc::new(followup_cfg); let followup = git_receive_pack( State(state.clone()), Path((owner.to_string(), name.to_string())), @@ -7184,6 +7192,39 @@ mod tests { assert!(b_ran.exists(), "push B ran its receive-pack once unblocked"); } + /// Backstop for the F1 wait loops below. It is a HANG detector, never the thing an + /// assertion rests on: every F1 conclusion is drawn from a state the loop actually + /// observed (a returned shed, a second reference on the lease entry), so a loaded + /// machine only spends more iterations getting there. It must stay comfortably under + /// `F1_HOLD_SECS` so push A is still holding the lease when the loop gives up. + #[cfg(unix)] + const F1_BACKSTOP: std::time::Duration = std::time::Duration::from_secs(30); + + /// How long `f1_hanging_first_git`'s first receive-pack holds the lease. Bounded so a + /// RED run leaks no permanent orphan, and well above the two sequential `F1_BACKSTOP` + /// windows the discriminator can spend, so the hold never expires mid-observation on + /// a loaded machine. The old bound was 10s, which is under the 19s the pair took on a + /// contended box: push A's git would exit and free the lease mid-test. + #[cfg(unix)] + const F1_HOLD_SECS: usize = 150; + + /// Poll `cond` until it holds, yielding between checks so spawned handlers make + /// progress. Returns false if `cap` elapses first. Callers assert on the state the + /// loop settled into, not on the elapsed time. + #[cfg(unix)] + async fn f1_wait_for(cap: std::time::Duration, mut cond: impl FnMut() -> bool) -> bool { + let deadline = std::time::Instant::now() + cap; + loop { + if cond() { + return true; + } + if std::time::Instant::now() >= deadline { + return false; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + } + /// Fake git for the U1 lease-park tests. The FIRST receive-pack invocation marks /// that it reached the pack and hangs in a BOUNDED loop (so a RED run leaks no /// permanent orphan); every later invocation marks that it got past the lease. @@ -7202,7 +7243,7 @@ mod tests { cat >/dev/null 2>/dev/null\n\ if mkdir \"{seq}\" 2>/dev/null; then\n\ echo 1 > \"{ainp}\"\n\ - i=0; while [ $i -lt 100 ]; do sleep 0.1; i=$((i+1)); done\n\ + i=0; while [ $i -lt {hold} ]; do sleep 1; i=$((i+1)); done\n\ else\n\ echo 1 > \"{later}\"\n\ fi ;;\n\ @@ -7213,6 +7254,7 @@ mod tests { seq = seq_a.display(), ainp = a_inpack.display(), later = later_ran.display(), + hold = F1_HOLD_SECS, ); (write_fake_git(tmp, &body), a_inpack, later_ran) } @@ -7228,10 +7270,13 @@ mod tests { /// them at `(max_concurrent_git_pushes / 8).max(1)` = 4 per source. /// /// Load-bearing: push A (source X) holds the lease and X's only per-caller slot. - /// Pre-fix push B (also X) parks on the lease and does not return inside the test - /// window (RED). Post-fix it sheds on the per-caller cap immediately (GREEN). The - /// elapsed assertion is the point: an error returned after `steal_after` would be - /// no fix at all. + /// Pre-fix push B (also X) reaches the lease and parks there (RED); post-fix it never + /// reaches the lease at all and sheds on the per-caller cap (GREEN). Shed-vs-park is + /// a question of WHICH PATH ran, so both outcomes are read as state, not as elapsed + /// time: the shed is B's returned `Overloaded`, and the park is the second reference + /// `RepoWriteLeases::acquire` takes on the lease entry (synchronously, before it + /// waits). The loop below waits for whichever of the two appears, so a machine that + /// is starved of CPU by a sibling test just iterates more times. #[cfg(unix)] #[sqlx::test] async fn f1_capped_source_sheds_instead_of_parking_on_a_contended_lease(pool: sqlx::PgPool) { @@ -7246,12 +7291,20 @@ mod tests { // Per-source write cap of 1 so push A alone exhausts source X's budget. The // global write pool stays generous (64), so nothing here sheds on it. state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + let repo_id = state + .db + .get_repo("z6f1cap", "c1") + .await + .unwrap() + .unwrap() + .id; let did = "did:key:z6MkF1CapPusherAAAAAAAAAAAAAAAAAAAAAAAAAA"; let src_x: SocketAddr = "203.0.113.61:5000".parse().unwrap(); - // Push A: drive its handler future in slices until it reaches receive-pack. It - // now holds the per-repo lease AND source X's single per-caller write slot. - let mut fut_a = Box::pin(git_receive_pack( + // Push A runs as its own task: the runtime schedules it, nothing here drives it in + // slices. It reaches receive-pack and then holds the per-repo lease AND source X's + // single per-caller write slot until it is aborted at the end. + let handle_a = tokio::spawn(git_receive_pack( State(state.clone()), Path(("z6f1cap".to_string(), "c1".to_string())), Extension(crate::auth::AuthenticatedDid(did.to_string())), @@ -7259,48 +7312,59 @@ mod tests { axum::http::HeaderMap::new(), axum::body::Bytes::from_static(b"0000"), )); - for _ in 0..1000 { - let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut_a).await; - if a_inpack.exists() { - break; - } - } assert!( - a_inpack.exists(), - "push A must reach receive-pack and hold the lease plus source X's write slot" + f1_wait_for(F1_BACKSTOP, || a_inpack.exists()).await, + "push A never reached receive-pack within {F1_BACKSTOP:?}" + ); + // Both preconditions read as state, so push B's shed below cannot be vacuous. + assert_eq!( + state.repo_write_leases.refs_for(&repo_id), + 1, + "push A must hold the repo write lease, and be the only handler on it" + ); + assert_eq!( + state.git_write_per_caller.tracked_keys(), + 1, + "push A must be holding a per-caller write permit, so the cap of 1 is spent" ); // Push B: SAME source, same repo. It must be turned away, not parked. - let started = std::time::Instant::now(); - let b = tokio::time::timeout( - std::time::Duration::from_secs(10), - git_receive_pack( - State(state.clone()), - Path(("z6f1cap".to_string(), "c1".to_string())), - Extension(crate::auth::AuthenticatedDid(did.to_string())), - crate::rate_limit::PeerAddr(Some(src_x)), - axum::http::HeaderMap::new(), - axum::body::Bytes::from_static(b"0000"), - ), - ) + let handle_b = tokio::spawn(git_receive_pack( + State(state.clone()), + Path(("z6f1cap".to_string(), "c1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(src_x)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + )); + // Wait for B to land in one of its two terminal states: returned (shed) or holding + // a reference on the lease entry (parked). They are mutually exclusive, and a + // parked waiter stays parked (steal_after is 1260s), so whichever the loop sees is + // stable rather than a sampled instant. + let settled = f1_wait_for(F1_BACKSTOP, || { + handle_b.is_finished() || state.repo_write_leases.refs_for(&repo_id) >= 2 + }) .await; - let elapsed = started.elapsed(); - let b = b.expect( + assert!( + settled, + "push B neither returned nor reached the repo write lease within {F1_BACKSTOP:?}; \ + it is stuck somewhere earlier in the handler" + ); + assert_eq!( + state.repo_write_leases.refs_for(&repo_id), + 1, "F1 RED: a source already at its per-source write cap PARKED on the contended \ repo lease holding a fully buffered pack body. The per-caller write permit \ must be taken BEFORE the lease so the request sheds instead of waiting out \ - steal_after (1260s at defaults)", + steal_after (1260s at defaults)" ); + let b = handle_b.await.expect("push B task must not panic"); assert!( matches!(b, Err(AppError::Overloaded(_))), "the capped source's second push must shed with Overloaded/503; got {b:?}" ); - assert!( - elapsed < std::time::Duration::from_secs(5), - "the shed must return promptly, not after the lease park; took {elapsed:?}" - ); - drop(fut_a); + handle_a.abort(); } /// #174 F1 (U1) THE DISCRIMINATOR: while source X sits at its per-source write cap @@ -7313,9 +7377,13 @@ mod tests { /// resolved source IP and Y sheds too, which is exactly the node-wide shed /// `f02c5e1` was written to prevent. /// - /// Load-bearing on both halves: X's second push sheds promptly (RED pre-fix, where - /// it parks) AND Y's push stays parked rather than shedding (RED under the - /// constant-key mutation). + /// Load-bearing on both halves: X's second push reaches the lease and parks (RED + /// pre-fix) AND Y's push parks rather than shedding (RED under the constant-key + /// mutation, where Y is denied at the per-caller gate and returns). Both halves are + /// read as state, never as elapsed time: parked is a reference on the lease entry, + /// shed is a returned `Overloaded`. The old `sleep(300ms)` + `!is_finished()` check + /// had the worse failure mode of the two, since a starved runtime that had not yet + /// polled push C would satisfy it vacuously and the mutation would go green. #[cfg(unix)] #[sqlx::test] async fn f1_capped_source_shed_leaves_a_different_source_parked_and_served(pool: sqlx::PgPool) { @@ -7328,12 +7396,19 @@ mod tests { let mut state = f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f1disc", "d1", false).await; state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + let repo_id = state + .db + .get_repo("z6f1disc", "d1") + .await + .unwrap() + .unwrap() + .id; let did = "did:key:z6MkF1DiscPusherAAAAAAAAAAAAAAAAAAAAAAAAA"; let src_x: SocketAddr = "203.0.113.63:5000".parse().unwrap(); let src_y: SocketAddr = "203.0.113.64:5000".parse().unwrap(); // Push A (source X) reaches receive-pack: holds the lease and X's only slot. - let mut fut_a = Box::pin(git_receive_pack( + let handle_a = tokio::spawn(git_receive_pack( State(state.clone()), Path(("z6f1disc".to_string(), "d1".to_string())), Extension(crate::auth::AuthenticatedDid(did.to_string())), @@ -7341,29 +7416,35 @@ mod tests { axum::http::HeaderMap::new(), axum::body::Bytes::from_static(b"0000"), )); - for _ in 0..1000 { - let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut_a).await; - if a_inpack.exists() { - break; - } - } - assert!(a_inpack.exists(), "push A must reach receive-pack"); + assert!( + f1_wait_for(F1_BACKSTOP, || a_inpack.exists()).await, + "push A never reached receive-pack within {F1_BACKSTOP:?}" + ); + assert_eq!( + state.repo_write_leases.refs_for(&repo_id), + 1, + "push A must hold the repo write lease, and be the only handler on it" + ); // Push C from source Y: it has its own budget, so it takes Y's permit and parks - // on the lease A holds. + // on the lease A holds. Wait for it to reach either terminal state. let state_c = state.clone(); - let handle_c = tokio::spawn(async move { - git_receive_pack( - State(state_c), - Path(("z6f1disc".to_string(), "d1".to_string())), - Extension(crate::auth::AuthenticatedDid(did.to_string())), - crate::rate_limit::PeerAddr(Some(src_y)), - axum::http::HeaderMap::new(), - axum::body::Bytes::from_static(b"0000"), - ) - .await - }); - tokio::time::sleep(std::time::Duration::from_millis(300)).await; + let handle_c = tokio::spawn(git_receive_pack( + State(state_c), + Path(("z6f1disc".to_string(), "d1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(src_y)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + )); + let c_settled = f1_wait_for(F1_BACKSTOP, || { + handle_c.is_finished() || state.repo_write_leases.refs_for(&repo_id) >= 2 + }) + .await; + assert!( + c_settled, + "push C neither returned nor reached the repo write lease within {F1_BACKSTOP:?}" + ); assert!( !handle_c.is_finished(), "F1 discriminator RED: a push from a DIFFERENT source was turned away while \ @@ -7371,28 +7452,34 @@ mod tests { source IP, so exhausting it denies only the exhausting source" ); - // Push B from source X: shed, promptly, while C stays parked. - let started = std::time::Instant::now(); - let b = tokio::time::timeout( - std::time::Duration::from_secs(10), - git_receive_pack( - State(state.clone()), - Path(("z6f1disc".to_string(), "d1".to_string())), - Extension(crate::auth::AuthenticatedDid(did.to_string())), - crate::rate_limit::PeerAddr(Some(src_x)), - axum::http::HeaderMap::new(), - axum::body::Bytes::from_static(b"0000"), - ), - ) - .await - .expect("the capped source must shed rather than park on the contended lease"); + // Push B from source X: shed, while C stays parked. A third reference on the + // lease entry would mean B parked too. + let handle_b = tokio::spawn(git_receive_pack( + State(state.clone()), + Path(("z6f1disc".to_string(), "d1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(src_x)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + )); + let b_settled = f1_wait_for(F1_BACKSTOP, || { + handle_b.is_finished() || state.repo_write_leases.refs_for(&repo_id) >= 3 + }) + .await; assert!( - matches!(b, Err(AppError::Overloaded(_))), - "source X's second push must shed with Overloaded/503; got {b:?}" + b_settled, + "push B neither returned nor reached the repo write lease within {F1_BACKSTOP:?}" + ); + assert_eq!( + state.repo_write_leases.refs_for(&repo_id), + 2, + "F1 RED: the capped source parked on the contended lease (A holding, C and B \ + waiting) instead of shedding on its per-source write cap" ); + let b = handle_b.await.expect("push B task must not panic"); assert!( - started.elapsed() < std::time::Duration::from_secs(5), - "source X's shed must be prompt" + matches!(b, Err(AppError::Overloaded(_))), + "source X's second push must shed with Overloaded/503; got {b:?}" ); assert!( !handle_c.is_finished(), @@ -7403,9 +7490,10 @@ mod tests { "no second receive-pack may run while A holds the lease" ); - // Free the lease: A disconnects. Source Y's parked push is served. - drop(fut_a); - let resp = tokio::time::timeout(std::time::Duration::from_secs(30), handle_c) + // Free the lease: A disconnects. Aborting drops its handler future at the child + // await, the same shape as a client hang-up. Source Y's parked push is served. + handle_a.abort(); + let resp = tokio::time::timeout(std::time::Duration::from_secs(60), handle_c) .await .expect("push C must complete once A's group is reaped and the lease frees") .expect("push C task must not panic") diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 16893e65..b7fb1492 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -589,6 +589,21 @@ impl RepoWriteLeases { pub fn is_empty(&self) -> bool { self.len() == 0 } + + /// How many handlers currently reference this repo's lease entry: the holder plus + /// every waiter parked in [`acquire`](Self::acquire). The ref is taken SYNCHRONOUSLY, + /// before the (cancellable) semaphore wait, so a waiter is visible here the moment it + /// reaches the lease. Tests use it to observe "this push parked" as a state rather + /// than inferring it from a deadline that never fired. + #[cfg(test)] + pub fn refs_for(&self, repo_id: &str) -> usize { + self.repos + .lock() + .expect("repo_write_leases mutex poisoned") + .get(repo_id) + .map(|slot| slot.refs) + .unwrap_or(0) + } } /// Shared-ownership handle to a held repo write lease (#174 U2/F3). `Clone` hands a From 8f0e9107a4044b6fa30a6ac8bf3aa3bf0ccb6342 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 28 Jul 2026 08:01:51 -0500 Subject: [PATCH 63/95] fix(node): bound lease waiters per repo, and stop spending cross-repo budget on a per-repo wait The previous attempt moved the per-source write permit above the lease park to bound buffered pack bodies. Two reviewers demonstrated that it denies far more than the source doing the parking. A source parked on one repo's lease was shed on repos with no contention at all, and because the trusted-proxy setting defaults to trusting nothing and keying on the socket peer, every pusher behind a proxy, NAT, or CI runner pool resolves to one key: four parked pushes shed every push from every pusher to every repo for up to twenty-one minutes. That is the node-wide shed the permit was moved below the lease to prevent in the first place, reintroduced at an eighth of the threshold. The permit goes back below the lease. What is actually unbounded is the waiter set, so that is what is bounded now, per repo, with a shed past the cap. The cap counts live waiters, never the entry refcount. The refcount includes the holder, so capping it would let a lease whose owner died without unwinding pin a slot forever, which is the permanent wedge the steal timeout exists to survive. The count is claimed in the same critical section as the refcount and held by a guard scoped to the cancellable wait, so a cancelled or shed waiter gives it back. An uncontended acquire takes the fast path and spends no waiter budget; that path does not barge a queued waiter, which was checked over two thousand rounds rather than assumed. Two tests now guard the defect that was just removed: a source parked on one repo is still served on another, and a parked push does not shed an unrelated pusher sharing its resolved key. Restoring the old ordering turns both red. Capping on the refcount instead of live waiters turns the steal-backstop test red. Holding the waiter slot past the wait turns the fast-path test red. What this bounds is the parked count, not the parked bytes. At a two gigabyte pack limit the byte figure stays large, and the total across repos, whose dominant term is the buffering the signature middleware does before any admission point at all, is tracked in #263. Past the cap, same-repo concurrency sheds instead of queueing; that is the trade the knob buys. --- crates/gitlawb-node/src/api/repos.rs | 489 ++++++++++++++---------- crates/gitlawb-node/src/auth/mod.rs | 2 +- crates/gitlawb-node/src/config.rs | 24 ++ crates/gitlawb-node/src/main.rs | 7 +- crates/gitlawb-node/src/state.rs | 372 ++++++++++++++++-- crates/gitlawb-node/src/test_support.rs | 2 +- 6 files changed, 643 insertions(+), 253 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 62516148..88138d0e 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1550,9 +1550,8 @@ pub async fn git_receive_pack( let name = smart_http_repo_name(&repo)?; // Fast-path shed BEFORE the DB lookup when the write pool is saturated, so a push // flood on a full pool does not hit Postgres per request. Best-effort (racy) and - // NON-holding: the authoritative, held GLOBAL write permit is taken after the per-repo - // lease below (so a lease-blocked waiter pins no global write slot, #174 F3 review; - // the per-SOURCE sub-cap is taken above the lease, see there). This peek only + // NON-holding: the authoritative, held permit is taken after the per-repo lease below + // (so a lease-blocked waiter pins no write slot — #174 F3 review). This peek only // restores the cheap pre-DB shed the permit reorder would otherwise have lost. if state.git_write_semaphore.available_permits() == 0 { return Err(AppError::Overloaded( @@ -1648,30 +1647,49 @@ pub async fn git_receive_pack( let lease_steal_after = std::time::Duration::from_secs(state.config.git_service_timeout_secs * 2 + 60); - // The two write admission permits are DELIBERATELY split around the lease park. - // - // Per-source sub-cap BEFORE the lease (#174 P1-d, and F1 review). One source IP - // cannot occupy the whole write pool via many slow pushes: owner enforcement defaults - // off, so any valid did:key is accepted (auth != authz) and the push rate limiter - // bounds arrival RATE, not in-flight concurrency. It is taken here, above the park, - // because `body: Bytes` means axum has ALREADY buffered the entire pack (up to - // max_pack_bytes) before this function runs, and `acquire` below is a block-and-wait - // serializer whose waiter set is unbounded (state.rs) and whose wait runs to - // steal_after (git_service_timeout_secs * 2 + 60 = 1260s at defaults). With both - // permits below the lease nothing accounted for parked waiters, so one source could - // stack unbounded buffered bodies on a contended repo. The sub-cap is - // (max_concurrent_git_pushes / 8).max(1) = 4 at defaults, so it bounds parked bodies - // at 4 per source. Keyed on the resolved source IP, NEVER the signed DID (a DID farm - // defeats a DID key); no resolvable key -> global write pool only. + // Parked waiters are bounded INSIDE the lease, not by an admission permit taken above + // it. `body: Bytes` means axum has already buffered the whole pack before this handler + // runs, and the park runs to steal_after (1260s at defaults), so an unbounded waiter + // set would let same-repo pushes stack buffered bodies. `acquire` therefore counts its + // LIVE WAITERS against GITLAWB_REPO_LEASE_MAX_WAITERS and returns None past the cap, + // which sheds here as a 503 + Retry-After like the other admission paths. The cap is + // per repo and counts only handlers actually parked, so it denies same-repo + // concurrency on the contended repo alone: a push to any other repo, from any source, + // is untouched. Taking a per-source or global admission permit above the park instead + // is what the F1 review rejected, since GITLAWB_TRUSTED_PROXY defaults to unset and + // every pusher behind a proxy/NAT then resolves to ONE key, turning one contended repo + // into a node-wide denial. + let lease = state + .repo_write_leases + .acquire(&record.id, lease_steal_after) + .await + .ok_or_else(|| { + tracing::warn!( + repo = %name, + "repo write-lease waiter cap reached; shedding with 503" + ); + AppError::Overloaded("repo is busy with another push, retry shortly".into()) + })?; + + // Admission permits are taken HERE, AFTER the per-repo lease and BEFORE acquire_write. + // Ordering is the fix (#174 P2 DoS): the lease is a block-and-wait serializer, so a + // second same-repo push can park on `acquire` above for up to steal_after. Taking the + // scarce write permits only once we own the lease means a lease-blocked waiter pins NO + // write-pool slot while it waits. Otherwise a few hostile sources could stack same-repo + // pushes, hold every global slot on zero-byte lease-waiters, and shed 503 on every push + // to every OTHER repo node-wide. The per-source sub-cap belongs below the park for the + // same reason: its key is the resolved source IP, which collapses to one key for every + // pusher when GITLAWB_TRUSTED_PROXY is unset (the default), so above the park it sheds + // cross-tenant too. Still before acquire_write, so the git op stays admission-gated + // (INV-10) and a saturated pool sheds 503 before spawning git. // - // The GLOBAL write permit deliberately stays BELOW the lease (#174 P2 DoS, f02c5e1). - // That is a scarce node-wide resource: a lease-blocked waiter holding one lets a few - // hostile sources stack same-repo pushes, pin every global slot on zero-byte - // lease-waiters, and shed 503 on every push to every OTHER repo node-wide. A - // per-source key is not global, so exhausting it denies only the exhausting source. - // Self-inflicted denial is the property that makes the sub-cap safe above the park and - // the global permit unsafe there. Both still land before acquire_write, so the git op - // stays admission-gated (INV-10) and a saturated pool sheds 503 before spawning git. + // Per-source sub-cap first (#174 P1-d): one source IP cannot occupy the whole write + // pool via many slow pushes. Owner enforcement defaults off, so any valid did:key is + // accepted (auth != authz) and the push rate limiter bounds arrival RATE, not in-flight + // concurrency. Keyed on the resolved source IP, NEVER the signed DID (a DID farm defeats + // a DID key); no resolvable key -> global write pool only. Then the global write permit: + // pushes draw from the dedicated WRITE pool, separate from reads, and it is held for the + // whole op (moved into the AdmissionGuard below). let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); let _caller_permit = acquire_read_caller_permit( &state.git_write_per_caller, @@ -1679,12 +1697,6 @@ pub async fn git_receive_pack( name, "receive-pack", )?; - - let lease = state - .repo_write_leases - .acquire(&record.id, lease_steal_after) - .await; - let _permit = git_permit(&state.git_write_semaphore)?; tracing::debug!(repo = %name, "acquiring write lock"); @@ -7259,134 +7271,148 @@ mod tests { (write_fake_git(tmp, &body), a_inpack, later_ran) } - /// #174 F1 (U1, RED-before/GREEN-after): a source already at its per-source WRITE - /// sub-cap must SHED on a contended repo rather than park on the per-repo write - /// lease. `git_receive_pack` takes `body: Bytes`, so axum has already buffered the - /// whole pack (up to `max_pack_bytes`, 2 GB by default) before the handler runs, - /// and the lease park runs to `lease_steal_after` = `git_service_timeout_secs * 2 - /// + 60` = 1260s at defaults. The waiter set in `RepoWriteLeases::acquire` is - /// unbounded, so with BOTH admission permits taken after the lease nothing accounts - /// for parked bodies. Taking the per-caller write permit BEFORE the lease bounds - /// them at `(max_concurrent_git_pushes / 8).max(1)` = 4 per source. + /// Add a second repo to a state built by `f4_state_with_repo`, returning its DB id. + /// The U1 tests need two repos in ONE state to show that shedding on a contended + /// lease is confined to that repo. + #[cfg(unix)] + async fn f1_add_repo(state: &AppState, owner: &str, name: &str) -> String { + state + .db + .upsert_mirror_repo(owner, name, &format!("/unused-{owner}-{name}"), None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + state + .repo_store + .init(&rec.owner_did, &rec.name) + .await + .unwrap(); + rec.id + } + + /// #174 U1 scenario 1 (RED-before/GREEN-after): parked pushes on one repo's write + /// lease are BOUNDED. `git_receive_pack` takes `body: Bytes`, so axum has already + /// buffered the whole pack (up to `max_pack_bytes`) before the handler runs, and the + /// park runs to `lease_steal_after` = `git_service_timeout_secs * 2 + 60` = 1260s at + /// defaults. An unbounded waiter set is therefore unbounded buffered memory held for + /// 21 minutes. With the cap at K, a holder plus K live waiters means the next push + /// sheds a 503 + Retry-After instead of joining the queue. /// - /// Load-bearing: push A (source X) holds the lease and X's only per-caller slot. - /// Pre-fix push B (also X) reaches the lease and parks there (RED); post-fix it never - /// reaches the lease at all and sheds on the per-caller cap (GREEN). Shed-vs-park is - /// a question of WHICH PATH ran, so both outcomes are read as state, not as elapsed - /// time: the shed is B's returned `Overloaded`, and the park is the second reference - /// `RepoWriteLeases::acquire` takes on the lease entry (synchronously, before it - /// waits). The loop below waits for whichever of the two appears, so a machine that - /// is starved of CPU by a sibling test just iterates more times. + /// Read as state, never as elapsed time: the shed is the returned `Overloaded`, and + /// "the queue did not grow" is the waiter count the loop polls to. The bound is on + /// LIVE WAITERS, so the holder is not counted; that is what keeps a leaked lease from + /// wedging the repo (see `steal_on_leaked_lease_still_works_under_the_waiter_cap`). #[cfg(unix)] #[sqlx::test] - async fn f1_capped_source_sheds_instead_of_parking_on_a_contended_lease(pool: sqlx::PgPool) { + async fn u1_push_past_the_lease_waiter_cap_sheds_with_503(pool: sqlx::PgPool) { use axum::extract::{Path, State}; + use axum::response::IntoResponse; use axum::Extension; use std::net::SocketAddr; let tmp = tempfile::TempDir::new().unwrap(); let (git_bin, a_inpack, _later_ran) = f1_hanging_first_git(tmp.path()); let mut state = - f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f1cap", "c1", false).await; - // Per-source write cap of 1 so push A alone exhausts source X's budget. The - // global write pool stays generous (64), so nothing here sheds on it. - state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6u1cap", "c1", false).await; + // Cap of ONE live waiter, so a holder plus one parked push fills the repo's queue. + state.repo_write_leases = crate::state::RepoWriteLeases::new(1); let repo_id = state .db - .get_repo("z6f1cap", "c1") + .get_repo("z6u1cap", "c1") .await .unwrap() .unwrap() .id; - let did = "did:key:z6MkF1CapPusherAAAAAAAAAAAAAAAAAAAAAAAAAA"; - let src_x: SocketAddr = "203.0.113.61:5000".parse().unwrap(); + let did = "did:key:z6MkU1CapPusherAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let push = |peer: SocketAddr| { + let st = state.clone(); + let did = did.to_string(); + async move { + git_receive_pack( + State(st), + Path(("z6u1cap".to_string(), "c1".to_string())), + Extension(crate::auth::AuthenticatedDid(did)), + crate::rate_limit::PeerAddr(Some(peer)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await + } + }; - // Push A runs as its own task: the runtime schedules it, nothing here drives it in - // slices. It reaches receive-pack and then holds the per-repo lease AND source X's - // single per-caller write slot until it is aborted at the end. - let handle_a = tokio::spawn(git_receive_pack( - State(state.clone()), - Path(("z6f1cap".to_string(), "c1".to_string())), - Extension(crate::auth::AuthenticatedDid(did.to_string())), - crate::rate_limit::PeerAddr(Some(src_x)), - axum::http::HeaderMap::new(), - axum::body::Bytes::from_static(b"0000"), - )); + // Push A holds the lease (its git hangs) until it is aborted below. + let handle_a = tokio::spawn(push("203.0.113.81:5000".parse().unwrap())); assert!( f1_wait_for(F1_BACKSTOP, || a_inpack.exists()).await, "push A never reached receive-pack within {F1_BACKSTOP:?}" ); - // Both preconditions read as state, so push B's shed below cannot be vacuous. assert_eq!( - state.repo_write_leases.refs_for(&repo_id), - 1, - "push A must hold the repo write lease, and be the only handler on it" + state.repo_write_leases.waiters_for(&repo_id), + 0, + "the uncontended holder must spend no waiter budget" ); - assert_eq!( - state.git_write_per_caller.tracked_keys(), - 1, - "push A must be holding a per-caller write permit, so the cap of 1 is spent" + + // Push B fills the single waiter slot. + let handle_b = tokio::spawn(push("203.0.113.82:5000".parse().unwrap())); + assert!( + f1_wait_for(F1_BACKSTOP, || state + .repo_write_leases + .waiters_for(&repo_id) + == 1) + .await, + "push B never parked on the contended lease within {F1_BACKSTOP:?}" ); - // Push B: SAME source, same repo. It must be turned away, not parked. - let handle_b = tokio::spawn(git_receive_pack( - State(state.clone()), - Path(("z6f1cap".to_string(), "c1".to_string())), - Extension(crate::auth::AuthenticatedDid(did.to_string())), - crate::rate_limit::PeerAddr(Some(src_x)), - axum::http::HeaderMap::new(), - axum::body::Bytes::from_static(b"0000"), - )); - // Wait for B to land in one of its two terminal states: returned (shed) or holding - // a reference on the lease entry (parked). They are mutually exclusive, and a - // parked waiter stays parked (steal_after is 1260s), so whichever the loop sees is - // stable rather than a sampled instant. - let settled = f1_wait_for(F1_BACKSTOP, || { - handle_b.is_finished() || state.repo_write_leases.refs_for(&repo_id) >= 2 - }) - .await; + // Push C is past the cap: it must be turned away, not queued. + let c = tokio::time::timeout(F1_BACKSTOP, push("203.0.113.83:5000".parse().unwrap())) + .await + .expect("a push past the waiter cap must return, not park"); assert!( - settled, - "push B neither returned nor reached the repo write lease within {F1_BACKSTOP:?}; \ - it is stuck somewhere earlier in the handler" + matches!(c, Err(AppError::Overloaded(_))), + "U1 RED: a push arriving past the repo's live-waiter cap joined the unbounded \ + park queue instead of shedding, holding its fully buffered pack for up to \ + steal_after (1260s at defaults); got {c:?}" ); + let resp = c.unwrap_err().into_response(); assert_eq!( - state.repo_write_leases.refs_for(&repo_id), + resp.status(), + 503, + "the shed must be a 503, consistent with the other admission paths" + ); + assert_eq!( + resp.headers().get("retry-after").unwrap().to_str().unwrap(), + "1", + "the shed must advertise Retry-After" + ); + assert_eq!( + state.repo_write_leases.waiters_for(&repo_id), 1, - "F1 RED: a source already at its per-source write cap PARKED on the contended \ - repo lease holding a fully buffered pack body. The per-caller write permit \ - must be taken BEFORE the lease so the request sheds instead of waiting out \ - steal_after (1260s at defaults)" + "the shed must not have joined the queue, and must leave no waiter residue" ); - let b = handle_b.await.expect("push B task must not panic"); - assert!( - matches!(b, Err(AppError::Overloaded(_))), - "the capped source's second push must shed with Overloaded/503; got {b:?}" + assert_eq!( + state.repo_write_leases.refs_for(&repo_id), + 2, + "only the holder and the one real waiter may reference the entry after a shed" ); handle_a.abort(); + let _ = tokio::time::timeout(std::time::Duration::from_secs(60), handle_b).await; } - /// #174 F1 (U1) THE DISCRIMINATOR: while source X sits at its per-source write cap - /// on a contended repo, a push from a DIFFERENT source Y must still park on the - /// lease and be served once the lease frees. This is what distinguishes moving the - /// PER-CALLER permit above the lease from the rejected alternative of moving the - /// GLOBAL write permit (or any node-wide waiter cap) above it: exhausting a - /// per-source key denies only the exhausting source, so the denial is - /// self-inflicted. Key the permit on a constant instead of `read_caller_key`'s - /// resolved source IP and Y sheds too, which is exactly the node-wide shed - /// `f02c5e1` was written to prevent. + /// #174 U1 scenario 2, THE REGRESSION GUARD for the rejected design. A source parked + /// on repo A's lease must still be served on an UNCONTENDED repo B. The rejected fix + /// bounded parked bodies by taking the per-source write permit ABOVE the lease, which + /// makes a parked push spend that source's node-wide budget: the same pusher is then + /// denied on every other repo, and (since `TrustedProxy` defaults to `None`, so every + /// pusher behind a proxy or NAT resolves to one key) so is everyone else. Moving the + /// per-caller permit back above the lease turns this red. /// - /// Load-bearing on both halves: X's second push reaches the lease and parks (RED - /// pre-fix) AND Y's push parks rather than shedding (RED under the constant-key - /// mutation, where Y is denied at the per-caller gate and returns). Both halves are - /// read as state, never as elapsed time: parked is a reference on the lease entry, - /// shed is a returned `Overloaded`. The old `sleep(300ms)` + `!is_finished()` check - /// had the worse failure mode of the two, since a starved runtime that had not yet - /// polled push C would satisfy it vacuously and the mutation would go green. + /// The per-source cap is 2 here: the holder spends one, so a parked push spending the + /// second is what denies repo B. Under the shipped ordering the parked push holds no + /// permit at all and repo B is served. #[cfg(unix)] #[sqlx::test] - async fn f1_capped_source_shed_leaves_a_different_source_parked_and_served(pool: sqlx::PgPool) { + async fn u1_a_source_parked_on_one_repo_is_still_served_on_another(pool: sqlx::PgPool) { use axum::extract::{Path, State}; use axum::Extension; use std::net::SocketAddr; @@ -7394,119 +7420,160 @@ mod tests { let tmp = tempfile::TempDir::new().unwrap(); let (git_bin, a_inpack, later_ran) = f1_hanging_first_git(tmp.path()); let mut state = - f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f1disc", "d1", false).await; - state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); - let repo_id = state + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6u1two", "r1", false).await; + state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(2, 100); + let repo1 = state .db - .get_repo("z6f1disc", "d1") + .get_repo("z6u1two", "r1") .await .unwrap() .unwrap() .id; - let did = "did:key:z6MkF1DiscPusherAAAAAAAAAAAAAAAAAAAAAAAAA"; - let src_x: SocketAddr = "203.0.113.63:5000".parse().unwrap(); - let src_y: SocketAddr = "203.0.113.64:5000".parse().unwrap(); + f1_add_repo(&state, "z6u1two", "r2").await; + let did = "did:key:z6MkU1TwoPusherAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let src: SocketAddr = "203.0.113.84:5000".parse().unwrap(); + let push = |repo: &'static str| { + let st = state.clone(); + let did = did.to_string(); + async move { + git_receive_pack( + State(st), + Path(("z6u1two".to_string(), repo.to_string())), + Extension(crate::auth::AuthenticatedDid(did)), + crate::rate_limit::PeerAddr(Some(src)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await + } + }; - // Push A (source X) reaches receive-pack: holds the lease and X's only slot. - let handle_a = tokio::spawn(git_receive_pack( - State(state.clone()), - Path(("z6f1disc".to_string(), "d1".to_string())), - Extension(crate::auth::AuthenticatedDid(did.to_string())), - crate::rate_limit::PeerAddr(Some(src_x)), - axum::http::HeaderMap::new(), - axum::body::Bytes::from_static(b"0000"), - )); + // Push A (this source) holds repo r1's lease; push B (same source) parks on it. + let handle_a = tokio::spawn(push("r1")); assert!( f1_wait_for(F1_BACKSTOP, || a_inpack.exists()).await, "push A never reached receive-pack within {F1_BACKSTOP:?}" ); - assert_eq!( - state.repo_write_leases.refs_for(&repo_id), - 1, - "push A must hold the repo write lease, and be the only handler on it" - ); - - // Push C from source Y: it has its own budget, so it takes Y's permit and parks - // on the lease A holds. Wait for it to reach either terminal state. - let state_c = state.clone(); - let handle_c = tokio::spawn(git_receive_pack( - State(state_c), - Path(("z6f1disc".to_string(), "d1".to_string())), - Extension(crate::auth::AuthenticatedDid(did.to_string())), - crate::rate_limit::PeerAddr(Some(src_y)), - axum::http::HeaderMap::new(), - axum::body::Bytes::from_static(b"0000"), - )); - let c_settled = f1_wait_for(F1_BACKSTOP, || { - handle_c.is_finished() || state.repo_write_leases.refs_for(&repo_id) >= 2 - }) - .await; + let handle_b = tokio::spawn(push("r1")); assert!( - c_settled, - "push C neither returned nor reached the repo write lease within {F1_BACKSTOP:?}" + f1_wait_for(F1_BACKSTOP, || state.repo_write_leases.waiters_for(&repo1) + == 1) + .await, + "push B never parked on r1's contended lease within {F1_BACKSTOP:?}" ); + + // Push C: same source, DIFFERENT repo, uncontended. It must be served. + let c = tokio::time::timeout(F1_BACKSTOP, push("r2")) + .await + .expect("a push to an uncontended repo must not park"); + let resp = c.unwrap_or_else(|e| { + panic!( + "U1 scenario 2 RED: a push to an UNCONTENDED repo was denied because the \ + same source had a push parked on a DIFFERENT repo's lease. A parked push \ + must hold no cross-repo admission budget; got {e:?}" + ) + }); + assert_eq!(resp.status(), 200, "the uncontended repo's push lands 200"); assert!( - !handle_c.is_finished(), - "F1 discriminator RED: a push from a DIFFERENT source was turned away while \ - source X was at its cap. The write sub-cap must be keyed on the resolved \ - source IP, so exhausting it denies only the exhausting source" + later_ran.exists(), + "the uncontended repo's push must have run its receive-pack" ); - // Push B from source X: shed, while C stays parked. A third reference on the - // lease entry would mean B parked too. - let handle_b = tokio::spawn(git_receive_pack( - State(state.clone()), - Path(("z6f1disc".to_string(), "d1".to_string())), - Extension(crate::auth::AuthenticatedDid(did.to_string())), - crate::rate_limit::PeerAddr(Some(src_x)), - axum::http::HeaderMap::new(), - axum::body::Bytes::from_static(b"0000"), + handle_a.abort(); + let _ = tokio::time::timeout(std::time::Duration::from_secs(60), handle_b).await; + } + + /// #174 U1 scenario 3, THE CROSS-TENANT GUARD. `GITLAWB_TRUSTED_PROXY` is unset by + /// default (`TrustedProxy::None`), so behind an edge proxy, a NAT, or a CI pool every + /// pusher resolves to the SAME source key. A push parked on one repo's lease must not + /// shed a DIFFERENT pusher's push to a DIFFERENT repo. This is the shape that made + /// the rejected design a cross-tenant denial rather than a self-inflicted one: with + /// the per-source permit above the park, four parked pushes deny every push on the + /// node for up to 1260s. + /// + /// Same one source IP for all three pushes (the collapsed-key shape), distinct pusher + /// DIDs, per-source cap 2 so the holder plus a parked push would exhaust it. + #[cfg(unix)] + #[sqlx::test] + async fn u1_parked_push_does_not_shed_another_pusher_behind_the_same_ip(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let tmp = tempfile::TempDir::new().unwrap(); + let (git_bin, a_inpack, later_ran) = f1_hanging_first_git(tmp.path()); + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6u1nat", "n1", false).await; + // The default proxy trust: the resolved key is the peer IP, which is the edge's. + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(2, 100); + let repo1 = state + .db + .get_repo("z6u1nat", "n1") + .await + .unwrap() + .unwrap() + .id; + f1_add_repo(&state, "z6u1nat", "n2").await; + // Every pusher arrives from the one edge IP, so they share a source key. + let edge: SocketAddr = "203.0.113.85:5000".parse().unwrap(); + let push = |pusher: &'static str, repo: &'static str| { + let st = state.clone(); + async move { + git_receive_pack( + State(st), + Path(("z6u1nat".to_string(), repo.to_string())), + Extension(crate::auth::AuthenticatedDid(pusher.to_string())), + crate::rate_limit::PeerAddr(Some(edge)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await + } + }; + + let handle_a = tokio::spawn(push( + "did:key:z6MkU1NatPusherOneAAAAAAAAAAAAAAAAAAAAAA", + "n1", )); - let b_settled = f1_wait_for(F1_BACKSTOP, || { - handle_b.is_finished() || state.repo_write_leases.refs_for(&repo_id) >= 3 - }) - .await; assert!( - b_settled, - "push B neither returned nor reached the repo write lease within {F1_BACKSTOP:?}" - ); - assert_eq!( - state.repo_write_leases.refs_for(&repo_id), - 2, - "F1 RED: the capped source parked on the contended lease (A holding, C and B \ - waiting) instead of shedding on its per-source write cap" - ); - let b = handle_b.await.expect("push B task must not panic"); - assert!( - matches!(b, Err(AppError::Overloaded(_))), - "source X's second push must shed with Overloaded/503; got {b:?}" - ); - assert!( - !handle_c.is_finished(), - "source Y must still be parked on the lease, unaffected by X hitting its cap" + f1_wait_for(F1_BACKSTOP, || a_inpack.exists()).await, + "pusher one never reached receive-pack within {F1_BACKSTOP:?}" ); + let handle_b = tokio::spawn(push( + "did:key:z6MkU1NatPusherTwoAAAAAAAAAAAAAAAAAAAAAA", + "n1", + )); assert!( - !later_ran.exists(), - "no second receive-pack may run while A holds the lease" + f1_wait_for(F1_BACKSTOP, || state.repo_write_leases.waiters_for(&repo1) + == 1) + .await, + "pusher two never parked on n1's contended lease within {F1_BACKSTOP:?}" ); - // Free the lease: A disconnects. Aborting drops its handler future at the child - // await, the same shape as a client hang-up. Source Y's parked push is served. - handle_a.abort(); - let resp = tokio::time::timeout(std::time::Duration::from_secs(60), handle_c) - .await - .expect("push C must complete once A's group is reaped and the lease frees") - .expect("push C task must not panic") - .expect("push C must succeed"); - assert_eq!( - resp.status(), - 200, - "the unaffected source's parked push lands 200 once the lease frees" - ); + // A third, unrelated pusher behind the same edge IP, on a different repo. + let c = tokio::time::timeout( + F1_BACKSTOP, + push("did:key:z6MkU1NatPusherThreeAAAAAAAAAAAAAAAAAA", "n2"), + ) + .await + .expect("an unrelated pusher's push to an uncontended repo must not park"); + let resp = c.unwrap_or_else(|e| { + panic!( + "U1 scenario 3 RED: one repo's parked push shed an UNRELATED pusher's push \ + to a DIFFERENT repo, because every pusher behind the proxy shares one \ + resolved source key. Contention on one repo must never deny another; \ + got {e:?}" + ) + }); + assert_eq!(resp.status(), 200, "the unrelated pusher lands 200"); assert!( later_ran.exists(), - "push C ran its receive-pack once the lease freed" + "the unrelated pusher must have run its receive-pack" ); + + handle_a.abort(); + let _ = tokio::time::timeout(std::time::Duration::from_secs(60), handle_b).await; } /// #174 F1 (U1) key shape: the write sub-cap is keyed on the SOURCE, not the repo. diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 431ac983..eee8fd8b 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -526,7 +526,7 @@ mod tests { git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), pin_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), encrypt_inflight: crate::state::EncryptInflight::new(), - repo_write_leases: crate::state::RepoWriteLeases::new(), + repo_write_leases: crate::state::RepoWriteLeases::new(8), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index d70be5ea..d1b98807 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -319,6 +319,30 @@ pub struct Config { )] pub max_concurrent_git_pushes: usize, + /// Maximum number of pushes that may be PARKED on one repo's in-process write lease + /// at once. Same-repo pushes are serialized (block-and-wait), and a parked push has + /// already had its entire pack buffered by axum, so an unbounded queue on a contended + /// repo is unbounded buffered memory held for up to `git_service_timeout_secs * 2 + + /// 60` (1260s at defaults). Past this cap the newest push sheds a clean 503 + + /// Retry-After instead of joining the queue. + /// + /// The trade: raising it lets more same-repo pushes wait their turn (fewer 503s for a + /// hot repo, more memory pinned by waiters); lowering it sheds sooner. Only pushes to + /// the SAME repo count, and only ones parked right now, so the cap can never deny a + /// push to a different repo. The holder is deliberately not counted: a holder whose + /// cleanup never ran would otherwise pin a slot forever and wedge the repo, which is + /// the failure the `steal_after` reclaim exists to survive. + /// + /// Default: 8, a quarter of the default `max_concurrent_git_pushes` (32). Raising the + /// push pool does not raise this; set it explicitly. Must be between 1 and 1_048_576. + #[arg( + long, + env = "GITLAWB_REPO_LEASE_MAX_WAITERS", + default_value_t = 8, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub repo_lease_max_waiters: usize, + /// Max concurrent post-push pin loops (`ipfs_pin` and `pinata` /// `pin_new_objects`) across all repos. `EncryptInflight` bounds the outstanding /// pin-task COUNT to one per repo, but each pin loop holds a full per-push diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index d6c0df46..a3e26753 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -412,9 +412,10 @@ async fn main() -> Result<()> { encrypt_inflight: crate::state::EncryptInflight::new(), // Per-repo in-process write-lease serializer (#174 U2/F3): supplements the pg // advisory lock so a disconnected push's still-reaping git group can't be raced - // by a second same-node push. Natural cap (one entry per contended repo, freed - // when unreferenced), no sized knob. - repo_write_leases: crate::state::RepoWriteLeases::new(), + // by a second same-node push. The map is naturally capped (one entry per contended + // repo, freed when unreferenced); the sized knob is how many pushes may PARK on + // one repo, since each parked push holds a fully buffered pack. + repo_write_leases: crate::state::RepoWriteLeases::new(config.repo_lease_max_waiters), git_read_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( config.max_concurrent_reads_per_caller, ), diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index b7fb1492..fa621aab 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -488,30 +488,54 @@ impl Drop for EncryptInflightGuard { /// block-and-wait, NOT coalesce. It mirrors [`EncryptInflight`]'s keyed-map + guard + /// Drop-frees-key STRUCTURE; the semantics differ (block-and-wait, so there is no /// lossy-coalesce degradation to fall back on). -#[derive(Clone, Default)] +#[derive(Clone)] pub struct RepoWriteLeases { // std::sync::Mutex: held only for O(1) map ops (get-or-create + refcount) in a sync // context, never across an await — the semaphore wait happens OUTSIDE this lock. repos: Arc>>, + /// Most handlers allowed to be PARKED on one repo's lease at once + /// (`GITLAWB_REPO_LEASE_MAX_WAITERS`). Past it `acquire` sheds instead of queueing. + max_waiters: usize, } -/// A per-repo lease entry: the one-permit semaphore plus a refcount of the handlers -/// currently referencing it (holding or waiting). While `refs > 0` every acquirer -/// shares the SAME semaphore, so mutual exclusion holds; the entry is removed only when -/// `refs` hits 0 (no one references it), so a fresh entry can never split serialization. +/// A per-repo lease entry: the one-permit semaphore, a refcount of the handlers +/// currently referencing it (holding or waiting), and a count of the ones actually +/// PARKED. While `refs > 0` every acquirer shares the SAME semaphore, so mutual exclusion +/// holds; the entry is removed only when `refs` hits 0 (no one references it), so a fresh +/// entry can never split serialization. +/// +/// `waiters` and `refs` are deliberately different counts, and the shed cap is on +/// `waiters`. `refs` includes the HOLDER, and a holder whose Drop never runs (task abort, +/// runtime teardown without unwind, `mem::forget`: precisely the leak `steal_after` exists +/// to survive) keeps its ref forever. Capping `refs` would let that leaked ref +/// permanently occupy a slot and wedge the repo, reintroducing the permanent wedge the +/// steal backstop was written to prevent. A waiter, by contrast, always leaves: it either +/// gets the permit, steals at `steal_after`, is shed, or is cancelled, and every one of +/// those paths drops the RAII waiter guard. struct LeaseSlot { sem: Arc, refs: usize, + waiters: usize, } impl RepoWriteLeases { - pub fn new() -> Self { - Self::default() + /// `max_waiters` is the per-repo live-waiter cap (see [`acquire`](Self::acquire)). + pub fn new(max_waiters: usize) -> Self { + Self { + repos: Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())), + max_waiters: max_waiters.max(1), + } } /// Acquire the per-repo write lease, blocking until it is free (a second same-repo /// writer waits). `steal_after` bounds that wait: past it the acquirer STEALS - /// (proceeds permit-less) rather than block forever. + /// (proceeds permit-less) rather than block forever. Returns `None` when this repo + /// already has `max_waiters` handlers parked, so the caller sheds (503) instead of + /// adding to an unbounded queue: `git_receive_pack` reaches here with the whole pack + /// already buffered by axum, and the park runs to `steal_after` (1260s at defaults), + /// so unbounded parking is unbounded buffered bytes. The cap is per repo and counts + /// only live waiters, so shedding is confined to the contended repo; a push to any + /// other repo, from any source, is unaffected. /// /// Why a bounded steal: block-and-wait has no degradation of its own (unlike the /// coalescing [`EncryptInflight`], whose lost key merely delays a best-effort copy), @@ -528,7 +552,11 @@ impl RepoWriteLeases { /// it. Nothing anywhere takes the pg lock before this lease, so the two serializers /// can never deadlock; taking the lease first also means a blocked second writer /// pins no pooled pg connection while it waits. - pub async fn acquire(&self, repo_id: &str, steal_after: std::time::Duration) -> RepoWriteLease { + pub async fn acquire( + &self, + repo_id: &str, + steal_after: std::time::Duration, + ) -> Option { // Take the entry refcount BEFORE the await, so the entry cannot be GC'd out from // under a waiter (a fresh entry for a new acquirer would split serialization). let sem = { @@ -536,6 +564,7 @@ impl RepoWriteLeases { let slot = map.entry(repo_id.to_string()).or_insert_with(|| LeaseSlot { sem: Arc::new(tokio::sync::Semaphore::new(1)), refs: 0, + waiters: 0, }); slot.refs += 1; Arc::clone(&slot.sem) @@ -550,30 +579,64 @@ impl RepoWriteLeases { repos: Arc::clone(&self.repos), repo_id: repo_id.to_string(), }; - let permit = match tokio::time::timeout(steal_after, Arc::clone(&sem).acquire_owned()).await - { - Ok(Ok(p)) => Some(p), - // The semaphore is never closed; treat the (unreachable) closed case as a - // steal so acquire always makes forward progress. - Ok(Err(_closed)) => None, - Err(_elapsed) => { - tracing::warn!( - repo = %repo_id, - steal_after_secs = steal_after.as_secs(), - "repo write-lease wait exceeded the steal bound; presuming a leaked \ - lease and proceeding permit-less (in-process serializer reclaim)" - ); - None + + // Uncontended fast path: take the free permit without spending waiter budget, so + // the cap bounds only handlers that actually park. `try_acquire_owned` on tokio's + // semaphore does NOT barge a queued FIFO waiter (it fails while anyone is queued), + // so the fast path is not a fairness hole; probed at 2000 rounds with a queued + // waiter present, 0 barges. + let permit = match Arc::clone(&sem).try_acquire_owned() { + Ok(p) => Some(p), + Err(_) => { + // Contended: park, holding a waiter slot for exactly the cancellable wait. + // Claimed in ONE critical section (check and increment together), so a + // burst of concurrent acquirers cannot all read an under-cap count and + // then all park. Past the cap, drop the reservation (freeing the ref, and + // the entry with it if nobody else holds one) and shed. + let waiter = match WaiterSlot::claim(&self.repos, repo_id, self.max_waiters) { + Some(w) => w, + None => { + tracing::warn!( + repo = %repo_id, + max_waiters = self.max_waiters, + "repo write-lease waiter cap reached; shedding this acquirer" + ); + drop(reservation); + return None; + } + }; + let parked = + tokio::time::timeout(steal_after, Arc::clone(&sem).acquire_owned()).await; + // Released HERE, at the end of the wait, not at the end of the push: past + // this point the handler is a holder, counted by `refs`, and a waiter slot + // it kept would be budget no parked request could ever use. + drop(waiter); + match parked { + Ok(Ok(p)) => Some(p), + // The semaphore is never closed; treat the (unreachable) closed case + // as a steal so acquire always makes forward progress. + Ok(Err(_closed)) => None, + Err(_elapsed) => { + tracing::warn!( + repo = %repo_id, + steal_after_secs = steal_after.as_secs(), + "repo write-lease wait exceeded the steal bound; presuming a \ + leaked lease and proceeding permit-less (in-process \ + serializer reclaim)" + ); + None + } + } } }; // Transfer the ref from the reservation to the guard: forget the reservation (so // it does NOT decrement) and let the guard own the single decrement on its Drop. std::mem::forget(reservation); - RepoWriteLease(Arc::new(LeaseGuardInner { + Some(RepoWriteLease(Arc::new(LeaseGuardInner { repos: Arc::clone(&self.repos), repo_id: repo_id.to_string(), _permit: permit, - })) + }))) } /// Number of repos with a live lease entry. Test/metrics observability. @@ -604,6 +667,19 @@ impl RepoWriteLeases { .map(|slot| slot.refs) .unwrap_or(0) } + + /// How many handlers are PARKED on this repo's lease right now (the count the shed + /// cap is enforced against, holder excluded). Tests use it to observe "this push + /// parked" and "the waiter slot was released" as state. + #[cfg(test)] + pub fn waiters_for(&self, repo_id: &str) -> usize { + self.repos + .lock() + .expect("repo_write_leases mutex poisoned") + .get(repo_id) + .map(|slot| slot.waiters) + .unwrap_or(0) + } } /// Shared-ownership handle to a held repo write lease (#174 U2/F3). `Clone` hands a @@ -654,6 +730,51 @@ impl Drop for RefReservation { } } +/// Holds one LIVE-WAITER slot on a lease entry for exactly the duration of the +/// cancellable park inside [`RepoWriteLeases::acquire`]. Every way out of that park +/// (permit acquired, steal on timeout, shed, or the acquire future being dropped on a +/// client disconnect) drops this guard, so the count can only reflect handlers that are +/// parked right now. That is why the shed cap is on this count and not on `refs`. +struct WaiterSlot { + repos: Arc>>, + repo_id: String, +} + +impl WaiterSlot { + /// Claim a waiter slot, or `None` when the repo already has `max_waiters` live + /// waiters. The entry always exists here: the caller took its `refs` reference in an + /// earlier critical section and still holds it. Check and increment share one + /// critical section, so concurrent acquirers cannot overshoot the cap. + fn claim( + repos: &Arc>>, + repo_id: &str, + max_waiters: usize, + ) -> Option { + if let Ok(mut map) = repos.lock() { + if let Some(slot) = map.get_mut(repo_id) { + if slot.waiters >= max_waiters { + return None; + } + slot.waiters += 1; + } + } + Some(Self { + repos: Arc::clone(repos), + repo_id: repo_id.to_string(), + }) + } +} + +impl Drop for WaiterSlot { + fn drop(&mut self) { + if let Ok(mut map) = self.repos.lock() { + if let Some(slot) = map.get_mut(&self.repo_id) { + slot.waiters = slot.waiters.saturating_sub(1); + } + } + } +} + /// Decrement a lease entry's refcount and remove it once no handler references it, so /// the map cannot grow without bound (Drop-frees-key, like `EncryptInflight`). Safe /// under block-and-wait: while `refs > 0` every acquirer shares the SAME semaphore, and @@ -713,11 +834,11 @@ mod repo_write_lease_tests { /// steal reclaim so a leaked (never-run Drop) holder cannot wedge the repo forever. #[tokio::test] async fn serializes_same_repo_frees_key_and_steals_on_leak() { - let leases = RepoWriteLeases::new(); + let leases = RepoWriteLeases::new(8); let big = Duration::from_secs(3600); // Block-and-wait: a second same-repo acquire waits while the first is held. - let a = leases.acquire("repo1", big).await; + let a = leases.acquire("repo1", big).await.expect("uncontended"); let blocked = tokio::time::timeout(Duration::from_millis(200), leases.acquire("repo1", big)).await; assert!( @@ -728,7 +849,8 @@ mod repo_write_lease_tests { drop(a); let b = tokio::time::timeout(Duration::from_millis(500), leases.acquire("repo1", big)) .await - .expect("the second acquire must proceed once the first lease frees"); + .expect("the second acquire must proceed once the first lease frees") + .expect("under the waiter cap, so it must not shed"); drop(b); // Drop-frees-key: with no holders the entry is removed (bounded map growth). @@ -738,35 +860,111 @@ mod repo_write_lease_tests { ); // Distinct repos never serialize against each other. - let x = leases.acquire("repoX", big).await; + let x = leases.acquire("repoX", big).await.expect("uncontended"); let _y = tokio::time::timeout(Duration::from_millis(200), leases.acquire("repoY", big)) .await - .expect("distinct repos must not serialize"); + .expect("distinct repos must not serialize") + .expect("a distinct repo has its own waiter budget"); drop(x); drop(_y); + } - // Steal-on-timeout reclaim: a leaked holder (never-run Drop, simulated by - // mem::forget) must NOT wedge the repo — the bounded wait proceeds permit-less. - let leaked = leases.acquire("repoZ", big).await; + /// Scenario 4: the steal backstop survives the waiter cap. A holder whose Drop never + /// runs (task abort, runtime teardown, `mem::forget`) keeps its entry `refs` forever; + /// the next acquirer must still proceed permit-less at `steal_after` and must not be + /// shed on the way in. This is why the cap counts LIVE WAITERS and never `refs`: + /// capping `refs` (which includes the holder) lets one leaked lease pin a slot + /// permanently and wedge the repo, the exact permanent wedge the steal exists to + /// prevent. `max_waiters` is 1 here, so a `refs`-based cap has no room at all. + #[tokio::test] + async fn steal_on_leaked_lease_still_works_under_the_waiter_cap() { + let leases = RepoWriteLeases::new(1); + let big = Duration::from_secs(3600); + + let leaked = leases.acquire("repoZ", big).await.expect("uncontended"); std::mem::forget(leaked); + assert_eq!( + leases.refs_for("repoZ"), + 1, + "the leaked holder keeps its entry reference forever (that is the leak)" + ); + let stolen = tokio::time::timeout( - Duration::from_secs(2), + Duration::from_secs(5), leases.acquire("repoZ", Duration::from_millis(150)), ) .await - .expect("a leaked lease must be reclaimed by the bounded-wait steal, not hang forever"); + .expect("a leaked lease must be reclaimed by the bounded-wait steal, not hang forever") + .expect( + "the waiter cap must not shed the stealer: it counts live waiters, and a leaked \ + HOLDER is not a waiter. Capping refs instead wedges the repo permanently", + ); + assert_eq!( + leases.waiters_for("repoZ"), + 0, + "the stealer must return its waiter slot when its wait ends" + ); drop(stolen); } + /// Scenario 5: a shed leaves no residue. Past the cap `acquire` returns `None` + /// without keeping either count, so the entry still GCs once the real holder and the + /// real waiter finish. A shed that stranded a ref would leak the map entry forever; + /// one that stranded a waiter slot would shrink the repo's budget permanently. + #[tokio::test] + async fn shed_waiter_leaves_no_ref_or_waiter_residue() { + let leases = RepoWriteLeases::new(1); + let big = Duration::from_secs(3600); + + let holder = leases.acquire("repoS", big).await.expect("uncontended"); + let waiting = leases.clone(); + let parked = tokio::spawn(async move { waiting.acquire("repoS", big).await }); + assert!( + wait_for(Duration::from_secs(5), || leases.waiters_for("repoS") == 1).await, + "the second acquire must park and be counted as a live waiter" + ); + + // At the cap (1 live waiter): the next acquire sheds instead of queueing. + let shed = tokio::time::timeout(Duration::from_secs(5), leases.acquire("repoS", big)) + .await + .expect("a shed must return immediately, not park"); + assert!( + shed.is_none(), + "past max_waiters the acquire must shed (None), not join the queue" + ); + assert_eq!( + leases.refs_for("repoS"), + 2, + "the shed must leave no refcount residue: only the holder and the real waiter" + ); + assert_eq!( + leases.waiters_for("repoS"), + 1, + "the shed must leave no waiter-count residue: only the real waiter" + ); + + // The entry still GCs once the real handlers finish. + drop(holder); + let promoted = parked + .await + .expect("the parked acquire task must not panic") + .expect("the parked acquire must be served once the holder frees"); + drop(promoted); + assert!( + wait_for(Duration::from_secs(5), || leases.is_empty()).await, + "the lease entry must still GC after a shed (no stranded ref)" + ); + } + /// Cancellation safety: dropping an acquire future while it is BLOCKED waiting for /// the lease (a client disconnect on a second same-repo push) must not strand the /// entry refcount — after the holder frees and the waiter is cancelled, the key GCs. #[tokio::test] async fn cancelled_waiter_does_not_strand_the_refcount() { - let leases = RepoWriteLeases::new(); + let leases = RepoWriteLeases::new(8); let big = Duration::from_secs(3600); - let a = leases.acquire("repoC", big).await; + let a = leases.acquire("repoC", big).await.expect("uncontended"); // A waiter blocks, then is cancelled (its acquire future dropped) mid-wait. let cancelled = tokio::time::timeout(Duration::from_millis(150), leases.acquire("repoC", big)).await; @@ -785,4 +983,104 @@ mod repo_write_lease_tests { "a cancelled waiter must not strand the entry refcount (key must GC)" ); } + + /// Scenario 6: a cancelled waiter returns its WAITER slot too, not just its ref. A + /// client that disconnects while parked is the common case, so a slot stranded here + /// would shrink the repo's waiter budget on every disconnect until the repo shed + /// every push. Observed as state: the count drops back to 0, and a later acquire on + /// the still-held lease parks rather than shedding, with the cap at 1. + #[tokio::test] + async fn cancelled_waiter_releases_its_waiter_slot() { + let leases = RepoWriteLeases::new(1); + let big = Duration::from_secs(3600); + + let holder = leases.acquire("repoX2", big).await.expect("uncontended"); + let cancelled = + tokio::time::timeout(Duration::from_millis(200), leases.acquire("repoX2", big)).await; + assert!( + cancelled.is_err(), + "the waiter must be parked, then cancelled mid-wait" + ); + assert_eq!( + leases.waiters_for("repoX2"), + 0, + "a cancelled waiter must release its waiter slot" + ); + + // The freed budget is usable: the next acquire parks (times out) instead of + // shedding (returning None immediately). + let next = + tokio::time::timeout(Duration::from_millis(300), leases.acquire("repoX2", big)).await; + assert!( + next.is_err(), + "the next acquire must be able to park on the freed waiter slot; it was shed \ + instead, so the cancelled waiter's slot was stranded" + ); + drop(holder); + } + + /// Scenario 7: an uncontended acquire spends no waiter budget, and a promoted waiter + /// hands its slot back at the END OF ITS WAIT rather than holding it for the whole + /// push. With `max_waiters` at 1: the uncontended holder leaves the count at 0, the + /// one waiter is counted while parked, and once that waiter is promoted to holder the + /// count returns to 0 so the NEXT push can still park. Holding the slot for the push + /// instead would let one waiter permanently occupy the only slot, shedding every + /// same-repo push behind it. + #[tokio::test] + async fn uncontended_acquire_spends_no_waiter_budget() { + let leases = RepoWriteLeases::new(1); + let big = Duration::from_secs(3600); + + // Fast path: the lease is free, so this takes it without parking. + let holder = leases.acquire("repoF", big).await.expect("uncontended"); + assert_eq!( + leases.waiters_for("repoF"), + 0, + "an uncontended acquire must take the free permit without spending waiter budget" + ); + + // One waiter parks behind it, spending the single slot. + let waiting = leases.clone(); + let parked = tokio::spawn(async move { waiting.acquire("repoF", big).await }); + assert!( + wait_for(Duration::from_secs(5), || leases.waiters_for("repoF") == 1).await, + "the second acquire must park and be counted" + ); + + // Promote it: the slot must come back at the end of its WAIT. + drop(holder); + let promoted = parked + .await + .expect("the parked acquire task must not panic") + .expect("the parked acquire must be served once the holder frees"); + assert!( + wait_for(Duration::from_secs(5), || leases.waiters_for("repoF") == 0).await, + "a promoted waiter must release its waiter slot when its wait ends, not when \ + its push finishes" + ); + + // ... so a third acquire can still park behind the new holder. + let third = + tokio::time::timeout(Duration::from_millis(300), leases.acquire("repoF", big)).await; + assert!( + third.is_err(), + "the freed waiter slot must be usable: this acquire was shed instead of parking" + ); + drop(promoted); + } + + /// Poll `cond` until it holds, yielding so spawned tasks progress. Returns false if + /// `cap` elapses first; callers assert on the state the loop settled into. + async fn wait_for(cap: Duration, mut cond: impl FnMut() -> bool) -> bool { + let deadline = std::time::Instant::now() + cap; + loop { + if cond() { + return true; + } + if std::time::Instant::now() >= deadline { + return false; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + } } diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 22bdf72d..831202ee 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -89,7 +89,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), pin_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), encrypt_inflight: crate::state::EncryptInflight::new(), - repo_write_leases: crate::state::RepoWriteLeases::new(), + repo_write_leases: crate::state::RepoWriteLeases::new(8), git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( 8, From d753198755450598b4d78e4d83424dd24701127f Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:14:54 -0500 Subject: [PATCH 64/95] fix(node): stop a failed walk from taking a pin permit, and bound the close on an errored unlock Three findings from the review round. Moving the Pinata gate to the rules-only predicate was necessary, because a coalesced push has no walk of its own to consult. But it also meant a push whose OWN walk failed now acquired a global pin permit and ran a second full re-derivation that failed the same way. Measured on a repo whose rev-list always fails: twelve git children and three walk attempts, against four and one before. With eight pin slots that defer rather than shed, eight such pushes stall pins for every repo on the node. The gate now excludes a non-coalesced push whose own walk failed, leaving the coalesced path on the rules-only predicate it needs, and the pin helper returns before taking a permit for an empty object list. The connection close on the failing-unlock path had no deadline, and release is awaited inline while the write lease and both admission permits are still held. The condition that produces that branch is a connection whose last statement errored, and a blackholed socket to Postgres is one way to get there, so the await could pin three admission resources for a TCP timeout. It now runs under a five second bound; on expiry the future is dropped, which closes the socket and ends the session anyway, which is what frees the lock. What is tested is the deadline. That sqlx's close is the thing that stalls in production is reasoned from its source, not reproduced, and the test says so. The two tests for that path waited a fixed four hundred milliseconds before asserting. Replacing the sleep with polling was not enough on its own: the test pool sets a one second idle timeout, so a connection returned to the pool is reaped shortly after and the session ends without any help from the code under test. Both tests passed under mutation once the sleep was long enough to reach that reaper, which means a straightforward flake fix would have quietly gutted them. They now run against a pool with no idle reaper, so release is the only thing that can end the session, and both mutations go red again. Adds tokio's test-util as a dev-dependency for paused-time assertions. Production dependencies are unchanged and the build stays --locked clean. --- crates/gitlawb-node/Cargo.toml | 3 + crates/gitlawb-node/src/api/repos.rs | 177 +++++++++++++++++++- crates/gitlawb-node/src/git/repo_store.rs | 193 +++++++++++++++++++--- 3 files changed, 342 insertions(+), 31 deletions(-) diff --git a/crates/gitlawb-node/Cargo.toml b/crates/gitlawb-node/Cargo.toml index 72748fae..82854803 100644 --- a/crates/gitlawb-node/Cargo.toml +++ b/crates/gitlawb-node/Cargo.toml @@ -77,3 +77,6 @@ libp2p-dns = { version = "0.44.0", features = ["tokio"] } [dev-dependencies] mockito = "1" tempfile = "3" +# `test-util` for `#[tokio::test(start_paused = true)]`: lets the deadline tests assert +# on which timer fired instead of on wall clock. +tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 88138d0e..b89ebc36 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1101,6 +1101,14 @@ async fn pin_new_objects_gated( object_list: Vec, db: &Arc, ) -> Vec<(String, String)> { + // Nothing to pin: answer before taking a permit (#174 F2b). The permit bounds how + // many MB-scale object lists are held at once and an empty list holds none, so + // parking here would spend a global pin slot on no work. The pool DEFERS rather + // than sheds, so those calls stall pins for every other repo. Empty is the normal + // shape for a push whose walk failed or that may replicate nothing. + if object_list.is_empty() { + return Vec::new(); + } let _permit = pin_sem .clone() .acquire_owned() @@ -1960,6 +1968,13 @@ async fn post_receive_replication_tail( .1 }; + // #174 F2b: did THIS push run its own walk and have it fail? An admitted push that + // could not be vetted must not go on to take a pin permit and re-run the same + // failing walk in the Pinata worker below. Only the COALESCED path needs the + // rules-only predicate there: it has no walk of its own, so the worker's + // re-derivation is its only fail-closed source. + let own_walk_failed = announce_at_root && !coalesced && withheld.is_none(); + // Resolve the per-push pin candidate set once, off the async worker, then // filter to what may actually replicate. Delta path: the reachable-only // `withheld` set suffices (delta objects are reachable). Full-scan path: the @@ -2106,7 +2121,13 @@ async fn post_receive_replication_tail( // `replication_withheld_set` inside the pin permit: it returns an empty // list AND announce=false when the walk fails or the repo may not // replicate, so neither blobs nor announcements escape an unvetted push. - let do_pinata_replication = announce_at_root; + // + // #174 F2b: except when THIS push already ran that walk and it failed. The + // re-derivation would fail the same way, so it buys nothing, and it would buy + // it at the price of a global pin permit plus a second round of git children. + // The pin pool DEFERS rather than sheds, so enough such pushes stall pins + // node-wide. A coalesced push never walked, so it is unaffected. + let do_pinata_replication = announce_at_root && !own_walk_failed; // #174 F2 / KTD-3: capture only the small inputs the re-derivation needs; the // MB-scale object list is NOT moved in. `pinata_object_list_for_refs` recomputes // it from these once a pin slot frees. rules/owner/is_public drive the fresh @@ -5445,10 +5466,12 @@ mod tests { let held = pin_sem.clone().acquire_owned().await.unwrap(); // Empty ipfs_api makes the pin itself a no-op, but the loop must still DEFER on - // the exhausted pin pool rather than run. + // the exhausted pin pool rather than run. The object list is non-empty because + // an empty one takes no permit at all by design (#174 F2b). + let objects = vec!["0123456789abcdef0123456789abcdef01234567".to_string()]; let blocked = tokio::time::timeout( std::time::Duration::from_millis(500), - pin_new_objects_gated(&pin_sem, "", tmp.path(), vec![], &db), + pin_new_objects_gated(&pin_sem, "", tmp.path(), objects.clone(), &db), ) .await; assert!( @@ -5460,13 +5483,47 @@ mod tests { drop(held); let out = tokio::time::timeout( std::time::Duration::from_secs(5), - pin_new_objects_gated(&pin_sem, "", tmp.path(), vec![], &db), + pin_new_objects_gated(&pin_sem, "", tmp.path(), objects, &db), ) .await .expect("the pin loop completes once admission frees"); assert!(out.is_empty(), "an empty ipfs_api pins nothing"); } + /// #174 F2b: the pin permit bounds how many MB-scale object lists are held at once, + /// so a call with NOTHING to pin must not take one. It otherwise spends a global pin + /// slot on no work, and the pool DEFERS rather than sheds, so those calls stall pins + /// for every other repo. The empty case is the normal shape for a push whose walk + /// failed or that may replicate nothing. + /// + /// Load-bearing: without the guard this call parks on the exhausted pool exactly like + /// the non-empty one above, and the completion assertion fails. + #[sqlx::test] + async fn pin_new_objects_gated_takes_no_permit_for_an_empty_object_list(pool: sqlx::PgPool) { + use std::sync::Arc; + use tokio::sync::Semaphore; + + let state = crate::test_support::test_state(pool).await; + let db = state.db.clone(); + let tmp = tempfile::TempDir::new().unwrap(); + let pin_sem = Arc::new(Semaphore::new(1)); + // Hold the only pin permit for the whole call. + let _held = pin_sem.clone().acquire_owned().await.unwrap(); + + let out = tokio::time::timeout( + std::time::Duration::from_millis(500), + pin_new_objects_gated(&pin_sem, "", tmp.path(), vec![], &db), + ) + .await + .expect("an empty object list must not wait on pin admission (#174 F2b)"); + assert!(out.is_empty(), "and it pins nothing"); + assert_eq!( + pin_sem.available_permits(), + 0, + "the test still holds the only permit, so the call never took one" + ); + } + /// Shared fixture for the F4 handler-layer tests: a state whose repo_store and /// git_bin point at the given tempdir/fake-git, plus a seeded on-disk repo, /// optionally with a path-scoped rule (so the post-receive walks actually run). @@ -8312,4 +8369,116 @@ mod tests { .expect("the admitted push fires its announce"); assert_eq!(broadcast.new_sha, c2); } + + // ---- #174 F2b: a failed OWN walk does not buy a pin permit and a second walk ---- + + /// Withheld-walk commit-enumeration attempts so far. Each `replication_withheld_set` + /// runs exactly one `rev-list --all`, so this counts the walks that were attempted + /// (the `ls-tree` counter above cannot: a walk whose enumeration fails never gets + /// to `ls-tree`). + fn f2b_walk_attempts(log: &std::path::Path) -> usize { + f2a_log(log) + .lines() + .filter(|l| l.starts_with("rev-list --all")) + .count() + } + + /// A NON-coalesced push whose own withheld walk failed must not take a global pin + /// permit and must not re-run the same failing walk in the Pinata worker. + /// + /// The F2a change moved the Pinata gate from `withheld.is_some()` to the rules-only + /// `announce_at_root`, which a coalesced push genuinely needs (it has no walk of its + /// own). But it also let an ADMITTED push whose walk failed acquire `pin_semaphore` + /// and re-derive `replication_withheld_set`, which fails the same way. With + /// `max_concurrent_pin_tasks` defaulting to 8 and the pin pool DEFERRING rather than + /// shedding, eight such pushes stall pins node-wide. + /// + /// Asserted on observable work, twice over, with the pin pool's only permit held for + /// the whole first phase: + /// * the walk attempts while the permit is held. Two: the tail's own (which fails) + /// and the recovery task's recipients walk, which must NOT park on the pin pool + /// for its empty object list. The Pinata worker's is the third and must not exist. + /// * the walk attempts after the permit is released. Still two: nothing was left + /// waiting on the pin pool, which is the permit assertion. + /// + /// Load-bearing both ways. With the gate back on plain `announce_at_root` the Pinata + /// worker parks on the held permit and then walks once it is freed (phase 2 sees 3). + /// Without the empty-list guard on `pin_new_objects_gated` the recovery task parks + /// too, so phase 1 sees 1. + #[cfg(unix)] + #[sqlx::test] + async fn f2b_failed_own_walk_takes_no_pin_permit_and_runs_no_second_walk(pool: sqlx::PgPool) { + let repo = tempfile::TempDir::new().unwrap(); + let bin = tempfile::TempDir::new().unwrap(); + u5_init_repo(repo.path()); + let c1 = u5_commit_file(repo.path(), "a.txt", "one\n"); + let c2 = u5_commit_file(repo.path(), "secret/s.txt", "two\n"); + let log = bin.path().join("git.log"); + // Every withheld walk fails, so this push can never be vetted. + let git_bin = write_fake_git( + bin.path(), + &format!( + "#!/bin/sh\n\ + printf '%s\\n' \"$*\" >> \"{log}\"\n\ + case \"$*\" in 'rev-list --all'*) exit 1 ;; esac\n\ + exec git \"$@\"\n", + log = log.display(), + ), + ); + let (mut state, rec) = f2a_state(pool, &git_bin, "z6f2bfail", "w1", true).await; + let (_server, _cid) = f2a_pinata(&mut state).await; + // One pin permit, held: anything that reaches a pin-admission acquire parks + // instead of running, which is what makes "took no permit" observable. + state.pin_semaphore = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + let held = state.pin_semaphore.clone().acquire_owned().await.unwrap(); + + // Nothing pre-takes the coalescing key, so this push is ADMITTED and runs its + // own walk. + post_receive_replication_tail( + state.clone(), + rec.clone(), + vec![RefUpdate { + old_sha: c1.clone(), + new_sha: c2.clone(), + ref_name: "refs/heads/main".to_string(), + }], + repo.path().to_path_buf(), + F2A_PUSHER.to_string(), + ) + .await; + + f2a_wait_for( + || f2b_walk_attempts(&log) >= 2, + "the tail's own walk and the recovery task's recipients walk", + ) + .await; + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + assert_eq!( + f2b_walk_attempts(&log), + 2, + "a failed own walk must not buy a third walk in the Pinata worker; log:\n{}", + f2a_log(&log) + ); + + // Release pin admission. A task that had parked on it now wakes and walks; + // nothing should have been parked. + drop(held); + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + assert_eq!( + f2b_walk_attempts(&log), + 2, + "nothing may be left waiting on the pin permit for a push that pins \ + nothing; log:\n{}", + f2a_log(&log) + ); + assert!( + state + .db + .list_branch_cids(&f2a_slug(&rec)) + .await + .unwrap() + .is_empty(), + "and the unvetted push still maps no CID" + ); + } } diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 2bc15e0a..69d1bb33 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -403,6 +403,42 @@ pub struct RepoWriteGuard { test_pre_unlock_gate: Option>, } +/// Deadline for tearing down the connection that saw a failing `pg_advisory_unlock`. +/// Long enough that a healthy socket always finishes well inside it, short enough that +/// a blackholed one does not pin admission resources for a TCP timeout. +const UNLOCK_ERROR_CLOSE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); + +/// Await `close` under a deadline (#174 F3c). +/// +/// `release` awaits this INLINE while the global write permit, the per-source permit +/// and the write lease are all still held, and sqlx puts no deadline on `close()`: +/// it writes Terminate and then tears the socket down. The branch that reaches here is +/// by definition a connection whose last statement errored, and a blackholed TCP path +/// to Postgres (a cloud failover that drops packets without an RST) is a plausible +/// cause, so an unbounded await here parks every later push to the repo behind three +/// pinned admission resources until the steal bound. +/// +/// On elapsed the future is simply dropped, which drops the `PoolConnection` it owns. +/// Dropping it closes the socket, and closing the socket is what actually ends the +/// session and makes Postgres release the lock, so the deadline costs nothing the +/// graceful path was buying. +async fn close_conn_bounded( + repo_name: &str, + close: impl std::future::Future>, +) { + match tokio::time::timeout(UNLOCK_ERROR_CLOSE_TIMEOUT, close).await { + Ok(Ok(())) => {} + Ok(Err(e)) => { + warn!(repo = %repo_name, err = %e, + "closing the write-lock connection failed, the session teardown still frees the lock server-side"); + } + Err(_) => { + warn!(repo = %repo_name, timeout_secs = UNLOCK_ERROR_CLOSE_TIMEOUT.as_secs(), + "closing the write-lock connection timed out, dropping it instead; the socket goes down either way, which is what frees the lock server-side"); + } + } +} + impl RepoWriteGuard { /// Path to the bare repo on local disk. pub fn path(&self) -> &Path { @@ -476,11 +512,10 @@ impl RepoWriteGuard { // whenever the returned `PgConnection` is dropped and its // background close completes. If this future is cancelled during // `close()`, the connection is dropped mid-close, which still - // tears the session down. - if let Err(e) = conn.close().await { - warn!(repo = %self.repo_name, err = %e, - "closing the write-lock connection failed, the session teardown still frees the lock server-side"); - } + // tears the session down. That last point is also why the await is + // safe to bound: see `close_conn_bounded`, which gives it the + // deadline sqlx does not. + close_conn_bounded(&self.repo_name, conn.close()).await; } } } @@ -893,6 +928,108 @@ mod tests { ); } + /// How long the polls below may wait for an asynchronous server-side effect. + /// Postgres frees a session advisory lock when the backend exits, which happens + /// asynchronously to our socket close, so these are polled to a generous deadline + /// rather than slept for a fixed interval: a constant sleep is a flake under load, + /// and one that is long enough to be safe is dead time on every run. + const RELEASE_POLL_DEADLINE: std::time::Duration = std::time::Duration::from_secs(20); + + /// Poll `cond` until it holds, failing at the deadline so a regression fails the + /// test rather than hanging the suite. + async fn wait_until(mut cond: impl FnMut() -> bool, what: &str) { + let deadline = std::time::Instant::now() + RELEASE_POLL_DEADLINE; + while !cond() { + assert!( + std::time::Instant::now() < deadline, + "timed out waiting for {what}" + ); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + } + + /// Poll the advisory lock from `checker` until it is free. `pg_try_advisory_lock` + /// ACQUIRES on success, so a true result both answers the question and leaves the + /// checker session holding the lock; the caller unlocks it. + async fn wait_until_lock_free(checker: &mut sqlx::PgConnection, key: i64, what: &str) { + let deadline = std::time::Instant::now() + RELEASE_POLL_DEADLINE; + loop { + let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *checker) + .await + .unwrap(); + if free { + return; + } + assert!( + std::time::Instant::now() < deadline, + "timed out waiting for {what}" + ); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + } + + /// A second pool over the same test database with the idle reaper DISABLED. + /// + /// `#[sqlx::test]`'s own pool sets `idle_timeout(1s)`, so a connection returned to + /// it is closed by the reaper about a second later, which ends the session and + /// frees the advisory lock all on its own. The old fixed 400ms sleep landed inside + /// that window by luck; polling to a deadline long enough to be flake-proof would + /// land outside it and go green whether or not `release` disposed of the + /// connection, so the tests below would stop testing anything (measured: with the + /// reaper in play, the disposal shows up at ~2s even with the fix reverted). With + /// no reaper, `release` is the only thing that can end that session, so the poll + /// measures exactly the property these two tests exist for. + async fn pool_without_idle_reaper(pool: &sqlx::PgPool) -> sqlx::PgPool { + sqlx::postgres::PgPoolOptions::new() + .max_connections(5) + .idle_timeout(None) + .max_lifetime(None) + .connect_with(pool.connect_options().as_ref().clone()) + .await + .expect("a second pool over the test database") + } + + /// F3c (P2): the connection teardown on the failing-unlock path must be BOUNDED. + /// `release` awaits it inline while the global write permit, the per-source permit + /// and the write lease are all still held, and sqlx's `close()` carries no deadline + /// of its own, so a blackholed socket would park every later push to that repo + /// behind three pinned admission resources. + /// + /// What this covers: the deadline itself. A close that never resolves still lets + /// `close_conn_bounded` return, which is the property `release` depends on. What it + /// does NOT cover, and is reasoned rather than run: that sqlx's own `close()` is + /// what stalls in production. Making a real `PgConnection::close` hang needs a + /// blackholed TCP path to Postgres, and the flip has to land after the unlock + /// statement round-trips but before the Terminate write, which is not a seam this + /// module exposes. A never-resolving future is the faithful stand-in for that + /// close, and the F3b tests above already cover that `release` really routes its + /// close through here. + /// + /// Time is paused, so nothing here depends on wall clock: the runtime auto-advances + /// to the next timer, and the assertion is on which timer fired, not on elapsed + /// time. The outer bound is what turns a removed deadline into a failure rather + /// than a hung suite. + /// + /// Load-bearing: drop the `tokio::time::timeout` in `close_conn_bounded` and the + /// inner future never resolves, so the outer bound fires and this fails. + #[tokio::test(start_paused = true)] + async fn unlock_error_connection_close_is_bounded() { + let hanging = std::future::pending::>(); + let outcome = tokio::time::timeout( + UNLOCK_ERROR_CLOSE_TIMEOUT * 4, + close_conn_bounded("boundedclosetest", hanging), + ) + .await; + assert!( + outcome.is_ok(), + "a connection close that never completes must not hold the write lease and \ + both admission permits open-endedly: close_conn_bounded must give up and \ + drop the connection" + ); + } + /// F3b (P1): when `pg_advisory_unlock` ERRORS while the session is still alive /// (statement timeout, admin cancel, aborted transaction), the lock must not /// survive `release`. The old code discarded the error with `let _ =` and set @@ -910,7 +1047,8 @@ mod tests { #[sqlx::test] async fn write_guard_release_with_failing_unlock_frees_the_lock(pool: sqlx::PgPool) { let dir = tempfile::TempDir::new().unwrap(); - let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone()); + let store_pool = pool_without_idle_reaper(&pool).await; + let store = RepoStore::for_testing(dir.path().to_path_buf(), store_pool.clone()); let owner = "did:key:z6MkUnlockErrorProofFFFFFFFFFFFFFFFFFFFFFF"; let name = "unlockerrtest"; let slug = owner.replace([':', '/'], "_"); @@ -935,19 +1073,18 @@ mod tests { ); guard.release(false).await; - // Give a background close a moment to reach the server. - tokio::time::sleep(std::time::Duration::from_millis(400)).await; - let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") - .bind(key) - .fetch_one(&mut *checker) - .await - .unwrap(); - assert!( - free, - "an errored pg_advisory_unlock must not leave the lock held: release \ - must dispose of the connection so the session ends" - ); + // Postgres drops the lock when the disposed session's backend exits, which is + // asynchronous to our socket close: poll for it rather than sleeping a + // constant. The deadline is what keeps this load-bearing: a release that + // leaves the lock held never satisfies the probe and fails here. + wait_until_lock_free( + &mut checker, + key, + "an errored pg_advisory_unlock must not leave the lock held: release must \ + dispose of the connection so the session ends", + ) + .await; let _ = sqlx::query("SELECT pg_advisory_unlock($1)") .bind(key) .execute(&mut *checker) @@ -963,24 +1100,26 @@ mod tests { pool: sqlx::PgPool, ) { let dir = tempfile::TempDir::new().unwrap(); - let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone()); + let store_pool = pool_without_idle_reaper(&pool).await; + let store = RepoStore::for_testing(dir.path().to_path_buf(), store_pool.clone()); let owner = "did:key:z6MkUnlockErrorPoolProofGGGGGGGGGGGGGGGGGG"; let name = "unlockerrpooltest"; let mut guard = store.acquire_write(owner, name).await.expect("acquire"); poison_guard_connection(&mut guard).await; - let size_before = pool.size(); + let size_before = store_pool.size(); assert!(size_before > 0, "the pool owns the guard's connection"); guard.release(false).await; - tokio::time::sleep(std::time::Duration::from_millis(400)).await; - assert_eq!( - pool.size(), - size_before - 1, - "the connection that saw the unlock error must be closed, not returned \ - to the pool still holding the session lock" - ); + // The pool's size drops when the closed connection's slot is given up, which + // is not synchronous with `release` returning: poll rather than sleep. + wait_until( + || store_pool.size() == size_before - 1, + "the connection that saw the unlock error to be closed rather than returned \ + to the pool still holding the session lock", + ) + .await; } /// F3b regression guard on the success path: a normal unlock keeps the From b9566b77503a28a70186aab91461e7d62c73aafd Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 28 Jul 2026 16:01:30 -0500 Subject: [PATCH 65/95] fix(node): sweep the /ipfs rate limiter and document the lease waiter cap The 300s cleanup task swept the DID, create-IP, push, sync-trigger and peer-write limiters but not ipfs_rate_limiter, so its expired keys sat until the map hit max_keys and the inline capacity sweep reclaimed them. Drive the sweep off AppState in a named sweep_rate_limiters so a limiter added to the state and forgotten here fails a test instead of leaking keys, and cover all six with a regression that goes red when any one is dropped from the sweep. GITLAWB_REPO_LEASE_MAX_WAITERS was also missing from .env.example and the README settings table, leaving operators without a way to discover or tune the cap behind the "repo is busy with another push" 503. --- .env.example | 9 ++++ README.md | 1 + crates/gitlawb-node/src/main.rs | 71 +++++++++++++++++++++++---- crates/gitlawb-node/src/rate_limit.rs | 7 +++ 4 files changed, 78 insertions(+), 10 deletions(-) diff --git a/.env.example b/.env.example index 9913b41b..5ed087d9 100644 --- a/.env.example +++ b/.env.example @@ -144,6 +144,15 @@ GITLAWB_MAX_CONCURRENT_GIT_OPS=128 # Retry-After. Default 32. GITLAWB_MAX_CONCURRENT_GIT_PUSHES=32 +# Max pushes PARKED at once waiting for the per-repo write lease of the SAME repo. +# A parked push holds its already-buffered pack body in memory, so this bounds that +# memory for a hot repo. Past the cap the newest push sheds a clean 503 + +# Retry-After ("repo is busy with another push") instead of joining the queue; a +# push to a DIFFERENT repo is never affected. The lease holder is not counted. +# Raising GITLAWB_MAX_CONCURRENT_GIT_PUSHES does not raise this; set it explicitly. +# Default 8. +GITLAWB_REPO_LEASE_MAX_WAITERS=8 + # Max concurrent post-push pin loops (IPFS + Pinata pin_new_objects) across all # repos. Each loop holds a full per-push object-id list while pinning, so this # bounds that MB-scale memory even though the per-repo encrypt-task set already diff --git a/README.md b/README.md index cc32bcf7..6789cd80 100644 --- a/README.md +++ b/README.md @@ -345,6 +345,7 @@ Important node settings: | `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. | | `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. | | `GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS` | Max seconds the storage-acquisition phase (Tigris HEAD/GET, push advisory-lock) of a served git op may run before the request is shed with a 503, separate from the git-run timeout. The concurrency permit is released on expiry so a stalled backend cannot pin the pool. Default 30. | +| `GITLAWB_REPO_LEASE_MAX_WAITERS` | Max pushes parked at once waiting for the same repo's write lease. Each waiter pins its buffered pack body, so this bounds that memory for a hot repo; past the cap the newest push sheds a 503 + Retry-After instead of queueing. Pushes to other repos are unaffected, and the lease holder is not counted. Default 8. | | `GITLAWB_MAX_CONCURRENT_IPFS_WALKS` | Max concurrent `GET /ipfs/{cid}` visibility walks across all callers (own pool, disjoint from the served-git pools); over-cap sheds 503. Default 32. | | `GITLAWB_IPFS_WALK_PER_SOURCE` | Max concurrent `/ipfs` walks a single source IP may hold. Default 4. | | `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max expensive path-scope visibility walks per `/ipfs/{cid}` request; over-cap repos are skipped and the scan continues, shedding a retryable 503 (not a false 404) if the object is then found nowhere. Default 64. | diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index a3e26753..1fd26979 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -481,22 +481,14 @@ async fn main() -> Result<()> { // Periodic cleanup of expired rate limit entries + consumed-proof ledger { - let rl = state.rate_limiter.clone(); - let create_ip_rl = state.create_ip_rate_limiter.clone(); - let push_rl = state.push_rate_limiter.clone(); - let sync_trigger_rl = state.sync_trigger_rate_limiter.clone(); - let peer_write_rl = state.peer_write_rate_limiter.clone(); + let sweep_state = state.clone(); let db = state.db.clone(); let mut shutdown_rx = state.subscribe_shutdown(); tokio::spawn(async move { loop { tokio::select! { _ = tokio::time::sleep(std::time::Duration::from_secs(300)) => { - rl.cleanup().await; - create_ip_rl.cleanup().await; - push_rl.cleanup().await; - sync_trigger_rl.cleanup().await; - peer_write_rl.cleanup().await; + sweep_rate_limiters(&sweep_state).await; let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|d| d.as_secs() as i64) @@ -1032,6 +1024,65 @@ async fn gossip_task( } } +#[cfg(test)] +mod rate_limiter_sweep_tests { + use crate::rate_limit::RateLimiter; + use std::time::Duration; + + // Every per-key limiter the router mounts must be swept by the periodic + // task, the `/ipfs` one included: a limiter left out keeps expired keys + // until its map fills and the inline capacity sweep fires. Fails on the + // pre-fix sweeper, which skipped `ipfs_rate_limiter`. + #[tokio::test] + async fn sweep_evicts_expired_keys_from_every_limiter() { + let window = Duration::from_millis(30); + let mut state = crate::test_support::test_state_lazy(); + state.rate_limiter = RateLimiter::new(10, window); + state.create_ip_rate_limiter = RateLimiter::new(10, window); + state.push_rate_limiter = RateLimiter::new(10, window); + state.sync_trigger_rate_limiter = RateLimiter::new(10, window); + state.peer_write_rate_limiter = RateLimiter::new(10, window); + state.ipfs_rate_limiter = RateLimiter::new(10, window); + + let limiters = |s: &crate::state::AppState| { + [ + s.rate_limiter.clone(), + s.create_ip_rate_limiter.clone(), + s.push_rate_limiter.clone(), + s.sync_trigger_rate_limiter.clone(), + s.peer_write_rate_limiter.clone(), + s.ipfs_rate_limiter.clone(), + ] + }; + for l in limiters(&state) { + assert!(l.check("1.2.3.4").await); + assert_eq!(l.tracked_keys().await, 1); + } + + tokio::time::sleep(window * 3).await; + super::sweep_rate_limiters(&state).await; + + for (i, l) in limiters(&state).into_iter().enumerate() { + assert_eq!(l.tracked_keys().await, 0, "limiter {i} was not swept"); + } + } +} + +/// Evict expired entries from every per-key rate limiter on the state. +/// +/// Named and driven off `AppState` so the periodic sweeper stays in step with +/// the limiters the router actually mounts: adding a limiter field and +/// forgetting it here leaves its keys pinned until the map hits `max_keys` and +/// the inline capacity sweep runs (the `/ipfs` limiter was missed this way). +async fn sweep_rate_limiters(state: &AppState) { + state.rate_limiter.cleanup().await; + state.create_ip_rate_limiter.cleanup().await; + state.push_rate_limiter.cleanup().await; + state.sync_trigger_rate_limiter.cleanup().await; + state.peer_write_rate_limiter.cleanup().await; + state.ipfs_rate_limiter.cleanup().await; +} + /// Build the shared node HTTP client used for every outbound fan-out (sync /// trigger, profile/repo fetches, gossip announce + peer pings). /// diff --git a/crates/gitlawb-node/src/rate_limit.rs b/crates/gitlawb-node/src/rate_limit.rs index 22281691..a0dc6137 100644 --- a/crates/gitlawb-node/src/rate_limit.rs +++ b/crates/gitlawb-node/src/rate_limit.rs @@ -121,6 +121,13 @@ impl RateLimiter { true } + /// Number of keys currently tracked. Tests use it to observe what a sweep + /// reclaimed; there is no production reader. + #[cfg(test)] + pub async fn tracked_keys(&self) -> usize { + self.state.lock().await.len() + } + pub async fn cleanup(&self) { let now = Instant::now(); let mut state = self.state.lock().await; From 1564302dd96ed57ede0ab7adc0349bed426071f4 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 02:16:04 -0500 Subject: [PATCH 66/95] fix(node): write coalesced-drain pin metadata under the re-fetched repo id resolve_drain_object_list already re-fetches the repo row by owner/name and deliberately reads visibility rules under record.id, because a delete+recreate under the same slug gives the row a new id while the bare repo on disk is reused. The encrypt write a few lines later still took ctx.repo_id, frozen when the task was spawned, so recovery-pin metadata for a coalesced drain landed against the deleted row and was unreachable from the live one. Return the re-fetched id from the resolver and pass it explicitly to pin_and_encrypt_objects. The parameter is explicit rather than read from ctx so the two callers stay honest about which id they mean: the snapshot iteration passes its own spawn-time id, the drain passes the fresh one. The Arweave anchor is unaffected - EncryptedManifest carries no repo id and builds its slug from owner and name, both stable across the rotation. --- crates/gitlawb-node/src/api/repos.rs | 118 +++++++++++++++++++++++++-- 1 file changed, 109 insertions(+), 9 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index b89ebc36..d174a26b 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -869,17 +869,28 @@ async fn run_encrypt_pin_task( snapshot_rules: Option>, snapshot_is_public: bool, ) { - pin_and_encrypt_objects(&ctx, snapshot_objects, snapshot_rules, snapshot_is_public).await; + // The snapshot is this push's own work, resolved before spawn, so it belongs + // to the id captured at spawn. + pin_and_encrypt_objects( + &ctx, + &ctx.repo_id, + snapshot_objects, + snapshot_rules, + snapshot_is_public, + ) + .await; let mut guard = guard; loop { match guard.finish_or_take_pending() { crate::state::FinishOutcome::Finished(_) => break, crate::state::FinishOutcome::Pending(g, pending) => { guard = g; - if let Some((object_list, rules, is_public)) = + // `drain_repo_id`, not ctx.repo_id: see resolve_drain_object_list. + if let Some((drain_repo_id, object_list, rules, is_public)) = resolve_drain_object_list(&ctx, pending).await { - pin_and_encrypt_objects(&ctx, object_list, rules, is_public).await; + pin_and_encrypt_objects(&ctx, &drain_repo_id, object_list, rules, is_public) + .await; } } } @@ -893,10 +904,19 @@ async fn run_encrypt_pin_task( /// re-read) pins nothing at all (`None`). Returns the filtered object list plus /// the fresh rules/is_public snapshot for the encrypt stage — the same /// resolution → withheld-filter pipeline the receive-pack tail runs. +/// The returned `String` is the repo id the drain's encrypt/anchor writes must +/// use: the id from the FRESH re-fetch, never `ctx.repo_id` frozen at task spawn. +/// A delete+recreate under the same slug gives the row a new id, and metadata +/// written against the dead id is invisible to readers on the live row (#174 U3). async fn resolve_drain_object_list( ctx: &EncryptTaskCtx, pending: crate::state::PendingWork, -) -> Option<(Vec, Option>, bool)> { +) -> Option<( + String, + Vec, + Option>, + bool, +)> { let record = match ctx.db.get_repo(&ctx.owner_did, &ctx.repo_name).await { Ok(Some(r)) => r, Ok(None) => { @@ -985,7 +1005,10 @@ async fn resolve_drain_object_list( } else { crate::git::visibility_pack::replicable_objects(pin_set.candidates, &withheld_set) }; - Some((object_list, rules_opt, record.is_public)) + // record.id, never ctx.repo_id: the record above is re-fetched fresh by + // owner/name, and a delete+re-create between spawn and drain gives the row a + // NEW id. This is the same rule the rules read a few lines up already follows. + Some((record.id, object_list, rules_opt, record.is_public)) } /// Re-derive the Pinata replication object set for a push from its ref-update @@ -1117,8 +1140,13 @@ async fn pin_new_objects_gated( crate::ipfs_pin::pin_new_objects(ipfs_api, repo_path, object_list, db).await } +/// `repo_id` is passed explicitly rather than read from `ctx` so the two callers +/// stay honest about which id they mean: the snapshot iteration passes +/// `ctx.repo_id` (its own push's row), while a coalesced drain passes the id from +/// its fresh re-fetch, which differs after a delete+recreate (#174 U3). async fn pin_and_encrypt_objects( ctx: &EncryptTaskCtx, + repo_id: &str, object_list: Vec, rules: Option>, is_public: bool, @@ -1163,7 +1191,7 @@ async fn pin_and_encrypt_objects( &ctx.ipfs_api, &ctx.repo_path, &ctx.db, - &ctx.repo_id, + repo_id, &node_seed, &recipients, ) @@ -6316,6 +6344,78 @@ mod tests { } } + /// #174 U3: a coalesced drain that runs after the repo row was deleted and + /// recreated under the same owner/name must resolve the LIVE row's id, not the + /// id frozen into the task ctx at spawn. Encrypted-pin metadata written under + /// the dead id is invisible to authorized readers on the live row. + /// + /// `resolve_drain_object_list` already re-fetches by owner/name and uses + /// `record.id` for the visibility-rule read; this binds the same id to the + /// encrypt write, which was still taking `ctx.repo_id`. + #[sqlx::test] + async fn u3_drain_resolves_the_refetched_repo_id_after_an_id_rotation(pool: sqlx::PgPool) { + let raw = pool.clone(); + let state = crate::test_support::test_state(pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + u5_init_repo(tmp.path()); + let c1 = u5_commit_file(tmp.path(), "a.txt", "one\n"); + + state + .db + .upsert_mirror_repo("z6u3rot", "r", "/u3-rotation", None, false) + .await + .unwrap(); + let before = state.db.get_repo("z6u3rot", "r").await.unwrap().unwrap(); + let ctx = u5_ctx( + &state, + &before, + tmp.path().to_path_buf(), + "git", + std::sync::Arc::new(tokio::sync::Semaphore::new(4)), + ); + assert_eq!(ctx.repo_id, before.id, "ctx captures the spawn-time id"); + + // Delete + recreate under the SAME owner/name. This is the rotation: a new + // row id over the same on-disk bare repo. + sqlx::query("DELETE FROM repos WHERE id = $1") + .bind(&before.id) + .execute(&raw) + .await + .unwrap(); + let after = crate::db::RepoRecord { + id: Uuid::new_v4().to_string(), + name: before.name.clone(), + owner_did: before.owner_did.clone(), + description: None, + is_public: true, + default_branch: "main".to_string(), + created_at: Utc::now(), + updated_at: Utc::now(), + disk_path: before.disk_path.clone(), + forked_from: None, + machine_id: None, + }; + state.db.create_repo(&after).await.unwrap(); + assert_ne!(after.id, before.id, "the recreate really did rotate the id"); + + let (drain_id, _list, _rules, _pub) = resolve_drain_object_list( + &ctx, + crate::state::PendingWork::Tips(vec![(ZERO_SHA.to_string(), c1.clone())]), + ) + .await + .expect("a public repo drains to a pin list"); + + assert_eq!( + drain_id, after.id, + "the drain must write its encrypted-pin metadata under the LIVE row id" + ); + assert_ne!( + drain_id, ctx.repo_id, + "the spawn-time id is the deleted row; metadata written there is \ + unreachable from the live repo" + ); + } + /// The drain resolves a coalesced push's tip pair to exactly that push's /// introduced objects (delta semantics — the F5 observable: push B's pins are /// recorded by the drain, and pre-existing objects are not re-listed). @@ -6341,7 +6441,7 @@ mod tests { ); // Push B advanced main c1 -> c2 and lost try_begin; its pair was coalesced. - let (list, _rules, is_public) = resolve_drain_object_list( + let (_drain_id, list, _rules, is_public) = resolve_drain_object_list( &ctx, crate::state::PendingWork::Tips(vec![(c1.clone(), c2.clone())]), ) @@ -6385,7 +6485,7 @@ mod tests { std::sync::Arc::new(tokio::sync::Semaphore::new(4)), ); - let (list, _rules, _pub) = + let (_drain_id, list, _rules, _pub) = resolve_drain_object_list(&ctx, crate::state::PendingWork::FullScan) .await .expect("a public repo drains to a pin list"); @@ -6441,7 +6541,7 @@ mod tests { ) .await .unwrap(); - let (list, _rules, _pub) = resolve_drain_object_list(&ctx, pending()) + let (_drain_id, list, _rules, _pub) = resolve_drain_object_list(&ctx, pending()) .await .expect("still announceable at root"); let got: std::collections::HashSet = list.into_iter().collect(); From 19eaf9fccdb3708a594933650029c71d595b1f59 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 02:39:09 -0500 Subject: [PATCH 67/95] fix(node): key the write lease and encrypt coalescing on the stable disk identity repo_write_leases and encrypt_inflight keyed on record.id, but RepoStore::local_path and the pg advisory lock key on the sanitized owner slug plus repo name, and the bare repo on disk is reused when a repo is deleted and recreated under the same slug. The row id rotates across that recreate, so both in-process serializers stopped serializing exactly where the shared objects/ directory still needed them: a disconnected push's reaper could still be tearing down while a new push took a different key and proceeded. Add repo_identity_key(owner_did, repo_name) and use it at both sites. Three details are load-bearing and carry tests: the sanitization is repo_disk_path's replace([':','/'],"_") rather than normalize_owner_key, which folds a did:key: prefix and would disagree with the disk path; the separator is a NUL, without which owner 'a' + name 'b_c' and owner 'a_b' + name 'c' collide; and both call sites pass record.owner_did / record.name rather than the request path segments, since get_repo normalizes DID aliases and the raw segments would mint two keys for one directory. The sanitization is deliberately not injective. Two did:web spellings can fold to one slug, which would misroute encrypt coalescing if it were reachable, but repos.disk_path is NOT NULL UNIQUE and holds exactly this derivation, so two such rows cannot coexist. A test pins that fold in place: making the key injective is what would let two owners sharing one objects/ directory push concurrently. --- crates/gitlawb-node/src/api/repos.rs | 137 ++++++++++++++++++++------- crates/gitlawb-node/src/state.rs | 100 +++++++++++++++++++ 2 files changed, 205 insertions(+), 32 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index d174a26b..553d8234 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1695,9 +1695,13 @@ pub async fn git_receive_pack( // is what the F1 review rejected, since GITLAWB_TRUSTED_PROXY defaults to unset and // every pusher behind a proxy/NAT then resolves to ONE key, turning one contended repo // into a node-wide denial. + // The stable disk identity, never record.id: the row id rotates on a + // delete+recreate under the same slug while the bare repo on disk is reused, + // and an id-keyed lease stops serializing exactly across that rotation. + let repo_key = crate::state::repo_identity_key(&record.owner_did, &record.name); let lease = state .repo_write_leases - .acquire(&record.id, lease_steal_after) + .acquire(&repo_key, lease_steal_after) .await .ok_or_else(|| { tracing::warn!( @@ -1962,7 +1966,11 @@ async fn post_receive_replication_tail( .iter() .map(|u| (u.old_sha.clone(), u.new_sha.clone())) .collect(); - match state.encrypt_inflight.try_begin(&record.id, tip_pairs) { + // Same stable disk identity as the lease above (#174 U2): keyed on + // record.id, a post-recreate push would take a fresh key and run a second + // encrypt task against the same on-disk repo instead of coalescing. + let coalesce_key = crate::state::repo_identity_key(&record.owner_did, &record.name); + match state.encrypt_inflight.try_begin(&coalesce_key, tip_pairs) { crate::state::BeginOutcome::Coalesced => { coalesced = true; tracing::debug!( @@ -7447,6 +7455,69 @@ mod tests { rec.id } + /// #174 U2: the write lease must register under the STABLE DISK IDENTITY + /// (sanitized owner slug + repo name, what `RepoStore::local_path` and the pg + /// advisory lock key on), not `record.id`. + /// + /// The row id rotates on delete+recreate under the same slug while the bare + /// repo on disk is reused, so an id-keyed lease silently stops serializing + /// across that rotation and lets two writers onto one `objects/` directory. + /// Asserting on the key the holder actually registers under binds the + /// production call site — a helper unit test alone would not. + #[cfg(unix)] + #[sqlx::test] + async fn u2_lease_registers_under_the_disk_identity_not_the_row_id(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let tmp = tempfile::TempDir::new().unwrap(); + let (git_bin, a_inpack, _later_ran) = f1_hanging_first_git(tmp.path()); + let state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6u2key", "k1", false).await; + let rec = state.db.get_repo("z6u2key", "k1").await.unwrap().unwrap(); + let identity = crate::state::repo_identity_key(&rec.owner_did, &rec.name); + assert_ne!( + identity, rec.id, + "the identity key must differ from the row id, or this test proves nothing" + ); + + let did = "did:key:z6MkU2KeyPusherAAAAAAAAAAAAAAAAAAAAAAAA"; + let peer: SocketAddr = "203.0.113.91:5000".parse().unwrap(); + let handle = tokio::spawn({ + let st = state.clone(); + let did = did.to_string(); + async move { + git_receive_pack( + State(st), + Path(("z6u2key".to_string(), "k1".to_string())), + Extension(crate::auth::AuthenticatedDid(did)), + crate::rate_limit::PeerAddr(Some(peer)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await + } + }); + assert!( + f1_wait_for(F1_BACKSTOP, || a_inpack.exists()).await, + "the push never reached receive-pack within {F1_BACKSTOP:?}" + ); + + assert_eq!( + state.repo_write_leases.refs_for(&identity), + 1, + "the lease holder must be registered under the stable disk identity" + ); + assert_eq!( + state.repo_write_leases.refs_for(&rec.id), + 0, + "and never under the rotating row id" + ); + + handle.abort(); + } + /// #174 U1 scenario 1 (RED-before/GREEN-after): parked pushes on one repo's write /// lease are BOUNDED. `git_receive_pack` takes `body: Bytes`, so axum has already /// buffered the whole pack (up to `max_pack_bytes`) before the handler runs, and the @@ -7473,13 +7544,11 @@ mod tests { f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6u1cap", "c1", false).await; // Cap of ONE live waiter, so a holder plus one parked push fills the repo's queue. state.repo_write_leases = crate::state::RepoWriteLeases::new(1); - let repo_id = state - .db - .get_repo("z6u1cap", "c1") - .await - .unwrap() - .unwrap() - .id; + // The lease keys on the stable disk identity (#174 U2), not the row id. + let repo_id = { + let r = state.db.get_repo("z6u1cap", "c1").await.unwrap().unwrap(); + crate::state::repo_identity_key(&r.owner_did, &r.name) + }; let did = "did:key:z6MkU1CapPusherAAAAAAAAAAAAAAAAAAAAAAAAAA"; let push = |peer: SocketAddr| { let st = state.clone(); @@ -7579,13 +7648,10 @@ mod tests { let mut state = f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6u1two", "r1", false).await; state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(2, 100); - let repo1 = state - .db - .get_repo("z6u1two", "r1") - .await - .unwrap() - .unwrap() - .id; + let repo1 = { + let r = state.db.get_repo("z6u1two", "r1").await.unwrap().unwrap(); + crate::state::repo_identity_key(&r.owner_did, &r.name) + }; f1_add_repo(&state, "z6u1two", "r2").await; let did = "did:key:z6MkU1TwoPusherAAAAAAAAAAAAAAAAAAAAAAAAAA"; let src: SocketAddr = "203.0.113.84:5000".parse().unwrap(); @@ -7664,13 +7730,10 @@ mod tests { // The default proxy trust: the resolved key is the peer IP, which is the edge's. state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(2, 100); - let repo1 = state - .db - .get_repo("z6u1nat", "n1") - .await - .unwrap() - .unwrap() - .id; + let repo1 = { + let r = state.db.get_repo("z6u1nat", "n1").await.unwrap().unwrap(); + crate::state::repo_identity_key(&r.owner_did, &r.name) + }; f1_add_repo(&state, "z6u1nat", "n2").await; // Every pusher arrives from the one edge IP, so they share a source key. let edge: SocketAddr = "203.0.113.85:5000".parse().unwrap(); @@ -8030,7 +8093,9 @@ mod tests { f2a_log(&log) ); assert_eq!( - state.encrypt_inflight.pending_for(&rec.id), + state + .encrypt_inflight + .pending_for(&crate::state::repo_identity_key(&rec.owner_did, &rec.name)), Some(crate::state::PendingWork::Tips(vec![( ZERO_SHA.to_string(), c1.clone() @@ -8099,9 +8164,10 @@ mod tests { let c3 = u5_commit_file(repo.path(), "b.txt", "three\n"); assert!( matches!( - state - .encrypt_inflight - .try_begin(&rec.id, vec![(c2.clone(), c3.clone())]), + state.encrypt_inflight.try_begin( + &crate::state::repo_identity_key(&rec.owner_did, &rec.name), + vec![(c2.clone(), c3.clone())], + ), crate::state::BeginOutcome::Coalesced ), "a push arriving while a task is in flight must coalesce" @@ -8179,9 +8245,10 @@ mod tests { // ADMITTED here instead, and this assertion is what catches that.) assert!( matches!( - state - .encrypt_inflight - .try_begin(&rec.id, vec![(c2.clone(), c3.clone())]), + state.encrypt_inflight.try_begin( + &crate::state::repo_identity_key(&rec.owner_did, &rec.name), + vec![(c2.clone(), c3.clone())], + ), crate::state::BeginOutcome::Coalesced ), "a push arriving mid-walk must coalesce, not start a second task" @@ -8265,7 +8332,10 @@ mod tests { let mut updates = state.ref_update_tx.subscribe(); // A task for this repo is already in flight, so the push below coalesces. - let _inflight = match state.encrypt_inflight.try_begin(&rec.id, Vec::new()) { + let _inflight = match state.encrypt_inflight.try_begin( + &crate::state::repo_identity_key(&rec.owner_did, &rec.name), + Vec::new(), + ) { crate::state::BeginOutcome::Admitted(g) => g, crate::state::BeginOutcome::Coalesced => panic!("the first begin must admit"), }; @@ -8336,7 +8406,10 @@ mod tests { let (_server, _cid) = f2a_pinata(&mut state).await; let mut updates = state.ref_update_tx.subscribe(); - let _inflight = match state.encrypt_inflight.try_begin(&rec.id, Vec::new()) { + let _inflight = match state.encrypt_inflight.try_begin( + &crate::state::repo_identity_key(&rec.owner_did, &rec.name), + Vec::new(), + ) { crate::state::BeginOutcome::Admitted(g) => g, crate::state::BeginOutcome::Coalesced => panic!("the first begin must admit"), }; diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index fa621aab..278aa6ae 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -498,6 +498,42 @@ pub struct RepoWriteLeases { max_waiters: usize, } +/// The stable per-repo identity that [`RepoWriteLeases`] and [`EncryptInflight`] +/// key on (#174 U2). +/// +/// NOT `record.id`. A repo deleted and recreated under the same owner/name gets a +/// new row id while the bare repo on disk is reused, so an id-keyed serializer +/// stops serializing across that rotation and lets two writers onto one +/// `objects/` directory. `RepoStore::local_path` and the pg advisory lock both +/// key on the sanitized owner slug plus repo name, and this reproduces exactly +/// that identity so the in-process serializers agree with them. +/// +/// Three details are load-bearing: +/// +/// - The sanitization is [`crate::git::store::repo_disk_path`]'s +/// (`replace([':', '/'], "_")`), NOT `db::normalize_owner_key`, which strips a +/// `did:key:` prefix instead and would map the same input to a different +/// string. The disk path is authoritative because the `objects/` directory is +/// the resource being serialized. +/// - The separator is `\0`, which cannot occur in either component. A plain join +/// would collide (owner `a` + name `b_c` against owner `a_b` + name `c`), +/// letting one repo's push park another's. +/// - Callers pass `record.owner_did` / `record.name`, never the request's path +/// segments: `db::get_repo` normalizes DID aliases, so a caller could otherwise +/// mint two keys for one directory just by varying the DID spelling. +/// +/// The sanitization is not injective (`did:web:example.com:alice` and +/// `did:web:example.com/alice` fold to one slug), which would be a cross-tenant +/// hazard for the coalescing map if it were reachable. It is not: `repos.disk_path` +/// is `NOT NULL UNIQUE` and holds exactly this derivation, so two rows that fold +/// to one key cannot coexist. Do not "fix" this by making the key injective — +/// that is precisely what would let two owners sharing one `objects/` directory +/// push concurrently. +pub fn repo_identity_key(owner_did: &str, repo_name: &str) -> String { + let owner_slug = owner_did.replace([':', '/'], "_"); + format!("{owner_slug}\0{repo_name}") +} + /// A per-repo lease entry: the one-permit semaphore, a refcount of the handlers /// currently referencing it (holding or waiting), and a count of the ones actually /// PARKED. While `refs > 0` every acquirer shares the SAME semaphore, so mutual exclusion @@ -824,6 +860,70 @@ pub async fn acquire_scan_permit( permit } +#[cfg(test)] +mod repo_identity_key_tests { + use super::repo_identity_key; + + /// The key must reproduce `repo_disk_path`'s slug derivation exactly, because + /// the whole point is to agree with the on-disk identity the store and the pg + /// advisory lock already use. + #[test] + fn matches_repo_disk_paths_sanitization() { + let owner = "did:key:z6Mkfoo"; + let name = "r"; + let expected_slug = owner.replace([':', '/'], "_"); + assert_eq!(repo_identity_key(owner, name), format!("{expected_slug}\0{name}")); + + // The disk path for the same pair must carry the same slug component. + let disk = crate::git::store::repo_disk_path(std::path::Path::new("/srv"), owner, name); + assert!( + disk.to_string_lossy().contains(&expected_slug), + "the key's slug must be the one repo_disk_path puts on disk: {disk:?}" + ); + } + + /// Stability across the rotation is the entire reason this key exists: the row + /// id changes on delete+recreate, the identity does not. + #[test] + fn is_stable_across_a_row_id_rotation() { + assert_eq!( + repo_identity_key("did:key:z6Mkfoo", "r"), + repo_identity_key("did:key:z6Mkfoo", "r"), + ); + } + + /// The `\0` separator is what stops one repo's push parking another's. With a + /// plain join these two pairs would produce a single key. + #[test] + fn separator_prevents_the_owner_name_boundary_collision() { + assert_ne!( + repo_identity_key("a", "b_c"), + repo_identity_key("a_b", "c"), + "owner/name boundary must not be ambiguous" + ); + } + + /// Distinct repos and distinct owners never share a key. + #[test] + fn distinct_repos_and_owners_do_not_share_a_key() { + assert_ne!(repo_identity_key("did:key:z6A", "r"), repo_identity_key("did:key:z6A", "s")); + assert_ne!(repo_identity_key("did:key:z6A", "r"), repo_identity_key("did:key:z6B", "r")); + } + + /// Documents the ONE collision the sanitization admits, and why it is safe + /// rather than fixed: `repos.disk_path` is UNIQUE and holds this derivation, + /// so two rows folding to one key cannot coexist. If this assertion ever + /// flips to `assert_ne!`, the key became injective and two owners sharing one + /// `objects/` directory could push concurrently — the defect U2 closes. + #[test] + fn folds_did_web_alias_spellings_together_which_the_unique_disk_path_makes_unreachable() { + assert_eq!( + repo_identity_key("did:web:example.com:alice", "r"), + repo_identity_key("did:web:example.com/alice", "r"), + ); + } +} + #[cfg(test)] mod repo_write_lease_tests { use super::RepoWriteLeases; From 38bce707b8b0e4eeb89e59e3b30d0f6c2c9e20c7 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 02:46:51 -0500 Subject: [PATCH 68/95] fix(node): hold /ipfs walk admission through the blocking walk, not the handler future The global and per-source walk permits were handler locals, so a client disconnect released them the moment the future dropped while the uncancellable spawn_blocking probe/walk/read kept running against a slot the pool now considered free. A disconnect flood could admit replacements while prior blocking git children still occupied threads and PIDs, weakening GITLAWB_MAX_CONCURRENT_IPFS_WALKS and GITLAWB_IPFS_WALK_PER_SOURCE on a surface any unauthenticated caller can reach. Hold the pair in an Arc and clone it into each of the three blocking closures. The permits release when the last clone drops, which collapses both failure directions into one case: a disconnect leaves the abandoned closure holding the slot, and a panicking closure leaves the handler holding it. No loop arm changes and the JoinError arms keep today's taint-and-continue. A move-and-return shuttle was implemented and rejected. It is correct only if every arm that continues the loop re-binds the returned value, which the compiler cannot enforce - the absent-object arm Ok(Ok(None)) => continue, taken on nearly every iteration of a random-CID scan, drops it - and it forces a second decision about the panic arms whose least-bad answer would let one broken repo shed every other owner's /ipfs lookups. The regression strengthens the existing admission test: it now asserts the slot is still held after the future is dropped and before the child is killed, with a child-still-alive precondition so it cannot pass vacuously. Only the walk site runs under state.git_bin and can be pinned that way; the probe and read deliberately shell to real git, so a source-scan gate in inv22_gates.rs binds all three clone sites instead. Removing any one clone turns that gate red. --- crates/gitlawb-node/src/api/ipfs.rs | 80 +++++++++++++++++++++--- crates/gitlawb-node/tests/inv22_gates.rs | 57 +++++++++++++++++ 2 files changed, 129 insertions(+), 8 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 5e7ae05a..d9f4c01b 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -78,6 +78,20 @@ use crate::visibility::{visibility_check, Decision}; /// Scope: this closes the direct unauthenticated scan, including the dangling /// case. A stale-public mirror row still serves withheld content (tracked /// separately, #124). +/// +/// One `/ipfs` request's walk admission: the global pool permit plus the +/// optional per-source sub-permit, both RAII (#174 U1). +/// +/// Held behind an `Arc` whose clones go into every `spawn_blocking` walk this +/// request runs, so the permits release only when the last clone drops — the +/// handler's, or an abandoned/panicking closure's, whichever outlives the other. +/// Admission therefore tracks real blocking-thread occupancy rather than the +/// lifetime of the future that requested it. +struct WalkAdmission { + _global: tokio::sync::OwnedSemaphorePermit, + _per_source: Option, +} + pub async fn get_by_cid( Path(cid_str): Path, State(state): State, @@ -131,7 +145,7 @@ pub async fn get_by_cid( // (`client_key`), never the DID (`/ipfs` admits any `did:key` unthrottled, so a DID // key would be free to mint around); a `None` key (no trusted header, no peer) is // bounded by the global pool only, never the per-source sub-cap. - let _ipfs_walk_permit = state + let global_permit = state .git_ipfs_walk_semaphore .clone() .try_acquire_owned() @@ -140,13 +154,29 @@ pub async fn get_by_cid( AppError::Overloaded("ipfs service at capacity, retry shortly".into()) })?; let source_key = crate::rate_limit::client_key(&req_headers, peer, state.push_limiter_trust); - let _ipfs_caller_permit = match &source_key { + let caller_permit = match &source_key { Some(ip) => Some(state.git_ipfs_walk_per_caller.try_acquire(ip).ok_or_else(|| { tracing::warn!(key = %ip, "/ipfs per-source walk cap reached; shedding request with 503"); AppError::Overloaded("ipfs service at capacity for this source, retry shortly".into()) })?), None => None, }; + // Share the admission rather than holding it as a handler local (#174 U1). A + // clone goes into every `spawn_blocking` closure below, so the permits release + // only when the LAST holder drops. That makes the two failure directions one + // case: a client disconnect drops the handler's clone while the abandoned + // closure's clone keeps the slot taken for as long as its git child runs, and a + // panicking closure drops its clone while the handler's keeps the slot taken. + // + // Deliberately NOT a move-and-return shuttle. That shape is correct only if + // every arm continuing the loop re-binds the returned value, which the compiler + // cannot enforce — including the absent-object arm `Ok(Ok(None)) => continue` + // that a random-CID scan takes on nearly every iteration — and it forces a + // second decision about the arms where a panic destroys the moved-in permits. + let admission = std::sync::Arc::new(WalkAdmission { + _global: global_permit, + _per_source: caller_permit, + }); // 2. Search all repos for an object with this SHA-256. // @@ -378,7 +408,11 @@ pub async fn get_by_cid( // independent of `state.git_bin` (which tests point at a fake walk git). let probe_path = repo_path.clone(); let probe_sha = sha256_hex.clone(); + let probe_admission = std::sync::Arc::clone(&admission); let obj_type = match tokio::task::spawn_blocking(move || { + // Admission clone (#174 U1): the slot stays taken until this blocking + // work returns, even if the handler future was dropped or this closure panics. + let _admission = probe_admission; store::object_type_bounded("git", &probe_path, &probe_sha, probe_deadline) }) .await @@ -455,7 +489,11 @@ pub async fn get_by_cid( ); // Full-history walk shells out to git — keep it off the async runtime, // bounded and reaped like the served-git ops (#174). + let walk_admission = std::sync::Arc::clone(&admission); let walk = tokio::task::spawn_blocking(move || { + // Admission clone (#174 U1): the slot stays taken until this blocking + // work returns, even if the handler future was dropped or this closure panics. + let _admission = walk_admission; allowed_blob_set_for_caller_bounded( &rp, &git_bin, @@ -525,7 +563,11 @@ pub async fn get_by_cid( let read_path = repo_path.clone(); let read_sha = sha256_hex.clone(); let read_type = obj_type.clone(); + let read_admission = std::sync::Arc::clone(&admission); let content = match tokio::task::spawn_blocking(move || { + // Admission clone (#174 U1): the slot stays taken until this blocking + // work returns, even if the handler future was dropped or this closure panics. + let _admission = read_admission; store::read_object_content_bounded( "git", &read_path, @@ -2331,13 +2373,35 @@ mod tests { "a replacement must shed 503 while the prior request's blocking walk still runs" ); - // Drop the in-flight request; the detached blocking walk keeps running (a - // spawn_blocking cannot be cancelled), but on the fix the permit is a handler - // local, so dropping the future releases it once the blocking join is abandoned. - // Either way, kill the sleeping child so the slot frees promptly and poll for - // recovery — the point already proven above is that the slot stayed held for the - // duration of the blocking work. + // Drop the in-flight request — a client disconnect. The detached blocking walk + // keeps running (a spawn_blocking cannot be cancelled) and its git child is still + // occupying a blocking thread and a PID, so the slot it was admitted under must + // STAY TAKEN. Admission is released by the blocking work finishing, never by the + // handler future going away (#174 U1). + // + // MUTATION (RED): make the admission a handler local again (drop the Arc clone + // moved into the spawn_blocking closures) and this assertion fails immediately — + // the permit count returns to 1 the moment the future is dropped, while the + // sleeping child is still alive. drop(fut); + // Give the runtime a chance to actually run the drop and any woken tasks, so + // this is not merely observing a not-yet-processed release. + for _ in 0..10 { + tokio::task::yield_now().await; + } + assert!( + unsafe { libc::kill(pid, 0) } == 0, + "precondition: the blocking walk's git child must still be alive, or this \ + assertion proves nothing" + ); + assert_eq!( + sem.available_permits(), + 0, + "a client disconnect must NOT release the walk slot while the uncancellable \ + blocking walk it admitted is still running" + ); + + // Now end the blocking work; the slot frees when the closure returns. unsafe { libc::kill(pid, libc::SIGKILL); } diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index 0ce54088..6dd414cc 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -356,3 +356,60 @@ fn f3_second_writer_leased_until_reap() { with_lease, so the lease rides the guard into KillGroupOnDrop's reaper" ); } + +/// #174 U1 — every blocking walk in the `/ipfs` scan carries the request's walk +/// admission. +/// +/// The admission is an `Arc` cloned into each `spawn_blocking` +/// closure, so the global + per-source permits release only when the last holder +/// drops: a client disconnect leaves the abandoned closure holding the slot, and a +/// panicking closure leaves the handler holding it. +/// +/// Only the walk site runs under `state.git_bin`, so only it can be pinned by the +/// fake-git harness and mutation-verified dynamically (see +/// `get_by_cid_walk_permit_held_through_blocking_walk`). The probe and the content +/// read deliberately shell to the real `git`, which is why this structural check +/// exists: it binds all three sites, and any blocking site added to this loop +/// later, without reversing that deliberate independence. +/// +/// MUTATION (RED): delete any one `Arc::clone(&admission)` binding, or drop the +/// clone from inside its closure, and the count falls below three. +#[test] +fn inv22_ipfs_walk_admission_reaches_every_blocking_site() { + let ipfs = src("api/ipfs.rs"); + + // The shared owner must exist and be built once per request. + assert!( + ipfs.contains("struct WalkAdmission") && ipfs.contains("Arc::new(WalkAdmission {"), + "U1 gate missing: the /ipfs walk admission must be a shared WalkAdmission, \ + not a handler-local permit pair" + ); + + // Every blocking site in the scan takes its own clone... + let clones = ipfs.matches("Arc::clone(&admission)").count(); + assert!( + clones >= 3, + "U1 gate bypassed: expected an admission clone for each of the three \ + /ipfs spawn_blocking sites (probe, walk, read); found {clones}. A blocking \ + walk that does not hold the admission lets a disconnect or a panic free the \ + slot while its git child is still running." + ); + + // ...and each clone is actually moved INTO the blocking closure, not merely + // created in the async frame (which would hold nothing across the join). + let held = ipfs.matches("let _admission = ").count(); + assert!( + held >= 3, + "U1 gate bypassed: each admission clone must be bound inside its \ + spawn_blocking closure so the blocking work owns it; found {held} of 3." + ); + + // The count of blocking sites is itself the thing being covered: if a fourth + // appears, it needs an admission clone too and this gate must be revisited. + let sites = ipfs.matches("spawn_blocking(move ||").count(); + assert_eq!( + sites, 3, + "the /ipfs scan grew or lost a spawn_blocking site ({sites} found, expected 3); \ + give any new blocking walk its own Arc::clone(&admission) and update this gate" + ); +} From 43d35ebd70613e9d5695a7bd7fd92320ac650c80 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 02:55:20 -0500 Subject: [PATCH 69/95] fix(node): hold the encrypt-scan permit in its closure and run info/refs under git_bin Two single-site wiring fixes from the same review round. withheld_recipients_gated held its git_encrypt_semaphore permit in the async frame and then awaited spawn_blocking, so dropping the future released the permit while the uncancellable walk kept running. Every other post-receive scan helper moves the permit into the closure - replication_withheld_set says so in as many words - so this aligns the last site with the F4 contract. The regression drops the future mid-walk and asserts the pool stays at zero while the git child is still alive. git_info_refs passed a literal "git" to smart_http::info_refs while upload-pack and receive-pack both pass state.git_bin, so no fake-git harness could drive the advertisement path. The regression points git_bin at a fake emitting a distinctive advertisement and asserts it reaches the response body; before the fix the body came back from real system git (agent=git/2.43.0). --- crates/gitlawb-node/src/api/repos.rs | 162 ++++++++++++++++++++++++++- 1 file changed, 160 insertions(+), 2 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 553d8234..400e10e1 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -728,7 +728,13 @@ pub async fn git_info_refs( // are moved in, so admission tracks the real process lifetime. let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); - smart_http::info_refs("git", &service, &disk_path, git_timeout, Some(admission)) + smart_http::info_refs( + &state.git_bin, + &service, + &disk_path, + git_timeout, + Some(admission), + ) .await .map_err(|e| { let app = git_service_app_error(&e); @@ -820,11 +826,15 @@ async fn withheld_recipients_gated( anyhow::Result>>, tokio::task::JoinError, > { - let _permit = encrypt_sem + let permit = encrypt_sem .acquire_owned() .await .expect("git_encrypt_semaphore is never closed"); tokio::task::spawn_blocking(move || { + // The permit lives inside the blocking closure (#174 U4, the F4 contract): a + // started walk always completes holding it, so a dropped future cannot free + // the slot while this uncancellable walk still occupies a thread and a PID. + let _permit = permit; crate::git::visibility_pack::withheld_blob_recipients_bounded( &repo_path, &git_bin, timeout, &rules, is_public, &owner_did, ) @@ -3029,6 +3039,154 @@ mod tests { p.to_str().unwrap().to_string() } + /// #174 U6: the `info/refs` advertisement must run the CONFIGURED git binary, + /// like upload-pack and receive-pack already do. It passed a literal "git", so a + /// fake-git harness could drive the pack paths but never the advertisement. + /// + /// MUTATION (RED): restore the `"git"` literal at the `smart_http::info_refs` + /// call and the fake's marker never reaches the response body. + #[cfg(unix)] + #[sqlx::test] + async fn u6_info_refs_runs_the_configured_git_binary(pool: sqlx::PgPool) { + use axum::extract::{Path, Query, State}; + use http_body_util::BodyExt; + + let tmp = tempfile::TempDir::new().unwrap(); + // Distinctive advertisement so the assertion cannot pass on real git's output. + let body = "#!/bin/sh\n\ + case \"$1\" in\n\ + upload-pack) printf 'U6-FAKE-ADVERTISEMENT' ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + let state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6u6adv", "a1", false).await; + + let resp = git_info_refs( + State(state), + Path(("z6u6adv".to_string(), "a1".to_string())), + Query(InfoRefsQuery { + service: Some("git-upload-pack".to_string()), + }), + crate::rate_limit::PeerAddr(Some("203.0.113.95:5000".parse().unwrap())), + axum::http::HeaderMap::new(), + None, + ) + .await + .expect("the advertisement must succeed"); + + let bytes = resp.into_body().collect().await.unwrap().to_bytes(); + let text = String::from_utf8_lossy(&bytes); + assert!( + text.contains("U6-FAKE-ADVERTISEMENT"), + "info/refs must run state.git_bin, not a hardcoded \"git\"; body was: {text:?}" + ); + } + + /// #174 U4: `withheld_recipients_gated` must hold its encrypt-scan permit INSIDE + /// the blocking closure, matching every other post-receive scan helper (see + /// `replication_withheld_set`, where the permit is moved in with "a started walk + /// always completes holding it"). + /// + /// Holding it in the async frame instead means dropping the future releases the + /// permit while the uncancellable `spawn_blocking` walk keeps running, so the + /// encrypt pool admits a replacement scan against a slot still occupied by a live + /// git child. + /// + /// MUTATION (RED): hoist `_permit` back out of the closure and the + /// still-held-after-drop assertion fails — the count returns to 1 immediately. + #[cfg(unix)] + #[tokio::test] + async fn u4_encrypt_scan_permit_is_held_through_the_blocking_walk() { + use std::time::Duration; + let tmp = tempfile::TempDir::new().unwrap(); + let pidfile = tmp.path().join("u4_walk.pid"); + // Hang on the first real walk command, whatever it is; only rev-parse answers. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + rev-parse) echo deadbeef ;;\n\ + *) echo $$ > \"{pid}\"; while true; do sleep 1; done ;;\n\ + esac\n\ + exit 0\n", + pid = pidfile.display(), + ); + let git_bin = write_fake_git(tmp.path(), &body); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + + let mut fut = Box::pin(withheld_recipients_gated( + sem.clone(), + tmp.path().to_path_buf(), + git_bin, + Duration::from_secs(600), + vec![vis_rule("/secret/**", &[])], + true, + "did:key:z6MkU4OwnerAAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + )); + + // Drive until the blocking walk's git child records its pid. + let mut walk_pid: Option = None; + for _ in 0..500 { + let _ = tokio::time::timeout(Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&pidfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + walk_pid = Some(p); + break; + } + } + let pid = walk_pid.expect("the fake git walk command must have spawned"); + struct ReapOnDrop(i32); + impl Drop for ReapOnDrop { + fn drop(&mut self) { + unsafe { + libc::kill(self.0, libc::SIGKILL); + } + } + } + let _cleanup = ReapOnDrop(pid); + + assert_eq!( + sem.available_permits(), + 0, + "the scan permit must be held while the blocking walk runs" + ); + + drop(fut); + for _ in 0..10 { + tokio::task::yield_now().await; + } + assert!( + unsafe { libc::kill(pid, 0) } == 0, + "precondition: the walk's git child must still be alive, or this \ + assertion proves nothing" + ); + assert_eq!( + sem.available_permits(), + 0, + "dropping the future must NOT release the encrypt-scan permit while the \ + uncancellable blocking walk it admitted is still running" + ); + + unsafe { + libc::kill(pid, libc::SIGKILL); + } + let mut freed = false; + for _ in 0..400 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + assert!( + freed, + "once the blocking walk ends the scan permit must return to the pool" + ); + } + /// #174 (write-pool twin, vetted by execution not reasoning): the receive-pack /// post-push replication walk is bounded. Drive `replication_withheld_set` with an /// injected fake git that hangs on `rev-list` and a short budget: it must RETURN From 9fc68e2fc9d27ea77b8263156eb606b50a27db6e Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 03:04:30 -0500 Subject: [PATCH 70/95] fix(node): spawn the replication tail at the durability boundary The post-receive replication tail was spawned at the end of git_receive_pack, after touch_repo, the trust-score update, per-ref certificate issuance and webhook fan-out. A client or proxy disconnect anywhere in that window dropped the tail, and with it this push's pins, recovery copy, and announcements - even though the push was already durable on disk. Spawn it immediately after the receive_result ? instead. That is the first point where the push is known to have landed: guard.release() a few lines up runs on failure too, so anchoring there would spawn a tail for a push git rejected, pinning and announcing a half-applied repo - the state release() deliberately refuses to upload to Tigris, and one a hostile pusher can produce on demand by aborting a pack mid-transfer. RefUpdate gains Clone so the tail takes the parsed updates while the certificate and webhook loops keep iterating their own copy. This makes the tail run concurrently with certificate issuance rather than after it, so a ref can be announced before its signed certificate exists. Accepted, and stated here rather than left for a reviewer to find: cert issuance already fails open (errors are logged and skipped) and the gossip event carries cert_id: None regardless, so no announce consumer reads a certificate out of it. Evidence is an ordering gate in inv22_gates.rs rather than a cancellation-race test, matching the F3 gate's instrument: forcing a disconnect to land inside the cert/webhook window is timing-dependent and would be flaky, while the ordering is precisely what changed. The gate is mutation-verified in both directions - moving the spawn back below the webhooks fails the upper bound, hoisting it above the ? fails the lower one. --- crates/gitlawb-node/src/api/repos.rs | 50 +++++++++++++-------- crates/gitlawb-node/tests/inv22_gates.rs | 56 ++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 18 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 400e10e1..470badc7 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1823,6 +1823,34 @@ pub async fn git_receive_pack( app })?; + // #174 F2/U5: the post-receive replication tail runs in an independently owned + // task. It parks on `git_encrypt_semaphore` (withheld / candidate / full-scan + // resolution), so leaving it in the request future means a client/proxy disconnect + // while parked silently drops this push's pins, recovery copy, and announcements. + // + // Spawned HERE, at the durability boundary, rather than at the end of the handler. + // This is the first point where the push is known to have landed: `guard.release()` + // above runs on failure too, so anchoring on it would spawn a tail that pins and + // announces a half-applied repo — the state release() deliberately refuses to + // upload, and one a hostile pusher can produce on demand by aborting a pack + // mid-transfer. Everything below (touch_repo, metrics, trust score, certificates, + // webhooks) stays in the cancellable handler, so a disconnect during that window no + // longer takes the tail with it. + // + // The tail now runs CONCURRENTLY with certificate issuance rather than after it, so + // a ref can be announced before its signed certificate exists. That window is + // accepted: cert issuance already fails open (errors are logged and skipped) and the + // gossip event carries `cert_id: None` regardless, so no announce consumer reads a + // certificate out of it. Each push owns its own tail, including its own + // always-spawned announce, so per-push announcements are never coalesced away. + tokio::spawn(post_receive_replication_tail( + state.clone(), + record.clone(), + ref_updates.clone(), + disk_path.clone(), + auth.0.to_string(), + )); + // Update the repo's updated_at timestamp after a successful push let _ = state.db.touch_repo(&record.id).await; @@ -1912,24 +1940,6 @@ pub async fn git_receive_pack( } } - // #174 F2: move the whole post-receive replication tail into an independently owned - // task. It parks on `git_encrypt_semaphore` (withheld / candidate / full-scan - // resolution), so leaving it in the request future means a client/proxy disconnect - // while parked silently drops this push's pins, recovery copy, and announcements — - // the residual `state.rs` documented, because the durable `try_begin` gate sat after - // the park. The request future now returns the git response immediately and the tail - // runs detached; a disconnect can no longer drop it. Each push owns its own tail, - // including its own always-spawned announce, so per-push announcements are never - // coalesced away (the announce spawn stays out of the per-repo encrypt coalescing). - let did = did.to_string(); - tokio::spawn(post_receive_replication_tail( - state, - record, - ref_updates, - disk_path, - did, - )); - Ok(result) } @@ -2638,6 +2648,10 @@ pub async fn get_icaptcha_proof( // ── Pkt-line parsing ────────────────────────────────────────────────────── +/// `Clone` so `git_receive_pack` can hand the parsed updates to the detached +/// replication tail at the durability boundary while the certificate and webhook +/// loops below still iterate their own copy (#174 U5). +#[derive(Clone)] struct RefUpdate { old_sha: String, new_sha: String, diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index 6dd414cc..82712c23 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -413,3 +413,59 @@ fn inv22_ipfs_walk_admission_reaches_every_blocking_site() { give any new blocking walk its own Arc::clone(&admission) and update this gate" ); } + +/// #174 U5: the post-receive replication tail is spawned at the DURABILITY BOUNDARY, +/// not at the end of the handler. +/// +/// The tail owes this push its pins, recovery copy, and announcements. Spawned below +/// the certificate and webhook work, a client disconnect anywhere in that window took +/// the tail with it. Spawned above it, the tail is already detached and survives. +/// +/// The lower bound matters just as much as the upper one: `guard.release()` runs on +/// failure too, so a spawn anchored there would fire for a push git rejected, pinning +/// and announcing a half-applied repo. The `?` on `receive_result` is the first point +/// where the push is known to have landed, which is why the spawn must sit between +/// that `?` and `touch_repo`. +/// +/// This is an ordering check rather than a cancellation-race test on purpose: forcing +/// a disconnect to land inside the cert/webhook window is timing-dependent and would +/// be flaky, while the ordering is exactly what the fix changes. Same instrument the +/// F3 gate above uses. +/// +/// MUTATION (RED): move the `tokio::spawn(post_receive_replication_tail` call back +/// below the webhook block and the ordering assertion fails; move it above the `?` +/// and the failed-push assertion fails. +#[test] +fn inv22_replication_tail_spawns_at_the_durability_boundary() { + let repos = src("api/repos.rs"); + // Production half only — the tests below name these identifiers too. + let production = repos + .split("#[cfg(test)]") + .next() + .expect("split always yields a first chunk"); + + let receive_ok = production + .find("let result = receive_result.map_err(") + .expect("U5 gate stale: git_receive_pack no longer maps receive_result with ?"); + let spawn = production + .find("tokio::spawn(post_receive_replication_tail(") + .expect("U5 gate missing: the replication tail must be spawned by git_receive_pack"); + let touch = production + .find("state.db.touch_repo(") + .expect("U5 gate stale: git_receive_pack no longer calls touch_repo"); + let webhook = production + .find("webhooks::fire_event(") + .expect("U5 gate stale: git_receive_pack no longer fires push webhooks"); + + assert!( + receive_ok < spawn, + "U5 gate bypassed: the tail must be spawned AFTER the receive_result `?`, or a \ + rejected push spawns a tail that pins and announces a half-applied repo" + ); + assert!( + spawn < touch && spawn < webhook, + "U5 gate bypassed: the tail must be spawned BEFORE touch_repo and the webhook \ + fan-out, so a disconnect in that window cannot drop this push's pins, \ + recovery copy, and announcements" + ); +} From a4960020d58c9fc087daed6012b23e76e682dc54 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 03:04:58 -0500 Subject: [PATCH 71/95] style: rustfmt the repo_identity_key tests --- crates/gitlawb-node/src/state.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 278aa6ae..ef093661 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -872,7 +872,10 @@ mod repo_identity_key_tests { let owner = "did:key:z6Mkfoo"; let name = "r"; let expected_slug = owner.replace([':', '/'], "_"); - assert_eq!(repo_identity_key(owner, name), format!("{expected_slug}\0{name}")); + assert_eq!( + repo_identity_key(owner, name), + format!("{expected_slug}\0{name}") + ); // The disk path for the same pair must carry the same slug component. let disk = crate::git::store::repo_disk_path(std::path::Path::new("/srv"), owner, name); @@ -906,8 +909,14 @@ mod repo_identity_key_tests { /// Distinct repos and distinct owners never share a key. #[test] fn distinct_repos_and_owners_do_not_share_a_key() { - assert_ne!(repo_identity_key("did:key:z6A", "r"), repo_identity_key("did:key:z6A", "s")); - assert_ne!(repo_identity_key("did:key:z6A", "r"), repo_identity_key("did:key:z6B", "r")); + assert_ne!( + repo_identity_key("did:key:z6A", "r"), + repo_identity_key("did:key:z6A", "s") + ); + assert_ne!( + repo_identity_key("did:key:z6A", "r"), + repo_identity_key("did:key:z6B", "r") + ); } /// Documents the ONE collision the sanitization admits, and why it is safe From 62ec882bcb39bfa7a1afeea41f5af3fc8dbd8ede Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 05:31:13 -0500 Subject: [PATCH 72/95] fix(node): keep the repo identity key printable The key is logged as the `repo` field on the lease waiter-cap shed and the steal-bound warning (state.rs), and it was joined with a NUL. Those are the two messages an operator reads to find a contended repo, and a NUL-hostile log sink truncates the field at the separator, rendering two different repos' warnings identically. Before this PR that field was a UUID or a mirror id, both printable, so the NUL was a regression this change introduced. Join with '/' instead. It cannot occur in the owner slug by construction - the slug is owner_did.replace([':','/'], "_") - and it mirrors the shape of the disk path the key exists to reproduce. A test asserts the key carries no control characters so this cannot regress silently. Rotating the U2 lease test's row to a UUID id is not incidental: the mirror path that seeds it uses 'owner_short/name' as the row id, which now coincides with the identity key and would leave the test unable to tell the two keys apart. The rotated id is also the real shape of the bug. Re-verified RED against the reverted call site after the change. --- crates/gitlawb-node/src/api/repos.rs | 15 ++++++++++++++ crates/gitlawb-node/src/state.rs | 30 +++++++++++++++++++++++----- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 470badc7..5871601e 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -7647,7 +7647,22 @@ mod tests { let (git_bin, a_inpack, _later_ran) = f1_hanging_first_git(tmp.path()); let state = f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6u2key", "k1", false).await; + // Rotate the row to a UUID id first. f4_state_with_repo creates the repo + // through the mirror path, whose id is literally `owner_short/name` and so + // coincides with the identity key — which would leave this test unable to + // tell the two keys apart. A UUID id is also the real shape of the bug: an + // API-created repo, or any repo whose row was recreated under the same slug. + let seeded = state.db.get_repo("z6u2key", "k1").await.unwrap().unwrap(); + let rotated_id = Uuid::new_v4().to_string(); + sqlx::query("UPDATE repos SET id = $1 WHERE id = $2") + .bind(&rotated_id) + .bind(&seeded.id) + .execute(&pool) + .await + .unwrap(); + let rec = state.db.get_repo("z6u2key", "k1").await.unwrap().unwrap(); + assert_eq!(rec.id, rotated_id, "the row really did take the new id"); let identity = crate::state::repo_identity_key(&rec.owner_did, &rec.name); assert_ne!( identity, rec.id, diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index ef093661..1f0f8c81 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -515,9 +515,14 @@ pub struct RepoWriteLeases { /// `did:key:` prefix instead and would map the same input to a different /// string. The disk path is authoritative because the `objects/` directory is /// the resource being serialized. -/// - The separator is `\0`, which cannot occur in either component. A plain join -/// would collide (owner `a` + name `b_c` against owner `a_b` + name `c`), -/// letting one repo's push park another's. +/// - The separator is `/`, which cannot occur in the owner slug by construction +/// (`replace([':', '/'], "_")` removes it) and mirrors the shape of the disk +/// path this key exists to reproduce. A plain join would collide (owner `a` + +/// name `b_c` against owner `a_b` + name `c`), letting one repo's push park +/// another's. It must stay PRINTABLE: this key is logged as the `repo` field +/// on the lease shed and steal-bound warnings below, and an unprintable +/// separator (a NUL, a unit separator) truncates at a NUL-hostile log sink and +/// renders two different repos' warnings identically. /// - Callers pass `record.owner_did` / `record.name`, never the request's path /// segments: `db::get_repo` normalizes DID aliases, so a caller could otherwise /// mint two keys for one directory just by varying the DID spelling. @@ -531,7 +536,7 @@ pub struct RepoWriteLeases { /// push concurrently. pub fn repo_identity_key(owner_did: &str, repo_name: &str) -> String { let owner_slug = owner_did.replace([':', '/'], "_"); - format!("{owner_slug}\0{repo_name}") + format!("{owner_slug}/{repo_name}") } /// A per-repo lease entry: the one-permit semaphore, a refcount of the handlers @@ -874,7 +879,7 @@ mod repo_identity_key_tests { let expected_slug = owner.replace([':', '/'], "_"); assert_eq!( repo_identity_key(owner, name), - format!("{expected_slug}\0{name}") + format!("{expected_slug}/{name}") ); // The disk path for the same pair must carry the same slug component. @@ -906,6 +911,21 @@ mod repo_identity_key_tests { ); } + /// The key is logged as the `repo` field on the lease waiter-cap shed and the + /// steal-bound warning, so it must contain no control characters. A NUL (or a + /// unit separator) truncates at a NUL-hostile log sink, which would render two + /// different repos' shed warnings identically — an observability lie on + /// exactly the messages an operator reads to find a contended repo. + #[test] + fn key_is_printable_so_the_lease_warnings_stay_readable() { + let k = repo_identity_key("did:web:example.com:alice", "my-repo.git"); + assert!( + !k.chars().any(|c| c.is_control()), + "the identity key is logged; it must carry no control characters: {k:?}" + ); + assert_eq!(k, "did_web_example.com_alice/my-repo.git"); + } + /// Distinct repos and distinct owners never share a key. #[test] fn distinct_repos_and_owners_do_not_share_a_key() { From 929256579c7bf38983ba7a8f43bd0a71e8048bec Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 12:37:39 -0500 Subject: [PATCH 73/95] fix(node): spawn the replication tail above guard.release() (#174) The post-receive tail was spawned after `guard.release(...)`, which on a successful push awaits the Tigris upload and then the advisory unlock while the handler future is still tied to the client connection. The pack has already landed on disk by then, so a disconnect in that window left a durable push whose pins, recovery copy and announce were never spawned: the dropped tail F2 closed, one step earlier in the handler. Spawn it above `release`, gated on an explicit `receive_result.is_ok()`. The `?` below the spawn can no longer be what keeps a rejected push from pinning and announcing a half-applied repo, so the success check is now a named flag that `release` consumes too. The tail is read-only on the repo directory and holds neither the lease nor the advisory lock, so overlapping it with the upload waits on nothing the handler still owns. Tests: a handler-level disconnect driven through a parked `release` (new store-level test seam), plus the must-not direction, a failed receive-pack that spawns no tail. The U5 source gate now binds the new boundary: spawn inside `if push_succeeded`, above `guard.release(push_succeeded)`. --- crates/gitlawb-node/src/api/repos.rs | 259 +++++++++++++++++++--- crates/gitlawb-node/src/git/repo_store.rs | 20 +- crates/gitlawb-node/tests/inv22_gates.rs | 68 +++--- 3 files changed, 292 insertions(+), 55 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 5871601e..866df30b 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1800,10 +1800,50 @@ pub async fn git_receive_pack( ) .await; + // #174 F2/U5: the post-receive replication tail runs in an independently owned + // task. It parks on `git_encrypt_semaphore` (withheld / candidate / full-scan + // resolution), so leaving it in the request future means a client/proxy disconnect + // while parked silently drops this push's pins, recovery copy, and announcements. + // + // Spawned HERE, ABOVE `guard.release()`, because `release` is itself cancellable: + // on success it awaits the Tigris upload and then the advisory unlock, both while + // this future is still tied to the client connection, and the pack has ALREADY + // landed on disk by then. Spawning below `release` left exactly that window open, + // where a disconnect meant a durable push with no tail (the same class F2 closed, + // one step earlier). + // + // The success gate is explicit rather than the `?` below, because that `?` now + // sits under this spawn: `release` runs on failure too, so anchoring the tail on + // it would pin and announce a half-applied repo, the state `release(false)` + // deliberately refuses to upload and one a hostile pusher can produce on demand + // by aborting a pack mid-transfer. + // + // The tail is read-only on `disk_path` (walk plus plumbing) and takes neither the + // write lease nor the advisory lock, so running it concurrently with the upload + // below waits on nothing this handler still holds. Everything after (touch_repo, + // metrics, trust score, certificates, webhooks) stays in the cancellable handler. + // + // The tail also runs CONCURRENTLY with certificate issuance rather than after it, + // so a ref can be announced before its signed certificate exists. That window is + // accepted: cert issuance already fails open (errors are logged and skipped) and + // the gossip event carries `cert_id: None` regardless, so no announce consumer + // reads a certificate out of it. Each push owns its own tail, including its own + // always-spawned announce, so per-push announcements are never coalesced away. + let push_succeeded = receive_result.is_ok(); + if push_succeeded { + tokio::spawn(post_receive_replication_tail( + state.clone(), + record.clone(), + ref_updates.clone(), + disk_path.clone(), + auth.0.to_string(), + )); + } + // Always release the advisory lock — even on error — to prevent stale locks // from blocking subsequent pushes. Only upload to Tigris when the push // succeeded; uploading a half-applied repo would propagate corruption. - guard.release(receive_result.is_ok()).await; + guard.release(push_succeeded).await; // Clean path: clone (a) already dropped inside run_git_service when the receive-pack // group was reaped; clone (b) held here spanned the success-only Tigris upload that // ran inside release() above. Drop it now so a second same-repo push proceeds the @@ -1823,34 +1863,6 @@ pub async fn git_receive_pack( app })?; - // #174 F2/U5: the post-receive replication tail runs in an independently owned - // task. It parks on `git_encrypt_semaphore` (withheld / candidate / full-scan - // resolution), so leaving it in the request future means a client/proxy disconnect - // while parked silently drops this push's pins, recovery copy, and announcements. - // - // Spawned HERE, at the durability boundary, rather than at the end of the handler. - // This is the first point where the push is known to have landed: `guard.release()` - // above runs on failure too, so anchoring on it would spawn a tail that pins and - // announces a half-applied repo — the state release() deliberately refuses to - // upload, and one a hostile pusher can produce on demand by aborting a pack - // mid-transfer. Everything below (touch_repo, metrics, trust score, certificates, - // webhooks) stays in the cancellable handler, so a disconnect during that window no - // longer takes the tail with it. - // - // The tail now runs CONCURRENTLY with certificate issuance rather than after it, so - // a ref can be announced before its signed certificate exists. That window is - // accepted: cert issuance already fails open (errors are logged and skipped) and the - // gossip event carries `cert_id: None` regardless, so no announce consumer reads a - // certificate out of it. Each push owns its own tail, including its own - // always-spawned announce, so per-push announcements are never coalesced away. - tokio::spawn(post_receive_replication_tail( - state.clone(), - record.clone(), - ref_updates.clone(), - disk_path.clone(), - auth.0.to_string(), - )); - // Update the repo's updated_at timestamp after a successful push let _ = state.db.touch_repo(&record.id).await; @@ -8563,6 +8575,195 @@ mod tests { assert_eq!(broadcast.ref_name, "refs/heads/main"); } + /// A push handler whose `release` parks at its pre-unlock point, so a test can + /// drop the future from inside the cancellable post-receive window. Returns the + /// state, the seeded record and the git-invocation log. + /// + /// Path-scoped on purpose: the tail's withheld walk is the observable, and + /// without a path-scoped rule `replication_withheld_set` takes the no-walk + /// shortcut and spawns no git at all. + #[cfg(unix)] + async fn p2_parked_release_state( + pool: sqlx::PgPool, + tmp: &std::path::Path, + owner: &str, + name: &str, + git_body: Option<&str>, + ) -> (AppState, std::path::PathBuf) { + let log = tmp.join("git.log"); + let git_bin = match git_body { + Some(body) => write_fake_git(tmp, body), + None => f2a_logging_git(tmp, &log), + }; + let repos_dir = tmp.join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + + let mut state = crate::test_support::test_state(pool.clone()).await; + state.git_bin = git_bin; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Armed and never notified: every guard this store hands out parks in + // `release` right before the advisory unlock, which is inside the window a + // client disconnect can hit. + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool.clone()) + .with_pre_unlock_gate(std::sync::Arc::new(tokio::sync::Notify::new())); + state + .db + .upsert_mirror_repo(owner, name, &format!("/unused-{owner}-{name}"), None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkP2TailReaderAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + state + .repo_store + .init(&rec.owner_did, &rec.name) + .await + .unwrap(); + (state, log) + } + + const P2_PUSHER: &str = "did:key:z6MkP2TailPusherAAAAAAAAAAAAAAAAAAAAAA"; + + fn p2_push( + state: &AppState, + owner: &str, + name: &str, + ) -> impl std::future::Future> { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + git_receive_pack( + State(state.clone()), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid(P2_PUSHER.to_string())), + crate::rate_limit::PeerAddr(Some("203.0.113.90:5000".parse::().unwrap())), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + } + + fn p2_logged(log: &std::path::Path, prefix: &str) -> bool { + f2a_log(log).lines().any(|l| l.starts_with(prefix)) + } + + /// #174 (jatmn P2, RED-before/GREEN-after): a client disconnect DURING + /// `guard.release()` must not take the replication tail with it. On a + /// successful push `release` awaits the Tigris upload and then the advisory + /// unlock, both cancellation points, while the pack has already landed on disk. + /// Spawning the tail below `release` means a disconnect in that window drops + /// this push's pins, recovery copy and announce: the F2 dropped-tail class, one + /// step earlier in the handler. + /// + /// Load-bearing: with the spawn below `release` the walk's `for-each-ref` never + /// appears after the disconnect (RED). With it above, gated on + /// `receive_result.is_ok()`, it does (GREEN). + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_tail_survives_a_disconnect_during_release(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let (state, log) = p2_parked_release_state(pool, tmp.path(), "z6p2tail", "t1", None).await; + + let mut fut = Box::pin(p2_push(&state, "z6p2tail", "t1")); + + // Drive until receive-pack has run. Nothing between it and `release` awaits, + // so a future that stops completing after that point is parked on the gate. + let mut ran = false; + for _ in 0..1000 { + let step = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + assert!( + step.is_err(), + "the handler must park inside release, not return" + ); + if p2_logged(&log, "receive-pack") { + ran = true; + break; + } + } + assert!(ran, "the push must reach receive-pack"); + // Settle into the parked state. Whether the tail's walk has already started + // by now is immaterial: pre-fix no tail is ever spawned, because `release` + // never returns, so the marker below can only come from a spawn above it. + for _ in 0..5 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + } + + // The disconnect: drop the handler future while `release` is still awaiting. + drop(fut); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while !p2_logged(&log, "for-each-ref") { + assert!( + std::time::Instant::now() < deadline, + "RED: the pack landed but its replication tail never ran. A disconnect \ + during guard.release() took the tail with the handler future — spawn it \ + above release, gated on receive_result.is_ok()" + ); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + } + + /// The must-not direction of the same reorder. Moving the spawn above + /// `release` moves it above the `?` that used to gate it, so the success check + /// has to be explicit: a FAILED receive-pack must still spawn no tail, or a + /// pusher who aborts a pack mid-transfer gets a half-applied repo pinned and + /// announced on demand. + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_failure_spawns_no_tail_even_when_the_client_disconnects( + pool: sqlx::PgPool, + ) { + // receive-pack fails; everything else the handler or a tail might run is + // logged, so any walk child would show up. + let tmp = tempfile::TempDir::new().unwrap(); + let log = tmp.path().join("git.log"); + let body = format!( + "#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"{}\"\n\ + case \"$1\" in receive-pack) cat >/dev/null 2>/dev/null; exit 1 ;; esac\n\ + exec git \"$@\"\n", + log.display() + ); + let (state, log) = + p2_parked_release_state(pool, tmp.path(), "z6p2fail", "t1", Some(&body)).await; + + let mut fut = Box::pin(p2_push(&state, "z6p2fail", "t1")); + let mut ran = false; + for _ in 0..1000 { + let step = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if p2_logged(&log, "receive-pack") { + ran = true; + // A failed push still parks in `release` (the lock is freed on + // failure too); drop it there, the same disconnect as above. + assert!( + step.is_err(), + "the failed push must still park inside release" + ); + break; + } + } + assert!(ran, "the push must reach receive-pack"); + for _ in 0..5 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + } + drop(fut); + + tokio::time::sleep(std::time::Duration::from_millis(750)).await; + assert!( + !p2_logged(&log, "for-each-ref"), + "a failed receive-pack must spawn no replication tail: pinning and \ + announcing a half-applied repo is exactly what release(false) refuses \ + to upload" + ); + } + /// Scenario 5 (trap 3, fail-closed). On a repo whose withheld walk is failing, a /// coalesced push must not publish. Before the gate moved, every push on such a /// repo got `announce = false` from its own walk; a coalesced push has no walk, so diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 69d1bb33..2677cea2 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -31,6 +31,12 @@ pub struct RepoStore { /// Tracks repos already confirmed to exist in Tigris — avoids redundant /// HEAD checks and background uploads for repos we've already migrated. migrated: Arc>>, + /// Test-only seam: armed here, copied into every `RepoWriteGuard` this store + /// hands out, so a test that only holds the `AppState` (not the guard) can + /// still park `release` at its pre-unlock point. See + /// `RepoWriteGuard::test_pre_unlock_gate`. Never set outside tests. + #[cfg(test)] + pre_unlock_gate: Option>, } impl RepoStore { @@ -41,15 +47,27 @@ impl RepoStore { tigris: None, pool, migrated: Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new())), + pre_unlock_gate: None, } } + /// Test-only: every guard from this store parks in `release` right before the + /// `pg_advisory_unlock` await, until `gate` is notified. Dropping the future + /// while it is parked reproduces a client disconnect inside `release`. + #[cfg(test)] + pub fn with_pre_unlock_gate(mut self, gate: Arc) -> Self { + self.pre_unlock_gate = Some(gate); + self + } + pub fn new(repos_dir: PathBuf, tigris: Option, pool: PgPool) -> Self { Self { repos_dir, tigris, pool, migrated: Arc::new(Mutex::new(HashSet::new())), + #[cfg(test)] + pre_unlock_gate: None, } } @@ -181,7 +199,7 @@ impl RepoStore { released: false, tigris: self.tigris.clone(), #[cfg(test)] - test_pre_unlock_gate: None, + test_pre_unlock_gate: self.pre_unlock_gate.clone(), }; // Acquire the advisory lock with retry, through the guard's OWN connection, diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index 82712c23..32e23dd0 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -415,26 +415,30 @@ fn inv22_ipfs_walk_admission_reaches_every_blocking_site() { } /// #174 U5: the post-receive replication tail is spawned at the DURABILITY BOUNDARY, -/// not at the end of the handler. +/// which is the moment receive-pack returns success, not the end of the handler and +/// not after `guard.release()`. /// -/// The tail owes this push its pins, recovery copy, and announcements. Spawned below -/// the certificate and webhook work, a client disconnect anywhere in that window took -/// the tail with it. Spawned above it, the tail is already detached and survives. +/// The tail owes this push its pins, recovery copy, and announcements. Everything +/// below the spawn stays in the cancellable request future, so anything the tail is +/// spawned after is a window where a client disconnect drops that work while the pack +/// is already durable on disk. `guard.release()` is such a window: on success it +/// awaits the Tigris upload and then the advisory unlock. /// -/// The lower bound matters just as much as the upper one: `guard.release()` runs on -/// failure too, so a spawn anchored there would fire for a push git rejected, pinning -/// and announcing a half-applied repo. The `?` on `receive_result` is the first point -/// where the push is known to have landed, which is why the spawn must sit between -/// that `?` and `touch_repo`. +/// The lower bound matters just as much as the upper one: `release` runs on failure +/// too, so an ungated spawn would fire for a push git rejected, pinning and announcing +/// a half-applied repo. Above `release` the `?` on `receive_result` can no longer be +/// what gates it, so the success check is explicit and this gate binds it: the spawn +/// must sit inside `if push_succeeded`, and `release` must consume the same flag so +/// the two cannot drift apart. /// -/// This is an ordering check rather than a cancellation-race test on purpose: forcing -/// a disconnect to land inside the cert/webhook window is timing-dependent and would -/// be flaky, while the ordering is exactly what the fix changes. Same instrument the -/// F3 gate above uses. +/// This is an ordering check rather than a cancellation-race test on purpose: it is +/// the companion to `receive_pack_tail_survives_a_disconnect_during_release`, which +/// drives the actual disconnect through a parked `release`. Same instrument the F3 +/// gate above uses. /// -/// MUTATION (RED): move the `tokio::spawn(post_receive_replication_tail` call back -/// below the webhook block and the ordering assertion fails; move it above the `?` -/// and the failed-push assertion fails. +/// MUTATION (RED): move the `tokio::spawn(post_receive_replication_tail` call below +/// `guard.release(` and the ordering assertion fails; take it out of the +/// `if push_succeeded` block and the failed-push assertion fails. #[test] fn inv22_replication_tail_spawns_at_the_durability_boundary() { let repos = src("api/repos.rs"); @@ -444,12 +448,20 @@ fn inv22_replication_tail_spawns_at_the_durability_boundary() { .next() .expect("split always yields a first chunk"); - let receive_ok = production - .find("let result = receive_result.map_err(") - .expect("U5 gate stale: git_receive_pack no longer maps receive_result with ?"); + let success_flag = production + .find("let push_succeeded = receive_result.is_ok();") + .expect( + "U5 gate missing: the tail's success gate must be bound from receive_result.is_ok()", + ); + let gate_open = production + .find("if push_succeeded {") + .expect("U5 gate missing: the tail spawn must be gated on the push having succeeded"); let spawn = production .find("tokio::spawn(post_receive_replication_tail(") .expect("U5 gate missing: the replication tail must be spawned by git_receive_pack"); + let release = production + .find("guard.release(push_succeeded)") + .expect("U5 gate stale: release must consume the same success flag as the tail gate"); let touch = production .find("state.db.touch_repo(") .expect("U5 gate stale: git_receive_pack no longer calls touch_repo"); @@ -458,14 +470,20 @@ fn inv22_replication_tail_spawns_at_the_durability_boundary() { .expect("U5 gate stale: git_receive_pack no longer fires push webhooks"); assert!( - receive_ok < spawn, - "U5 gate bypassed: the tail must be spawned AFTER the receive_result `?`, or a \ + success_flag < gate_open && gate_open < spawn, + "U5 gate bypassed: the tail must be spawned inside `if push_succeeded`, or a \ rejected push spawns a tail that pins and announces a half-applied repo" ); + // Still inside that block: no `}` may close it between the gate and the spawn. + assert!( + !production[gate_open + "if push_succeeded {".len()..spawn].contains('}'), + "U5 gate bypassed: the tail spawn left the `if push_succeeded` block, so a \ + rejected push now spawns a tail" + ); assert!( - spawn < touch && spawn < webhook, - "U5 gate bypassed: the tail must be spawned BEFORE touch_repo and the webhook \ - fan-out, so a disconnect in that window cannot drop this push's pins, \ - recovery copy, and announcements" + spawn < release && spawn < touch && spawn < webhook, + "U5 gate bypassed: the tail must be spawned BEFORE guard.release, touch_repo \ + and the webhook fan-out, so a disconnect in any of those windows cannot drop \ + this push's pins, recovery copy, and announcements" ); } From 13af73fc3e55288fc0b411cd42c29431d48b7eb5 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 13:46:33 -0500 Subject: [PATCH 74/95] test(node): record why the handler path cannot probe the stranded-lock case A handler-level "the next acquire_write succeeds" check does not discriminate: it still passes with the RepoWriteGuard::Drop backstop disabled. Measured with the backstop off, the lock is held at the drop and reads free about two seconds later from a session outside the pool. #[sqlx::test] builds its pool with idle_timeout(1s), so the guard's connection is reaped once the handler future drops it, the session ends, and postgres frees the lock with no help from the code under test. acquire_write retries for far longer than that window, so it waits the release out and reports success either way. write_guard_release_cancelled_mid_unlock_frees_the_lock is the real proof: it probes from a connection held out of the pool, 400ms after the drop, inside that window rather than past it. --- crates/gitlawb-node/src/api/repos.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 866df30b..3236fa99 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -8709,6 +8709,19 @@ mod tests { ); tokio::time::sleep(std::time::Duration::from_millis(25)).await; } + + // Not asserted here: that the same disconnect leaves the session advisory + // lock free. A handler-level "the next acquire_write succeeds" probe does not + // discriminate. It still passes with the `Drop` backstop disabled, because the + // guard's `PoolConnection` goes back to the pool when this future is dropped + // and `#[sqlx::test]` builds the pool with `idle_timeout(1s)`, so the session + // ends on its own a beat later and postgres frees the lock with no help from + // the code under test. Measured with the backstop disabled: held at the drop, + // free ~2s later, observed from a session outside the pool. `acquire_write` + // retries for far longer than that, so it waits the release out and reports + // success either way. `write_guard_release_cancelled_mid_unlock_frees_the_lock` + // is the real proof: it probes from a connection held OUT of the pool, 400ms + // after the drop, which is inside that window rather than past it. } /// The must-not direction of the same reorder. Moving the spawn above From a9d8906fa03415a749aa2927aedb948eb3cf829d Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 14:31:51 -0500 Subject: [PATCH 75/95] docs(node): record the Tigris ordering residual the tail reorder introduces Spawning the replication tail above guard.release() also moves it above the Tigris upload, so a ref can be announced while the shared durable copy is still the old one, and a disconnect there cancels the upload while the detached tail still pins and announces. Accepted rather than fixed. Upload-then-announce was never guaranteed: release tolerates a failed upload by design, warning and continuing to the unlock, so an announce over a stale Tigris copy was already reachable before this reorder, and it self-heals through acquire_fresh's local fallback plus the next push. The alternative, detaching release and the tail together, would return 200 before the durable copy lands, which changes the client contract more than the window it closes. --- crates/gitlawb-node/src/api/repos.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 3236fa99..c60f8c19 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1829,6 +1829,19 @@ pub async fn git_receive_pack( // the gossip event carries `cert_id: None` regardless, so no announce consumer // reads a certificate out of it. Each push owns its own tail, including its own // always-spawned announce, so per-push announcements are never coalesced away. + // + // ACCEPTED RESIDUAL, and it is the cost of this ordering: the tail also runs + // concurrently with the Tigris upload inside `release` below, where before it ran + // after. So a ref can be announced while the shared durable copy is still the old + // one, and on a disconnect here the upload is cancelled outright while the + // detached tail still pins and announces. What makes that acceptable is that + // upload-then-announce was never actually guaranteed: `release` tolerates a failed + // upload by design (it warns and continues to the unlock), so an announce over a + // stale Tigris copy was already reachable before this reorder, and it self-heals, + // since `acquire_fresh` falls back to the local copy and the next push re-uploads. + // The alternative, detaching `release` and the tail together to keep the ordering, + // would return 200 to the pusher before the durable copy lands, which is a larger + // change to the client contract than the window it closes. let push_succeeded = receive_result.is_ok(); if push_succeeded { tokio::spawn(post_receive_replication_tail( From 4f3278f28c69777471bb12c95ce97c0834c39422 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 21:44:28 -0500 Subject: [PATCH 76/95] fix(node): share one deadline across the withheld walk and the pack serve (#62) The path-scoped git-upload-pack branch spent git_service_timeout_secs twice: once on the withheld-blob classification walk, then a second full budget on the serve. One clone could hold a read-pool permit for ~2x the configured budget, which is the occupancy hole this PR otherwise exists to close. Thread one Instant deadline through both phases, mirroring fail_closed_full_scan_objects and build_filtered_pack: the walk runs against the remainder and the serve gets what the walk left, computed after the blocking task's await so it cannot pick up a fresh budget. A walk that consumes the whole budget saturates the serve to zero, which surfaces as GitServiceTimeout -> 504 rather than running unbounded. Under-serving one slow clone is the safe direction versus over-holding the pool. Both serve arms are covered because a fix that reached only the plain arm would leave the ~2x hold reachable by any clone of a repo that actually withholds a blob. Each test was observed RED before the change (status 200, the serve completing on a fresh budget) and GREEN after, and reverting the filtered arm alone flips only its own test, so neither guard is a proxy for the other. The knob's meaning changed for this path, so .env.example now says the value bounds the two phases combined rather than each separately. --- .env.example | 5 + crates/gitlawb-node/src/api/repos.rs | 192 ++++++++++++++++++++++++++- 2 files changed, 194 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index 5ed087d9..a4d66a7a 100644 --- a/.env.example +++ b/.env.example @@ -115,6 +115,11 @@ GITLAWB_MAX_PACK_BYTES=2147483648 # worker and, on push, the repo write lock. Also bounds both info/refs # advertisements, the withheld-blob pack build, and the push-side candidate # discovery (rev-list / cat-file), all reaped via process-group teardown (#174). +# On the path-scoped upload-pack path the withheld-blob classification walk and +# the pack serve share ONE deadline, so this value bounds their COMBINED duration +# rather than giving each stage a full budget: a walk that consumes it leaves the +# serve nothing and the clone gets a 504. Serving large path-scoped repos may +# need a higher value here than when each stage was budgeted separately. # Default 600. GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600 diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index c60f8c19..aedf03e6 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1361,6 +1361,16 @@ pub async fn git_upload_pack( // task hands the permits back so the serve phase below keeps them; on a // dropped future the returned tuple (with the permits) is discarded only when // the blocking task completes, so admission tracks the real git work. + // ONE deadline spans the walk AND the serve below (#174 U1 follow-up). A fresh + // `git_timeout` for the serve let a slow-but-successful walk plus a full serve + // hold this read permit ~2x the configured budget. Sharing the deadline caps + // that at ~1x: the walk runs against the remaining budget, and if it consumes + // the budget the serve gets what is left and is reaped rather than over-holding + // — the safe direction, same tradeoff as `fail_closed_full_scan_objects` and + // `build_filtered_pack`. The cost is honest: a genuinely slow walk on a large + // repo 504s this clone instead of silently holding the pool for ~2x, so size + // `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` so both phases normally fit. + let deadline = std::time::Instant::now() + git_timeout; let (withheld, _permit, _caller_permit) = { let path = disk_path.clone(); let rules = rules.clone(); @@ -1368,11 +1378,12 @@ pub async fn git_upload_pack( let caller_owned = caller.map(str::to_string); let is_public = record.is_public; let git_bin = state.git_bin.clone(); + let walk_budget = deadline.saturating_duration_since(std::time::Instant::now()); tokio::task::spawn_blocking(move || { let withheld = visibility_pack::withheld_blob_oids_bounded( &path, &git_bin, - git_timeout, + walk_budget, &rules, is_public, &owner_did, @@ -1393,10 +1404,22 @@ pub async fn git_upload_pack( // The handler keeps no copy (F1: handler-local permits would drop the // instant a disconnect drops this future, mid-reap). let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + // Computed AFTER the walk's await, so the serve gets what the walk left, not a + // second full budget. A walk that consumed the whole budget saturates this to + // zero, which the serve surfaces as GitServiceTimeout -> 504 rather than + // running unbounded. + let serve_budget = deadline.saturating_duration_since(std::time::Instant::now()); if withheld.is_empty() { // No blobs to withhold: serve the plain pack (the walk already held the // permits per be0cdd6; the guard hands them to the serve). - smart_http::upload_pack(&state.git_bin, &disk_path, body, git_timeout, Some(admission)).await + smart_http::upload_pack( + &state.git_bin, + &disk_path, + body, + serve_budget, + Some(admission), + ) + .await } else { tracing::info!(repo = %name, caller = ?caller, withheld = withheld.len(), "serving filtered pack"); // The guard threads through both filtered-pack stages (rev-list, then @@ -1407,7 +1430,7 @@ pub async fn git_upload_pack( &disk_path, body, &withheld, - git_timeout, + serve_budget, Some(admission), ) .await @@ -3332,6 +3355,169 @@ mod tests { ); } + /// #174 U1 follow-up (RED-before/GREEN-after): the path-scoped upload-pack branch + /// shares ONE deadline across the withheld-blob walk and the pack serve, so one + /// clone cannot hold a read permit for ~2x `git_service_timeout_secs`. A walk that + /// consumes most of the budget must leave the serve only the REMAINDER, so the + /// serve is reaped and the request is a 504. + /// + /// Load-bearing: give the serve a fresh `git_timeout` instead of the remainder and + /// the fake `upload-pack` (1.2s) fits inside a fresh 2s budget, completes, and the + /// status is no longer 504 (RED). This is the ~2x-budget hold the unit removes. + /// + /// Plain-serve arm: the fake git lists no refs and fails `rev-parse`, so the walk + /// yields an empty withheld set and the branch takes `upload_pack`. `rev-list` + /// carries the walk's cost. + #[cfg(unix)] + #[sqlx::test] + async fn upload_pack_shares_one_deadline_across_walk_and_plain_serve(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + // Walk: no refs (for-each-ref empty), HEAD does not resolve (rev-parse exit 1), + // rev-list burns 1.2s of the 2s budget and lists no commits -> empty withheld. + // Serve: upload-pack needs 1.2s, which does NOT fit the ~0.8s remainder but + // WOULD fit a fresh 2s budget. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-parse) exit 1 ;;\n rev-list) sleep 1.2 ;;\n upload-pack) sleep 1.2 ;;\n *) : ;;\nesac\nexit 0\n"; + let fake = write_fake_git(tmp.path(), body); + + let mut state = crate::test_support::test_state(pool).await; + state.git_bin = fake; + let mut cfg = (*state.config).clone(); + cfg.git_service_timeout_secs = 2; + state.config = std::sync::Arc::new(cfg); + state + .db + .upsert_mirror_repo("z6shared1", "sv", "/tmp/z6shared1-sv", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6shared1", "sv").await.unwrap().unwrap(); + // Path-scoped rule so has_path_scoped_rule() is true and the walk runs. + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &[], + OWNER_DID, + ) + .await + .unwrap(); + let disk = std::path::Path::new("/tmp/z6shared1/sv.git"); + std::fs::create_dir_all(disk).unwrap(); + + let peer: SocketAddr = "203.0.113.92:7000".parse().unwrap(); + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::POST) + .uri("/z6shared1/sv/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let status = router.oneshot(req).await.unwrap().status(); + let _ = std::fs::remove_dir_all("/tmp/z6shared1"); + assert_eq!( + status, + StatusCode::GATEWAY_TIMEOUT, + "the walk and the serve must share ONE deadline, so a walk that burns most \ + of the budget leaves the serve only the remainder and the serve is reaped; \ + a non-504 here means the serve got a fresh full budget (the ~2x hold)" + ); + } + + /// #174 U1 follow-up, FILTERED arm (RED-before/GREEN-after): the shared deadline + /// must reach `upload_pack_excluding` too, not just the plain `upload_pack`. Same + /// property as the plain-arm test, different serve function, because the branch + /// threads the remainder into both arms and a fix that missed one would leave the + /// ~2x hold reachable by any clone of a repo that actually withholds something. + /// + /// The fake git yields a NON-EMPTY withheld set: one ref peeling to a commit, a + /// resolvable HEAD, one commit, and an `ls-tree -rz` record placing a blob under + /// `/secret/`, which the path-scoped rule denies to an anonymous caller. `ls-tree` + /// carries the walk's cost (walk-only), and `pack-objects` carries the serve's, so + /// the two phases are independently attributable. + /// + /// Load-bearing: hand the serve a fresh `git_timeout` and `pack-objects` (1.2s) + /// fits a fresh 2s budget, completes, and the status is no longer 504 (RED). + #[cfg(unix)] + #[sqlx::test] + async fn upload_pack_shares_one_deadline_across_walk_and_filtered_serve(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + let commit = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + let blob = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + // for-each-ref lists one ref; cat-file peels it to a commit (the fail-closed + // ref check); rev-parse resolves HEAD; rev-list lists the one commit; ls-tree + // emits " blob \t" (NUL-delimited) under secret/ and burns + // 1.2s of the 2s budget; pack-objects is the serve's 1.2s cost. + let body = format!( + "#!/bin/sh\ncase \"$1\" in\n \ + for-each-ref) echo refs/heads/main ;;\n \ + cat-file) echo commit ;;\n \ + rev-parse) echo {commit} ;;\n \ + rev-list) echo {commit} ;;\n \ + ls-tree) printf '100644 blob {blob}\\tsecret/f.txt' ; sleep 1.2 ;;\n \ + pack-objects) sleep 1.2 ;;\n \ + *) : ;;\nesac\nexit 0\n" + ); + let fake = write_fake_git(tmp.path(), &body); + + let mut state = crate::test_support::test_state(pool).await; + state.git_bin = fake; + let mut cfg = (*state.config).clone(); + cfg.git_service_timeout_secs = 2; + state.config = std::sync::Arc::new(cfg); + state + .db + .upsert_mirror_repo("z6shared2", "sv", "/tmp/z6shared2-sv", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6shared2", "sv").await.unwrap().unwrap(); + // Denies /secret/** to an anonymous caller, so the blob above is withheld and + // the branch takes upload_pack_excluding rather than the plain serve. + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &[], + OWNER_DID, + ) + .await + .unwrap(); + let disk = std::path::Path::new("/tmp/z6shared2/sv.git"); + std::fs::create_dir_all(disk).unwrap(); + + let peer: SocketAddr = "203.0.113.93:7000".parse().unwrap(); + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::POST) + .uri("/z6shared2/sv/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let status = router.oneshot(req).await.unwrap().status(); + let _ = std::fs::remove_dir_all("/tmp/z6shared2"); + assert_eq!( + status, + StatusCode::GATEWAY_TIMEOUT, + "the FILTERED serve must also take the shared deadline's remainder; a \ + non-504 here means upload_pack_excluding got a fresh full budget and the \ + ~2x hold is still reachable whenever a repo withholds a blob" + ); + } + /// #174 (F2 sizing edge, vetted by execution): the receive-pack advertisement /// per-source cap is derived in main.rs as `(max_concurrent_git_pushes / 8).max(1)`, /// so it is never 0 even at the minimum write-pool size (1). A 0 cap would make From 6b76600755e184cedb19db0a443876060e82d039 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 21:44:53 -0500 Subject: [PATCH 77/95] docs(node): correct stale admission-bound comments and document derived caps (#62) Four prose corrections on this PR's surfaces, no behavior change. The receive-pack advertisement no longer draws from the write pool, it draws from a dedicated advert pool, but five comments still described the old shape: the info/refs per-source block, the cap construction in main.rs, the State field doc, and a test doc plus its assertion message. The state.rs and main.rs ones mattered most because they contradicted the operator text being written in this same commit. The /ipfs probe and content-read subprocesses each run under their own deadline now (the lesser of git_service_timeout_secs and the remaining budget, reaped by process-group teardown), but four surfaces still claimed they had no duration bound past the pre-start budget check. The correction stays scoped rather than declaring the route safe: the probe's object_store_readable check is a synchronous filesystem sweep with nothing to reap, so a wedged filesystem can still hold the walk slot past the deadline, and that residual is now stated where the old warning used to be. A grep for the full phrase misses two of these surfaces, since one wraps across doc-comment lines and one says "duration clamp" instead, so the sweep uses the fragment. Both push-side per-source concurrency caps are derived as max_concurrent_git_pushes / 8 with a floor of 1 and have no environment variable, so neither was documented anywhere an operator reads. The write-side one is load-bearing for push availability: it is acquired before the global write permit, and owner enforcement defaults off, so without it one host minting disposable did:key identities could monopolize the write pool. They are described in .env.example beside the read cap rather than in README's table, which is keyed by variable name. The hung-walk assertion in visibility_pack printed {err}, showing only the top anyhow context and dropping the underlying io error. A beta-lane failure surfaced as "failed to spawn git for-each-ref" with no errno, which made it undiagnosable. {err:#} prints the chain. The underlying flake is still undiagnosed: it did not reproduce in 123 local runs and the stable lane passed on the same commit, so this only improves the next occurrence. --- .env.example | 26 ++++++++++++++++--- crates/gitlawb-node/src/api/ipfs.rs | 21 ++++++++++----- crates/gitlawb-node/src/api/repos.rs | 21 ++++++++------- crates/gitlawb-node/src/config.rs | 9 ++++--- .../gitlawb-node/src/git/visibility_pack.rs | 5 +++- crates/gitlawb-node/src/main.rs | 8 +++--- crates/gitlawb-node/src/state.rs | 13 ++++++---- 7 files changed, 71 insertions(+), 32 deletions(-) diff --git a/.env.example b/.env.example index a4d66a7a..aea17f61 100644 --- a/.env.example +++ b/.env.example @@ -175,6 +175,23 @@ GITLAWB_MAX_CONCURRENT_PIN_TASKS=8 # Default 16. GITLAWB_MAX_CONCURRENT_READS_PER_CALLER=16 +# Two further per-source concurrency caps exist on the PUSH side but have NO +# environment variable of their own: both are DERIVED as +# GITLAWB_MAX_CONCURRENT_GIT_PUSHES / 8, with a floor of 1. They are not settable +# independently, so raising the push pool raises both proportionally, and the +# default push pool of 32 gives each a cap of 4. +# * The anon receive-pack info/refs advertisement cap: one source IP may hold at +# most this many slots in the dedicated advert pool, so saturating that pool +# takes ~8 distinct source IPs. +# * The authenticated receive-pack POST cap: one source IP may hold at most this +# many WRITE-pool slots. This one is load-bearing for push availability, since +# it is acquired before the global write permit: without it, one host minting +# disposable did:key identities could open enough slow pushes to monopolize the +# write pool and 503 every other source (owner enforcement defaults off, and +# the push rate limiter caps arrival rate, not in-flight concurrency). +# Keyed on the resolved source IP, never the DID, so a DID farm does not defeat +# them; keying granularity follows GITLAWB_TRUSTED_PROXY like the read cap above. + # ── Push rate limiting (git-receive-pack flood brake) ───────────────────── # Max receive-pack requests (info/refs advertisement + push POST) per client # IP per hour. 0 disables. Default 600. @@ -207,9 +224,12 @@ GITLAWB_IPFS_MAX_REPO_VISITS=1024 # lifetime (all stages of the whole scan). Per-stage clamps bound the acquire # and walk stages to the remaining budget, and no stage starts once it is # exhausted (the scan then sheds a retryable 503). The object-type probe and -# content-read cat-file subprocesses are budget-checked before starting but -# have NO duration bound of their own: a hung git probe (corrupt pack, stuck -# filesystem) holds the request's walk slot for the full duration of the hang. +# content-read cat-file subprocesses are budget-checked before starting AND each +# run under their own deadline (the lesser of GITLAWB_GIT_SERVICE_TIMEOUT_SECS and +# the remaining budget), reaped via process-group teardown, so a hung cat-file +# cannot hold the request's walk slot past it. One hang path is still unbounded: +# the probe's object-store readability check is a plain filesystem sweep with +# nothing to reap, so a wedged filesystem can hold the slot past the deadline. # Default 600. GITLAWB_IPFS_REQUEST_BUDGET_SECS=600 # Max /ipfs/{cid} requests per client IP per hour (route flood brake, distinct diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index d9f4c01b..f5420324 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -71,9 +71,16 @@ use crate::visibility::{visibility_check, Decision}; /// Request budget (F3): one absolute clock (`ipfs_request_budget_secs`) spans /// the whole admitted request. No stage (acquire, probe, walk, content read) /// starts once it is exhausted, and the acquire wait and walk deadline are -/// clamped to the remainder. The probe and content-read subprocesses have no -/// duration bound of their own past their pre-start budget check, so a hung -/// git probe holds the request's walk slot for the full duration of the hang. +/// clamped to the remainder. The probe and content-read subprocesses each ALSO +/// run under their own deadline, the lesser of `git_service_timeout_secs` and the +/// remaining budget, reaped by process-group teardown at that deadline, so a hung +/// `cat-file` cannot hold the request's walk slot past it. +/// +/// Residual, still true: the probe's object-store readability check +/// (`store::object_store_readable`, reached on the `missing` branch that a +/// random-CID spray drives) is a synchronous `read_dir` + `File::open` sweep with +/// no deadline and nothing to reap, so a wedged filesystem can still hold the +/// walk slot past the deadline. Same class as the D-state git survivor residual. /// /// Scope: this closes the direct unauthenticated scan, including the dangling /// case. A stale-public mirror row still serves withheld content (tracked @@ -274,10 +281,10 @@ pub async fn get_by_cid( // knob and tainting "budget"; the call site only breaks (the scan STOPS, // leaving this and every later candidate unproven, never a false 404). A // stage is never started with zero remaining; the probe and read - // subprocesses carry no internal duration clamp, so this pre-start check is - // their entire bound (a hung one holds the request's walk slot for the - // duration of the hang). The acquire and walk stages clamp their deadlines - // to the returned remainder. + // subprocesses additionally carry their own deadline (the lesser of + // git_service_timeout_secs and the remainder, reaped by process-group + // teardown), so this pre-start check is a gate, not their entire bound. The + // acquire and walk stages clamp their deadlines to the returned remainder. fn budget_gate( truncated_by: &mut Vec<&'static str>, deadline: std::time::Instant, diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index aedf03e6..3244bc29 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -647,11 +647,12 @@ pub async fn git_info_refs( // acquired AFTER the visibility + push-rate gates (KTD7) so a denied or // rate-limited request never consumes a slot; held for the whole op. The // upload-pack advertisement is bounded on the read pool (git_read_per_caller). - // The receive-pack advertisement draws from the write pool, so it is bounded per - // source (git_push_advert_per_caller) instead: without this, an anonymous - // multi-source flood of push-handshake advertisements could hold the write pool's - // slots across acquire_fresh and shed authenticated pushes, since the per-IP push - // rate limiter caps rate, not concurrency (#174 review fix). + // The receive-pack advertisement draws from its own dedicated advert pool + // (git_push_advert_semaphore, see the _permit block below), so it is bounded per + // source by git_push_advert_per_caller instead: without this, an anonymous + // multi-source flood of push-handshake advertisements could hold every advert-pool + // slot across acquire_fresh and shed other sources' advertisements, since the + // per-IP push rate limiter caps rate, not concurrency (#174 review fix). let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); let _caller_permit = if service == "git-receive-pack" { acquire_read_caller_permit( @@ -4316,10 +4317,10 @@ mod tests { ); } - /// #174 (review fix): the anon-reachable receive-pack advertisement draws from - /// the write pool, so it is bounded per source by `git_push_advert_per_caller` to - /// stop one source from monopolizing the write pool and shedding authenticated - /// pushes. Fill one source IP's advert slot; its next receive-pack advertisement + /// #174 (review fix): the anon-reachable receive-pack advertisement draws from its + /// own dedicated advert pool, so it is bounded per source by + /// `git_push_advert_per_caller` to stop one source from monopolizing that pool and + /// shedding other sources' advertisements. Fill one source IP's advert slot; its next receive-pack advertisement /// sheds 503, while a different source and the upload-pack advertisement are /// unaffected. Remove the advert-cap acquisition and the same-source assertion /// goes green-not-503. @@ -4358,7 +4359,7 @@ mod tests { assert_eq!( router.oneshot(req).await.unwrap().status(), StatusCode::SERVICE_UNAVAILABLE, - "a source at its receive-pack advertisement cap must shed 503, so it cannot monopolize the write pool" + "a source at its receive-pack advertisement cap must shed 503, so it cannot monopolize the advert pool" ); // A DIFFERENT source keeps its own advert budget -> not shed. diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index d1b98807..5ba10584 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -470,9 +470,12 @@ pub struct Config { /// only the acquire and walk stages (overshoot there is the walk watchdog's /// SIGTERM grace + SIGKILL settle); the `object_type` / /// `read_object_content` probe subprocesses are budget-checked before they - /// start but have NO duration bound of their own, so a hung git probe - /// (corrupt pack, stuck filesystem) holds the request's walk slot for the - /// full duration of the hang. + /// start AND each run under their own deadline (the lesser of + /// `git_service_timeout_secs` and the remaining budget), reaped by + /// process-group teardown, so a hung `cat-file` cannot hold the request's walk + /// slot past it. Still unbounded: the probe's `object_store_readable` check is a + /// synchronous filesystem sweep with nothing to reap, so a wedged filesystem can + /// hold the slot past the deadline. /// Must be positive. Default: 600s (10 min), matching /// `git_service_timeout_secs` so a single full-length walk still fits. #[arg( diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index e69c4ceb..2d94b84d 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -801,7 +801,10 @@ mod tests { assert!( err.downcast_ref::() .is_some(), - "a hung walk must abort with GitServiceTimeout (mapped to 504), got: {err}" + // `{err:#}` prints the whole anyhow chain. Plain `{err}` shows only the top + // context ("failed to spawn git for-each-ref") and drops the underlying io + // error, which left a real beta-lane CI failure undiagnosable. + "a hung walk must abort with GitServiceTimeout (mapped to 504), got: {err:#}" ); // The recorded process-group leader must be gone: the watchdog reaps the diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 1fd26979..5843d0ab 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -420,9 +420,11 @@ async fn main() -> Result<()> { config.max_concurrent_reads_per_caller, ), // Per-source cap on the receive-pack advertisement, sized to an eighth of the - // write pool (min 1): a single source can hold at most this many write-pool - // slots via the anon advertisement, so saturating the pool takes ~8 distinct - // source IPs, each also rate-limited (#174). + // write pool (min 1): a single source can hold at most this many slots in the + // DEDICATED advert pool (git_push_advert_semaphore, disjoint from the write + // pool), so saturating that pool takes ~8 distinct source IPs, each also + // rate-limited (#174). Sized off the write pool only because the advert pool + // is created at the same size; an advert flood cannot touch a write permit. git_push_advert_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( (config.max_concurrent_git_pushes / 8).max(1), ), diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 1f0f8c81..77755932 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -169,11 +169,14 @@ pub struct AppState { /// advertisement. pub git_read_per_caller: crate::rate_limit::PerCallerConcurrency, /// Per-source concurrency sub-cap on the anon-reachable receive-pack `info/refs` - /// advertisement: each source IP may hold at most a small share of the write - /// pool, so a multi-source flood of push-handshake advertisements cannot - /// saturate `git_write_semaphore` and shed authenticated pushes (#174). Sized as - /// a fraction of `max_concurrent_git_pushes`, so filling the write pool takes many - /// distinct source IPs (each also braked by the per-IP push rate limiter). + /// advertisement: each source IP may hold at most a small share of the DEDICATED + /// advert pool (`git_push_advert_semaphore`), so a multi-source flood of + /// push-handshake advertisements cannot saturate that pool and shed other sources' + /// advertisements (#174). An advert flood cannot reach `git_write_semaphore` at + /// all, since the two pools are disjoint. Sized as a fraction of + /// `max_concurrent_git_pushes` because the advert pool is created at the same size, + /// so filling it takes many distinct source IPs (each also braked by the per-IP + /// push rate limiter). pub git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency, /// Per-source concurrency sub-cap on the authenticated `git-receive-pack` POST: /// each source IP may hold at most a small share of `git_write_semaphore`, so one From 701a5e3428e1c1e758147d8bb63bbe41c272da74 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 21:48:49 -0500 Subject: [PATCH 78/95] docs: normalize README punctuation and record the two admission doc changes (#62) Two table rows needed correcting for the changes in this PR: the GITLAWB_GIT_SERVICE_TIMEOUT_SECS row now says the classification walk and the pack serve share one deadline on the path-scoped upload-pack path, so the value bounds them combined; and the GITLAWB_IPFS_REQUEST_BUDGET_SECS row no longer claims the probe and content-read subprocesses are unbounded, while keeping the object-store-readability sweep named as a live residual. The other thirteen changed lines are punctuation only, no wording changes: em dashes replaced with colons, commas, or sentence breaks, and one pair of curly quotes straightened. The repo's prose-style hook validates a whole file rather than a diff, so these pre-existing characters were rejecting every edit to README.md, including the two rows above. --- README.md | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 6789cd80..f4ba0f9e 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Gitlawb Node is the open-source node software behind the Gitlawb network. It lets anyone run a self-hosted node, publish repositories under a DID, sign writes with Ed25519 HTTP signatures, replicate git activity across peers, and move toward a resilient app-delivery network where code and build assets can be served closer to users. -Gitlawb is not trying to be only “another git host.” The long-term direction is: +Gitlawb is not trying to be only "another git host." The long-term direction is: ```txt Decentralized GitHub @@ -34,12 +34,12 @@ This is a Rust workspace with four crates: Most git hosting today depends on a small number of centralized platforms. Gitlawb Nodes are designed for a different model: -- **Own your identity** — every user, agent, and node is an Ed25519 keypair represented as `did:key:z6Mk...`. -- **Signed writes by default** — write requests use RFC 9421 HTTP Signatures instead of passwords. -- **Git-native transport** — repositories are still real git repositories served over smart HTTP. -- **Agent-native workflows** — the `gl` CLI and MCP server expose repo, issue, task, PR, and UCAN flows to AI agents. -- **Peer-aware delivery** — nodes can announce, discover, gossip, and sync with each other. -- **App CDN direction** — the network can evolve from decentralized code storage into code + asset + app delivery. +- **Own your identity**: every user, agent, and node is an Ed25519 keypair represented as `did:key:z6Mk...`. +- **Signed writes by default**: write requests use RFC 9421 HTTP Signatures instead of passwords. +- **Git-native transport**: repositories are still real git repositories served over smart HTTP. +- **Agent-native workflows**: the `gl` CLI and MCP server expose repo, issue, task, PR, and UCAN flows to AI agents. +- **Peer-aware delivery**: nodes can announce, discover, gossip, and sync with each other. +- **App CDN direction**: the network can evolve from decentralized code storage into code + asset + app delivery. --- @@ -189,14 +189,14 @@ For public-network use, make sure `GITLAWB_NODE` points to the node you want. Th Public nodes (e.g. `node.gitlawb.com`) require two things on writes: -1. **RFC 9421 HTTP Signatures** — every write is signed by your identity key. `gl` +1. **RFC 9421 HTTP Signatures**: every write is signed by your identity key. `gl` and the `git-remote-gitlawb` helper do this automatically. An old/unsigned CLI fails with `401 not_an_agent`; `gl` will tell you to upgrade and register. 2. **An iCaptcha proof** on the spam-gated writes (**repo create, fork, register**). `gl` solves this for you: on the node's `403 icaptcha_proof_required` it reads the `x-icaptcha-url` / `x-icaptcha-level` hints, requests a challenge, solves it locally (arithmetic / algebra / sequence), and **retries the same signed request** - with the `x-icaptcha-proof` header — no manual steps, no env vars. + with the `x-icaptcha-proof` header. No manual steps, no env vars. ```bash gl identity new # create did:key identity @@ -215,14 +215,14 @@ Notes: - **Proofs are short-lived (~5 min TTL) and single-use.** If one expires between solving and use, the client transparently solves a fresh one and retries. - **What needs what:** create / fork / register are signed **and** iCaptcha-gated; - `git push` is **signed-only** (owner signature is the gate — no per-push challenge); + `git push` is **signed-only** (owner signature is the gate, no per-push challenge); reads (clone / fetch / `repo info`) need no proof. A non-existent repo returns a clear `404`, never a placeholder. - **API-key iCaptcha deployments:** set `GITLAWB_ICAPTCHA_URL` to your iCaptcha origin and `GITLAWB_ICAPTCHA_API_KEY` to its key. The client only talks to an `https` origin whose host is allowlisted (that URL or the public default), and - sends the bearer token **only** to your configured origin — never to a URL a - node advertises — so a hostile node can't capture the key or redirect the solve. + sends the bearer token **only** to your configured origin, never to a URL a + node advertises, so a hostile node can't capture the key or redirect the solve. --- @@ -343,14 +343,14 @@ Important node settings: | `GITLAWB_REQUIRE_SIGNED_PEER_WRITES` | Require signed peer announce/sync writes. | | `GITLAWB_AUTO_SYNC` | Enable automatic sync from known peers. | | `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. | -| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. | +| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. On the path-scoped upload-pack path the classification walk and the pack serve share ONE deadline, so this value bounds their combined duration rather than granting each stage a full budget: a walk that consumes it leaves the serve nothing and the clone gets a 504. Serving large path-scoped repos may therefore need a higher value than they did when each stage was budgeted separately. | | `GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS` | Max seconds the storage-acquisition phase (Tigris HEAD/GET, push advisory-lock) of a served git op may run before the request is shed with a 503, separate from the git-run timeout. The concurrency permit is released on expiry so a stalled backend cannot pin the pool. Default 30. | | `GITLAWB_REPO_LEASE_MAX_WAITERS` | Max pushes parked at once waiting for the same repo's write lease. Each waiter pins its buffered pack body, so this bounds that memory for a hot repo; past the cap the newest push sheds a 503 + Retry-After instead of queueing. Pushes to other repos are unaffected, and the lease holder is not counted. Default 8. | | `GITLAWB_MAX_CONCURRENT_IPFS_WALKS` | Max concurrent `GET /ipfs/{cid}` visibility walks across all callers (own pool, disjoint from the served-git pools); over-cap sheds 503. Default 32. | | `GITLAWB_IPFS_WALK_PER_SOURCE` | Max concurrent `/ipfs` walks a single source IP may hold. Default 4. | | `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max expensive path-scope visibility walks per `/ipfs/{cid}` request; over-cap repos are skipped and the scan continues, shedding a retryable 503 (not a false 404) if the object is then found nowhere. Default 64. | -| `GITLAWB_IPFS_MAX_REPO_VISITS` | Ceiling on repos one `/ipfs/{cid}` request may visit (acquire + probe) past the visibility gate — also the worst-case per-request Tigris fetch count. On exhaustion the scan stops with a retryable 503. Default 1024. | -| `GITLAWB_IPFS_REQUEST_BUDGET_SECS` | Absolute wall-clock budget for one admitted `/ipfs/{cid}` request's acquire+walk lifetime. Per-stage clamps bound the acquire and walk stages to the remaining budget, and no stage starts once it is exhausted; the scan then stops with a retryable 503. The object-type probe and content-read `cat-file` subprocesses are budget-checked before starting but have no duration bound of their own, so a hung git probe (corrupt pack, stuck filesystem) holds the request's walk slot for the full duration of the hang. Default 600. | +| `GITLAWB_IPFS_MAX_REPO_VISITS` | Ceiling on repos one `/ipfs/{cid}` request may visit (acquire + probe) past the visibility gate. Also the worst-case per-request Tigris fetch count. On exhaustion the scan stops with a retryable 503. Default 1024. | +| `GITLAWB_IPFS_REQUEST_BUDGET_SECS` | Absolute wall-clock budget for one admitted `/ipfs/{cid}` request's acquire+walk lifetime. Per-stage clamps bound the acquire and walk stages to the remaining budget, and no stage starts once it is exhausted; the scan then stops with a retryable 503. The object-type probe and content-read `cat-file` subprocesses are budget-checked before starting and each also run under their own deadline (the lesser of `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` and the remaining budget), reaped via process-group teardown, so a hung `cat-file` cannot hold the request's walk slot past it. One hang path is still unbounded: the probe's object-store readability check is a plain filesystem sweep with nothing to reap, so a wedged filesystem can hold the slot past the deadline. Default 600. | | `GITLAWB_IPFS_RATE_LIMIT` | Max `/ipfs/{cid}` requests per client IP per hour (route flood brake). 0 disables. Default 600. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | From 28a6ca4082f2ccff8e0ddac11cb8caf9b2703dff Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 22:11:43 -0500 Subject: [PATCH 79/95] fix(node): charge blocking-queue delay against the shared upload-pack deadline (#62) The shared deadline computed the walk's budget on the async side, before the task was queued on the blocking pool. withheld_blob_oids_bounded starts its own clock when it actually runs, so a budget measured at queue time handed the walk a full budget from whenever the pool got to it: the read permit was held for queue delay PLUS the budget, and only the serve phase saw the outer deadline already spent. Computing the remainder inside the closure charges that delay against the shared deadline, which is what makes the ~1x bound in the docs true rather than approximate. Not a regression from the previous commit (the old code passed a constant git_timeout, so the delay went uncharged there too, and the combined hold was queue delay plus two full budgets), but the claim this PR now prints in README and .env.example should be exact. The two shared-deadline tests still pass and still fail on revert; they cover the budget arithmetic, not the queue-saturation path. Reproducing that needs a runtime with the blocking pool pinned and parked, which the sqlx::test harness does not give a seam for, so the queue-delay path is reasoned, not executed. --- crates/gitlawb-node/src/api/repos.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 3244bc29..de357059 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1379,8 +1379,16 @@ pub async fn git_upload_pack( let caller_owned = caller.map(str::to_string); let is_public = record.is_public; let git_bin = state.git_bin.clone(); - let walk_budget = deadline.saturating_duration_since(std::time::Instant::now()); tokio::task::spawn_blocking(move || { + // Derive the walk's budget from the shared deadline HERE, inside the + // closure, not on the async side before the task is queued. The walk + // starts its own clock when it runs, so a budget computed at queue time + // would hand it a full budget measured from whenever the blocking pool + // got to it, leaving the permit held for queue-delay PLUS the budget. + // Computing it at task start charges that queue delay against the + // shared deadline, which is what makes the ~1x bound true rather than + // approximate. + let walk_budget = deadline.saturating_duration_since(std::time::Instant::now()); let withheld = visibility_pack::withheld_blob_oids_bounded( &path, &git_bin, From 4b69c450070c20d30244f25871a0c1c6632e6947 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Wed, 29 Jul 2026 22:56:47 -0500 Subject: [PATCH 80/95] fix(node): skip an /ipfs absence re-probe the budget cannot pay for (#62) The absent-CID path disambiguates a clean `missing` by re-running the probe, but the re-probe took the same deadline with no check that any budget was left. A first probe that nearly exhausted the budget still spawned a second child that could only be reaped, and the watchdog's SIGTERM grace plus SIGKILL settle carried the whole call to roughly twice the budget. Measured on object_type_bounded with a 1s budget and a probe that burns 0.9s of it: two spawns and 2021ms elapsed. That is the same ~2x occupancy shape this PR exists to remove, on a different handler, and it is worse here than on the served-git path: /ipfs/{cid} carries only optional_signature, and an absent CID drives this branch once per repo, so an unauthenticated caller spraying well-formed absent CIDs pays the overshoot per repo. It also degraded an honest absence into a retryable 503. Re-probe only when the remaining budget covers what the first probe actually took, since the re-probe runs the same command. Below that, taint to Transient rather than spawn: an inconclusive disambiguation is not an absence verdict, so it must not become a false 404 either. The same 1s case is now one spawn and finishes inside the budget. Both introduced by this PR (d39fc8d, the F5 disambiguation); neither object_type_bounded nor the re-probe exists at the merge base, so this is not a pre-existing condition. The companion test pins that an ample budget still re-probes, so the check cannot silently disable the disambiguation it guards. --- crates/gitlawb-node/src/git/store.rs | 127 +++++++++++++++++++++++++-- 1 file changed, 122 insertions(+), 5 deletions(-) diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index a18ab472..084fa20e 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -438,6 +438,7 @@ pub fn object_type_bounded( sha256_hex: &str, deadline: std::time::Instant, ) -> std::result::Result, ProbeError> { + let probe_started = std::time::Instant::now(); match batch_check_probe(git_bin, repo_path, sha256_hex, deadline)? { BatchProbe::Present(ty) => Ok(Some(ty)), BatchProbe::Fault(detail) => Err(classify_store_fault(repo_path, detail)), @@ -453,11 +454,27 @@ pub fn object_type_bounded( repo_path.display() ))); } - // Store readable: re-probe once. Still `missing` on a confirmed-readable - // store is very likely truly absent (Ok(None)); a mid-repack race that - // resolved returns the type. This narrows, but cannot fully close, the - // concurrent-repack window (the readability check samples a different - // instant than the failing probe). + // Only re-probe if the budget can actually pay for it. The re-probe runs the + // SAME command, so it needs roughly what the first probe took; with less than + // that left, the child is spawned only to be reaped, and the watchdog's + // SIGTERM grace plus SIGKILL settle carries this call well past `deadline` + // (measured ~2x a 1s budget before this check existed). `/ipfs/{cid}` is + // anon-reachable and an absent CID drives this branch once per repo, so an + // unaffordable re-probe is pure overshoot on a permissionless path. An + // inconclusive disambiguation is NOT an absence verdict, so taint to a + // retryable Transient rather than spawn or return a false Ok(None). + let first_probe_took = probe_started.elapsed(); + if deadline.saturating_duration_since(std::time::Instant::now()) < first_probe_took { + return Err(ProbeError::Transient(anyhow::anyhow!( + "git cat-file inconclusive: no budget left for the confirming re-probe at {} (not an absence verdict)", + repo_path.display() + ))); + } + // Store readable and budget available: re-probe once. Still `missing` on a + // confirmed-readable store is very likely truly absent (Ok(None)); a + // mid-repack race that resolved returns the type. This narrows, but cannot + // fully close, the concurrent-repack window (the readability check samples a + // different instant than the failing probe). match batch_check_probe(git_bin, repo_path, sha256_hex, deadline)? { BatchProbe::Present(ty) => Ok(Some(ty)), BatchProbe::Fault(detail) => Err(classify_store_fault(repo_path, detail)), @@ -1021,4 +1038,104 @@ mod tests { never a false Ok(None) 404; got {res:?}" ); } + + /// #174 U1 follow-up (RED-before/GREEN-after): the absent-CID path must not spawn a + /// confirming re-probe it cannot afford. `object_type_bounded` disambiguates a clean + /// `missing` by re-running the probe, but the re-probe took the SAME deadline with no + /// check that any budget was left, so a first probe that nearly exhausted the budget + /// still spawned a second child that could only be reaped. The watchdog's SIGTERM + /// grace plus SIGKILL settle then carried the whole call to ~2x the budget, on a + /// route an unauthenticated caller drives for every repo by spraying absent CIDs. + /// + /// Load-bearing: remove the affordability check and this goes RED on both assertions + /// (a second spawn appears, and elapsed crosses 2x the budget). Measured before the + /// fix: spawns=2, elapsed 2021ms against a 1000ms budget. + #[cfg(unix)] + #[test] + fn absent_probe_skips_a_reprobe_it_cannot_afford() { + use std::os::unix::fs::PermissionsExt; + let td = tempfile::TempDir::new().unwrap(); + let bare = td.path().join("bare.git"); + std::fs::create_dir_all(bare.join("objects/pack")).unwrap(); + let log = td.path().join("spawns.log"); + let fake = td.path().join("fakegit"); + // Burns 0.9s of a 1s budget, then reports the structured absence token cleanly. + std::fs::write( + &fake, + format!( + "#!/bin/sh\necho call >> {}\nsleep 0.9\necho 'deadbeef missing'\nexit 0\n", + log.display() + ), + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + + let budget = std::time::Duration::from_millis(1000); + let deadline = std::time::Instant::now() + budget; + let started = std::time::Instant::now(); + let res = super::object_type_bounded(fake.to_str().unwrap(), &bare, "deadbeef", deadline); + let elapsed = started.elapsed(); + let spawns = std::fs::read_to_string(&log) + .map(|s| s.lines().count()) + .unwrap_or(0); + + assert_eq!( + spawns, 1, + "a re-probe with no remaining budget must not be spawned at all; a second \ + spawn here is a child created only to be reaped, and its teardown grace is \ + what pushes this call past the deadline" + ); + assert!( + elapsed < budget + std::time::Duration::from_millis(400), + "the call must not overshoot its deadline by the reap grace; got {elapsed:?} \ + against a {budget:?} budget (pre-fix this was ~2x the budget)" + ); + assert!( + matches!(res, Err(super::ProbeError::Transient(_))), + "an unaffordable disambiguation is NOT an absence verdict: it must taint to \ + a retryable Transient, never a false Ok(None) 404; got {res:?}" + ); + } + + /// Companion must-not-regress case for the affordability check above: with an ample + /// budget the confirming re-probe MUST still run, so the #174 F5 disambiguation is + /// intact and the check did not simply disable it. Two spawns, and a clean absence. + #[cfg(unix)] + #[test] + fn absent_probe_still_reprobes_when_the_budget_allows() { + use std::os::unix::fs::PermissionsExt; + let td = tempfile::TempDir::new().unwrap(); + let bare = td.path().join("bare.git"); + std::fs::create_dir_all(bare.join("objects/pack")).unwrap(); + let log = td.path().join("spawns.log"); + let fake = td.path().join("fakegit"); + std::fs::write( + &fake, + format!( + "#!/bin/sh\necho call >> {}\necho 'deadbeef missing'\nexit 0\n", + log.display() + ), + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + let res = super::object_type_bounded(fake.to_str().unwrap(), &bare, "deadbeef", deadline); + let spawns = std::fs::read_to_string(&log) + .map(|s| s.lines().count()) + .unwrap_or(0); + assert_eq!( + spawns, 2, + "with budget to spare the confirming re-probe must still run, or the \ + absence-vs-unreadable-pack disambiguation is gone" + ); + assert!( + matches!(res, Ok(None)), + "a clean `missing` twice on a readable store is a genuine absence; got {res:?}" + ); + } } From 28cbb50ec0de8f83675ca4b62d791772e6ff5d3d Mon Sep 17 00:00:00 2001 From: t Date: Thu, 30 Jul 2026 10:49:00 -0500 Subject: [PATCH 81/95] fix(node): bound the IPFS pin batch and let a slow healthy upload finish (#174) pin_new_objects runs under a pin_semaphore permit and that pool defers rather than sheds. Both sinks in ipfs_pin.rs built a bare reqwest::Client::new(), which carries no timeout, so a silent endpoint parked the pool indefinitely. Both now use the shared no-redirect client from build_http_client() through a module-local OnceLock, rather than the hand-rolled twin its docstring forbids. The client alone does not bound the hold: without a ceiling the loop is O(N) with N chosen by the pusher. A 120s batch budget is taken once at loop start and gated at the top of every iteration, so no object's work begins with zero budget left. The remainder is also passed to each add as a per-request timeout, which is what lets one large healthy upload run past the client's 10s default without letting the batch run forever. RequestBuilder::timeout replaces the client value for that request only and leaks nothing to other callers, verified by execution against the pinned reqwest. An earlier version of this commit classified a dead endpoint by reqwest::Error::is_timeout() || is_connect() and broke the batch. That was wrong in both directions and is gone. It missed the ordinary dead-peer shapes, since a peer closing mid-request yields IncompleteMessage and a peer RST mid-upload yields ConnectionReset, both observed with is_timeout and is_connect false, so a load-balancer idle timeout evaded it entirely. And it fired on a slow but alive endpoint, because .timeout() is a total deadline covering body upload, so a healthy multi-megabyte add was read as a dead endpoint and abandoned the rest of the push. A wall-clock bound needs neither judgement. The docstring now states what is bounded and what is not, rather than implying the hold is closed. Not bounded: the git read, since store::read_object shells two bare Command::output() calls with no timeout or process-group reaping and this loop does not run it under spawn_blocking; the DB round-trips; and the pool itself, since the Pinata replication task holds the same semaphore across a full git re-derivation with no deadline. Truncation is stated plainly: a batch stopped at the deadline leaves the rest unpinned and nothing sweeps them up, so recovery is opportunistic through a later full-scan push. Tests written first, both RED by assertion rather than by a harness timeout: pin_new_objects_stops_the_batch_at_its_deadline ... FAILED the batch must stop partway, not pin all five and not stall on the first: pinned 5 pin_new_objects_does_not_abandon_the_batch_on_a_slow_but_alive_endpoint ... FAILED a slow but progressing endpoint must pin both objects: an upload past the client's 10s default is not a dead endpoint left: 1, right: 2 Both guards mutation-proved load-bearing against present-but-wrong mutants, each red on its own message: the loop-top break widened to continue, and the per-request timeout replaced with None. Also bounds encrypted_pin.rs's pin_git_object call, where a hang wedged a repo's EncryptInflight key. Its Drop backstop covers unwind, not hang. Full suite 730 passed, 0 failed. fmt and clippy clean. --- crates/gitlawb-node/src/api/ipfs.rs | 24 +- crates/gitlawb-node/src/api/repos.rs | 9 +- crates/gitlawb-node/src/encrypted_pin.rs | 2 +- crates/gitlawb-node/src/ipfs_pin.rs | 522 ++++++++++++++++++++++- crates/gitlawb-node/src/sync.rs | 2 +- crates/gitlawb-node/src/test_support.rs | 19 + 6 files changed, 542 insertions(+), 36 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index f5420324..a539ff34 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -804,24 +804,6 @@ mod tests { .to_string() } - /// A local endpoint whose TCP accept succeeds instantly but that never writes - /// an HTTP response, so a Tigris HEAD against it stalls deterministically - /// until the caller's timeout. (A non-routable address hangs only if the - /// network blackholes the SYN — a fast RST would end the stall early.) The - /// accepted sockets are parked in the spawned task, which dies with the - /// test's runtime, so the peer never sees a close mid-test. - async fn silent_tigris_endpoint() -> String { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let endpoint = format!("http://{}", listener.local_addr().unwrap()); - tokio::spawn(async move { - let mut held = Vec::new(); - while let Ok((sock, _)) = listener.accept().await { - held.push(sock); - } - }); - endpoint - } - /// Fake git for the WALK only (`state.git_bin`): empty refs, `rev-parse` /// resolves, and each `rev-list` appends one line to `log` and prints nothing — /// every walked repo yields an EMPTY allowed-set (path-gate deny verdict) and @@ -1090,7 +1072,7 @@ mod tests { // Endpoint-pinned test client (no AWS_* env reads — env is racy under a // parallel test run); the silent local endpoint stalls the HEAD // deterministically. - let endpoint = silent_tigris_endpoint().await; + let endpoint = crate::test_support::silent_http_endpoint().await; let tigris = crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) .await; @@ -1154,7 +1136,7 @@ mod tests { // repo stays a fast local hit) and add a NEWER ghost row with no local // copy: its acquire consults the silent local endpoint and stalls to the // 1s timeout (endpoint-pinned test client, no AWS_* env reads). - let endpoint = silent_tigris_endpoint().await; + let endpoint = crate::test_support::silent_http_endpoint().await; let tigris = crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) .await; @@ -1679,7 +1661,7 @@ mod tests { // repo stays a fast local hit) and add a NEWER ghost row with no local // copy: its acquire consults the silent local endpoint and stalls past // the budget (endpoint-pinned test client, no AWS_* env reads). - let endpoint = silent_tigris_endpoint().await; + let endpoint = crate::test_support::silent_http_endpoint().await; let tigris = crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) .await; diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index de357059..67c2ddd0 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1148,7 +1148,14 @@ async fn pin_new_objects_gated( .acquire_owned() .await .expect("pin_semaphore is never closed"); - crate::ipfs_pin::pin_new_objects(ipfs_api, repo_path, object_list, db).await + crate::ipfs_pin::pin_new_objects( + ipfs_api, + repo_path, + object_list, + db, + crate::ipfs_pin::PIN_BATCH_BUDGET, + ) + .await } /// `repo_id` is passed explicitly rather than read from `ctx` so the two callers diff --git a/crates/gitlawb-node/src/encrypted_pin.rs b/crates/gitlawb-node/src/encrypted_pin.rs index 40e444e5..19b80651 100644 --- a/crates/gitlawb-node/src/encrypted_pin.rs +++ b/crates/gitlawb-node/src/encrypted_pin.rs @@ -170,7 +170,7 @@ pub async fn encrypt_and_pin( continue; } }; - let cid = match crate::ipfs_pin::pin_git_object(ipfs_api, oid, &envelope).await { + let cid = match crate::ipfs_pin::pin_git_object(ipfs_api, oid, &envelope, None).await { Ok(c) if !c.is_empty() => c, Ok(_) => { tracing::warn!(oid = %oid, "pin_git_object returned no cid; skipping"); diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 3b346190..765bd6b0 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -9,6 +9,36 @@ use anyhow::Result; use gitlawb_core::cid::Cid; +use std::time::{Duration, Instant}; + +/// Wall-clock ceiling on one [`pin_new_objects`] batch. +/// +/// The loop runs under a `pin_semaphore` permit and that pool defers rather than +/// sheds, so without a ceiling the hold is O(N) with N (the push's object count) +/// chosen by the pusher. This bounds the drain of a saturated pool instead. +/// +/// 120s is 12x the shared client's 10s whole-request ceiling, so a single large +/// healthy upload that needs more than the client default still has room to +/// finish (the per-request timeout is set to the remainder, not the default), +/// while a batch of them still cannot hold the permit indefinitely. Deliberately +/// a constant and not a config knob: the value only has to be large enough to be +/// uninteresting on a healthy node, and a knob is operator surface that would +/// have to be documented, validated, and kept meaningful. +pub const PIN_BATCH_BUDGET: Duration = Duration::from_secs(120); + +/// The shared outbound client for both IPFS sinks. +/// +/// `pin_new_objects` runs while holding a `pin_semaphore` permit and that pool +/// defers rather than sheds, so an unbounded await here parks the pool. A bare +/// `reqwest::Client::new()` has no timeout, which is exactly that. Built from +/// `crate::build_http_client` rather than a local builder: its docstring forbids +/// hand-rolling an equivalent, so that the redirect and timeout guarantees the +/// node's tests bind stay bound to the client every outbound path actually uses. +fn http_client() -> &'static reqwest::Client { + static CLIENT: std::sync::OnceLock = std::sync::OnceLock::new(); + CLIENT + .get_or_init(|| crate::build_http_client().expect("failed to build production http client")) +} /// Pin a single git object to the local IPFS/Kubo node. /// @@ -16,9 +46,20 @@ use gitlawb_core::cid::Cid; /// If empty the function returns `Ok("")` immediately. /// - `sha256_hex`: the git SHA-256 hex object ID (used only for logging). /// - `data`: raw git object content bytes (same bytes used for CID computation). +/// - `request_timeout`: overrides the shared client's whole-request timeout for +/// THIS request only. `RequestBuilder::timeout` replaces the client-level value +/// per request and leaks nothing to other calls on the same client, so the +/// batch loop can hand each add whatever is left of its budget without +/// loosening or tightening any other outbound path. `None` keeps the client's +/// own ceiling. /// /// Returns the CID string on success, or `""` when IPFS is not configured. -pub async fn pin_git_object(ipfs_api: &str, sha256_hex: &str, data: &[u8]) -> Result { +pub async fn pin_git_object( + ipfs_api: &str, + sha256_hex: &str, + data: &[u8], + request_timeout: Option, +) -> Result { if ipfs_api.is_empty() { return Ok(String::new()); } @@ -37,13 +78,26 @@ pub async fn pin_git_object(ipfs_api: &str, sha256_hex: &str, data: &[u8]) -> Re .mime_str("application/octet-stream")?; let form = reqwest::multipart::Form::new().part("file", part); - let client = reqwest::Client::new(); - let resp = client - .post(&url) - .multipart(form) + let mut req = http_client().post(&url).multipart(form); + if let Some(t) = request_timeout { + req = req.timeout(t); + } + + let resp = req .send() .await - .map_err(|e| anyhow::anyhow!("IPFS add request failed: {e}"))?; + // Keep the `reqwest::Error` as this error's source rather than + // formatting it away. Operators reading a pin failure want the concrete + // transport cause in the logged chain, not a single flattened line, and + // this module's tests downcast to it to prove a silent endpoint really + // surfaces as a timeout rather than as some other failure that happens + // to arrive in time. + // The context keeps the old message verbatim so the callers that log + // this at `%e` (here, `sync.rs`, `encrypted_pin.rs`) read the same. + .map_err(|e| { + let msg = format!("IPFS add request failed: {e}"); + anyhow::Error::new(e).context(msg) + })?; if !resp.status().is_success() { let status = resp.status(); @@ -55,7 +109,10 @@ pub async fn pin_git_object(ipfs_api: &str, sha256_hex: &str, data: &[u8]) -> Re // Kubo returns newline-delimited JSON; we only care about the last object // (there's typically just one for a single-file add). - let body = resp.text().await?; + let body = resp + .text() + .await + .map_err(|e| anyhow::anyhow!("IPFS add response body read failed: {e}"))?; let cid = body .lines() .filter(|l| !l.trim().is_empty()) @@ -76,13 +133,33 @@ pub async fn cat(ipfs_api: &str, cid: &str) -> Result> { return Err(anyhow::anyhow!("IPFS not configured")); } let url = format!("{}/api/v0/cat?arg={}", ipfs_api.trim_end_matches('/'), cid); - let resp = reqwest::Client::new().post(&url).send().await?; + let resp = http_client().post(&url).send().await?; if !resp.status().is_success() { return Err(anyhow::anyhow!("ipfs cat {cid}: {}", resp.status())); } Ok(resp.bytes().await?.to_vec()) } +/// The batch's remaining wall-clock, or `None` once it is spent, after logging +/// the truncation exactly once. +/// +/// Shaped like `api::ipfs`'s `budget_gate` on purpose: the nonzero-ness rides in +/// the returned value, so every call site must consume it as +/// `let Some(x) = ... else { break }` and a zero `Duration` can never reach a +/// request as its timeout. +fn batch_budget_gate(deadline: Instant, pinned: usize, unattempted: usize) -> Option { + let left = deadline.saturating_duration_since(Instant::now()); + if left.is_zero() { + tracing::warn!( + pinned, + unattempted, + "IPFS pin batch deadline reached; the remaining objects are left unpinned" + ); + return None; + } + Some(left) +} + /// Pin any of the given candidate git objects that are not yet recorded in /// `pinned_cids`. /// @@ -90,8 +167,50 @@ pub async fn cat(ipfs_api: &str, cid: &str) -> Result> { /// applies `visibility_pack::replicable_objects` on the delta path or the /// `..._fail_closed` filter on the full-scan path before calling, so this /// function never sees a withheld blob. `repo_path` is still needed to read each -/// object's bytes. The twin in `pinata.rs` mirrors this shape — change both in -/// lockstep. +/// object's bytes. +/// +/// # What `batch_budget` does and does not bound +/// +/// The loop holds a `pin_semaphore` permit and that pool defers rather than +/// sheds, so the hold has to be bounded by something other than the pusher's +/// object count. Two things here are: +/// +/// - this loop's own wall-clock: the deadline is taken once at loop start and +/// checked at the top of every iteration, so no object's work begins with zero +/// budget left. It is a gate, not a hard ceiling, since a started iteration +/// still runs to completion; +/// - each HTTP add: `pin_git_object` is handed the remainder measured at the top +/// of the iteration as its per-request timeout, which is what lets one large +/// healthy upload run past the shared client's 10s default without letting the +/// batch run forever. The DB check and the git read spend a little of that +/// remainder before the request starts, so the add can finish marginally after +/// the deadline. +/// +/// Three things are NOT bounded, and the gate cannot fix any of them: +/// +/// - the git read. `store::read_object` composes two bare +/// `std::process::Command::output()` calls with no timeout and no +/// process-group reaping, and this loop does not run it under +/// `spawn_blocking`, so one hung `git cat-file` overruns the deadline and +/// blocks a runtime worker thread while it does. +/// - the DB round-trips (`is_pinned`, `record_pinned_cid`). +/// - the pool. `api::repos` acquires the same `pin_semaphore` for the Pinata +/// replication task and holds it across a full git re-derivation plus +/// `pinata::pin_new_objects`, neither of which has a deadline. This change +/// bounds this loop's hold, not the semaphore's worst-case queue. +/// +/// # Truncation semantics +/// +/// A batch stopped at the deadline leaves its remaining objects unpinned, and +/// nothing sweeps them up afterwards. There is no reconciliation pass over +/// `pinned_cids`; recovery is opportunistic, happening only if some later push +/// on the repo takes the full-scan fallback (`push_delta::list_all_objects`) and +/// re-derives the whole object set, which then re-offers the skipped OIDs. +/// +/// The twin in `pinata.rs` mirrors this loop's shape but NOT its bound: it has +/// no batch deadline and no per-request override, so the two are no longer at +/// parity here. Everything else about the shape (the skip-if-pinned check, the +/// warn-and-continue arm, the returned pairs) still changes in lockstep. /// /// Returns a list of `(sha256_hex, cid)` pairs for objects pinned this call. pub async fn pin_new_objects( @@ -99,14 +218,23 @@ pub async fn pin_new_objects( repo_path: &std::path::Path, object_list: Vec, db: &crate::db::Db, + batch_budget: Duration, ) -> Vec<(String, String)> { if ipfs_api.is_empty() { return vec![]; } + let deadline = Instant::now() + batch_budget; + let total = object_list.len(); let mut pinned = Vec::new(); - for sha in object_list { + for (attempted, sha) in object_list.into_iter().enumerate() { + // Top of the iteration, before any of this object's work: an object is + // never started with zero budget left. The remainder becomes this add's + // request timeout below. + let Some(budget_left) = batch_budget_gate(deadline, pinned.len(), total - attempted) else { + break; + }; // Skip if already pinned match db.is_pinned(&sha).await { Ok(true) => continue, @@ -128,7 +256,7 @@ pub async fn pin_new_objects( }; // Pin to IPFS - match pin_git_object(ipfs_api, &sha, &data).await { + match pin_git_object(ipfs_api, &sha, &data, Some(budget_left)).await { Ok(cid) if !cid.is_empty() => { if let Err(e) = db.record_pinned_cid(&sha, &cid).await { tracing::warn!(sha = %sha, err = %e, "failed to record pinned CID in DB"); @@ -144,3 +272,373 @@ pub async fn pin_new_objects( pinned } + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + /// Write `n` loose blobs into a fresh bare repo and return their oids. + /// `read_object` shells to `git cat-file`, so the objects must genuinely + /// exist on disk — a fabricated oid would `continue` past the pin call and + /// the loop scenario below would prove nothing. + fn seed_loose_blobs(repo_path: &std::path::Path, n: usize) -> Vec { + crate::git::store::init_bare(repo_path).expect("init bare repo"); + (0..n) + .map(|i| { + let mut cmd = std::process::Command::new("git"); + cmd.args(["hash-object", "-w", "--stdin"]) + .current_dir(repo_path) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()); + let mut child = cmd.spawn().expect("spawn git hash-object"); + { + use std::io::Write; + child + .stdin + .as_mut() + .expect("stdin") + .write_all(format!("pin loop object {i}\n").as_bytes()) + .expect("write stdin"); + } + let out = child.wait_with_output().expect("hash-object output"); + assert!( + out.status.success(), + "git hash-object: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }) + .collect() + } + + /// A live endpoint that answers every add with `500`. Counts the requests it + /// received so a test can tell "the loop kept going" from "the loop stopped", + /// which the returned pin list cannot (it is empty either way). Reads the + /// full request, headers plus the `Content-Length` body, before answering: + /// responding early and closing would surface as a write failure on the + /// client and turn a rejection into something else. + async fn rejecting_endpoint( + requests: std::sync::Arc, + ) -> String { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { + while let Ok((mut sock, _)) = listener.accept().await { + requests.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + tokio::spawn(async move { + let mut acc = Vec::new(); + let mut buf = [0u8; 4096]; + loop { + let n = match sock.read(&mut buf).await { + Ok(0) | Err(_) => break, + Ok(n) => n, + }; + acc.extend_from_slice(&buf[..n]); + // Once the headers are complete, keep reading until the + // declared body has arrived. + if let Some(hdr_end) = + acc.windows(4).position(|w| w == b"\r\n\r\n").map(|p| p + 4) + { + let headers = String::from_utf8_lossy(&acc[..hdr_end]).to_lowercase(); + let len: usize = headers + .lines() + .find_map(|l| l.strip_prefix("content-length:")) + .and_then(|v| v.trim().parse().ok()) + .unwrap_or(0); + if acc.len() >= hdr_end + len { + break; + } + } + } + let _ = sock + .write_all( + b"HTTP/1.1 500 Internal Server Error\r\nContent-Length: 0\r\n\r\n", + ) + .await; + let _ = sock.flush().await; + }); + } + }); + endpoint + } + + /// A sleeping-but-live endpoint. Answers `200` with an empty body after + /// `delays[i]` for the i-th request it accepts (the last entry repeats), so + /// a test can make one add slow and the next fast. Drains the full request, + /// headers plus the declared `Content-Length` body, before sleeping: exactly + /// as in `rejecting_endpoint`, answering early and closing would surface as + /// a write failure on the client and turn a slow-but-healthy add into a + /// different failure shape. + /// + /// An empty body is a successful pin: `pin_git_object` falls back to the CID + /// it computed from the bytes when the response carries no `Hash`. + async fn delaying_endpoint(delays: Vec) -> String { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { + let mut seen = 0usize; + while let Ok((mut sock, _)) = listener.accept().await { + let delay = *delays + .get(seen) + .or_else(|| delays.last()) + .unwrap_or(&Duration::ZERO); + seen += 1; + tokio::spawn(async move { + let mut acc = Vec::new(); + let mut buf = [0u8; 4096]; + loop { + let n = match sock.read(&mut buf).await { + Ok(0) | Err(_) => break, + Ok(n) => n, + }; + acc.extend_from_slice(&buf[..n]); + if let Some(hdr_end) = + acc.windows(4).position(|w| w == b"\r\n\r\n").map(|p| p + 4) + { + let headers = String::from_utf8_lossy(&acc[..hdr_end]).to_lowercase(); + let len: usize = headers + .lines() + .find_map(|l| l.strip_prefix("content-length:")) + .and_then(|v| v.trim().parse().ok()) + .unwrap_or(0); + if acc.len() >= hdr_end + len { + break; + } + } + } + tokio::time::sleep(delay).await; + let _ = sock + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n") + .await; + let _ = sock.flush().await; + }); + } + }); + endpoint + } + + /// A `tracing` sink a test can read back, so the deadline warn can be + /// asserted on rather than assumed. Installed with `set_default`, which is + /// thread-local and scoped to the guard, so it cannot bleed into any other + /// test in the binary. + #[derive(Clone, Default)] + struct CapturedLogs(std::sync::Arc>>); + + impl CapturedLogs { + fn text(&self) -> String { + String::from_utf8_lossy(&self.0.lock().unwrap()).to_string() + } + } + + impl std::io::Write for CapturedLogs { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for CapturedLogs { + type Writer = CapturedLogs; + fn make_writer(&'a self) -> Self::Writer { + self.clone() + } + } + + fn capture_logs() -> (CapturedLogs, tracing::subscriber::DefaultGuard) { + let logs = CapturedLogs::default(); + let subscriber = tracing_subscriber::fmt() + .with_writer(logs.clone()) + .with_max_level(tracing::Level::WARN) + .with_ansi(false) + .finish(); + let guard = tracing::subscriber::set_default(subscriber); + (logs, guard) + } + + /// The add sink must be built from the shared no-redirect client, and the + /// per-request override must actually reach the request. Against a silent + /// endpoint (accept succeeds, no response ever written) a bare + /// `reqwest::Client::new()` blocks forever. With `Some(2s)` the call must + /// come back well inside that, and as a reqwest timeout: the elapsed + /// assertion is the real RED signal, and the outer `tokio::time::timeout` + /// is only a wedge guard so a regression fails the suite instead of hanging + /// it (`cargo test` has no per-test timeout). The old "no elapsed assertion + /// because the timeout is a process-global `OnceLock`" caveat no longer + /// holds now that `request_timeout` overrides it per call. + #[tokio::test] + async fn pin_git_object_against_silent_endpoint_errors_within_its_own_timeout() { + let endpoint = crate::test_support::silent_http_endpoint().await; + let started = std::time::Instant::now(); + let inner = tokio::time::timeout( + Duration::from_secs(30), + pin_git_object( + &endpoint, + "deadbeef", + b"some object bytes\n", + Some(Duration::from_secs(2)), + ), + ) + .await + .expect("wedge guard: pin_git_object must return long before 30s"); + let elapsed = started.elapsed(); + let err = inner.expect_err("a silent endpoint must not surface as a successful pin"); + assert!( + elapsed < Duration::from_secs(5), + "the 2s per-request override must bound this call, not the client's own ceiling (took {elapsed:?})" + ); + assert!( + err.downcast_ref::() + .is_some_and(|e| e.is_timeout()), + "a silent endpoint must surface as a reqwest timeout, preserved as the error's source: {err:#}" + ); + } + + /// The second unhardened sink, reached from `sync.rs`. Same shape as above. + #[tokio::test] + async fn cat_against_silent_endpoint_errors_within_its_own_timeout() { + let endpoint = crate::test_support::silent_http_endpoint().await; + let inner = tokio::time::timeout(Duration::from_secs(30), cat(&endpoint, "bafkqaaa")) + .await + .expect( + "cat must return before the outer 30s timeout — an unbounded client hangs here", + ); + assert!( + inner.is_err(), + "a silent endpoint must surface as a transport error, not successful bytes" + ); + } + + /// The permit-hold bound. `pin_new_objects` runs under a deferring + /// `pin_semaphore`, so without a batch deadline the hold is O(N) with N + /// chosen by the pusher. Five objects against an endpoint that takes 2s + /// each, under a 5.5s budget, must stop partway: only the first two can + /// finish inside the budget, so the batch is truncated and the remainder is + /// left unattempted with one warn naming how many. + /// + /// The windows are deliberately loose. Three pins would need every add to + /// answer in under 1.83s, which the endpoint's own 2s sleep forbids, and one + /// pin needs only the first add to land inside 5.5s, so both bounds hold + /// with more than a second of slack on a loaded box. + #[sqlx::test] + async fn pin_new_objects_stops_the_batch_at_its_deadline(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("slow.git"); + let oids = seed_loose_blobs(&repo_path, 5); + let endpoint = delaying_endpoint(vec![Duration::from_secs(2)]).await; + + let (logs, _guard) = capture_logs(); + let pinned = tokio::time::timeout( + Duration::from_secs(30), + pin_new_objects( + &endpoint, + &repo_path, + oids, + &db, + Duration::from_millis(5500), + ), + ) + .await + .expect("wedge guard: a 5.5s budget cannot take 30s"); + + assert!( + (1..=3).contains(&pinned.len()), + "the batch must stop partway, not pin all five and not stall on the first: pinned {}", + pinned.len() + ); + let text = logs.text(); + let warns: Vec<&str> = text + .lines() + .filter(|l| l.contains("pin batch deadline reached")) + .collect(); + assert_eq!( + warns.len(), + 1, + "the deadline must be reported exactly once for the batch, not per object: {text}" + ); + let unattempted: usize = warns[0] + .split("unattempted=") + .nth(1) + .and_then(|s| { + s.split(|c: char| !c.is_ascii_digit()) + .next() + .and_then(|d| d.parse().ok()) + }) + .unwrap_or_else(|| panic!("the deadline warn must name the unattempted count: {text}")); + assert!( + unattempted >= 1 && unattempted + pinned.len() <= 5, + "unattempted={unattempted} with {} pinned is not a partial batch of five", + pinned.len() + ); + } + + /// The must-not case, and the regression that killed the old transport + /// classifier: an endpoint that is slow but genuinely alive must NOT cost + /// the rest of the batch. The first add takes 13s, past the shared client's + /// 10s ceiling, which is exactly what the classifier used to read as a dead + /// endpoint; the second is immediate. Under a 90s budget both must pin, so + /// this fails if the per-request timeout is left at the client default and + /// fails if any error arm breaks the loop. Two objects, not one, because + /// with one object "did not abandon the rest" would be vacuous. + #[sqlx::test] + async fn pin_new_objects_does_not_abandon_the_batch_on_a_slow_but_alive_endpoint( + pool: sqlx::PgPool, + ) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("slow_alive.git"); + let oids = seed_loose_blobs(&repo_path, 2); + let endpoint = + delaying_endpoint(vec![Duration::from_secs(13), Duration::from_millis(0)]).await; + + let pinned = tokio::time::timeout( + Duration::from_secs(60), + pin_new_objects(&endpoint, &repo_path, oids, &db, Duration::from_secs(90)), + ) + .await + .expect("wedge guard: a 13s add plus an immediate one cannot take 60s"); + assert_eq!( + pinned.len(), + 2, + "a slow but progressing endpoint must pin both objects: an upload past the client's \ + 10s default is not a dead endpoint" + ); + } + + /// The must-not case for the warn-and-continue arm: a live endpoint + /// rejecting each object with `500` is a per-object failure, so the loop + /// must still warn and continue and every object must be attempted. Without + /// this, a `break` arm could be reintroduced and the deadline test above + /// would not notice. + #[sqlx::test] + async fn pin_new_objects_continues_past_a_per_object_rejection(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("rejecting.git"); + let oids = seed_loose_blobs(&repo_path, 4); + let requests = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let endpoint = rejecting_endpoint(std::sync::Arc::clone(&requests)).await; + + let pinned = tokio::time::timeout( + Duration::from_secs(30), + pin_new_objects(&endpoint, &repo_path, oids, &db, Duration::from_secs(60)), + ) + .await + .expect("a rejecting endpoint answers immediately, so this cannot take 30s"); + assert!(pinned.is_empty(), "every add was rejected"); + assert_eq!( + requests.load(std::sync::atomic::Ordering::SeqCst), + 4, + "a non-2xx rejection is per-object: all four objects must still be attempted" + ); + } +} diff --git a/crates/gitlawb-node/src/sync.rs b/crates/gitlawb-node/src/sync.rs index 5c6864fd..ab97747f 100644 --- a/crates/gitlawb-node/src/sync.rs +++ b/crates/gitlawb-node/src/sync.rs @@ -605,7 +605,7 @@ async fn replicate_encrypted_blobs( continue; } }; - match crate::ipfs_pin::pin_git_object(ipfs_api, &blob.oid, &envelope).await { + match crate::ipfs_pin::pin_git_object(ipfs_api, &blob.oid, &envelope, None).await { Ok(cid) if !cid.is_empty() => { if cid != blob.cid { warn!(oid = %blob.oid, expected = %blob.cid, got = %cid, "replicated envelope CID mismatch; skipping record"); diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 831202ee..0eb11481 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -129,6 +129,25 @@ pub(crate) fn signed_request_as(did: &str, method: Method, uri: &str, body: Body .expect("request builder") } +/// A local endpoint whose TCP accept succeeds instantly but that never writes an +/// HTTP response, so any request against it stalls deterministically until the +/// caller's own timeout. (A non-routable address hangs only if the network +/// blackholes the SYN — a fast RST would end the stall early and make a timeout +/// test pass for the wrong reason.) The accepted sockets are parked in the +/// spawned task, which dies with the test's runtime, so the peer never sees a +/// close mid-test. +pub(crate) async fn silent_http_endpoint() -> String { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { + let mut held = Vec::new(); + while let Ok((sock, _)) = listener.accept().await { + held.push(sock); + } + }); + endpoint +} + #[cfg(test)] mod tests { use super::*; From 17cbdf7ff318a7fbbb4a7dc76c043338ce9028a1 Mon Sep 17 00:00:00 2001 From: t Date: Thu, 30 Jul 2026 10:59:27 -0500 Subject: [PATCH 82/95] fix(node): charge the /ipfs walk budget for blocking-queue delay (#174) The walk's deadline clamp was computed on the async side, before spawn_blocking was called, then used inside the closure. Time spent queued for a blocking thread went uncharged, so the walk ran with a budget larger than the request's true remainder. The clamp is now derived inside the closure at task start. This is the sibling of the upload-pack site fixed in 28a6ca4, and unlike that one it sits on the anonymously reachable route. A queue delay that eats the whole remainder saturates the clamp to zero and the bounded walk fails closed through the existing taint arm, no verdict, which is the safe direction. Testing gap, recorded at the site rather than papered over: the queue-delay path is reasoned, not executed. Observing it needs a runtime with the blocking pool pinned and parked, and the sqlx::test harness gives no seam for that. 28a6ca4 shipped with the same gap for the same reason, verified against its diff and message rather than assumed. The tests in this file saturate the admission semaphores, never the blocking pool, so there was nothing to reuse. An assertion on the arithmetic would only restate the implementation. cargo test ipfs::: 24 passed, 0 failed. --- crates/gitlawb-node/src/api/ipfs.rs | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index a539ff34..f9e501ff 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -460,7 +460,7 @@ pub async fn get_by_cid( // started walk runs its git children under a deadline clamped // to the remainder (the min below), so a walk can never // complete past the budget. - let Some(budget_left) = budget_gate( + let Some(_budget_left) = budget_gate( &mut truncated_by, request_deadline, state.config.ipfs_request_budget_secs, @@ -490,10 +490,9 @@ pub async fn get_by_cid( let owner = repo.owner_did.clone(); let caller_for_walk = caller_owned.clone(); let git_bin = state.git_bin.clone(); - let walk_timeout = std::cmp::min( - std::time::Duration::from_secs(state.config.git_service_timeout_secs), - budget_left, - ); + let git_service_timeout = + std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let walk_deadline = request_deadline; // Full-history walk shells out to git — keep it off the async runtime, // bounded and reaped like the served-git ops (#174). let walk_admission = std::sync::Arc::clone(&admission); @@ -501,6 +500,24 @@ pub async fn get_by_cid( // Admission clone (#174 U1): the slot stays taken until this blocking // work returns, even if the handler future was dropped or this closure panics. let _admission = walk_admission; + // Derive the walk's budget from the request deadline HERE, inside the + // closure, not on the async side before the task is queued. The walk + // starts its own clock when it runs, so a budget computed at queue time + // would hand it the full remainder measured from whenever the blocking + // pool got to it — the queue delay would go uncharged and the walk could + // finish past the request budget. Computing it at task start charges the + // delay against the deadline; a queue delay that eats the whole remainder + // saturates this to zero and the walk fails closed (no verdict, taint), + // which is the safe direction. Same fix as the upload-pack walk in + // `api/repos.rs`, and this route is anonymously reachable. + // TESTING GAP: the queue-delay path is reasoned, not executed. Observing + // it needs a runtime with the blocking pool pinned and parked, and the + // `#[sqlx::test]` harness gives no seam for that (the sibling fix, + // 28a6ca4, shipped with the same gap for the same reason). + let walk_timeout = std::cmp::min( + git_service_timeout, + walk_deadline.saturating_duration_since(std::time::Instant::now()), + ); allowed_blob_set_for_caller_bounded( &rp, &git_bin, From 2413a71aa3b4c29b2148826a539185bec8cc1176 Mon Sep 17 00:00:00 2001 From: t Date: Thu, 30 Jul 2026 11:01:14 -0500 Subject: [PATCH 83/95] test(node): bind the repo-key boundary test to the separator it names (#174) separator_prevents_the_owner_name_boundary_collision asserted that ("a","b_c") and ("a_b","c") produce different keys. Those differ under any join at all, bare included (ab_c against a_bc), so deleting the separator from repo_identity_key left the test green. It could not fail for the reason it named. The inputs are now ("a","bc") against ("ab","c"), both abc under a bare join. The test docstring said the separator was \0; the code joins with /. Corrected. The docstring on repo_identity_key itself carried the same wrong collision example, so it is corrected too. Proven by mutation rather than argued, joining with no separator: old inputs, injected, re-running ... still green VACUOUS the guard's own defect was injected and nothing failed 0/1 load-bearing new inputs, injected, re-running ... red, on the expected message LOAD-BEARING matched 'owner/name boundary must not be ambiguous' 1/1 load-bearing Separator restored, no mutation residue. All 6 repo_identity_key_tests pass. --- crates/gitlawb-node/src/state.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 77755932..a189548b 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -521,9 +521,9 @@ pub struct RepoWriteLeases { /// - The separator is `/`, which cannot occur in the owner slug by construction /// (`replace([':', '/'], "_")` removes it) and mirrors the shape of the disk /// path this key exists to reproduce. A plain join would collide (owner `a` + -/// name `b_c` against owner `a_b` + name `c`), letting one repo's push park -/// another's. It must stay PRINTABLE: this key is logged as the `repo` field -/// on the lease shed and steal-bound warnings below, and an unprintable +/// name `bc` against owner `ab` + name `c`, both `abc`), letting one repo's +/// push park another's. It must stay PRINTABLE: this key is logged as the +/// `repo` field on the lease shed and steal-bound warnings below, and an unprintable /// separator (a NUL, a unit separator) truncates at a NUL-hostile log sink and /// renders two different repos' warnings identically. /// - Callers pass `record.owner_did` / `record.name`, never the request's path @@ -903,13 +903,14 @@ mod repo_identity_key_tests { ); } - /// The `\0` separator is what stops one repo's push parking another's. With a - /// plain join these two pairs would produce a single key. + /// The `/` separator is what stops one repo's push parking another's. These + /// two pairs both concatenate to `abc`, so with a plain join they would + /// produce a single key. #[test] fn separator_prevents_the_owner_name_boundary_collision() { assert_ne!( - repo_identity_key("a", "b_c"), - repo_identity_key("a_b", "c"), + repo_identity_key("a", "bc"), + repo_identity_key("ab", "c"), "owner/name boundary must not be ambiguous" ); } From a56f094affad2901f68b701f60b750070a99b5b8 Mon Sep 17 00:00:00 2001 From: t Date: Thu, 30 Jul 2026 11:06:21 -0500 Subject: [PATCH 84/95] test(node): bind the per-source cap test to the production derivation (#174) advert_per_caller_cap_sizing_is_never_zero opened with a local closure re-implementing (pushes / 8).max(1), so it bound nothing. Production derives that expression twice, and the second site was added after the test was written. per_source_push_cap now lives in rate_limit.rs next to PerCallerConcurrency, the type the value feeds, and both main.rs sites and the test call it. The finding was confirmed by execution before the fix. Deleting .max(1) from both production sites with the closure still in place: injected (2 sites), re-running ... still green VACUOUS the guard's own defect was injected and nothing failed 0/1 load-bearing And after, deleting .max(1) from the helper: injected (1 site), re-running ... red, on the expected message LOAD-BEARING matched 'advert cap must be >= 1 for pushes=1' 1/1 load-bearing The first attempt at that second proof came back RED-WRONG-REASON against 'minimum write pool must derive cap 1, not 0': the loop's assertion for pushes=1 fires first and aborts before that line is reached. Same property, earlier assertion. No mutation residue. Test green, fmt and clippy clean. --- crates/gitlawb-node/src/api/repos.rs | 9 +++++---- crates/gitlawb-node/src/main.rs | 4 ++-- crates/gitlawb-node/src/rate_limit.rs | 10 ++++++++++ 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 67c2ddd0..b3051db3 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -3535,12 +3535,13 @@ mod tests { } /// #174 (F2 sizing edge, vetted by execution): the receive-pack advertisement - /// per-source cap is derived in main.rs as `(max_concurrent_git_pushes / 8).max(1)`, - /// so it is never 0 even at the minimum write-pool size (1). A 0 cap would make - /// PerCallerConcurrency shed EVERY receive-pack advertisement and break all pushes. + /// per-source cap comes from `rate_limit::per_source_push_cap`, the same helper + /// both main.rs derivation sites call, so it is never 0 even at the minimum + /// write-pool size (1). A 0 cap would make PerCallerConcurrency shed EVERY + /// receive-pack advertisement and break all pushes. #[test] fn advert_per_caller_cap_sizing_is_never_zero() { - let cap = |pushes: usize| (pushes / 8).max(1); + let cap = crate::rate_limit::per_source_push_cap; for pushes in [1usize, 4, 8, 32, 256] { assert!( cap(pushes) >= 1, diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 5843d0ab..30c2ba37 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -426,14 +426,14 @@ async fn main() -> Result<()> { // rate-limited (#174). Sized off the write pool only because the advert pool // is created at the same size; an advert flood cannot touch a write permit. git_push_advert_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( - (config.max_concurrent_git_pushes / 8).max(1), + rate_limit::per_source_push_cap(config.max_concurrent_git_pushes), ), // Per-source cap on the authenticated receive-pack POST, sized like the advert // cap: one source IP can hold at most this many write-pool slots, so // monopolizing the pool takes ~8 distinct source IPs, each also rate-limited // (#174 P1-d). git_write_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( - (config.max_concurrent_git_pushes / 8).max(1), + rate_limit::per_source_push_cap(config.max_concurrent_git_pushes), ), // Bounds concurrent /ipfs visibility walks — a distinct public cost center, so // its own pool + per-source sub-cap + per-IP rate limiter, never a git pool diff --git a/crates/gitlawb-node/src/rate_limit.rs b/crates/gitlawb-node/src/rate_limit.rs index a0dc6137..6e2a7f3c 100644 --- a/crates/gitlawb-node/src/rate_limit.rs +++ b/crates/gitlawb-node/src/rate_limit.rs @@ -139,6 +139,16 @@ impl RateLimiter { } } +/// Per-source concurrency cap derived from the write-pool size: one source may hold +/// at most an eighth of the pool, so saturating it takes ~8 distinct source IPs. +/// +/// Floored at 1 because the value feeds [`PerCallerConcurrency`], where a cap of 0 +/// would shed EVERY receive-pack advertisement and break all pushes. The floor is +/// load-bearing at the minimum write-pool size (1), which integer-divides to 0. +pub(crate) fn per_source_push_cap(max_concurrent_git_pushes: usize) -> usize { + (max_concurrent_git_pushes / 8).max(1) +} + /// A bounded per-caller CONCURRENCY limiter — distinct from [`RateLimiter`], which /// caps request RATE. Each caller key may hold at most `per_caller` in-flight /// permits at once; beyond that [`try_acquire`](Self::try_acquire) returns `None` From 15aceb462d884fc6b69821adaa9162203418af2d Mon Sep 17 00:00:00 2001 From: t Date: Thu, 30 Jul 2026 11:08:43 -0500 Subject: [PATCH 85/95] docs(node): drop a backstop claim with no backstop behind it (#174) The pinata_object_list_for_refs docstring said a visibility tightening that lands after tail-start is covered by the reconciliation sweep. No such sweep exists; state.rs says so directly, that a dropped job would be lost forever precisely because there is none. The claim is dropped and nothing replaces it. An earlier draft wanted to substitute a next-push claim, which no site establishes either, and swapping one unbacked claim for another is the same defect. Four pre-existing sites make the same claim (repos.rs:30, push_delta.rs at 8, 189 and 281 with warns at 349 and 358). They predate this PR and are left for a follow-up rather than widening a nine-round diff. --- crates/gitlawb-node/src/api/repos.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index b3051db3..72650086 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1047,10 +1047,10 @@ async fn resolve_drain_object_list( /// and the filter always fails closed (a newly-withheld blob is not pinned; a /// no-longer-announceable repo pins nothing). A tightening AFTER tail-start — before /// a slow re-derivation runs — is not reflected, matching the old retained-list -/// behavior; the reconciliation sweep is the backstop. Every git child runs -/// through the same INV-22 bounded, process-group-reaped helpers the sibling -/// post-receive scans use (`replication_withheld_set`, -/// `resolve_candidates_for_push`, `fail_closed_full_scan_objects`). +/// behavior. Every git child runs through the same INV-22 bounded, +/// process-group-reaped helpers the sibling post-receive scans use +/// (`replication_withheld_set`, `resolve_candidates_for_push`, +/// `fail_closed_full_scan_objects`). #[allow(clippy::too_many_arguments)] async fn pinata_object_list_for_refs( encrypt_sem: Arc, From 43d2dd786044f589a92acc9eee1bd6f66989bee5 Mon Sep 17 00:00:00 2001 From: t Date: Thu, 30 Jul 2026 11:09:08 -0500 Subject: [PATCH 86/95] docs(node): stop the per-source cap comments claiming a bound the keying lacks (#174) Both new comments said saturating the pool takes ~8 distinct source IPs, each also rate-limited. Neither half holds for an IPv6 caller with a routed /64: the sub-cap and the rate limiter both key on rate_limit::client_key, which with TrustedProxy::None falls through to the full peer address with no prefix folding, so one caller has 2^64 keys. max_keys does not compensate, since keys are freed on permit drop and live keys never approach the 100,000 default. The comments now say what the cap actually bounds, concurrent slots per resolved client key, and name the residual. The keying fix stays deferred as a design call, and that deferral is defensible: full-IP keying predates this PR, so these caps help IPv4 and single-address callers without regressing anything. Landing a comment that overstates the guarantee is not defensible, in a round that corrects a weaker overclaim one screen away. The per_source_push_cap docstring added earlier in this round repeated the same claim, so it is corrected here rather than shipping fresh. Comments only, no behaviour change. check, fmt clean. --- crates/gitlawb-node/src/main.rs | 20 ++++++++++++-------- crates/gitlawb-node/src/rate_limit.rs | 8 ++++++-- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 30c2ba37..486bfa0e 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -420,18 +420,22 @@ async fn main() -> Result<()> { config.max_concurrent_reads_per_caller, ), // Per-source cap on the receive-pack advertisement, sized to an eighth of the - // write pool (min 1): a single source can hold at most this many slots in the - // DEDICATED advert pool (git_push_advert_semaphore, disjoint from the write - // pool), so saturating that pool takes ~8 distinct source IPs, each also - // rate-limited (#174). Sized off the write pool only because the advert pool - // is created at the same size; an advert flood cannot touch a write permit. + // write pool (min 1): one resolved client key (rate_limit::client_key) can hold + // at most this many slots in the DEDICATED advert pool (git_push_advert_semaphore, + // disjoint from the write pool), so saturating that pool takes ~8 distinct keys + // (#174). That bounds an IPv4 or single-address caller; a caller controlling many + // addresses (an IPv6 /64 is 2^64 keys) still gets one cap per address, since + // client_key uses the full IP with no prefix folding. Narrowing the keying is a + // deferred design call, not something these caps claim to solve. Sized off the + // write pool only because the advert pool is created at the same size; an advert + // flood cannot touch a write permit. git_push_advert_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( rate_limit::per_source_push_cap(config.max_concurrent_git_pushes), ), // Per-source cap on the authenticated receive-pack POST, sized like the advert - // cap: one source IP can hold at most this many write-pool slots, so - // monopolizing the pool takes ~8 distinct source IPs, each also rate-limited - // (#174 P1-d). + // cap: one resolved client key can hold at most this many write-pool slots, so + // monopolizing the pool takes ~8 distinct keys (#174 P1-d). Same residual as + // above: keys are full IPs, so a caller with many addresses has many caps. git_write_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( rate_limit::per_source_push_cap(config.max_concurrent_git_pushes), ), diff --git a/crates/gitlawb-node/src/rate_limit.rs b/crates/gitlawb-node/src/rate_limit.rs index 6e2a7f3c..d203f69f 100644 --- a/crates/gitlawb-node/src/rate_limit.rs +++ b/crates/gitlawb-node/src/rate_limit.rs @@ -139,8 +139,12 @@ impl RateLimiter { } } -/// Per-source concurrency cap derived from the write-pool size: one source may hold -/// at most an eighth of the pool, so saturating it takes ~8 distinct source IPs. +/// Per-source concurrency cap derived from the write-pool size: one resolved client +/// key (see [`client_key`]) may hold at most an eighth of the pool, so saturating it +/// takes ~8 distinct keys. Real for an IPv4 or single-address caller; a caller with a +/// routed IPv6 /64 has 2^64 keys, because `client_key` returns the full address with +/// no prefix folding. Narrowing that keying is a deferred decision, so this cap is a +/// bound per key, not a bound per operator. /// /// Floored at 1 because the value feeds [`PerCallerConcurrency`], where a cap of 0 /// would shed EVERY receive-pack advertisement and break all pushes. The floor is From b9d901f978fe2f29be68e2b56cdef0d9a0192501 Mon Sep 17 00:00:00 2001 From: t Date: Thu, 30 Jul 2026 11:15:17 -0500 Subject: [PATCH 87/95] fix(node): dispose the write-guard connection when the Drop unlock errors (#174) Review finding from jatmn, round nine. RepoWriteGuard::release closes the connection when pg_advisory_unlock returns Err (F3b): the await resolved, so the session is alive and still holds the lock, and returning that connection to the pool hands the next caller a lock nobody tracks. The Drop backstop carried the same hazard and only logged. Both arms are fixed. The spawned detached unlock now closes the connection through close_conn_bounded on the error path, instead of letting the async block end and return it to the pool. The off-runtime arm was worse than the finding described. It dropped an already-taken connection with no unlock attempted at all, and this turned out to panic rather than leak: PoolConnection::drop calls rt::spawn for its return-to-pool task, whose no-runtime fallback panics with "this functionality requires a Tokio context". So any off-runtime guard drop panicked in a destructor today. It now detaches, which gives up the pool slot and yields a plain PgConnection whose drop closes the socket, ending the session that holds the lock. Drop cannot await, so detach is the whole disposal available here. Confirmed against the vendored sqlx 0.8.6 source rather than assumed, and min_connections is 0 everywhere in this repo. Three tests, modeled on the release F3b cases, on a pool with the idle reaper disabled so a reaped session cannot pass the assertion for us. Observed RED first: write_guard_dropped_off_runtime_disposes_the_connection ... FAILED panicked at sqlx-core-0.8.6/src/pool/connection.rs:208: this functionality requires a Tokio context then: dropping a write guard off a Tokio runtime must not panic write_guard_drop_with_failing_unlock_does_not_return_the_connection ... FAILED timed out waiting for the connection whose detached unlock errored to be closed rather than returned to the pool still holding the session lock write_guard_drop_with_successful_unlock_keeps_the_connection ... ok The success case is green on both sides on purpose: it is the must-not case, so it only ever catches a widening of disposal onto the healthy path. GREEN after, and all 12 guard tests pass, so the release coverage did not regress. Both new guards mutation-proved load-bearing, each red on its own message, 2/2. Full suite 729 passed, 0 failed. fmt and clippy clean. --- crates/gitlawb-node/src/git/repo_store.rs | 188 +++++++++++++++++++++- 1 file changed, 180 insertions(+), 8 deletions(-) diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index afc0d6e8..6fcc5a92 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -674,8 +674,8 @@ impl Drop for RepoWriteGuard { /// error: that case is closed inside `release` by disposing of the connection, /// because `Drop` early-returns on `conn == None` and the two mechanisms cannot /// both apply (#174 F3b). `Drop` cannot await, so spawn a detached unlock — it runs on the - /// same session (connection-affine). An off-runtime drop falls back to a log; - /// the ~60s stale-lock retry loop in `acquire_write` reclaims it. On runtime + /// same session (connection-affine). An off-runtime drop has nothing to spawn onto, + /// so it disposes of the connection instead. On runtime /// SHUTDOWN the spawned unlock task may be dropped before it polls, so the unlock /// may not run — but shutdown tears down the pool, and closing the connection /// releases the session-level advisory lock server-side, so this too is bounded. @@ -691,20 +691,38 @@ impl Drop for RepoWriteGuard { match tokio::runtime::Handle::try_current() { Ok(handle) => { handle.spawn(async move { - if let Err(e) = sqlx::query("SELECT pg_advisory_unlock($1)") + let unlock = sqlx::query("SELECT pg_advisory_unlock($1)") .bind(lock_key) .execute(&mut *conn) - .await - { - warn!(repo = %repo_name, err = %e, "detached advisory-unlock on write-guard drop failed"); + .await; + // Same failure as `release`'s (#174 F3b), one level down: the await + // RESOLVED with an error, so the session is alive and still holds + // the lock. Letting this async block end here would drop `conn` and + // RETURN it to the pool, handing the next caller a connection + // holding a lock nobody tracks. Close it instead, which both keeps + // it out of the pool and ends the session that holds the lock. + if let Err(e) = unlock { + warn!(repo = %repo_name, err = %e, "detached advisory-unlock on write-guard drop failed, closing the connection so the session ends and postgres drops the lock"); + close_conn_bounded(&repo_name, conn.close()).await; } }); } Err(_) => { + // No runtime to spawn the unlock onto, and the connection is already + // out of the guard, so there is no path that unlocks on this session. + // Returning it to the pool would hand the next caller a connection + // still holding the lock. `PoolConnection`'s own drop also spawns its + // return-to-pool task, which panics with no runtime. `detach` gives up + // the pool slot and yields a plain `PgConnection`; dropping that closes + // the socket, which ends the session and is what frees the lock + // server-side. `Drop` cannot await, so this is the whole disposal: + // `close_conn_bounded` is not available here. + drop(conn.detach()); warn!( repo = %repo_name, - "RepoWriteGuard dropped off a Tokio runtime; advisory lock not released \ - synchronously — the stale-lock retry loop will reclaim it" + "RepoWriteGuard dropped off a Tokio runtime; no detached unlock is \ + possible, so the pinned connection is disposed of instead: ending \ + the session is what releases the advisory lock" ); } } @@ -1544,6 +1562,160 @@ mod tests { .await; } + // ── the Drop backstop disposes its connection too (#174 U8) ───────────── + + /// U8 (P2): the `Drop` backstop carries the same hazard F3b closed in `release`. + /// When the detached `pg_advisory_unlock` ERRORS on a live session, the async + /// block ends and drops the moved `PoolConnection`, which RETURNS it to the pool + /// while that session may still hold the lock, handing the next caller a + /// connection holding a lock nobody tracks. The errored connection must be closed + /// instead: that keeps it out of the pool and ends the session, which is what + /// frees the lock server-side. + /// + /// Run against a pool with the idle reaper disabled, for the reason spelled out on + /// `pool_without_idle_reaper`: with the reaper in play the session dies on its own + /// about a second later and the assertion stops measuring the disposal. + /// + /// Load-bearing: RED before the fix (the connection goes back to the pool, so the + /// size never drops and this times out), GREEN after. + #[sqlx::test] + async fn write_guard_drop_with_failing_unlock_does_not_return_the_connection( + pool: sqlx::PgPool, + ) { + let dir = tempfile::TempDir::new().unwrap(); + let store_pool = pool_without_idle_reaper(&pool).await; + let store = RepoStore::for_testing(dir.path().to_path_buf(), store_pool.clone()); + let owner = "did:key:z6MkDropUnlockErrProofIIIIIIIIIIIIIIIIIIII"; + let name = "dropunlockerrtest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + // Distinct session for the probe, held out of the store's pool entirely. + let mut checker = pool.acquire().await.expect("checker connection"); + + let mut guard = store.acquire_write(owner, name).await.expect("acquire"); + poison_guard_connection(&mut guard).await; + let size_before = store_pool.size(); + assert!(size_before > 0, "the pool owns the guard's connection"); + + // The backstop shape: dropped without release(), with an unlock that errors. + drop(guard); + + wait_until( + || store_pool.size() == size_before - 1, + "the connection whose detached unlock errored to be closed rather than \ + returned to the pool still holding the session lock", + ) + .await; + wait_until_lock_free( + &mut checker, + key, + "an errored detached unlock must not leave the lock held: Drop must dispose \ + of the connection so the session ends", + ) + .await; + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } + + /// U8 regression guard on the success path: a detached unlock that SUCCEEDS must + /// still return the connection to the pool. Without this, "close the connection on + /// Drop" could be widened to "always close" and the test above would not notice. + #[sqlx::test] + async fn write_guard_drop_with_successful_unlock_keeps_the_connection(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let store_pool = pool_without_idle_reaper(&pool).await; + let store = RepoStore::for_testing(dir.path().to_path_buf(), store_pool.clone()); + let owner = "did:key:z6MkDropUnlockOkProofJJJJJJJJJJJJJJJJJJJJ"; + let name = "dropunlockoktest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + let mut checker = pool.acquire().await.expect("checker connection"); + let guard = store.acquire_write(owner, name).await.expect("acquire"); + let size_before = store_pool.size(); + assert!(size_before > 0, "the pool owns the guard's connection"); + + drop(guard); + + // The connection goes back only once the detached unlock task has finished. + wait_until( + || store_pool.num_idle() > 0, + "the detached unlock to finish and hand the connection back", + ) + .await; + assert_eq!( + store_pool.size(), + size_before, + "a successful detached unlock must leave the connection in the pool" + ); + wait_until_lock_free( + &mut checker, + key, + "the Drop backstop's successful unlock to free the lock", + ) + .await; + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } + + /// U8, the off-runtime arm: with no Tokio runtime there is nothing to spawn the + /// unlock onto, and the connection has already been taken out of the guard, so the + /// old code dropped it with no unlock attempted at all, back to the pool, session + /// lock still held. Dropping a `PoolConnection` off a runtime is worse than that: + /// sqlx's return-to-pool path spawns, and its no-runtime fallback panics, so the + /// old arm also panicked in a destructor. + /// + /// Reached by dropping the guard on a plain `std::thread`, where + /// `Handle::try_current()` fails. + /// + /// Load-bearing: RED before the fix (the join sees the "requires a Tokio context" + /// panic from sqlx's return-to-pool spawn), GREEN after (`detach` gives up the + /// pool slot, so nothing is spawned and dropping the detached connection closes + /// the socket, which ends the session and frees the lock). + #[sqlx::test] + async fn write_guard_dropped_off_runtime_disposes_the_connection(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let store_pool = pool_without_idle_reaper(&pool).await; + let store = RepoStore::for_testing(dir.path().to_path_buf(), store_pool.clone()); + let owner = "did:key:z6MkDropOffRuntimeProofKKKKKKKKKKKKKKKKKK"; + let name = "dropoffruntimetest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + let mut checker = pool.acquire().await.expect("checker connection"); + let guard = store.acquire_write(owner, name).await.expect("acquire"); + let size_before = store_pool.size(); + assert!(size_before > 0, "the pool owns the guard's connection"); + + let dropped = std::thread::spawn(move || drop(guard)).join(); + assert!( + dropped.is_ok(), + "dropping a write guard off a Tokio runtime must not panic" + ); + + wait_until( + || store_pool.size() == size_before - 1, + "the connection of a guard dropped off a runtime to be disposed of rather \ + than returned to the pool with no unlock attempted", + ) + .await; + wait_until_lock_free( + &mut checker, + key, + "a guard dropped off a runtime to end its session so postgres drops the lock", + ) + .await; + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } + /// F4: releasing a guard that never took the lock (`locked == false`, the state /// acquire_write leaves after a failed acquisition) must not unlock or panic. #[sqlx::test] From fa9bf08739f0ced4de8d2f36b8ebfbba28370030 Mon Sep 17 00:00:00 2001 From: t Date: Thu, 30 Jul 2026 17:28:25 -0500 Subject: [PATCH 88/95] fix(node): saturate the derived lease steal bound instead of overflowing it (#174) GITLAWB_GIT_SERVICE_TIMEOUT_SECS accepts every positive u64, so deriving the per-repo write-lease steal bound as `* 2 + 60` panics every push in a debug build across the top of the accepted range, and in release wraps to a short Duration that would let a waiter steal a live push's lease. Saturating keeps the knob's documented meaning: a timeout that large means the service bound is effectively off, so the steal bound should be effectively infinite rather than tiny. The regression drives git_receive_pack rather than the arithmetic, so the call-site wiring is what is under test. RED before at repos.rs:1733 with "attempt to multiply with overflow", GREEN after; re-confirmed load-bearing by reinstating the unchecked expression. --- crates/gitlawb-node/src/api/repos.rs | 71 +++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 2 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 72650086..61cc42cf 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1729,8 +1729,19 @@ pub async fn git_receive_pack( // bound — on the non-disconnect path the retained pg advisory lock still serializes // the stealer at acquire_write (a spurious 503, not a race); the only corruption-capable // overlap is the ~4s disconnect/reap window, which the reaper-carried clone (a) covers. - let lease_steal_after = - std::time::Duration::from_secs(state.config.git_service_timeout_secs * 2 + 60); + // Saturating, not unchecked: `git_service_timeout_secs` accepts every positive u64 and + // its help text explicitly permits setting it very large to disable the service bound, + // so `* 2 + 60` overflows across the top of the accepted range — a debug panic on every + // push, and in release a WRAPPED bound short enough for a waiter to steal a live push's + // lease. Saturating keeps the documented semantic intact: a timeout that large means no + // steal, which is what an effectively-disabled service bound implies. + let lease_steal_after = std::time::Duration::from_secs( + state + .config + .git_service_timeout_secs + .saturating_mul(2) + .saturating_add(60), + ); // Parked waiters are bounded INSIDE the lease, not by an admission permit taken above // it. `body: Bytes` means axum has already buffered the whole pack before this handler @@ -7933,6 +7944,62 @@ mod tests { handle.abort(); } + /// #174 (RED-before/GREEN-after): a large-but-valid `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` + /// must not overflow the lease steal bound derived from it. The config help explicitly + /// permits setting the timeout very large to disable the bound and clap accepts every + /// positive `u64`, so an unchecked `* 2 + 60` panics the push in a debug build and + /// wraps to a short `Duration` in release — a wrapped bound would let a waiter steal + /// the lease out from under a live push. Drives the handler rather than the + /// arithmetic, so the call-site wiring is what is under test. + #[cfg(unix)] + #[sqlx::test] + async fn push_survives_a_git_service_timeout_that_overflows_the_lease_bound( + pool: sqlx::PgPool, + ) { + use axum::extract::{Path, State}; + use axum::response::IntoResponse; + use axum::Extension; + use std::net::SocketAddr; + + let tmp = tempfile::TempDir::new().unwrap(); + let git_bin = write_fake_git( + tmp.path(), + "#!/bin/sh\n\ + case \"$1\" in\n\ + receive-pack) cat > /dev/null 2>/dev/null ;;\n\ + rev-parse) echo deadbeef ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + ); + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6ovflow", "o1", false).await; + // The largest value clap accepts. Any value above (u64::MAX - 60) / 2 overflows the + // derived bound, so this is the whole tail of the accepted range, not a corner. + let mut cfg = (*state.config).clone(); + cfg.git_service_timeout_secs = u64::MAX; + state.config = std::sync::Arc::new(cfg); + + let resp = git_receive_pack( + State(state), + Path(("z6ovflow".to_string(), "o1".to_string())), + Extension(crate::auth::AuthenticatedDid( + "did:key:z6MkOverflowPusherAAAAAAAAAAAAAAAAAAAAAA".to_string(), + )), + crate::rate_limit::PeerAddr(Some("203.0.113.90:5000".parse::().unwrap())), + axum::http::HeaderMap::new(), + ref_update_body("1111111111111111111111111111111111111111"), + ) + .await + .expect("push must succeed under a maximal git_service_timeout_secs") + .into_response(); + assert_eq!( + resp.status(), + 200, + "a maximal service timeout must disable the steal bound, not break the push" + ); + } + /// #174 U1 scenario 1 (RED-before/GREEN-after): parked pushes on one repo's write /// lease are BOUNDED. `git_receive_pack` takes `body: Bytes`, so axum has already /// buffered the whole pack (up to `max_pack_bytes`) before the handler runs, and the From 88f5e938fdaeea91f5ab1745586a8ed319c22b8f Mon Sep 17 00:00:00 2001 From: t Date: Thu, 30 Jul 2026 17:28:36 -0500 Subject: [PATCH 89/95] fix(node): bound git_service_timeout_secs to a range its deadlines can represent (#174) Tracing the lease-bound overflow to its other consumers turned up a worse instance of the same defect. This PR routed git_service_timeout_secs into build_filtered_pack and blob_paths, and both build a deadline as `Instant::now() + Duration::from_secs(..)`. That addition panics on overflow in RELEASE as well as debug, so a large-but-accepted value crashes the serve path rather than merely disabling a timeout. Hardening each arithmetic site as it is found would leave the next one open, so the bound goes where the value enters: clap now rejects anything above 365 days at parse time, which keeps every derived duration in range at once and fails fast at boot instead of at the first path-scoped clone. A year is many orders of magnitude above any real clone or push, so it remains the practical way to disable the bound. The test pins both derivations at the maximum and rejects the value one past it and u64::MAX. RED when the upper bound is dropped from the value_parser. --- .env.example | 3 +- README.md | 2 +- crates/gitlawb-node/src/config.rs | 72 ++++++++++++++++++++++++++++--- 3 files changed, 69 insertions(+), 8 deletions(-) diff --git a/.env.example b/.env.example index aea17f61..c719fa1f 100644 --- a/.env.example +++ b/.env.example @@ -120,7 +120,8 @@ GITLAWB_MAX_PACK_BYTES=2147483648 # rather than giving each stage a full budget: a walk that consumes it leaves the # serve nothing and the clone gets a 504. Serving large path-scoped repos may # need a higher value here than when each stage was budgeted separately. -# Default 600. +# Must be 1..=31536000 (365 days): the node derives deadlines from this value, +# and a larger one cannot be represented. Default 600. GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600 # Max seconds the storage-ACQUISITION phase of a served git op may run before the diff --git a/README.md b/README.md index f4ba0f9e..73c8127d 100644 --- a/README.md +++ b/README.md @@ -343,7 +343,7 @@ Important node settings: | `GITLAWB_REQUIRE_SIGNED_PEER_WRITES` | Require signed peer announce/sync writes. | | `GITLAWB_AUTO_SYNC` | Enable automatic sync from known peers. | | `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. | -| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. On the path-scoped upload-pack path the classification walk and the pack serve share ONE deadline, so this value bounds their combined duration rather than granting each stage a full budget: a walk that consumes it leaves the serve nothing and the clone gets a 504. Serving large path-scoped repos may therefore need a higher value than they did when each stage was budgeted separately. | +| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. On the path-scoped upload-pack path the classification walk and the pack serve share ONE deadline, so this value bounds their combined duration rather than granting each stage a full budget: a walk that consumes it leaves the serve nothing and the clone gets a 504. Serving large path-scoped repos may therefore need a higher value than they did when each stage was budgeted separately. Accepted range is 1 to 31536000 (365 days), since the node derives deadlines from this value and a larger one cannot be represented. | | `GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS` | Max seconds the storage-acquisition phase (Tigris HEAD/GET, push advisory-lock) of a served git op may run before the request is shed with a 503, separate from the git-run timeout. The concurrency permit is released on expiry so a stalled backend cannot pin the pool. Default 30. | | `GITLAWB_REPO_LEASE_MAX_WAITERS` | Max pushes parked at once waiting for the same repo's write lease. Each waiter pins its buffered pack body, so this bounds that memory for a hot repo; past the cap the newest push sheds a 503 + Retry-After instead of queueing. Pushes to other repos are unaffected, and the lease holder is not counted. Default 8. | | `GITLAWB_MAX_CONCURRENT_IPFS_WALKS` | Max concurrent `GET /ipfs/{cid}` visibility walks across all callers (own pool, disjoint from the served-git pools); over-cap sheds 503. Default 32. | diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 5ba10584..fbe894db 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -1,6 +1,20 @@ use clap::Parser; use std::path::PathBuf; +/// Upper bound on `git_service_timeout_secs`, in seconds (365 days). +/// +/// The knob is not just stored, it is arithmetic input: the write path derives the +/// per-repo lease steal bound from it (`* 2 + 60`), and #174 routed it into +/// `build_filtered_pack` and `blob_paths`, which each build a deadline as +/// `Instant::now() + Duration::from_secs(this)`. That addition panics on overflow in +/// RELEASE as well as debug, so an unbounded `u64` here turns an operator typo into a +/// serve-path crash rather than a very long timeout. Bounding at parse time keeps every +/// derived duration in range at once, instead of hardening each site as it is found. +/// +/// A year is many orders of magnitude above any real clone or push, so this remains the +/// practical way to disable the bound. +pub const GIT_SERVICE_TIMEOUT_SECS_MAX: u64 = 365 * 24 * 60 * 60; + #[derive(Parser, Debug, Clone)] #[command(name = "gitlawb-node", about = "gitlawb node daemon", version)] pub struct Config { @@ -170,16 +184,18 @@ pub struct Config { /// Maximum wall-clock time a single served git operation (upload-pack / /// receive-pack through `run_git_service`) may run before it is aborted and /// its process group torn down, in seconds. Bounds a git that neither - /// finishes nor disconnects. Must be positive; set it very large to - /// effectively disable the bound. Default: 600s (10 min), generous for large - /// clones. Also bounds the ref advertisement (`info/refs`) and the withheld-blob - /// pack build (`upload_pack_excluding`'s pack-objects stage), which now share the - /// same timeout + process-group teardown (#174). + /// finishes nor disconnects. Must be positive and at most + /// [`GIT_SERVICE_TIMEOUT_SECS_MAX`] (365 days), which is far above any real git + /// operation and is the practical way to disable the bound. Default: 600s + /// (10 min), generous for large clones. Also bounds the ref advertisement + /// (`info/refs`) and the withheld-blob pack build (`upload_pack_excluding`'s + /// pack-objects stage), which now share the same timeout + process-group + /// teardown (#174). #[arg( long, env = "GITLAWB_GIT_SERVICE_TIMEOUT_SECS", default_value_t = 600, - value_parser = clap::value_parser!(u64).range(1..) + value_parser = clap::value_parser!(u64).range(1..=GIT_SERVICE_TIMEOUT_SECS_MAX) )] pub git_service_timeout_secs: u64, @@ -563,6 +579,50 @@ mod tests { ); } + /// #174 (RED-before/GREEN-after): the upper bound is what keeps every duration + /// derived from this knob in range — the lease steal bound's `* 2 + 60` on the write + /// path, and the `Instant::now() + Duration::from_secs(..)` deadlines in + /// `build_filtered_pack` / `blob_paths` on the serve path, which panic on overflow in + /// release builds too. Checked at parse time so no reachable configuration can carry a + /// value those sites cannot represent. + #[test] + fn git_service_timeout_rejects_values_no_derived_duration_can_represent() { + // At the bound: accepted, and every derived duration still fits. + let at_max = Config::try_parse_from([ + "gitlawb-node", + "--git-service-timeout-secs", + &GIT_SERVICE_TIMEOUT_SECS_MAX.to_string(), + ]) + .expect("the documented maximum must parse"); + assert_eq!( + at_max.git_service_timeout_secs, + GIT_SERVICE_TIMEOUT_SECS_MAX + ); + assert!(at_max + .git_service_timeout_secs + .checked_mul(2) + .and_then(|v| v.checked_add(60)) + .is_some()); + assert!(std::time::Instant::now() + .checked_add(std::time::Duration::from_secs( + at_max.git_service_timeout_secs + )) + .is_some()); + + // One past the bound, and the top of the u64 range clap used to accept. + for over in [GIT_SERVICE_TIMEOUT_SECS_MAX + 1, u64::MAX] { + assert!( + Config::try_parse_from([ + "gitlawb-node", + "--git-service-timeout-secs", + &over.to_string(), + ]) + .is_err(), + "{over} exceeds the derivable range and must be rejected at parse time" + ); + } + } + #[test] fn max_concurrent_pin_tasks_defaults_and_rejects_out_of_range() { assert_eq!( From 578ae5b99d4275898d546bc15dd21823353cf0e2 Mon Sep 17 00:00:00 2001 From: t Date: Thu, 30 Jul 2026 18:07:16 -0500 Subject: [PATCH 90/95] fix(node): set the git_service_timeout ceiling by representability, not tidiness (#174) A second-model pass on the previous two commits caught an upgrade footgun in the first and stale prose in the second. The 365-day ceiling was tighter than the defect required. The help text has always told operators to set this very large to disable the bound, and clap accepted every positive u64, so values like 999999999 (~31 years) are working production "off" settings that neither overflow `* 2 + 60` nor exceed an Instant deadline. A node running one of those would have failed to start after this upgrade, over a value that was never the defect. The ceiling moves to 100 years, which clears every such setting and stays an order of magnitude under the ~584-year limit of a u64-nanosecond Instant, the tightest representation on any platform we build for. Every value that worked before still parses; only the ones that would have panicked are rejected. The test now pins the pre-existing disable values alongside the rejections so the distinction cannot quietly regress. Clamp-and-warn was the suggested remedy and is not what landed: it would keep accepting a value the node cannot honor and silently substitute a different one, where the range says what is representable and fails fast, matching how every other bound in this file behaves. The lease-bound comment and its test still described the pre-ceiling world. Both now say what is true, and the test deliberately keeps u64::MAX rather than moving to the ceiling as suggested: Config is reachable by direct construction, and at the ceiling the arithmetic does not overflow, so a test pinned there would pass with the saturation removed and prove nothing. Re-confirmed load-bearing after the rewording. --- .env.example | 2 +- README.md | 2 +- crates/gitlawb-node/src/api/repos.rs | 37 +++++++++------ crates/gitlawb-node/src/config.rs | 70 +++++++++++++++++++--------- 4 files changed, 72 insertions(+), 39 deletions(-) diff --git a/.env.example b/.env.example index c719fa1f..5a3a5acc 100644 --- a/.env.example +++ b/.env.example @@ -120,7 +120,7 @@ GITLAWB_MAX_PACK_BYTES=2147483648 # rather than giving each stage a full budget: a walk that consumes it leaves the # serve nothing and the clone gets a 504. Serving large path-scoped repos may # need a higher value here than when each stage was budgeted separately. -# Must be 1..=31536000 (365 days): the node derives deadlines from this value, +# Must be 1..=3153600000 (100 years): the node derives deadlines from this value, # and a larger one cannot be represented. Default 600. GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600 diff --git a/README.md b/README.md index 73c8127d..d5b9c6b9 100644 --- a/README.md +++ b/README.md @@ -343,7 +343,7 @@ Important node settings: | `GITLAWB_REQUIRE_SIGNED_PEER_WRITES` | Require signed peer announce/sync writes. | | `GITLAWB_AUTO_SYNC` | Enable automatic sync from known peers. | | `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. | -| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. On the path-scoped upload-pack path the classification walk and the pack serve share ONE deadline, so this value bounds their combined duration rather than granting each stage a full budget: a walk that consumes it leaves the serve nothing and the clone gets a 504. Serving large path-scoped repos may therefore need a higher value than they did when each stage was budgeted separately. Accepted range is 1 to 31536000 (365 days), since the node derives deadlines from this value and a larger one cannot be represented. | +| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. On the path-scoped upload-pack path the classification walk and the pack serve share ONE deadline, so this value bounds their combined duration rather than granting each stage a full budget: a walk that consumes it leaves the serve nothing and the clone gets a 504. Serving large path-scoped repos may therefore need a higher value than they did when each stage was budgeted separately. Accepted range is 1 to 3153600000 (100 years), since the node derives deadlines from this value and a larger one cannot be represented. | | `GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS` | Max seconds the storage-acquisition phase (Tigris HEAD/GET, push advisory-lock) of a served git op may run before the request is shed with a 503, separate from the git-run timeout. The concurrency permit is released on expiry so a stalled backend cannot pin the pool. Default 30. | | `GITLAWB_REPO_LEASE_MAX_WAITERS` | Max pushes parked at once waiting for the same repo's write lease. Each waiter pins its buffered pack body, so this bounds that memory for a hot repo; past the cap the newest push sheds a 503 + Retry-After instead of queueing. Pushes to other repos are unaffected, and the lease holder is not counted. Default 8. | | `GITLAWB_MAX_CONCURRENT_IPFS_WALKS` | Max concurrent `GET /ipfs/{cid}` visibility walks across all callers (own pool, disjoint from the served-git pools); over-cap sheds 503. Default 32. | diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 61cc42cf..22a216fe 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1729,12 +1729,14 @@ pub async fn git_receive_pack( // bound — on the non-disconnect path the retained pg advisory lock still serializes // the stealer at acquire_write (a spurious 503, not a race); the only corruption-capable // overlap is the ~4s disconnect/reap window, which the reaper-carried clone (a) covers. - // Saturating, not unchecked: `git_service_timeout_secs` accepts every positive u64 and - // its help text explicitly permits setting it very large to disable the service bound, - // so `* 2 + 60` overflows across the top of the accepted range — a debug panic on every - // push, and in release a WRAPPED bound short enough for a waiter to steal a live push's - // lease. Saturating keeps the documented semantic intact: a timeout that large means no - // steal, which is what an effectively-disabled service bound implies. + // Saturating, not unchecked. `GIT_SERVICE_TIMEOUT_SECS_MAX` now keeps every parsed + // value inside this arithmetic, so on the configured path this cannot overflow; it is + // deliberate defense in depth for the construction paths clap does not cover (tests + // build `Config` by mutation, and nothing stops a future caller doing the same). The + // failure it holds off is not cosmetic: unchecked, `* 2 + 60` panics the push in debug + // and in release WRAPS to a bound short enough for a waiter to steal a live push's + // lease. Saturating also states the intent — a timeout that large means no steal, which + // is what an effectively-disabled service bound implies. let lease_steal_after = std::time::Duration::from_secs( state .config @@ -7944,13 +7946,18 @@ mod tests { handle.abort(); } - /// #174 (RED-before/GREEN-after): a large-but-valid `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` - /// must not overflow the lease steal bound derived from it. The config help explicitly - /// permits setting the timeout very large to disable the bound and clap accepts every - /// positive `u64`, so an unchecked `* 2 + 60` panics the push in a debug build and - /// wraps to a short `Duration` in release — a wrapped bound would let a waiter steal - /// the lease out from under a live push. Drives the handler rather than the - /// arithmetic, so the call-site wiring is what is under test. + /// #174 (RED-before/GREEN-after): the lease steal bound derived from + /// `git_service_timeout_secs` must not overflow. Unchecked, `* 2 + 60` panics the push + /// in a debug build and wraps to a short `Duration` in release, and a wrapped bound + /// would let a waiter steal the lease out from under a live push. Drives the handler + /// rather than the arithmetic, so the call-site wiring is what is under test. + /// + /// `GIT_SERVICE_TIMEOUT_SECS_MAX` means clap no longer admits a value this large, and + /// the test keeps `u64::MAX` anyway rather than moving to the ceiling. Two reasons, and + /// the second is the load-bearing one: `Config` is reachable by direct construction, + /// which is how this test and every other one build it; and at the ceiling the + /// arithmetic does not overflow, so a test pinned there would pass with the saturation + /// removed and prove nothing about this line. #[cfg(unix)] #[sqlx::test] async fn push_survives_a_git_service_timeout_that_overflows_the_lease_bound( @@ -7974,8 +7981,8 @@ mod tests { ); let mut state = f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6ovflow", "o1", false).await; - // The largest value clap accepts. Any value above (u64::MAX - 60) / 2 overflows the - // derived bound, so this is the whole tail of the accepted range, not a corner. + // Every value above (u64::MAX - 60) / 2 overflows the derived bound; u64::MAX is + // the top of that tail, and the value clap accepted before the ceiling landed. let mut cfg = (*state.config).clone(); cfg.git_service_timeout_secs = u64::MAX; state.config = std::sync::Arc::new(cfg); diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index fbe894db..c038a525 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -1,7 +1,7 @@ use clap::Parser; use std::path::PathBuf; -/// Upper bound on `git_service_timeout_secs`, in seconds (365 days). +/// Upper bound on `git_service_timeout_secs`, in seconds (100 years). /// /// The knob is not just stored, it is arithmetic input: the write path derives the /// per-repo lease steal bound from it (`* 2 + 60`), and #174 routed it into @@ -11,9 +11,16 @@ use std::path::PathBuf; /// serve-path crash rather than a very long timeout. Bounding at parse time keeps every /// derived duration in range at once, instead of hardening each site as it is found. /// -/// A year is many orders of magnitude above any real clone or push, so this remains the -/// practical way to disable the bound. -pub const GIT_SERVICE_TIMEOUT_SECS_MAX: u64 = 365 * 24 * 60 * 60; +/// The ceiling is representability, NOT a policy view of a sane timeout, and that +/// distinction is what sets the number. The help text has always told operators to set +/// this very large to disable the bound, so values like `999999999` (~31 years) are +/// working production "off" settings; a tighter, tidier cap would fail those nodes at +/// boot on upgrade over a value that was never the defect. 100 years clears every such +/// setting while staying an order of magnitude under the ~584-year ceiling of a +/// `u64`-nanosecond `Instant`, which is the tightest representation on any platform we +/// build for. Every value that worked before still parses; only the ones that would have +/// panicked are rejected. +pub const GIT_SERVICE_TIMEOUT_SECS_MAX: u64 = 100 * 365 * 24 * 60 * 60; #[derive(Parser, Debug, Clone)] #[command(name = "gitlawb-node", about = "gitlawb node daemon", version)] @@ -185,9 +192,10 @@ pub struct Config { /// receive-pack through `run_git_service`) may run before it is aborted and /// its process group torn down, in seconds. Bounds a git that neither /// finishes nor disconnects. Must be positive and at most - /// [`GIT_SERVICE_TIMEOUT_SECS_MAX`] (365 days), which is far above any real git - /// operation and is the practical way to disable the bound. Default: 600s - /// (10 min), generous for large clones. Also bounds the ref advertisement + /// [`GIT_SERVICE_TIMEOUT_SECS_MAX`] (100 years, the largest value every derived + /// deadline can represent); setting it very large is still the way to disable the + /// bound. Default: 600s (10 min), generous for large clones. Also bounds the ref + /// advertisement /// (`info/refs`) and the withheld-blob pack build (`upload_pack_excluding`'s /// pack-objects stage), which now share the same timeout + process-group /// teardown (#174). @@ -585,15 +593,34 @@ mod tests { /// `build_filtered_pack` / `blob_paths` on the serve path, which panic on overflow in /// release builds too. Checked at parse time so no reachable configuration can carry a /// value those sites cannot represent. + /// + /// The pre-existing "set it very large to disable the bound" settings are asserted + /// alongside the rejections on purpose. The bound exists to exclude unrepresentable + /// values, not to impose a view of a reasonable timeout, so a node that has been + /// running on ~31 years must not start failing at boot on upgrade. #[test] fn git_service_timeout_rejects_values_no_derived_duration_can_represent() { - // At the bound: accepted, and every derived duration still fits. - let at_max = Config::try_parse_from([ - "gitlawb-node", - "--git-service-timeout-secs", - &GIT_SERVICE_TIMEOUT_SECS_MAX.to_string(), - ]) - .expect("the documented maximum must parse"); + let parse = |secs: u64| { + Config::try_parse_from([ + "gitlawb-node", + "--git-service-timeout-secs", + &secs.to_string(), + ]) + }; + + // Large "disable the bound" values that predate the ceiling still parse. + for disable in [1_000_000_000, 999_999_999] { + assert_eq!( + parse(disable) + .unwrap_or_else(|e| panic!("{disable} was a working setting: {e}")) + .git_service_timeout_secs, + disable + ); + } + + // At the ceiling: accepted, and every derived duration still fits. + let at_max = + parse(GIT_SERVICE_TIMEOUT_SECS_MAX).expect("the documented maximum must parse"); assert_eq!( at_max.git_service_timeout_secs, GIT_SERVICE_TIMEOUT_SECS_MAX @@ -609,18 +636,17 @@ mod tests { )) .is_some()); - // One past the bound, and the top of the u64 range clap used to accept. + // Past the ceiling, and the top of the u64 range clap used to accept — the value + // that panics `Instant::now() + Duration::from_secs(..)` outright. for over in [GIT_SERVICE_TIMEOUT_SECS_MAX + 1, u64::MAX] { assert!( - Config::try_parse_from([ - "gitlawb-node", - "--git-service-timeout-secs", - &over.to_string(), - ]) - .is_err(), - "{over} exceeds the derivable range and must be rejected at parse time" + parse(over).is_err(), + "{over} is past the representable ceiling and must be rejected at parse time" ); } + assert!(std::time::Instant::now() + .checked_add(std::time::Duration::from_secs(u64::MAX)) + .is_none()); } #[test] From 44efb996674aba19f8a23d9b734387a398f6c9e8 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 31 Jul 2026 00:56:04 -0500 Subject: [PATCH 91/95] fix(node): bound GITLAWB_IPFS_REQUEST_BUDGET_SECS by representability ipfs_request_budget_secs parsed range(1..), so every positive u64 was accepted, but get_by_cid derives the request deadline as Instant::now() + Duration::from_secs(that). That addition is an explicit overflow check rather than a debug-only one, so an oversized value aborts every /ipfs/{cid} request in a release build instead of setting a very long budget. The route is anon-reachable, so an operator typo is felt publicly. Reuse GIT_SERVICE_TIMEOUT_SECS_MAX, the ceiling the sibling knob already carries: the bound is representability, not a policy view of a sane budget, so the large "set it very large to disable" values operators were told to use still parse. Reject rather than clamp, matching the other bounds in this config. Execution evidence: - RED before the change: the new test failed with "3153600001 is past the representable ceiling and must be rejected at parse time" while its disable-value assertions passed, so the test is not vacuous in either direction. - GREEN after: config::tests 13 passed, including the sibling git_service_timeout regression test. Sweep for the same defect class, re-run with a widened shape because the original grep missed a site. Passes: Instant::now() sites; from_secs() fed by config fields; config fields reaching a Duration through an arithmetic expression; config fields reaching a tokio timer constructor. The arithmetic and timer passes each return exactly one hit, heartbeat_interval_hours (config.rs 150, main.rs 916, operator.rs 161), which has no range at all and multiplies by 3600 before the Duration. That knob is a separate defect on a path this branch does not touch, so it is recorded for a follow-up rather than widened into here. Every other Instant addition derives from the already-bounded git_service_timeout_secs or from a constant. --- crates/gitlawb-node/src/config.rs | 76 +++++++++++++++++++++++++++++-- 1 file changed, 73 insertions(+), 3 deletions(-) diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index c038a525..2d4299c2 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -500,13 +500,21 @@ pub struct Config { /// slot past it. Still unbounded: the probe's `object_store_readable` check is a /// synchronous filesystem sweep with nothing to reap, so a wedged filesystem can /// hold the slot past the deadline. - /// Must be positive. Default: 600s (10 min), matching - /// `git_service_timeout_secs` so a single full-length walk still fits. + /// Must be positive, and no larger than `GIT_SERVICE_TIMEOUT_SECS_MAX`. The ceiling is + /// representability, NOT a policy view of a sane budget: `get_by_cid` derives the + /// request deadline as `Instant::now() + Duration::from_secs(this)`, and that addition + /// is an explicit overflow check rather than a debug-only one, so a value past the + /// ceiling aborts every `/ipfs/{cid}` request in a release build instead of setting a + /// very long budget. Rejecting at parse time keeps that value out of every reachable + /// configuration. Setting it very large is still the way to effectively disable the + /// budget; the ceiling clears every such setting (see the constant's own note). + /// Default: 600s (10 min), matching `git_service_timeout_secs` so a single full-length + /// walk still fits. #[arg( long, env = "GITLAWB_IPFS_REQUEST_BUDGET_SECS", default_value_t = 600, - value_parser = clap::value_parser!(u64).range(1..) + value_parser = clap::value_parser!(u64).range(1..=GIT_SERVICE_TIMEOUT_SECS_MAX) )] pub ipfs_request_budget_secs: u64, @@ -819,6 +827,68 @@ mod tests { ); } + /// #174 (RED-before/GREEN-after): the upper bound is what keeps the deadline derived + /// from this knob in range. `get_by_cid` builds the request budget as + /// `Instant::now() + Duration::from_secs(this)` (api/ipfs.rs), and that addition is an + /// explicit overflow check rather than a debug-only one, so an oversized value aborts + /// every `/ipfs/{cid}` request in a release build instead of setting a very long budget. + /// The route is anon-reachable, so the failure is operator-triggered but publicly felt. + /// Checked at parse time so no reachable configuration can carry a value the deadline + /// cannot represent. + /// + /// The large "disable the bound" settings are asserted alongside the rejections on + /// purpose, the same way the `git_service_timeout_secs` sibling does it. The ceiling + /// exists to exclude unrepresentable values, not to impose a view of a reasonable + /// budget, so a node already running on such a value must not start failing at boot on + /// upgrade. A test that only checked the boundary would pass with a far tighter cap. + #[test] + fn ipfs_request_budget_rejects_values_no_derived_duration_can_represent() { + let parse = |secs: u64| { + Config::try_parse_from([ + "gitlawb-node", + "--ipfs-request-budget-secs", + &secs.to_string(), + ]) + }; + + // Large "disable the bound" values that predate the ceiling still parse. + for disable in [1_000_000_000, 999_999_999] { + assert_eq!( + parse(disable) + .unwrap_or_else(|e| panic!("{disable} was a working setting: {e}")) + .ipfs_request_budget_secs, + disable + ); + } + + // At the ceiling: accepted, and the derived deadline still fits. This knob feeds + // only the `Instant` addition (no multiply derivation like the lease steal bound), + // so there is no `checked_mul` clause to carry over from the sibling test. + let at_max = + parse(GIT_SERVICE_TIMEOUT_SECS_MAX).expect("the documented maximum must parse"); + assert_eq!( + at_max.ipfs_request_budget_secs, + GIT_SERVICE_TIMEOUT_SECS_MAX + ); + assert!(std::time::Instant::now() + .checked_add(std::time::Duration::from_secs( + at_max.ipfs_request_budget_secs + )) + .is_some()); + + // Past the ceiling, and the top of the u64 range clap used to accept — the value + // that panics `Instant::now() + Duration::from_secs(..)` outright. + for over in [GIT_SERVICE_TIMEOUT_SECS_MAX + 1, u64::MAX] { + assert!( + parse(over).is_err(), + "{over} is past the representable ceiling and must be rejected at parse time" + ); + } + assert!(std::time::Instant::now() + .checked_add(std::time::Duration::from_secs(u64::MAX)) + .is_none()); + } + #[test] fn max_concurrent_reads_per_caller_defaults_and_rejects_out_of_range() { assert_eq!( From 87bdc0cebab6ad603a8625e53efadd3b7fbfce76 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 31 Jul 2026 00:57:53 -0500 Subject: [PATCH 92/95] docs(node): state what the pin-admission pool actually bounds The pin pool's docs claimed it bounded "how many MB-scale lists are held at once across all repos". It does not. On the local IPFS path the caller materializes the object list before pin_new_objects_gated acquires, so a task parked waiting for a permit is still holding its full list, and how many tasks park is capped only per repo by EncryptInflight. Across distinct repos that retained memory is unbounded, which is the gap a reviewer raised against this claim. Correct the claim at each site rather than implying the pool closes an exposure it cannot see. The pool does bound concurrent pin loops, and therefore lists held while being pinned; that part was always true and is kept. The residual is now named where each claim lives, including on the config knob, so an operator does not read the knob as a memory cap it cannot deliver. Bounding the retention is a real change to the capture shape (the Pinata twin already avoids it by acquiring before it derives) and is not attempted here. Sites corrected: pin_new_objects_gated's docstring and its F2b empty-list comment, the empty-list test's docstring, AppState::pin_semaphore and AppState::encrypt_inflight field docs, the max_concurrent_pin_tasks knob doc, and the construction comment in main. Sweep over pin_semaphore, F6, and "held at once". Left alone with reason: ipfs_pin.rs 16/31/174-200/518 describe the loop that genuinely holds the permit, so their claims are about hold duration and are true; repos.rs 1029-1036 and 2276-2294 describe the Pinata path, which acquires before materializing, so its "at most pin_semaphore permits' worth exist at once" holds; repos.rs 5898 claims concurrent loops are bounded, which is the true concurrency claim; the F6 hits in api/ipfs.rs are that file's own review numbering for walk-permit ordering, unrelated to this pool; constructions and test fixtures carry no claim. No behavior change: the diff is comments and docstrings only, checked by filtering the diff for non-comment lines. --- crates/gitlawb-node/src/api/repos.rs | 27 ++++++++++++++++++--------- crates/gitlawb-node/src/config.rs | 10 +++++++--- crates/gitlawb-node/src/main.rs | 6 ++++-- crates/gitlawb-node/src/state.rs | 13 +++++++++++-- 4 files changed, 40 insertions(+), 16 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 22a216fe..5f4ab9b2 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1126,8 +1126,17 @@ async fn pinata_object_list_for_refs( /// Pin `object_list` to the local IPFS node under the global pin-admission permit /// (#174 F6). `EncryptInflight` bounds the pin-task COUNT to one per repo, but each /// pin loop holds a full per-push object-id list while walking it; this permit bounds -/// how many such MB-scale lists are held at once across all repos. DEFERS (waits) when -/// the pool is full — never drops, since a dropped pin loses the replication copy. +/// how many pin loops RUN CONCURRENTLY, and therefore how many such MB-scale lists are +/// held WHILE BEING PINNED. DEFERS (waits) when the pool is full — never drops, since a +/// dropped pin loses the replication copy. +/// +/// What it does NOT bound, stated plainly rather than implied closed: on this path the +/// caller materializes `object_list` BEFORE this function acquires, so a task that is +/// parked here waiting for a permit is still holding its full list. The parked-task +/// count is capped only per repo by `EncryptInflight`, so across distinct repos the +/// retained list memory is not bounded by this pool. Bounding that is a real change to +/// the capture shape and is deliberately not attempted here; the Pinata twin below +/// avoids it by acquiring BEFORE it derives its list. async fn pin_new_objects_gated( pin_sem: &Arc, ipfs_api: &str, @@ -1136,8 +1145,8 @@ async fn pin_new_objects_gated( db: &Arc, ) -> Vec<(String, String)> { // Nothing to pin: answer before taking a permit (#174 F2b). The permit bounds how - // many MB-scale object lists are held at once and an empty list holds none, so - // parking here would spend a global pin slot on no work. The pool DEFERS rather + // many pin loops run concurrently, and an empty list does no pinning, so parking + // here would spend a global pin slot on no work. The pool DEFERS rather // than sheds, so those calls stall pins for every other repo. Empty is the normal // shape for a push whose walk failed or that may replicate nothing. if object_list.is_empty() { @@ -5939,11 +5948,11 @@ mod tests { assert!(out.is_empty(), "an empty ipfs_api pins nothing"); } - /// #174 F2b: the pin permit bounds how many MB-scale object lists are held at once, - /// so a call with NOTHING to pin must not take one. It otherwise spends a global pin - /// slot on no work, and the pool DEFERS rather than sheds, so those calls stall pins - /// for every other repo. The empty case is the normal shape for a push whose walk - /// failed or that may replicate nothing. + /// #174 F2b: the pin permit bounds how many pin loops run concurrently, so a call + /// with NOTHING to pin must not take one. It otherwise spends a global pin slot on no + /// work, and the pool DEFERS rather than sheds, so those calls stall pins for every + /// other repo. The empty case is the normal shape for a push whose walk failed or + /// that may replicate nothing. /// /// Load-bearing: without the guard this call parks on the exhausted pool exactly like /// the non-empty one above, and the completion assertion fails. diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 2d4299c2..d01fdaa6 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -372,9 +372,13 @@ pub struct Config { /// pin-task COUNT to one per repo, but each pin loop holds a full per-push /// object-id list (up to `git_max_pack_bytes` worth of OIDs) while it walks it, /// so N distinct repos could hold N such lists at once. This caps how many run - /// concurrently, bounding that MB-scale memory regardless of how many repos an - /// authenticated actor pushes to (#174 F6). Beyond it a pin loop DEFERS (waits), - /// never drops — a dropped pin would lose the object's replication copy. + /// concurrently (#174 F6). Beyond it a pin loop DEFERS (waits), never drops — a + /// dropped pin would lose the object's replication copy. + /// + /// It does not cap the memory itself: the local IPFS path builds its list before + /// taking a permit, so tasks parked on this pool still hold theirs, and how many + /// park is capped only per repo. Lowering this knob bounds concurrent pinning, not + /// how much an actor pushing to many repos can retain. /// /// Default: 8. Must be between 1 and 1_048_576. #[arg( diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 486bfa0e..4c6cf905 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -402,8 +402,10 @@ async fn main() -> Result<()> { git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new( config.max_concurrent_git_pushes, )), - // Bounds concurrent post-push pin loops (their MB-scale object-id lists) across - // all repos (#174 F6), independent of the per-repo encrypt-task coalescing below. + // Bounds how many post-push pin loops run concurrently across all repos (#174 F6), + // independent of the per-repo encrypt-task coalescing below. Not a bound on the + // MB-scale object-id lists themselves: parked tasks still hold theirs (see the + // field doc on AppState::pin_semaphore). pin_semaphore: Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_pin_tasks)), // Coalesces the DETACHED post-push encryption tasks per repo so a rapid pusher // cannot grow the outstanding parked-waiter set past one task per repo (#174 diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index a189548b..df78668e 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -133,13 +133,22 @@ pub struct AppState { /// per repo, but each pin loop holds a full per-push object-id list while walking /// it, so N distinct repos could hold N such MB-scale lists at once. This caps how /// many run concurrently; a loop DEFERS (waits) when the pool is full, never drops. + /// + /// It does NOT bound the lists held by tasks PARKED on it. The local IPFS path + /// materializes its list before acquiring, so a parked task still holds one, and the + /// parked-task count is capped only per repo by `encrypt_inflight`. Cross-repo + /// retained memory is therefore not bounded by this pool. The Pinata twin acquires + /// before it derives and does not carry that residual. pub pin_semaphore: Arc, /// Bounds the outstanding post-push encryption-task set to at most one PER REPO by /// coalescing (#174 P2-2). This is NOT a global cap: N distinct repos still admit N /// tasks; the cross-repo residual (an authenticated actor pushing to many repos /// leaves many parked tasks) is throttled by auth plus the per-IP/per-DID rate - /// limits, and its real cost — the MB-scale per-push object-id list each pin loop - /// holds — is bounded by `pin_semaphore`, not this. `git_encrypt_semaphore` caps + /// limits. Its real cost, the MB-scale per-push object-id list each parked task + /// holds, is NOT bounded by `pin_semaphore` either: on the local IPFS path the list + /// is materialized before that permit is acquired, so a task parked on the pin pool + /// still holds one. That pool bounds concurrent pin loops, not parked retention, so + /// nothing currently bounds this memory across repos. `git_encrypt_semaphore` caps /// *active* walks; this caps duplicate SPAWNS per repo. Before spawning a per-push /// encryption task, the receive-pack handler consults this set: if the repo already /// has a task in flight it coalesces (skips the duplicate spawn) rather than parking From 41325c34c4106f57fd877496d0245f02fcf57036 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 31 Jul 2026 00:59:51 -0500 Subject: [PATCH 93/95] docs(node): state what the permit-less load-shed peek actually delivers The three handler peeks said they shed "before any DB/disk work", which reads as an admission bound on the DB window. It is not one. The peek holds no permit, and permits are only held from the authoritative acquire below, after the visibility and rate gates, so a burst arriving while permits are free all proceeds into the DB and none of it sheds. What the peek actually gives is a cheap 503 for one request once the pool has ALREADY filled, sparing that request's DB work. The gate ordering that makes it a peek is deliberate and stays: a denied or rate-limited request must consume no slot, and one source must not hold global slots through the DB/visibility window. Reordering to bound the window would undo that. So this corrects the claims and changes no behavior; bounding the DB window would need an admission mechanism the peek is not. test_support.rs's receive-pack docstring additionally called the peek "the permit" and "the first statement". Both were wrong at this head: it is a non-holding peek, and the authoritative write-pool acquire sits after the per-repo lease. Corrected along with the shed claim. The "remove the block and this goes red" sentences stay, since those tests drive an already-saturated pool, which is the case the peek does handle. Sweep over "before any DB", "before DB", "pre-DB". Left alone with reason: api/ipfs.rs:142 describes a HELD walk-admission acquire, so its claim is true; graphql/mutation.rs:201 and test_support.rs:528 are about auth ordering, unrelated; the repos.rs test docstrings at 4194/4213/4556/ 4618/4682 describe test mechanics against a saturated pool, which is the case the peek delivers. No behavior change: the diff is comments and docstrings only, checked by filtering the diff for non-comment lines. --- crates/gitlawb-node/src/api/repos.rs | 35 +++++++++++++++---------- crates/gitlawb-node/src/test_support.rs | 34 +++++++++++++++--------- 2 files changed, 42 insertions(+), 27 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 5f4ab9b2..7cec9914 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -567,12 +567,16 @@ pub async fn git_info_refs( "unsupported git service: {service}" ))); } - // #62 cheap pre-DB load shed: if the pool this service draws from is already - // saturated, shed with 503 before any DB/disk work. Best-effort (holds no - // permit); the authoritative hold is `git_permit` below, after the per-source - // cap. Restores the shed-before-DB property the reordered held acquire alone - // would drop, while the reorder still prevents one source from occupying global - // slots during the DB/visibility window. + // #62 cheap load shed: if the pool this service draws from is ALREADY saturated, + // shed this request with a 503 before it does any DB/disk work. Best-effort and + // permit-less, so it is a snapshot, not admission: it spares THIS request's DB + // work once the pool has filled, and nothing more. It is NOT a bound on the DB + // window. Permits are only held from `git_permit` below, after the visibility and + // rate gates, so a burst arriving while permits are free all proceeds into the DB + // and none of it sheds here. That ordering is deliberate (a denied or rate-limited + // request must consume no slot, and one source must not hold global slots through + // the DB/visibility window); bounding the DB window itself would need an admission + // mechanism this peek is not. { // The receive-pack advertisement peeks its DEDICATED advert pool, not the // write pool the authenticated POST uses (#174) — matching the held acquire @@ -1291,9 +1295,10 @@ pub async fn git_upload_pack( headers: axum::http::HeaderMap, body: Bytes, ) -> Result { - // #62 cheap pre-DB load shed (see git_info_refs): shed before DB when the read - // pool is saturated; the authoritative hold is `git_permit` below, after the - // per-source cap. + // #62 cheap load shed (see git_info_refs for the full contract): spares THIS + // request's DB work when the read pool is ALREADY saturated. Permit-less snapshot, + // not admission, so it does not bound the DB window; the authoritative hold is + // `git_permit` below, after the per-source cap. if state.git_read_semaphore.available_permits() == 0 { tracing::warn!("served-git concurrency cap reached; shedding request with 503 (pre-DB)"); return Err(AppError::Overloaded( @@ -1642,11 +1647,13 @@ pub async fn git_receive_pack( body: Bytes, ) -> Result { let name = smart_http_repo_name(&repo)?; - // Fast-path shed BEFORE the DB lookup when the write pool is saturated, so a push - // flood on a full pool does not hit Postgres per request. Best-effort (racy) and - // NON-holding: the authoritative, held permit is taken after the per-repo lease below - // (so a lease-blocked waiter pins no write slot — #174 F3 review). This peek only - // restores the cheap pre-DB shed the permit reorder would otherwise have lost. + // Fast-path shed before the DB lookup when the write pool is ALREADY saturated, so a + // push flood against a full pool does not hit Postgres per request. Best-effort + // (racy) and NON-holding: a snapshot, not admission. It spares this request's DB + // work once the pool has filled; pushes arriving while permits are free all proceed + // into the DB, so it does not bound that window. The authoritative, held permit is + // taken after the per-repo lease below (so a lease-blocked waiter pins no write slot + // — #174 F3 review). if state.git_write_semaphore.available_permits() == 0 { return Err(AppError::Overloaded( "git service at capacity, retry shortly".into(), diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 0eb11481..a93c882d 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -228,10 +228,12 @@ mod tests { /// PR3 (#62): the served-git concurrency cap sheds at the HTTP layer before the /// DB. The held `git_permit` acquire now sits after the per-source cap, so the - /// shed-before-DB property is carried by an explicit `available_permits() == 0` - /// early check at the top of the handler (the held permit remains the - /// authoritative bound further down). DB-free: an exhausted semaphore sheds - /// before any DB/disk access, so a lazy state works. Remove the early-shed block + /// cheap early shed is carried by an explicit `available_permits() == 0` check at + /// the top of the handler (the held permit remains the authoritative bound further + /// down). That check is a permit-less snapshot: it spares a request's DB work once + /// the pool is ALREADY saturated, which is the case this test drives, and it does + /// not bound the DB window in general. DB-free here because an exhausted semaphore + /// sheds before any DB/disk access, so a lazy state works. Remove the early-shed block /// from git_info_refs and this goes red (the request falls through to the DB and /// returns something other than 503). #[tokio::test] @@ -267,10 +269,11 @@ mod tests { } /// PR3 (#62) sibling of the info/refs shed test: git-upload-pack carries the same - /// explicit `available_permits() == 0` early-shed check at the top, so an - /// exhausted semaphore must shed it with a 503 before any DB/disk work. - /// Anonymous-reachable, so no auth injection is needed. Remove the early-shed - /// block from git_upload_pack and this goes red. + /// explicit `available_permits() == 0` early check at the top, so an ALREADY + /// exhausted semaphore must shed the request with a 503 before its DB/disk work. + /// That is the case the permit-less snapshot does deliver; it is not an admission + /// bound on the DB window. Anonymous-reachable, so no auth injection is needed. + /// Remove the early-shed block from git_upload_pack and this goes red. #[tokio::test] async fn git_upload_pack_sheds_with_503_when_semaphore_exhausted() { let mut state = test_state_lazy(); @@ -305,8 +308,10 @@ mod tests { /// PR3 (#62) receive-pack sibling of the info/refs shed test: the early shed /// selects the dedicated ADVERT pool for a git-receive-pack advertisement (#174), - /// so an exhausted advert pool sheds the advert with 503 before any DB/disk work - /// — while the write pool (reserved for authenticated POSTs) is left free here. + /// so an ALREADY exhausted advert pool sheds the advert with 503 before its DB/disk + /// work (the case the permit-less snapshot delivers, not an admission bound on the + /// DB window) — while the write pool (reserved for authenticated POSTs) is left + /// free here. /// Flip the pool selection back to the write pool, or remove the early-shed /// block, and this goes red. #[tokio::test] @@ -344,9 +349,12 @@ mod tests { /// PR3 (#62) sibling for the push path: git-receive-pack requires an /// AuthenticatedDid extension (production: require_signature injects it), so the /// request carries one via signed_request_as — without it the Extension - /// extractor 500s before the handler body reaches git_permit. The permit is the - /// first statement, so an exhausted semaphore still sheds 503 before any DB - /// work. Remove the permit line from git_receive_pack and this goes red. + /// extractor 500s before the handler body reaches the shed. What sits at the top of + /// the handler is a permit-less `available_permits() == 0` peek, NOT the permit + /// itself: the authoritative held acquire is taken after the per-repo lease, so a + /// lease-blocked waiter pins no write slot. An ALREADY exhausted pool is the case + /// the peek delivers, so the request sheds 503 before its DB work here. Remove the + /// early-shed block from git_receive_pack and this goes red. #[tokio::test] async fn git_receive_pack_sheds_with_503_when_semaphore_exhausted() { let mut state = test_state_lazy(); From 807bc935b362bab129a3a8a78bcbb4682e0df6a7 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 31 Jul 2026 01:24:46 -0500 Subject: [PATCH 94/95] fix(review): correct the budget-knob claims and mirror the range in operator docs Review round on the three preceding commits. The knob's doc said a value past the ceiling "aborts every /ipfs/{cid} request in a release build". That is wrong, and three reviewers plus an outside model landed on it independently: the ceiling is 100 years, the Instant addition does not actually overflow until roughly 584 years on a u64-nanosecond Instant, so values in between are representable and are rejected as a deliberate safety margin, not because they panic. Reworded to say that, since a confidently wrong comment is worse than the vague one it replaced. Also on the knob: - GIT_SERVICE_TIMEOUT_SECS_MAX has two consumers now, so its own doc says so, and notes this knob derives only the Instant addition and not the lease-steal multiply. - The "clears every disable setting" phrasing overclaimed: u32::MAX sits above the ceiling. Now names the documented sentinels instead. Operator-facing docs: the sibling knob's ceiling commit updated both README.md and .env.example in the same change, and this one had not. Both now carry the accepted range, matching that precedent. House style: five comment lines added by the preceding commits used em dashes, which this project does not allow. Four were carried over from the text being edited and one was copied from the sibling test. All replaced. Checked mechanically over the added lines rather than by eye. Two comments restated an argument in full immediately after pointing at the canonical site for it. Trimmed to the pointer plus the local delta. Scoping correction worth recording: the two preceding commits claimed "no behavior change". That holds for repos.rs, state.rs, main.rs and test_support.rs, but a config.rs field doc IS clap --help output, so the pin-knob doc edit changes operator-visible help text. No runtime behavior changes either way. Verification: full suite 733 passed, fmt clean, clippy clean under the CI form. The range guard was separately proven load-bearing in both directions by mutation: reverting to range(1..) goes RED on the rejection assertions, and tightening the ceiling to 500_000_000 goes RED on the disable-value assertions. Known residual, not fixed here: the bound is parse-side only, so a Config built by direct construction rather than through clap can still hold an unrepresentable value. --- .env.example | 3 ++- README.md | 2 +- crates/gitlawb-node/src/api/repos.rs | 15 ++++++------- crates/gitlawb-node/src/config.rs | 30 ++++++++++++++++--------- crates/gitlawb-node/src/state.rs | 6 ++--- crates/gitlawb-node/src/test_support.rs | 2 +- 6 files changed, 33 insertions(+), 25 deletions(-) diff --git a/.env.example b/.env.example index 5a3a5acc..1d728f88 100644 --- a/.env.example +++ b/.env.example @@ -231,7 +231,8 @@ GITLAWB_IPFS_MAX_REPO_VISITS=1024 # cannot hold the request's walk slot past it. One hang path is still unbounded: # the probe's object-store readability check is a plain filesystem sweep with # nothing to reap, so a wedged filesystem can hold the slot past the deadline. -# Default 600. +# Must be 1..=3153600000 (100 years): the node derives an Instant deadline from +# this value, and a larger one cannot be represented. Default 600. GITLAWB_IPFS_REQUEST_BUDGET_SECS=600 # Max /ipfs/{cid} requests per client IP per hour (route flood brake, distinct # from the concurrency caps above). 0 disables. Default 600. diff --git a/README.md b/README.md index d5b9c6b9..2c382d7c 100644 --- a/README.md +++ b/README.md @@ -350,7 +350,7 @@ Important node settings: | `GITLAWB_IPFS_WALK_PER_SOURCE` | Max concurrent `/ipfs` walks a single source IP may hold. Default 4. | | `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max expensive path-scope visibility walks per `/ipfs/{cid}` request; over-cap repos are skipped and the scan continues, shedding a retryable 503 (not a false 404) if the object is then found nowhere. Default 64. | | `GITLAWB_IPFS_MAX_REPO_VISITS` | Ceiling on repos one `/ipfs/{cid}` request may visit (acquire + probe) past the visibility gate. Also the worst-case per-request Tigris fetch count. On exhaustion the scan stops with a retryable 503. Default 1024. | -| `GITLAWB_IPFS_REQUEST_BUDGET_SECS` | Absolute wall-clock budget for one admitted `/ipfs/{cid}` request's acquire+walk lifetime. Per-stage clamps bound the acquire and walk stages to the remaining budget, and no stage starts once it is exhausted; the scan then stops with a retryable 503. The object-type probe and content-read `cat-file` subprocesses are budget-checked before starting and each also run under their own deadline (the lesser of `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` and the remaining budget), reaped via process-group teardown, so a hung `cat-file` cannot hold the request's walk slot past it. One hang path is still unbounded: the probe's object-store readability check is a plain filesystem sweep with nothing to reap, so a wedged filesystem can hold the slot past the deadline. Default 600. | +| `GITLAWB_IPFS_REQUEST_BUDGET_SECS` | Absolute wall-clock budget for one admitted `/ipfs/{cid}` request's acquire+walk lifetime. Per-stage clamps bound the acquire and walk stages to the remaining budget, and no stage starts once it is exhausted; the scan then stops with a retryable 503. The object-type probe and content-read `cat-file` subprocesses are budget-checked before starting and each also run under their own deadline (the lesser of `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` and the remaining budget), reaped via process-group teardown, so a hung `cat-file` cannot hold the request's walk slot past it. One hang path is still unbounded: the probe's object-store readability check is a plain filesystem sweep with nothing to reap, so a wedged filesystem can hold the slot past the deadline. Default 600. Accepted range is 1 to 3153600000 (100 years), since the node derives a deadline from this value and a larger one cannot be represented. | | `GITLAWB_IPFS_RATE_LIMIT` | Max `/ipfs/{cid}` requests per client IP per hour (route flood brake). 0 disables. Default 600. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 7cec9914..8ce60e7e 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -1131,8 +1131,8 @@ async fn pinata_object_list_for_refs( /// (#174 F6). `EncryptInflight` bounds the pin-task COUNT to one per repo, but each /// pin loop holds a full per-push object-id list while walking it; this permit bounds /// how many pin loops RUN CONCURRENTLY, and therefore how many such MB-scale lists are -/// held WHILE BEING PINNED. DEFERS (waits) when the pool is full — never drops, since a -/// dropped pin loses the replication copy. +/// held WHILE BEING PINNED. DEFERS (waits) when the pool is full and never drops, since +/// a dropped pin loses the replication copy. /// /// What it does NOT bound, stated plainly rather than implied closed: on this path the /// caller materializes `object_list` BEFORE this function acquires, so a task that is @@ -1295,10 +1295,9 @@ pub async fn git_upload_pack( headers: axum::http::HeaderMap, body: Bytes, ) -> Result { - // #62 cheap load shed (see git_info_refs for the full contract): spares THIS - // request's DB work when the read pool is ALREADY saturated. Permit-less snapshot, - // not admission, so it does not bound the DB window; the authoritative hold is - // `git_permit` below, after the per-source cap. + // #62 cheap load shed. Permit-less snapshot, not admission; see git_info_refs for + // what it does and does not bound. The authoritative hold is `git_permit` below, + // after the per-source cap. if state.git_read_semaphore.available_permits() == 0 { tracing::warn!("served-git concurrency cap reached; shedding request with 503 (pre-DB)"); return Err(AppError::Overloaded( @@ -1652,8 +1651,8 @@ pub async fn git_receive_pack( // (racy) and NON-holding: a snapshot, not admission. It spares this request's DB // work once the pool has filled; pushes arriving while permits are free all proceed // into the DB, so it does not bound that window. The authoritative, held permit is - // taken after the per-repo lease below (so a lease-blocked waiter pins no write slot - // — #174 F3 review). + // taken after the per-repo lease below, so a lease-blocked waiter pins no write slot + // (#174 F3 review). if state.git_write_semaphore.available_permits() == 0 { return Err(AppError::Overloaded( "git service at capacity, retry shortly".into(), diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index d01fdaa6..aab8e758 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -1,7 +1,13 @@ use clap::Parser; use std::path::PathBuf; -/// Upper bound on `git_service_timeout_secs`, in seconds (100 years). +/// Upper bound on `git_service_timeout_secs` and `ipfs_request_budget_secs`, in seconds +/// (100 years). +/// +/// Two consumers now, so a future tightening moves both. `ipfs_request_budget_secs` +/// derives only the `Instant` addition in `get_by_cid`, not the lease-steal multiply +/// below, but it shares this ceiling because the defect class and the "set it very large +/// to disable" contract are the same. /// /// The knob is not just stored, it is arithmetic input: the write path derives the /// per-repo lease steal bound from it (`* 2 + 60`), and #174 routed it into @@ -372,8 +378,8 @@ pub struct Config { /// pin-task COUNT to one per repo, but each pin loop holds a full per-push /// object-id list (up to `git_max_pack_bytes` worth of OIDs) while it walks it, /// so N distinct repos could hold N such lists at once. This caps how many run - /// concurrently (#174 F6). Beyond it a pin loop DEFERS (waits), never drops — a - /// dropped pin would lose the object's replication copy. + /// concurrently (#174 F6). Beyond it a pin loop DEFERS (waits) and never drops, + /// since a dropped pin would lose the object's replication copy. /// /// It does not cap the memory itself: the local IPFS path builds its list before /// taking a permit, so tasks parked on this pool still hold theirs, and how many @@ -507,11 +513,14 @@ pub struct Config { /// Must be positive, and no larger than `GIT_SERVICE_TIMEOUT_SECS_MAX`. The ceiling is /// representability, NOT a policy view of a sane budget: `get_by_cid` derives the /// request deadline as `Instant::now() + Duration::from_secs(this)`, and that addition - /// is an explicit overflow check rather than a debug-only one, so a value past the - /// ceiling aborts every `/ipfs/{cid}` request in a release build instead of setting a - /// very long budget. Rejecting at parse time keeps that value out of every reachable - /// configuration. Setting it very large is still the way to effectively disable the - /// budget; the ceiling clears every such setting (see the constant's own note). + /// is an explicit overflow check rather than a debug-only one, so a value near the top + /// of the `u64` range aborts every `/ipfs/{cid}` request in a release build instead of + /// setting a very long budget. The ceiling sits an order of magnitude below where that + /// starts (see the constant's own note), so it is a conservative margin rather than the + /// exact overflow point; rejecting at parse time keeps the unrepresentable values out + /// of every reachable configuration. Setting it very large is still the way to + /// effectively disable the budget, and the documented sentinels (`999999999`, + /// `1000000000`) are well inside the range. /// Default: 600s (10 min), matching `git_service_timeout_secs` so a single full-length /// walk still fits. #[arg( @@ -880,8 +889,9 @@ mod tests { )) .is_some()); - // Past the ceiling, and the top of the u64 range clap used to accept — the value - // that panics `Instant::now() + Duration::from_secs(..)` outright. + // Past the ceiling, and the top of the u64 range clap used to accept. Only the + // latter actually panics `Instant::now() + Duration::from_secs(..)`; the ceiling + // sits well below that, which is the conservative margin the constant documents. for over in [GIT_SERVICE_TIMEOUT_SECS_MAX + 1, u64::MAX] { assert!( parse(over).is_err(), diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index df78668e..c1086841 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -145,10 +145,8 @@ pub struct AppState { /// tasks; the cross-repo residual (an authenticated actor pushing to many repos /// leaves many parked tasks) is throttled by auth plus the per-IP/per-DID rate /// limits. Its real cost, the MB-scale per-push object-id list each parked task - /// holds, is NOT bounded by `pin_semaphore` either: on the local IPFS path the list - /// is materialized before that permit is acquired, so a task parked on the pin pool - /// still holds one. That pool bounds concurrent pin loops, not parked retention, so - /// nothing currently bounds this memory across repos. `git_encrypt_semaphore` caps + /// holds, is NOT bounded by `pin_semaphore` either; see that field's doc above for + /// why. Nothing currently bounds this memory across repos. `git_encrypt_semaphore` caps /// *active* walks; this caps duplicate SPAWNS per repo. Before spawning a per-push /// encryption task, the receive-pack handler consults this set: if the repo already /// has a task in flight it coalesces (skips the duplicate spawn) rather than parking diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index a93c882d..2b5aef95 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -310,7 +310,7 @@ mod tests { /// selects the dedicated ADVERT pool for a git-receive-pack advertisement (#174), /// so an ALREADY exhausted advert pool sheds the advert with 503 before its DB/disk /// work (the case the permit-less snapshot delivers, not an admission bound on the - /// DB window) — while the write pool (reserved for authenticated POSTs) is left + /// DB window), while the write pool (reserved for authenticated POSTs) is left /// free here. /// Flip the pool selection back to the write pool, or remove the early-shed /// block, and this goes red. From b0482480a873d7436c032b3de5fccbb41fe46a9c Mon Sep 17 00:00:00 2001 From: t Date: Fri, 31 Jul 2026 02:09:32 -0500 Subject: [PATCH 95/95] docs(node): correct the ceiling margin, it is 5.85x not an order of magnitude Both doc comments describing GIT_SERVICE_TIMEOUT_SECS_MAX said the 100-year ceiling sits "an order of magnitude" under the ~584-year limit of a u64-nanosecond Instant. The real margin is 3,153,600,000 seconds against 18,446,744,074, a factor of about 5.85. An order of magnitude means ten. Worth noting how it spread, since the fix is trivial and the pattern is not. The phrase was written once on the constant, then copied into the ipfs_request_budget_secs doc by the commit that was at that moment correcting a different false claim in the same file. Nobody re-ran the division at either site, because quoting an existing sentence feels like citation rather than assertion. The direction of the error is the part that matters: a vague "well under" would have been true and would have invited a reader to check. The precise-sounding "order of magnitude" was false and discouraged checking. Comments only; the diff carries no executable line, checked by filtering it for non-comment changes. fmt and clippy clean under the CI form. --- crates/gitlawb-node/src/config.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index aab8e758..fefa063e 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -22,7 +22,7 @@ use std::path::PathBuf; /// this very large to disable the bound, so values like `999999999` (~31 years) are /// working production "off" settings; a tighter, tidier cap would fail those nodes at /// boot on upgrade over a value that was never the defect. 100 years clears every such -/// setting while staying an order of magnitude under the ~584-year ceiling of a +/// setting while staying a factor of about 5.85 under the ~584-year ceiling of a /// `u64`-nanosecond `Instant`, which is the tightest representation on any platform we /// build for. Every value that worked before still parses; only the ones that would have /// panicked are rejected. @@ -515,8 +515,8 @@ pub struct Config { /// request deadline as `Instant::now() + Duration::from_secs(this)`, and that addition /// is an explicit overflow check rather than a debug-only one, so a value near the top /// of the `u64` range aborts every `/ipfs/{cid}` request in a release build instead of - /// setting a very long budget. The ceiling sits an order of magnitude below where that - /// starts (see the constant's own note), so it is a conservative margin rather than the + /// setting a very long budget. The ceiling sits well below where that starts, about a + /// factor of 5.85 (see the constant's own note), so it is a conservative margin rather than the /// exact overflow point; rejecting at parse time keeps the unrepresentable values out /// of every reachable configuration. Setting it very large is still the way to /// effectively disable the budget, and the documented sentinels (`999999999`,