diff --git a/.env.example b/.env.example index bbd9a342..5028d53b 100644 --- a/.env.example +++ b/.env.example @@ -24,8 +24,11 @@ DATABASE_URL=postgresql://gitlawb:changeme@localhost:5432/gitlawb # ── Database pool & startup resilience ──────────────────────────────────── # Maximum connections in the PostgreSQL pool. A cap, not a floor — # connections open lazily. Size against the DB server's max_connections, -# remembering admin tooling opens its own pool. -GITLAWB_DB_MAX_CONNECTIONS=20 +# remembering admin tooling opens its own pool. Each concurrent write pins one +# connection for its whole duration (the connection-affine advisory lock), so the +# node REJECTS at boot any value below GITLAWB_MAX_CONCURRENT_GIT_PUSHES + 8 +# headroom — keep this comfortably above that (default 48 for pushes 32). +GITLAWB_DB_MAX_CONNECTIONS=48 # Seconds a request waits for a pool connection before failing with 503. GITLAWB_DB_ACQUIRE_TIMEOUT_SECS=5 # Upper bound on each startup connect+migrate attempt, in seconds. Keep it @@ -109,15 +112,136 @@ GITLAWB_MAX_PACK_BYTES=2147483648 # Max seconds a served git upload-pack / receive-pack (clone / push) may run # before it is aborted with a 504. Bounds a hung git that would otherwise pin a -# worker and, on push, the repo write lock. Does NOT cover the info/refs -# advertisement or the withheld-blob path, which remain unbounded. Default 600. +# worker and, on push, the repo write lock. Also bounds both info/refs +# advertisements, the withheld-blob pack build, and the push-side candidate +# discovery (rev-list / cat-file), all reaped via process-group teardown (#174). +# On the path-scoped upload-pack path the withheld-blob classification walk and +# the pack serve share ONE deadline, so this value bounds their COMBINED duration +# rather than giving each stage a full budget: a walk that consumes it leaves the +# serve nothing and the clone gets a 504. Serving large path-scoped repos may +# need a higher value here than when each stage was budgeted separately. +# Must be 1..=3153600000 (100 years): the node derives deadlines from this value, +# and a larger one cannot be represented. Default 600. GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600 +# Max seconds the storage-ACQUISITION phase of a served git op may run before the +# request is shed with a 503, separate from the git-run timeout above. A +# concurrency permit is taken before this phase and GITLAWB_GIT_SERVICE_TIMEOUT_SECS +# only starts once git spawns, so without this a stalled backend (a hung Tigris +# HEAD/GET, or a hung pg advisory-lock iteration on push) pins the permit and drains +# the pool until every later request 503s. On expiry the permit is released +# (fail-closed). Kept separate because acquisition and git execution are distinct +# cost centers. Must be positive; set very large to effectively disable. Default 30. +GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS=30 + +# Max concurrent git READ ops (upload-pack + the upload-pack info/refs +# advertisement) served at once, a global pool separate from the push pool below. +# The anon receive-pack info/refs advertisement has its OWN pool (see below), not +# this one. Over-cap sheds a clean 503 + Retry-After. Anonymous reads draw from +# here, so pair it with GITLAWB_MAX_CONCURRENT_READS_PER_CALLER (below) so one +# caller cannot monopolize the pool. Default 128. +GITLAWB_MAX_CONCURRENT_GIT_OPS=128 + +# Max concurrent git-receive-pack (push) POST operations, in a pool separate from +# the read pool (GITLAWB_MAX_CONCURRENT_GIT_OPS) so anonymous reads cannot shed an +# authenticated push at admission. The anon receive-pack info/refs advertisement +# runs in a SEPARATE pool of the same size (disjoint from this one), so an +# advertisement flood cannot shed a push either. Over-cap sheds a 503 + +# Retry-After. Default 32. +GITLAWB_MAX_CONCURRENT_GIT_PUSHES=32 + +# Max pushes PARKED at once waiting for the per-repo write lease of the SAME repo. +# A parked push holds its already-buffered pack body in memory, so this bounds that +# memory for a hot repo. Past the cap the newest push sheds a clean 503 + +# Retry-After ("repo is busy with another push") instead of joining the queue; a +# push to a DIFFERENT repo is never affected. The lease holder is not counted. +# Raising GITLAWB_MAX_CONCURRENT_GIT_PUSHES does not raise this; set it explicitly. +# Default 8. +GITLAWB_REPO_LEASE_MAX_WAITERS=8 + +# Max concurrent post-push pin loops (IPFS + Pinata pin_new_objects) across all +# repos. Each loop holds a full per-push object-id list while pinning, so this +# bounds that MB-scale memory even though the per-repo encrypt-task set already +# caps the task COUNT. A loop DEFERS (waits) when the pool is full, never drops a +# pin. Default 8. +GITLAWB_MAX_CONCURRENT_PIN_TASKS=8 + +# Max concurrent read ops (upload-pack + the upload-pack info/refs advertisement) +# a single caller may hold, so one caller cannot monopolize the read pool. Keyed +# on the resolved SOURCE IP, never the DID: a signature does not move a caller off +# this cap. The source-IP key is only as granular as GITLAWB_TRUSTED_PROXY below: +# left unset, a node behind an edge/NAT keys all callers on the edge IP and this +# collapses to one global cap. Set GITLAWB_TRUSTED_PROXY for per-client keying; a +# high-fanout caller (CI behind one NAT) then needs the operator to raise this. +# Default 16. +GITLAWB_MAX_CONCURRENT_READS_PER_CALLER=16 + +# Two further per-source concurrency caps exist on the PUSH side but have NO +# environment variable of their own: both are DERIVED as +# GITLAWB_MAX_CONCURRENT_GIT_PUSHES / 8, with a floor of 1. They are not settable +# independently, so raising the push pool raises both proportionally, and the +# default push pool of 32 gives each a cap of 4. +# * The anon receive-pack info/refs advertisement cap: one source IP may hold at +# most this many slots in the dedicated advert pool, so saturating that pool +# takes ~8 distinct source IPs. +# * The authenticated receive-pack POST cap: one source IP may hold at most this +# many WRITE-pool slots. This one is load-bearing for push availability, since +# it is acquired before the global write permit: without it, one host minting +# disposable did:key identities could open enough slow pushes to monopolize the +# write pool and 503 every other source (owner enforcement defaults off, and +# the push rate limiter caps arrival rate, not in-flight concurrency). +# Keyed on the resolved source IP, never the DID, so a DID farm does not defeat +# them; keying granularity follows GITLAWB_TRUSTED_PROXY like the read cap above. + # ── Push rate limiting (git-receive-pack flood brake) ───────────────────── # Max receive-pack requests (info/refs advertisement + push POST) per client # IP per hour. 0 disables. Default 600. GITLAWB_PUSH_RATE_LIMIT=600 +# ── /ipfs/{cid} visibility-walk admission (#174) ────────────────────────── +# GET /ipfs/{cid} runs a per-repo full-history git walk in a blocking thread to +# decide whether the caller may read a path-scoped blob. It is publicly reachable, +# so it is bounded to keep a permissionless caller from fanning out unbounded +# concurrent walks and exhausting blocking-pool threads + PIDs. +# Max concurrent /ipfs walks across all callers (a pool of its own, disjoint from +# the served-git pools). Over-cap sheds a 503. Default 32. +GITLAWB_MAX_CONCURRENT_IPFS_WALKS=32 +# Max concurrent /ipfs walks a single SOURCE IP may hold (keyed like the git +# per-caller caps via GITLAWB_TRUSTED_PROXY; reject-before-insert bounded map). +# Default 4. +GITLAWB_IPFS_WALK_PER_SOURCE=4 +# Max legacy (NULL-provenance) repos probed per single /ipfs request, bounding the +# scan-fallback fan-out (git cat-file per candidate repo) for an anonymous caller. A +# truncated scan sheds a retryable 503, never a false 404. Default 256. +GITLAWB_IPFS_MAX_LEGACY_PROBES=256 +# Max EXPENSIVE path-scope visibility walks per single /ipfs request (only a +# blob in a path-scoped repo costs a full-history walk). Over-cap repos are +# skipped without a verdict and the scan continues; if the object is then found +# nowhere the request sheds a retryable 503 instead of a false 404. Default 64. +GITLAWB_IPFS_MAX_REPOS_WALKED=64 +# Ceiling on repos one /ipfs request may VISIT past the visibility gate. Each +# visit costs a repo acquire — on a Tigris cache miss a full archive download, +# so this is also the worst-case object-store fetch count per request — plus a +# cat-file probe. On exhaustion the scan stops and sheds a retryable 503. +# Default 1024. +GITLAWB_IPFS_MAX_REPO_VISITS=1024 +# Absolute wall-clock budget for one admitted /ipfs request's acquire+walk +# lifetime (all stages of the whole scan). Per-stage clamps bound the acquire +# and walk stages to the remaining budget, and no stage starts once it is +# exhausted (the scan then sheds a retryable 503). The object-type probe and +# content-read cat-file subprocesses are budget-checked before starting AND each +# run under their own deadline (the lesser of GITLAWB_GIT_SERVICE_TIMEOUT_SECS and +# the remaining budget), reaped via process-group teardown, so a hung cat-file +# cannot hold the request's walk slot past it. One hang path is still unbounded: +# the probe's object-store readability check is a plain filesystem sweep with +# nothing to reap, so a wedged filesystem can hold the slot past the deadline. +# Must be 1..=3153600000 (100 years): the node derives an Instant deadline from +# this value, and a larger one cannot be represented. Default 600. +GITLAWB_IPFS_REQUEST_BUDGET_SECS=600 +# Max /ipfs/{cid} requests per client IP per hour (route flood brake, distinct +# from the concurrency caps above). 0 disables. Default 600. +GITLAWB_IPFS_RATE_LIMIT=600 + # ── Creation rate limiting (repo/agent/issue/PR flood brake) ────────────── # Max creation requests (POST /api/v1/repos, /api/register, fork, issues, # pulls) per client IP per hour, in addition to the per-DID limit. The per-DID diff --git a/README.md b/README.md index 57ce2885..44057906 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Gitlawb Node is the open-source node software behind the Gitlawb network. It lets anyone run a self-hosted node, publish repositories under a DID, sign writes with Ed25519 HTTP signatures, replicate git activity across peers, and move toward a resilient app-delivery network where code and build assets can be served closer to users. -Gitlawb is not trying to be only “another git host.” The long-term direction is: +Gitlawb is not trying to be only "another git host." The long-term direction is: ```txt Decentralized GitHub @@ -34,12 +34,12 @@ This is a Rust workspace with four crates: Most git hosting today depends on a small number of centralized platforms. Gitlawb Nodes are designed for a different model: -- **Own your identity** — every user, agent, and node is an Ed25519 keypair represented as `did:key:z6Mk...`. -- **Signed writes by default** — write requests use RFC 9421 HTTP Signatures instead of passwords. -- **Git-native transport** — repositories are still real git repositories served over smart HTTP. -- **Agent-native workflows** — the `gl` CLI and MCP server expose repo, issue, task, PR, and UCAN flows to AI agents. -- **Peer-aware delivery** — nodes can announce, discover, gossip, and sync with each other. -- **App CDN direction** — the network can evolve from decentralized code storage into code + asset + app delivery. +- **Own your identity**: every user, agent, and node is an Ed25519 keypair represented as `did:key:z6Mk...`. +- **Signed writes by default**: write requests use RFC 9421 HTTP Signatures instead of passwords. +- **Git-native transport**: repositories are still real git repositories served over smart HTTP. +- **Agent-native workflows**: the `gl` CLI and MCP server expose repo, issue, task, PR, and UCAN flows to AI agents. +- **Peer-aware delivery**: nodes can announce, discover, gossip, and sync with each other. +- **App CDN direction**: the network can evolve from decentralized code storage into code + asset + app delivery. --- @@ -189,14 +189,14 @@ For public-network use, make sure `GITLAWB_NODE` points to the node you want. Th Public nodes (e.g. `node.gitlawb.com`) require two things on writes: -1. **RFC 9421 HTTP Signatures** — every write is signed by your identity key. `gl` +1. **RFC 9421 HTTP Signatures**: every write is signed by your identity key. `gl` and the `git-remote-gitlawb` helper do this automatically. An old/unsigned CLI fails with `401 not_an_agent`; `gl` will tell you to upgrade and register. 2. **An iCaptcha proof** on the spam-gated writes (**repo create, fork, register**). `gl` solves this for you: on the node's `403 icaptcha_proof_required` it reads the `x-icaptcha-url` / `x-icaptcha-level` hints, requests a challenge, solves it locally (arithmetic / algebra / sequence), and **retries the same signed request** - with the `x-icaptcha-proof` header — no manual steps, no env vars. + with the `x-icaptcha-proof` header. No manual steps, no env vars. ```bash gl identity new # create did:key identity @@ -215,14 +215,14 @@ Notes: - **Proofs are short-lived (~5 min TTL) and single-use.** If one expires between solving and use, the client transparently solves a fresh one and retries. - **What needs what:** create / fork / register are signed **and** iCaptcha-gated; - `git push` is **signed-only** (owner signature is the gate — no per-push challenge); + `git push` is **signed-only** (owner signature is the gate, no per-push challenge); reads (clone / fetch / `repo info`) need no proof. A non-existent repo returns a clear `404`, never a placeholder. - **API-key iCaptcha deployments:** set `GITLAWB_ICAPTCHA_URL` to your iCaptcha origin and `GITLAWB_ICAPTCHA_API_KEY` to its key. The client only talks to an `https` origin whose host is allowlisted (that URL or the public default), and - sends the bearer token **only** to your configured origin — never to a URL a - node advertises — so a hostile node can't capture the key or redirect the solve. + sends the bearer token **only** to your configured origin, never to a URL a + node advertises, so a hostile node can't capture the key or redirect the solve. --- @@ -343,13 +343,24 @@ Important node settings: | `GITLAWB_REQUIRE_SIGNED_PEER_WRITES` | Require signed peer announce/sync writes. | | `GITLAWB_AUTO_SYNC` | Enable automatic sync from known peers. | | `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. | -| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack/receive-pack may run before it is aborted (504). Default 600. Does not bound `info/refs` or the withheld-blob path. | +| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. On the path-scoped upload-pack path the classification walk and the pack serve share ONE deadline, so this value bounds their combined duration rather than granting each stage a full budget: a walk that consumes it leaves the serve nothing and the clone gets a 504. Serving large path-scoped repos may therefore need a higher value than they did when each stage was budgeted separately. Accepted range is 1 to 3153600000 (100 years), since the node derives deadlines from this value and a larger one cannot be represented. | +| `GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS` | Max seconds the storage-acquisition phase (Tigris HEAD/GET, push advisory-lock) of a served git op may run before the request is shed with a 503, separate from the git-run timeout. The concurrency permit is released on expiry so a stalled backend cannot pin the pool. Default 30. | +| `GITLAWB_REPO_LEASE_MAX_WAITERS` | Max pushes parked at once waiting for the same repo's write lease. Each waiter pins its buffered pack body, so this bounds that memory for a hot repo; past the cap the newest push sheds a 503 + Retry-After instead of queueing. Pushes to other repos are unaffected, and the lease holder is not counted. Default 8. | +| `GITLAWB_MAX_CONCURRENT_IPFS_WALKS` | Max concurrent `GET /ipfs/{cid}` visibility walks across all callers (own pool, disjoint from the served-git pools); over-cap sheds 503. Default 32. | +| `GITLAWB_IPFS_WALK_PER_SOURCE` | Max concurrent `/ipfs` walks a single source IP may hold. Default 4. | +| `GITLAWB_IPFS_MAX_LEGACY_PROBES` | Max legacy (NULL-provenance) repos probed per `/ipfs/{cid}` request, bounding the scan-fallback fan-out. A truncated scan returns a retryable 503, not a false 404. Default 256. | +| `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max expensive path-scope visibility walks per `/ipfs/{cid}` request; over-cap repos are skipped and the scan continues, shedding a retryable 503 (not a false 404) if the object is then found nowhere. Raised to `MAX_PIN_SOURCES + 1` if set below it, so a provenanced request is never truncated before its full source set is tried. Default 64. | +| `GITLAWB_IPFS_MAX_REPO_VISITS` | Ceiling on repos one `/ipfs/{cid}` request may visit (acquire + probe) past the visibility gate. Also the worst-case per-request Tigris fetch count. On exhaustion the scan stops with a retryable 503. Default 1024. | +| `GITLAWB_IPFS_REQUEST_BUDGET_SECS` | Absolute wall-clock budget for one admitted `/ipfs/{cid}` request's acquire+walk lifetime. Per-stage clamps bound the acquire and walk stages to the remaining budget, and no stage starts once it is exhausted; the scan then stops with a retryable 503. The object-type probe and content-read `cat-file` subprocesses are budget-checked before starting and each also run under their own deadline (the lesser of `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` and the remaining budget), reaped via process-group teardown, so a hung `cat-file` cannot hold the request's walk slot past it. One hang path is still unbounded: the probe's object-store readability check is a plain filesystem sweep with nothing to reap, so a wedged filesystem can hold the slot past the deadline. Default 600. Accepted range is 1 to 3153600000 (100 years), since the node derives a deadline from this value and a larger one cannot be represented. | +| `GITLAWB_IPFS_RATE_LIMIT` | Max `/ipfs/{cid}` requests per client IP per hour (route flood brake). 0 disables. Default 600. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | | `GITLAWB_IRYS_URL` | Optional Irys/Arweave permanent anchoring. | Production note: change the default Postgres password before exposing a node publicly. +Legacy-pin window: releases before the CID-resolver work stored the provider CID (Kubo dag-pb / Pinata) as a pinned object's resolver key. The `/ipfs/{cid}` resolver now recomputes the raw-content CID from the object bytes and refuses to serve a key that does not match, so `GET /api/v1/ipfs/pins` can still advertise an unrepaired legacy CID that 404s. Such a row is repaired opportunistically the next time a push carries the object again (its key is rewritten to the raw CID, the old value kept in `legacy_provider_cid`), but git negotiation omits objects the node already has, so most legacy rows never re-enter a push delta. A deferred one-shot startup sweep, not this opportunistic path, is what fully retires the advertise-then-404 window. Rows whose object bytes are gone stay withheld. + --- ## Optional node staking diff --git a/crates/gitlawb-core/src/cid.rs b/crates/gitlawb-core/src/cid.rs index b7993cc4..2071d478 100644 --- a/crates/gitlawb-core/src/cid.rs +++ b/crates/gitlawb-core/src/cid.rs @@ -64,6 +64,19 @@ impl Cid { } } +/// True when `s` parses as a CIDv1 with the raw codec — the exact shape +/// [`Cid::from_git_object_bytes`] produces and the `/ipfs` resolver looks up. +/// A legacy provider CID (Kubo dag-pb, Pinata CIDv0) parses to a different +/// version or codec and returns `false`, marking it an opportunistic-repair +/// candidate. Decidable from the string alone (no object bytes), so the pin path +/// can gate the byte-read/recompute cost on it and leave non-legacy rows at the +/// existing DB-only skip cost. An unparseable string is non-canonical (`false`). +pub fn is_raw_cidv1(s: &str) -> bool { + s.parse::>() + .map(|c| c.version() == cid::Version::V1 && c.codec() == RAW) + .unwrap_or(false) +} + impl fmt::Display for Cid { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.0) @@ -177,6 +190,38 @@ mod tests { assert!(result.is_err()); } + #[test] + fn is_raw_cidv1_classifies_codec_from_string() { + // The canonical resolver key: CIDv1 + raw codec → not a repair candidate. + let raw = Cid::from_git_object_bytes(b"blob 5\0hello"); + assert!( + is_raw_cidv1(raw.as_str()), + "from_git_object_bytes output is CIDv1/raw" + ); + + // A CIDv0 (Pinata dag-pb legacy shape) → repair candidate. + assert!( + !is_raw_cidv1("QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG"), + "a CIDv0 dag-pb value is a legacy-repair candidate" + ); + + // A CIDv1 with the dag-pb codec (the Kubo above-block-size root) over the + // same multihash → still a repair candidate (codec, not just version). + let parsed = raw.as_str().parse::>().unwrap(); + const DAG_PB: u64 = 0x70; + let dagpb = CidGeneric::<64>::new_v1(DAG_PB, *parsed.hash()).to_string(); + assert!( + !is_raw_cidv1(&dagpb), + "a CIDv1 dag-pb value is a legacy-repair candidate" + ); + + // Garbage is non-canonical. + assert!( + !is_raw_cidv1("not-a-cid"), + "an unparseable string is non-canonical" + ); + } + #[test] fn sha256_hex_of_empty_input_is_well_known() { // SHA-256("") is a fixed constant; verifies the hasher is wired correctly. diff --git a/crates/gitlawb-node/Cargo.toml b/crates/gitlawb-node/Cargo.toml index 72748fae..82854803 100644 --- a/crates/gitlawb-node/Cargo.toml +++ b/crates/gitlawb-node/Cargo.toml @@ -77,3 +77,6 @@ libp2p-dns = { version = "0.44.0", features = ["tokio"] } [dev-dependencies] mockito = "1" tempfile = "3" +# `test-util` for `#[tokio::test(start_paused = true)]`: lets the deadline tests assert +# on which timer fired instead of on wall clock. +tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index f3de7570..04cf75c6 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -1,14 +1,17 @@ //! GET /ipfs/{cid} — content-addressed retrieval of git objects by CIDv1. //! -//! Every git object stored on this node is addressable by its IPFS CIDv1. +//! Every git object pinned on this node is addressable by its IPFS CIDv1. //! The CID is computed as: //! //! CIDv1(codec=raw, multihash=sha2-256(content_bytes)) //! //! where `content_bytes` is the raw object content as returned by -//! `git cat-file ` (i.e. without the git framing header). -//! This is consistent with how `gitlawb_core::cid::Cid::from_git_object_bytes` -//! computes CIDs when objects are pushed. +//! `git cat-file ` (i.e. without the git framing header) — the +//! same bytes `gitlawb_core::cid::Cid::from_git_object_bytes` hashes when the +//! object is pinned. That digest is NOT the object's git oid: git frames the +//! content with a `" \0"` header before hashing, so `sha2-256(content)` +//! and the git oid differ. The handler therefore maps the CID back to its oid via +//! the `pinned_cids` table rather than treating the digest as an oid (#173). //! //! Serving is access-controlled: an object is returned only from a repo row the //! requesting caller is permitted to read (per-caller path-scoped visibility, @@ -27,14 +30,75 @@ use std::str::FromStr; use crate::auth::AuthenticatedDid; use crate::error::{AppError, Result}; use crate::git::store; -use crate::git::visibility_pack::{allowed_blob_set_for_caller, has_path_scoped_rule}; +use crate::git::visibility_pack::{ + allowed_blob_set_for_caller_bounded, allowed_tree_set_for_caller_bounded, has_path_scoped_rule, + reachable_commit_tag_oids_bounded, +}; use crate::state::AppState; use crate::visibility::{visibility_check, Decision}; +/// Hard ceiling on the number of full-history reachability walks a single +/// `GET /ipfs/{cid}` request may spawn. The route brake (`ipfs_rate_limiter`, charged +/// once per request by the middleware) caps request RATE, and the per-walk charge on +/// the separate `ipfs_work_rate_limiter` bounds the walk work across requests, but +/// within ONE request the object can exist under path-scoped rules in many repos, and +/// each distinct repo pays its own `spawn_blocking` walk (the memo only dedups the same +/// repo). Without a ceiling a single request fans out to O(repos) walks — an +/// amplification sink (INV-10). Once this many walks have run, no further walk is +/// spawned for the rest of the request: any remaining candidate that still needs +/// a walk is skipped (and, with nothing else readable, the request falls through +/// to the opaque 404). The bound is deliberately generous: a legitimate caller +/// serves on the first repo that grants them, so reaching it requires being +/// denied by this many path-scoped repos first, which real traffic effectively +/// never does. Tunable if that assumption stops holding. +/// +/// Kept at `MAX_PIN_SOURCES + 1` so the ceiling can never truncate a request +/// BEFORE its whole bounded provenance source set (first-pinner + up to +/// `MAX_PIN_SOURCES` additional) has been tried: an authorizing public source that +/// sorts after `MAX_PIN_SOURCES` path-scoped denials must still be reached and +/// served, not falsely 503'd as a truncated search. The legacy scan's fan-out is +/// separately bounded by `MAX_LEGACY_PROBES_PER_REQUEST`, so widening this by one +/// does not loosen that path. +pub(crate) const MAX_HISTORY_WALKS_PER_REQUEST: u32 = crate::db::MAX_PIN_SOURCES as u32 + 1; + +/// Hard per-request ceiling on how many legacy (NULL-provenance) repositories +/// the CID resolver's scan fallback may PROBE (`acquire` + `git cat-file -t`). +/// The provenance path targets one repo; the legacy scan, absent this bound, +/// fans one anonymous request out to O(repos) subprocess spawns and cold-cache +/// Tigris fetches for a CID enumerable from the public pins index (#173 round 3, +/// F1, INV-10). Deliberately generous: a normal node has far fewer repos than +/// this, so a genuine miss still completes the whole scan and returns a truthful +/// 404; only a node larger than the cap truncates, and a truncated search +/// surfaces as a retryable 503 (never a false "absent"). Legacy pins are a +/// shrinking set — each re-pin backfills provenance — so this fallback is a +/// transitional path, not the steady state. Tunable via `AppState`. +pub(crate) const MAX_LEGACY_PROBES_PER_REQUEST: u32 = 256; + +/// Hard ceiling on the byte size of an object `GET /ipfs/{cid}` buffers and serves +/// (#173 round 8, F6, INV-10). The serve reads via a blocking `git cat-file` and +/// buffers the whole object; unbounded, a large public blob (enumerable from the pins +/// index) could exhaust memory or block a runtime worker. A content-addressed serve +/// must verify the whole object hashes to the requested CID before any byte egresses +/// (F2), so it cannot stream — it buffers up to this cap and withholds anything larger +/// (raise the cap if a class of legitimate objects legitimately exceeds it; never +/// stream unverified). 32 MiB is generous for git blobs/trees/commits. Tunable via +/// `AppState` for the test seam, like the sibling caps. +pub(crate) const MAX_SERVED_OBJECT_BYTES: u64 = 32 * 1024 * 1024; + +/// Lazily-loaded context for the legacy (NULL-provenance) scan fallback in +/// `get_by_cid`: all repos, their visibility rules keyed by repo id, and the set of +/// quarantined repo ids. Loaded once per request only if a legacy pin is hit. +type LegacyScanCtx = ( + Vec, + HashMap>, + HashSet, +); + /// GET /ipfs/{cid} /// -/// Search all repos on the node for a git object whose SHA-256 hash matches -/// the given CIDv1, returning its raw content if the caller may read it. +/// Resolve the CIDv1 to its git oid via the `pinned_cids` table, then search all +/// repos on the node for that object, returning its raw content if the caller may +/// read it. /// /// Visibility (#110, #126): the object is served only from a repo row the /// caller passes. For each iterated row we gate against that row's OWN rules @@ -43,23 +107,74 @@ use crate::visibility::{visibility_check, Decision}; /// row than the one read (KTD2a). We check object existence via /// `store::object_type` *before* the expensive reachability walk so random-CID /// spray cannot trigger full-history git walks on repos that don't carry the -/// object. When the row carries path-scoped rules (KTD4) the served object -/// must be either a non-blob (trees/commits are structural; KTD3) OR a blob -/// in the caller's *reachable* allowed-set (`allowed_blob_set_for_caller`). -/// The reachable allowed-set excludes dangling blobs — a blob written via -/// `git hash-object -w` and never committed has no path to gate, so it is -/// fail-closed 404'd under path-scoped rules (#126). Denial and genuine -/// not-found both fall through to an opaque 404. +/// object. When the row carries path-scoped rules (KTD4) the served object is +/// gated by type: a `blob`/`tree` must be in the caller's *reachable* allowed-set +/// (`allowed_blob_set_for_caller` / `allowed_tree_set_for_caller`), and a +/// `commit`/`tag` must be in the repo's *reachable* commit/tag set +/// (`reachable_commit_tag_oids`, #173). A withheld subtree's tree object is denied +/// here exactly as `get_tree` denies its path, so its child names and oids cannot +/// leak by CID (#135). All these sets exclude dangling objects — a blob, tree, +/// commit, or tag written via plumbing and never referenced has no reachable path, +/// so it is fail-closed 404'd under path-scoped rules (#126, #173). Denial and +/// genuine not-found both fall through to an opaque 404. +/// +/// Scan completeness (F2): the 404 above is returned ONLY when every candidate +/// repo reached a VERDICT — visibility deny, probe-says-absent, walk-gate deny, +/// or served. A candidate skipped WITHOUT a verdict (acquire failure/timeout, +/// probe error, walk failure/panic, content-read error, or truncation by +/// `ipfs_max_repos_walked` / `ipfs_max_repo_visits` / +/// `ipfs_request_budget_secs`) taints the scan, and a +/// tainted scan that found nothing sheds a retryable 503 + Retry-After naming +/// the truncation sources — existing content is never misreported absent +/// because of unrelated repos or transient faults. +/// +/// Deterministic fault (F5/U4): a candidate repo that is persistently broken (a +/// corrupt repo, a bad `.git/config`) also yields no absence verdict, but a retry +/// cannot fix it, so a scan that found nothing sheds a TERMINAL, non-retryable 500 +/// (opaque body) rather than the retryable 503 — checked first so a deterministic +/// fault is never downgraded, and gated on nothing-served so a healthy repo that +/// carries the object still serves. +/// +/// Request budget (F3): one absolute clock (`ipfs_request_budget_secs`) spans +/// the whole admitted request. No stage (acquire, probe, walk, content read) +/// starts once it is exhausted, and the acquire wait and walk deadline are +/// clamped to the remainder. The probe and content-read subprocesses each ALSO +/// run under their own deadline, the lesser of `git_service_timeout_secs` and the +/// remaining budget, reaped by process-group teardown at that deadline, so a hung +/// `cat-file` cannot hold the request's walk slot past it. +/// +/// Residual, still true: the probe's object-store readability check +/// (`store::object_store_readable`, reached on the `missing` branch that a +/// random-CID spray drives) is a synchronous `read_dir` + `File::open` sweep with +/// no deadline and nothing to reap, so a wedged filesystem can still hold the +/// walk slot past the deadline. Same class as the D-state git survivor residual. /// /// Scope: this closes the direct unauthenticated scan, including the dangling /// case. A stale-public mirror row still serves withheld content (tracked /// separately, #124). +/// +/// One `/ipfs` request's walk admission: the global pool permit plus the +/// optional per-source sub-permit, both RAII (#174 U1). +/// +/// Held behind an `Arc` whose clones go into every `spawn_blocking` walk this +/// request runs, so the permits release only when the last clone drops — the +/// handler's, or an abandoned/panicking closure's, whichever outlives the other. +/// Admission therefore tracks real blocking-thread occupancy rather than the +/// lifetime of the future that requested it. +struct WalkAdmission { + _global: tokio::sync::OwnedSemaphorePermit, + _per_source: Option, +} + pub async fn get_by_cid( Path(cid_str): Path, State(state): State, + crate::rate_limit::PeerAddr(peer): crate::rate_limit::PeerAddr, + headers: HeaderMap, auth: Option>, ) -> Result { - // 1. Decode the CID and extract the SHA-256 digest + // 1. Decode and validate the CID (uniform 400 on a malformed / non-sha2-256 + // CID, before any DB or git work). let cid = CidGeneric::<64>::from_str(&cid_str) .map_err(|e| AppError::BadRequest(format!("invalid CID: {e}")))?; @@ -72,143 +187,1055 @@ pub async fn get_by_cid( )); } - let sha256_hex = hex::encode(mh.digest()); - let caller = auth.as_ref().map(|e| e.0 .0.as_str()); - let caller_owned = caller.map(|c| c.to_string()); + // Canonicalize the CID for the pinned_cids lookup. Pins are stored under the + // canonical base32 `cid.to_string()`, but a client may send any equivalent + // multibase spelling (base58/base64) of the same CID; those parse and pass + // the sha2-256 check yet miss the canonical key, so they must be normalized + // before the DB lookup (#173). Response headers and error messages still echo + // the original `cid_str` the client sent. + let canonical_cid = cid.to_string(); - // 2. Search all repos for an object with this SHA-256 - let repos = state - .db - .list_all_repos() - .await - .map_err(AppError::Internal)?; + // One absolute budget bounds this request's whole acquire+walk lifetime (F3), + // captured before admission so the clock covers everything the walk permit + // holds. Each stage below (acquire, probe, walk, read) starts only while + // budget remains, and the acquire wait + walk deadline run clamped to the + // remainder, so an admitted request cannot hold its scarce walk slot for + // hours by drawing a fresh per-stage timeout every iteration. The budget + // NEVER aborts a running spawn_blocking walk: the clamped git deadline + // inside the walk is what ends it (a tokio timeout around the walk future + // would free the walk permit while the blocking thread still runs, the + // exact hole the held permit closes). + let request_deadline = std::time::Instant::now() + + std::time::Duration::from_secs(state.config.ipfs_request_budget_secs); - // Fetch every repo's visibility rules in one query rather than one per row - // (the gate runs each row against its OWN rules — KTD2a). A row absent from - // the map has no rules. - let repo_ids: Vec = repos.iter().map(|r| r.id.clone()).collect(); - let rules_by_repo = state + // Bounded walk admission (#174 P1-3), taken before any DB/git work so a flood sheds + // cheaply. The per-repo `spawn_blocking` walk below is a full-history git walk with + // no served-git admission of its own; a permissionless caller could otherwise fan + // out concurrent walks past every git pool, exhausting the blocking pool + PIDs. + // Acquire the global permit (and, for a resolvable source, the per-source + // sub-permit) ONCE here and hold BOTH for the whole request — across every + // `spawn_blocking` walk below — so the slot reflects real blocking-thread + // occupancy (a tokio walk-timeout cannot free it while the blocking work still runs) + // and one request cannot open more than its share of concurrent walks. Holding a + // slot across a walk is only safe because every walk child is duration-bounded + // (`*_bounded` + `run_bounded_git` teardown), so a hung git cannot pin the slot + // past `git_service_timeout_secs`. On unavailability shed a clean 503. The + // per-source key is the resolved source IP (`client_key`), never the DID (`/ipfs` + // admits any `did:key` unthrottled, so a DID key would be free to mint around); a + // `None` key (no trusted header, no peer) is bounded by the global pool only, + // never the per-source sub-cap. + let global_permit = state + .git_ipfs_walk_semaphore + .clone() + .try_acquire_owned() + .map_err(|_| { + tracing::warn!("/ipfs walk concurrency cap reached; shedding request with 503"); + AppError::Overloaded("ipfs service at capacity, retry shortly".into()) + })?; + let source_key = crate::rate_limit::client_key(&headers, peer, state.push_limiter_trust); + let caller_permit = match &source_key { + Some(ip) => Some(state.git_ipfs_walk_per_caller.try_acquire(ip).ok_or_else(|| { + tracing::warn!(key = %ip, "/ipfs per-source walk cap reached; shedding request with 503"); + AppError::Overloaded("ipfs service at capacity for this source, retry shortly".into()) + })?), + None => None, + }; + // Share the admission rather than holding it as a handler local (#174 U1). A + // clone goes into every `spawn_blocking` closure below, so the permits release + // only when the LAST holder drops. That makes the two failure directions one + // case: a client disconnect drops the handler's clone while the abandoned + // closure's clone keeps the slot taken for as long as its git child runs, and a + // panicking closure drops its clone while the handler's keeps the slot taken. + // + // Deliberately NOT a move-and-return shuttle. That shape is correct only if + // every arm continuing the loop re-binds the returned value, which the compiler + // cannot enforce — including the absent-object arm `Ok(Ok(None)) => continue` + // that a random-CID scan takes on nearly every iteration — and it forces a + // second decision about the arms where a panic destroys the moved-in permits. + let admission = std::sync::Arc::new(WalkAdmission { + _global: global_permit, + _per_source: caller_permit, + }); + + // Caller DID (owned): the `spawn_blocking` closures below cannot borrow the + // handler's `auth` extension, so resolve it once here. + let caller_owned = auth.as_ref().map(|e| e.0 .0.as_str().to_string()); + + // Resolve the content-addressed CID to the object's git oid(s). A real pin + // CID digests the raw object content (`Cid::from_git_object_bytes`), NOT the + // git oid (git frames content with a `" \0"` header first), so we + // map it back through `pinned_cids` rather than treating the digest as an oid + // (#173). The cid index is non-unique, so one CID can map to several oids (a + // tree and a blob whose raw bytes collide, or content pinned under two oids); + // we try each candidate below rather than pick one arbitrarily and false-404 + // when the chosen one is withheld or absent while another is readable (#173). + // An empty result is an opaque 404, uniform with a genuine not-found and a + // visibility denial. + let oids = state .db - .list_visibility_rules_for_repos(&repo_ids) + .oids_for_cid(&canonical_cid) .await .map_err(AppError::Internal)?; + if oids.is_empty() { + return Err(AppError::RepoNotFound(format!( + "no git object found for CID {cid_str}" + ))); + } + let caller = caller_owned.as_deref(); - // Request-scoped memo of the per-repo allowed-blob set (KTD1, #126). The - // caller is constant for one request, so `repo.id` alone is a safe, - // sufficient key — never a coarse caller "class", which - // `visibility_check`'s exact full-DID reader match would make unsafe. - // - // We flipped from a deny-set (`withheld_blob_oids`) to an allowed-set - // (`allowed_blob_set_for_caller`) so dangling blobs — never enumerated by - // the reachable walk — fail closed instead of slipping through an empty - // deny entry (#126). - let mut allowed_memo: HashMap> = HashMap::new(); - - for repo in &repos { - // Repo-level read gate against THIS row's own rules (KTD2a). - let rules: &[crate::db::VisibilityRule] = rules_by_repo - .get(&repo.id) - .map(Vec::as_slice) - .unwrap_or(&[]); - if visibility_check(rules, repo.is_public, &repo.owner_did, caller, "/") == Decision::Deny { - continue; - } - - let repo_path = match state.repo_store.acquire(&repo.owner_did, &repo.name).await { - Ok(p) => p, - Err(_) => continue, - }; + // Per-request walk budget + memos + throttle flag, shared by the provenance path + // and the legacy scan so both honor the same fan-out ceiling, per-repo memo, and + // IP brake. The caller is constant for one request, so `repo.id` alone keys the memo. + let mut walk = WalkState { + walks: 0, + probes: 0, + visits: 0, + truncated_by: Vec::new(), + deterministic_fault: false, + allowed_blob_memo: HashMap::new(), + allowed_tree_memo: HashMap::new(), + reachable_ct_memo: HashMap::new(), + }; + // Set when a walk-requiring candidate is skipped because the source IP's walk quota + // is spent (#173 review, F-C): the scan keeps going so a later walk-free copy still + // serves; only if nothing is servable is it turned into the 429. + let mut throttled = false; + let rctx = ResolveCtx { + caller, + caller_owned: &caller_owned, + headers: &headers, + peer, + cid_str: &cid_str, + canonical_cid: &canonical_cid, + request_deadline, + admission: &admission, + }; - // Check whether the object exists in this repo before any expensive - // reachability walk. This prevents random-CID spray from triggering - // full-history git walks on repos that don't carry the object. - let obj_type = match store::object_type(&repo_path, &sha256_hex) { - Ok(Some(t)) => t, - Ok(None) => continue, - Err(e) => { - tracing::warn!(repo = %repo.name, err = %e, "error checking git object type"); + // Legacy scan context (repos + rules + quarantined ids), loaded LAZILY only when a + // legacy NULL-provenance pin is hit — the provenance path must never trigger the + // O(repos) load (that fan-out is exactly what provenance removes, #173 round 2). + let mut scan_ctx: Option = None; + + for sha256_hex in &oids { + // A pinned object records EVERY repo it was pinned from (#173 round 8, F1). + // Resolve a PROVENANCED pin by trying each source repo (bounded to + // MAX_PIN_SOURCES) through the SAME gate; the first that authorizes serves — no + // scan fan-out. A shared object first pinned from a private/quarantined repo + // still serves from a later PUBLIC source. Deterministic (ORDER BY on the + // union), so no ordering can turn an authorized copy into a 404. + let sources = state + .db + .pin_sources_for_oid(sha256_hex) + .await + .map_err(AppError::Internal)?; + // Provenance fast-path: try each recorded source repo through the SAME gate + // (bounded to first-pinner + MAX_PIN_SOURCES). Empty for a legacy NULL-provenance + // pin. The first source that authorizes serves — no scan fan-out on the common + // path. + for repo_id in &sources { + // These three per-source lookups run while the scarce walk permits are + // ALREADY held, exactly like the legacy scan's preload below, so they carry + // the same clamp (#174 F6/KTD-5). The pool sets no statement_timeout, so an + // unclamped query blocked in Postgres would pin a walk slot for the whole + // stall, past the request budget, and capacity-503 later requests. Returning + // here drops the permits. The quarantine bit and the visibility rules are + // both access control, so a timeout must DENY rather than fall through with + // an empty answer (FAIL CLOSED). + let budget_shed = || { + AppError::Overloaded(format!( + "ipfs scan incomplete (budget) for CID {cid_str}; retry shortly" + )) + }; + let remaining = + || request_deadline.saturating_duration_since(std::time::Instant::now()); + let repo = match tokio::time::timeout(remaining(), state.db.get_repo_by_id(repo_id)) + .await + { + Ok(Ok(Some(r))) => r, + // A source repo is gone: skip it; a later source or the scan fallback + // below may still resolve. + Ok(Ok(None)) => continue, + Ok(Err(e)) => return Err(AppError::Internal(e)), + Err(_elapsed) => { + tracing::warn!( + budget_secs = state.config.ipfs_request_budget_secs, + "/ipfs get_repo_by_id exceeded the request budget \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); shedding a retryable 503 and freeing the walk permit" + ); + return Err(budget_shed()); + } + }; + let quarantined = match tokio::time::timeout( + remaining(), + state.db.is_repo_quarantined(repo_id), + ) + .await + { + Ok(Ok(q)) => q, + Ok(Err(e)) => return Err(AppError::Internal(e)), + Err(_elapsed) => { + tracing::warn!( + budget_secs = state.config.ipfs_request_budget_secs, + "/ipfs is_repo_quarantined exceeded the request budget \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); denying (fail closed) and freeing the walk permit" + ); + return Err(budget_shed()); + } + }; + let rules_map = match tokio::time::timeout( + remaining(), + state + .db + .list_visibility_rules_for_repos(std::slice::from_ref(repo_id)), + ) + .await + { + Ok(Ok(rules)) => rules, + Ok(Err(e)) => return Err(AppError::Internal(e)), + Err(_elapsed) => { + tracing::warn!( + budget_secs = state.config.ipfs_request_budget_secs, + "/ipfs per-source list_visibility_rules_for_repos exceeded the request budget \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); denying (fail closed) and freeing the walk permit" + ); + return Err(budget_shed()); + } + }; + let rules = rules_map.get(repo_id).map(Vec::as_slice).unwrap_or(&[]); + match gate_and_serve( + &state, + &repo, + rules, + quarantined, + sha256_hex, + &rctx, + &mut walk, + false, + ) + .await + { + GateOutcome::Served(resp) => return Ok(resp), + GateOutcome::Throttled => { + throttled = true; + continue; + } + GateOutcome::Skip => continue, + } + } + + // Bounded legacy-scan fallback. Run it when the provenance set could not have + // served the caller AND may be INCOMPLETE: + // - empty -> a legacy NULL-provenance pin (recorded before provenance existed), or + // - at_cap -> `record_pin_source` stops inserting at MAX_PIN_SOURCES and drops + // later sources SILENTLY, so a full table may hide a servable source + // (e.g. a later PUBLIC pinner buried by 16 attacker sources — the + // pin-source griefing hole). The scan gates every repo through the + // real per-caller gate, so it finds that copy. + // - marked -> a `record_pin_source` for this object failed outright (U3, #173). + // `record_pin_source` is best effort at every pin call site, so a + // non-empty below-cap set is NOT self-evidently complete: an object + // first pinned from a PRIVATE repo and later pushed from a PUBLIC + // one whose record failed names only the private source. The + // durable `pin_sources_incomplete` marker is the node's own record + // that a source is missing, so the fallback stays available for + // exactly those objects instead of 404ing a servable public copy. + // Only a set with NONE of these three signals is treated as complete (every + // recorded source was just tried), so it skips the scan and lets the tail 404, and + // ordinary denials never fan out to O(repos) (INV-10 / F3). Both extra queries run + // only on a provenance MISS (we return above on Served) by a caller that still has + // work budget, so neither costs the serve path nor a shed caller, and the fallback + // is not an authorization bypass: the scan gates every repo + // through the SAME per-caller gate, so a caller who may not read the object is + // still denied. + // + // F3 (#173, INV-10/INV-15): peek the per-IP WORK-budget limiter WITHOUT + // consuming a token so an already-throttled source is shed BEFORE the + // O(repos) preload; the consuming per-probe charge inside gate_and_serve is + // left UNCHANGED (it is load-bearing for the across-request bound), so this + // adds no double-charge. This peeks `ipfs_work_rate_limiter`, the SAME bucket + // the per-probe charge below debits — NOT the route limiter (`ipfs_rate_limiter`, + // charged once per request by the middleware): peeking the route bucket here + // would re-shed a request the route already admitted (R6, U5). + // + // The peek runs BEFORE the two marker queries (#173 round 11, F5): shedding is + // the whole point of a peek, so a spent-budget caller should not pay two + // lookups per request first. It stays AFTER the provenance walk, so no caller + // who could have been served is shed. The one caller this moves: a spent-budget + // caller whose source set turns out COMPLETE now takes the 429 tail instead of + // the 404 tail. That is the honest answer (its search never ran), and it drops + // an oracle, since the old order let a throttled caller tell a complete source + // set from an incomplete one by 404 vs 429. + if let Some(key) = + crate::rate_limit::client_key(rctx.headers, rctx.peer, state.push_limiter_trust) + { + if state.ipfs_work_rate_limiter.is_throttled(&key).await { + throttled = true; continue; } + } + let needs_scan = sources.is_empty() || { + #[cfg(test)] + bump_marker_queries(); + state + .db + .pin_sources_at_cap(sha256_hex) + .await + .map_err(AppError::Internal)? + || state + .db + .pin_sources_incomplete(sha256_hex) + .await + .map_err(AppError::Internal)? }; - - // Per-blob gating only applies when a path-scoped rule exists (KTD4). - // Without any path-scoped rule, the "/" gate above is the whole story. - // Trees/commits are always served under path-scoped rules (KTD3). - let path_scoped = has_path_scoped_rule(rules); - if path_scoped && obj_type == "blob" { - if !allowed_memo.contains_key(&repo.id) { - let rp = repo_path.clone(); - let r = rules.to_vec(); - let is_public = repo.is_public; - let owner = repo.owner_did.clone(); - let caller_for_walk = caller_owned.clone(); - // Full-history walk shells out to git — keep it off the async runtime. - let walk = tokio::task::spawn_blocking(move || { - allowed_blob_set_for_caller( - &rp, - &r, - is_public, - &owner, - caller_for_walk.as_deref(), - ) - }) - .await; - // Fail closed on EITHER a task panic (JoinError) or a walk error: - // we cannot prove the caller may read here, so skip this repo and - // let a public copy (if any) serve. Never serve on an unproven gate. - let set = match walk { - Ok(Ok(set)) => set, - Ok(Err(e)) => { - tracing::warn!(repo = %repo.name, err = %e, "allowed-blob walk failed; skipping repo"); - continue; + if needs_scan { + // Load the scan context once, lazily (shared across oid candidates). + if scan_ctx.is_none() { + #[cfg(test)] + bump_preload_queries(); + // F6/KTD-5 (#174): the preload queries run while the scarce walk permits + // are ALREADY held, and the pool sets no statement_timeout, so a query + // blocked in Postgres would pin those slots for the whole stall — past the + // request budget — capacity-503'ing later requests. Clamp each to the + // remaining budget; a timeout returns the same retryable budget 503 the + // later stages shed, and returning here drops the permits. + // `list_visibility_rules_for_repos` is the access-control query, so its + // timeout returns BEFORE the loop: the scan can never run with an empty + // rule map and serve an unfiltered listing that exposes private repos + // (FAIL CLOSED). + let budget_secs = state.config.ipfs_request_budget_secs; + let budget_shed = || { + AppError::Overloaded(format!( + "ipfs scan incomplete (budget) for CID {cid_str}; retry shortly" + )) + }; + let repos = match tokio::time::timeout( + request_deadline.saturating_duration_since(std::time::Instant::now()), + state.db.list_all_repos(), + ) + .await + { + Ok(Ok(repos)) => repos, + Ok(Err(e)) => return Err(AppError::Internal(e)), + Err(_elapsed) => { + tracing::warn!( + budget_secs, + "/ipfs list_all_repos exceeded the request budget \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); shedding a retryable 503 and freeing the walk permit" + ); + return Err(budget_shed()); } - Err(e) => { - tracing::warn!(repo = %repo.name, err = %e, "allowed-blob walk task panicked; skipping repo"); - continue; + }; + let repo_ids: Vec = repos.iter().map(|r| r.id.clone()).collect(); + let rules_by_repo = match tokio::time::timeout( + request_deadline.saturating_duration_since(std::time::Instant::now()), + state.db.list_visibility_rules_for_repos(&repo_ids), + ) + .await + { + Ok(Ok(rules)) => rules, + Ok(Err(e)) => return Err(AppError::Internal(e)), + Err(_elapsed) => { + tracing::warn!( + budget_secs, + "/ipfs list_visibility_rules_for_repos exceeded the request budget \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); denying (fail closed) and freeing the walk permit" + ); + return Err(budget_shed()); } }; - allowed_memo.insert(repo.id.clone(), set); + let quarantined: HashSet = match tokio::time::timeout( + request_deadline.saturating_duration_since(std::time::Instant::now()), + state.db.list_quarantined_repos(), + ) + .await + { + // The quarantine set is also access control (INV-11), so a timeout + // must deny rather than scan with an empty set. + Ok(Ok(rows)) => rows.into_iter().map(|r| r.id).collect(), + Ok(Err(e)) => return Err(AppError::Internal(e)), + Err(_elapsed) => { + tracing::warn!( + budget_secs, + "/ipfs list_quarantined_repos exceeded the request budget \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); denying (fail closed) and freeing the walk permit" + ); + return Err(budget_shed()); + } + }; + scan_ctx = Some((repos, rules_by_repo, quarantined)); } - let in_allowed = allowed_memo - .get(&repo.id) - .is_some_and(|set| set.contains(&sha256_hex)); - if !in_allowed { - continue; + let (repos, rules_by_repo, quarantined) = scan_ctx.as_ref().unwrap(); + for repo in repos { + let rules = rules_by_repo + .get(&repo.id) + .map(Vec::as_slice) + .unwrap_or(&[]); + let is_quar = quarantined.contains(&repo.id); + match gate_and_serve( + &state, repo, rules, is_quar, sha256_hex, &rctx, &mut walk, true, + ) + .await + { + GateOutcome::Served(resp) => return Ok(resp), + // A throttled walk-requiring candidate is skipped, not fatal: + // keep scanning for a later walk-free copy (#173 review, F-C). + GateOutcome::Throttled => throttled = true, + GateOutcome::Skip => {} + } } } + } - // Now that we've passed the gate, read the content. - let content = match store::read_object_content(&repo_path, &sha256_hex, &obj_type) { - Ok(c) => c, - Err(e) => { - tracing::warn!(repo = %repo.name, err = %e, "error reading git object content"); - continue; - } - }; + // Nothing served — four distinct tails, in precedence order: + // 1. A candidate repo is persistently broken (a corrupt repo, a bad `.git/config`), + // and that was the SOLE reason nothing served → terminal, non-retryable 500 + // (#174 F5/U4). A retry cannot fix it, and a 503 here would invite a conformant + // client to retry-storm a fresh `cat-file` per attempt against the broken repo. + // Gated on nothing else having tainted: when a transient skip co-occurs, the + // object may live in the repo that was skipped transiently, so a retry CAN + // surface it and the retryable 503 below is the honest answer. The body is + // opaque; the raw git detail was logged at the probe and never reaches the client. + // 2. The scan was cut short (a cap, the request budget, or a transient stage + // failure), so the object was NOT proven absent/unreadable everywhere → 503, + // retryable, and explicitly NOT a definitive not-found (#173 F2). This outranks + // the throttle: an incomplete search must not masquerade as a clean rate-limit + // outcome. The message names the truncation sources so an operator can map the + // shed to the right knob or backend, and carries no object/OID/metadata. + // 3. A walk-requiring candidate was skipped for a spent IP quota while the scan + // otherwise completed → 429 (the brake bit; a cheaper copy was sought first). + // 4. A full scan under the caps found nothing readable → opaque 404, uniform with + // a genuine not-found and a visibility denial. + if walk.deterministic_fault && walk.truncated_by.is_empty() { + return Err(AppError::Git( + "ipfs object probe could not complete: a candidate repository is corrupt".into(), + )); + } + if !walk.truncated_by.is_empty() { + return Err(AppError::SearchIncomplete(format!( + "CID {cid_str} search incomplete ({}) — retry", + walk.truncated_by.join("+") + ))); + } + if throttled { + return Err(AppError::TooManyRequests( + "ipfs retrieval rate limit exceeded — try again later".into(), + )); + } + Err(AppError::RepoNotFound(format!( + "no git object found for CID {cid_str}" + ))) +} - // 3. Return the content with IPFS-style headers - let mut headers = HeaderMap::new(); - headers.insert( - HeaderName::from_static("content-type"), - HeaderValue::from_static("application/octet-stream"), - ); - headers.insert( - HeaderName::from_static("x-content-cid"), - HeaderValue::from_str(&cid_str).unwrap_or_else(|_| HeaderValue::from_static("invalid")), +/// Outcome of gating one repo for one candidate oid. +enum GateOutcome { + /// The object passed the gate; serve this response. + Served(Response), + /// This repo does not serve the object (absent, denied, quarantined, walk-capped, + /// or a walk error) — try the next candidate. + Skip, + /// A walk-requiring candidate hit the per-IP walk quota; skip it but let the caller + /// record the throttle so a later walk-free copy can still serve. + Throttled, +} + +/// Outcome of the bounded, off-worker object read for one gated candidate (F6, #173). +enum ServedRead { + /// Verified: the object's bytes hash to the requested CID; serve them. + Ok(Vec), + /// The bytes do not hash to the requested CID (a legacy provider-CID row); withhold. + Mismatch(String), + /// The object exceeds the served-object size cap; withhold rather than buffer it. + TooLarge(u64), + /// The object is genuinely absent (git reported it does not exist); try the next + /// candidate. Distinct from `ReadErr` so an infra failure is never silently rendered + /// as a clean not-found. + Gone, + /// A git subprocess failed to run (spawn/IO error, not a "no such object"). Logged at + /// the handler layer and skipped — an infra failure must surface as an error, not a + /// silent 404 for an authorized caller (INV-25 spirit, #173). + ReadErr(String), +} + +/// Immutable per-request context threaded into the gate. +struct ResolveCtx<'a> { + caller: Option<&'a str>, + caller_owned: &'a Option, + headers: &'a HeaderMap, + peer: Option, + cid_str: &'a str, + /// Canonical base32 form of the requested CID (`cid.to_string()`), used by the + /// serve-side integrity check to confirm the served bytes actually hash to the + /// requested content address (F2, #173). Compared against the recomputed CID, NOT + /// `cid_str` — a client may send an equivalent non-canonical multibase spelling. + canonical_cid: &'a str, + /// One absolute clock for the whole admitted request (#174 F3). No stage starts + /// once it is exhausted, and the acquire wait plus the probe/walk/read child + /// deadlines clamp to the remainder, so an admitted request cannot hold its scarce + /// walk slot by drawing a fresh per-stage timeout on every candidate. + request_deadline: std::time::Instant, + /// The request's walk admission (#174 U1). A clone goes into every `spawn_blocking` + /// below, so the permits release only when the last holder drops — the handler's + /// clone, or an abandoned or panicking closure's, whichever outlives the other. + admission: &'a std::sync::Arc, +} + +/// Per-request walk budget + memos, shared across the provenance path and the legacy +/// scan so the fan-out ceiling and per-repo memoization span the whole request. +struct WalkState { + walks: u32, + /// Count of legacy (NULL-provenance) repos actually probed this request, so the + /// scan can stop at `ipfs_max_legacy_probes` instead of fanning out to O(repos) + /// `acquire` + `cat-file` (#173, F1, INV-10). Only the legacy path bumps it. + probes: u32, + /// Count of repos this request has VISITED: every candidate that got past the + /// visibility gate and reached the acquire stage, on the provenance path as well + /// as the legacy scan (#174 F2). Every visit costs an acquire (worst case a full + /// Tigris archive download on a cache miss) plus a `cat-file` probe, so one + /// request can trigger at most `ipfs_max_repo_visits` object-store fetches. This + /// is the broader of the two ceilings: the probe ceiling above bounds only the + /// legacy scan's fan-out, and a provenance-only request never reaches it. + visits: usize, + /// Why the scan reached no verdict on one or more candidates: a cap cut it short + /// (the legacy probe ceiling, the walk ceiling, the request budget) or a stage + /// failed transiently (acquire, probe, walk, read). A truncated scan did NOT prove + /// the object absent/unreadable everywhere, so the tail returns a retryable 503 + /// rather than a definitive 404 (#173 F2), and the sources name the knob or backend + /// the operator should look at (#174 F2). Deduplicated, so one source appears once + /// however many candidates hit it. + truncated_by: Vec<&'static str>, + /// Set when a candidate repo is persistently broken (a corrupt repo, a bad + /// `.git/config`; #174 F5/U4). It yields no absence verdict either, but a retry + /// cannot fix it, so the tail sheds a terminal 500 instead of the retryable 503 — + /// and only when nothing else tainted, so one broken repo never converts a + /// retryable outcome into a terminal one. + deterministic_fault: bool, + allowed_blob_memo: HashMap>, + allowed_tree_memo: HashMap>, + reachable_ct_memo: HashMap>, +} + +impl WalkState { + /// Record that a candidate was skipped WITHOUT a verdict, naming the stage. + fn taint(&mut self, source: &'static str) { + if !self.truncated_by.contains(&source) { + self.truncated_by.push(source); + } + } + + /// Remaining request budget, or `None` once it is spent. A stage is never started + /// with zero remaining: the call site taints "budget" and stops, leaving this and + /// every later candidate unproven rather than reporting a false absence. + fn budget_left( + &mut self, + ctx: &ResolveCtx<'_>, + budget_secs: u64, + repo_name: &str, + stage: &'static str, + ) -> Option { + let left = ctx + .request_deadline + .saturating_duration_since(std::time::Instant::now()); + if left.is_zero() { + tracing::warn!( + repo = %repo_name, + stage, + budget_secs, + "/ipfs request budget exhausted before the stage \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); stopping without a verdict" + ); + self.taint("budget"); + return None; + } + Some(left) + } +} + +/// Gate ONE repo for ONE candidate oid and, if the caller may read it, serve it. The +/// SINGLE gate both the provenance path and the legacy scan call, so INV-11 (quarantine +/// hard-drops before visibility), INV-2 (the repo's own "/" gate), and the per-object +/// reachability walk hold identically on both paths (KTD5). Never re-resolves via +/// `authorize_repo_read`, whose fuzzy match could authorize a different physical row +/// than the one read (KTD2a). +// The per-repo gate genuinely needs the row, its rules, its quarantine bit, the oid, +// the request context, the shared walk budget, and whether this is the fan-out-bounded +// legacy scan; bundling them buys nothing over the existing threshold. +#[allow(clippy::too_many_arguments)] +async fn gate_and_serve( + state: &AppState, + repo: &crate::db::RepoRecord, + rules: &[crate::db::VisibilityRule], + quarantined: bool, + sha256_hex: &str, + ctx: &ResolveCtx<'_>, + walk: &mut WalkState, + // True only for the legacy NULL-provenance scan, which iterates every repo. The + // provenance path targets one repo (no fan-out) and passes false, so it does not + // consume the per-request probe budget below. + legacy_scan: bool, +) -> GateOutcome { + // Quarantine gate (INV-11): a quarantined mirror is hidden from every reader, owner + // included, BEFORE any visibility check — so an owner whom visibility would Allow + // still 404s. + if quarantined { + return GateOutcome::Skip; + } + // Repo-level "/" read gate against THIS row's own rules (INV-2, KTD2a). + if visibility_check(rules, repo.is_public, &repo.owner_did, ctx.caller, "/") == Decision::Deny { + return GateOutcome::Skip; + } + // Legacy-scan fan-out control (#173, F1/F3, INV-10). The legacy path probes every + // root-visible repo, and the probe below (`acquire` — a possible cold-cache + // Tigris fetch — plus a `git cat-file -t` subprocess) is the expensive part. + // Cap it per request BEFORE that work runs, so an anonymous caller wielding a + // CID from the public pins index cannot amplify one request into O(repos) + // subprocesses. A legacy scan is inherently fan-out (unlike a targeted + // provenance fetch), so EVERY legacy probe is charged to the source IP from the + // first one, not just the ones past a free budget. A per-request-only budget + // reset each request, leaving a NULL-provenance CID open to unbounded ACROSS- + // request amplification: N requests spending N x budget cold `acquire` calls + // against Tigris with zero limiter contact (#173, F3, jatmn). Charging the first + // probe makes those requests accumulate against the per-IP `ipfs_work_rate_limiter` + // (the resolver's WORK bucket, separate from the once-per-request route brake + // `ipfs_rate_limiter` — R6, U5), closing that path. The per-request cap below stays + // as the second bound (a single request's ceiling). A spent quota is the same non-fatal Throttled as the + // walk brake: keep scanning for a walk-free copy, and only a wholly-unservable + // request becomes the 429. No resolvable key (a test oneshot with no peer/header) + // skips the brake, as the walk brake does. The provenance path targets one repo + // (no fan-out) and is exempt (`legacy_scan == false`). + if legacy_scan { + if walk.probes >= state.ipfs_max_legacy_probes { + // Budget spent: stop probing and mark the scan truncated so the tail + // reports an incomplete search (503), not a false 404 (#173, F2). + walk.taint("probe-ceiling"); + return GateOutcome::Skip; + } + if let Some(key) = + crate::rate_limit::client_key(ctx.headers, ctx.peer, state.push_limiter_trust) + { + if !state.ipfs_work_rate_limiter.check(&key).await { + return GateOutcome::Throttled; + } + } + walk.probes += 1; + } + // Visit ceiling (#174 F2), checked before the acquire it bounds. On exhaustion the + // scan STOPS on this candidate without a verdict: there is no cheaper way to reach + // one, since a verdict needs the acquire and probe this ceiling is refusing. + if walk.visits >= state.config.ipfs_max_repo_visits { + tracing::warn!( + ceiling = state.config.ipfs_max_repo_visits, + repo = %repo.name, + "/ipfs request hit the per-request repo-visit ceiling \ + (GITLAWB_IPFS_MAX_REPO_VISITS); skipping repo without a verdict" ); - headers.insert( - HeaderName::from_static("x-git-hash"), - HeaderValue::from_str(&sha256_hex) - .unwrap_or_else(|_| HeaderValue::from_static("invalid")), + walk.taint("visit-ceiling"); + return GateOutcome::Skip; + } + walk.visits += 1; + + // Budget gate for the acquire stage (#174 F3). + let Some(acquire_budget) = walk.budget_left( + ctx, + state.config.ipfs_request_budget_secs, + &repo.name, + "repo acquire", + ) else { + return GateOutcome::Skip; + }; + // Bound the per-repo acquire under `git_acquire_timeout_secs`: this gate runs while + // the /ipfs walk permit is held (F5), so a hung or cold-Tigris acquire would otherwise + // pin the global walk slot for the whole request. On expiry skip the repo (a public + // copy may still serve) and mark the search truncated so a wholly-unserved request + // tails to a retryable 503, never a false 404 (reopened the #174 P1-2 stall vector on + // this path otherwise). Clamped to the remaining request budget so per-repo acquires + // cannot each draw a fresh full timeout past it (#174 F3). + let acquire_deadline = std::cmp::min( + std::time::Duration::from_secs(state.config.git_acquire_timeout_secs), + acquire_budget, + ); + let repo_path = match tokio::time::timeout( + acquire_deadline, + state.repo_store.acquire(&repo.owner_did, &repo.name), + ) + .await + { + Ok(Ok(p)) => p, + // An acquire FAILURE is not an absence verdict either: the repo may well hold the + // object, we just could not open it. + Ok(Err(e)) => { + tracing::warn!(repo = %repo.name, err = %e, "repo acquire failed during /ipfs gate; skipping repo without a verdict"); + walk.taint("acquire"); + return GateOutcome::Skip; + } + Err(_elapsed) => { + tracing::warn!(repo = %repo.name, "repo acquire timed out during /ipfs gate; skipping repo without a verdict"); + walk.taint("acquire"); + return GateOutcome::Skip; + } + }; + + // Existence probe before any walk (random-CID spray must not trigger a walk on a + // repo that lacks the object). Off the async runtime — it shells out to + // `git cat-file -t`. Fail closed (skip) on a task panic. + let Some(probe_budget) = walk.budget_left( + ctx, + state.config.ipfs_request_budget_secs, + &repo.name, + "object-type probe", + ) else { + return GateOutcome::Skip; + }; + let obj_type = { + let rp = repo_path.clone(); + let sha = sha256_hex.to_string(); + // The probe shells to the REAL `git`, as the unbounded `object_type` always + // did, independent of `state.git_bin`. That knob is the WALK binary: tests + // point it at a fake that answers `rev-list` and friends, and routing the + // existence probe through it would ask that fake to impersonate + // `cat-file --batch-check` as well (#174). + let git_bin = "git".to_string(); + // Bound the probe CHILD itself (process-group teardown via + // `object_type_bounded` -> `run_bounded_git`), not just an outer tokio timeout + // racing an uncancellable `spawn_blocking`: this probe runs while the /ipfs walk + // permit is held, so a wedged cat-file (corrupt pack, NFS stall) must be REAPED + // at the deadline rather than left to linger and delay admission release + // (#173 round-10, KTD2). No outer timeout, mirroring the bounded walk below. The + // child's own deadline is the lesser of `git_service_timeout_secs` and the + // remaining request budget (#174 F3), so a started probe cannot finish past it. + let probe_timeout = std::cmp::min( + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + probe_budget, ); + let probe_admission = std::sync::Arc::clone(ctx.admission); + match tokio::task::spawn_blocking(move || { + // Admission clone (#174 U1): the slot stays taken until this blocking work + // returns, even if the handler future was dropped or this closure panics. + let _admission = probe_admission; + store::object_type_bounded(&git_bin, &rp, &sha, probe_timeout) + }) + .await + { + Ok(Ok(Some(t))) => t, + // Absence is the one verdict the probe can reach on its own. + Ok(Ok(None)) => return GateOutcome::Skip, + // Transient fault (an unreadable or mid-repack store, or the reaped + // deadline): unproven and retryable, so taint rather than 404. + Ok(Err(store::ProbeError::Transient(e))) => { + tracing::warn!(repo = %repo.name, err = %e, "object-type probe hit a transient store fault under the /ipfs walk permit; skipping repo without a verdict"); + walk.taint("probe"); + return GateOutcome::Skip; + } + // Deterministic fault (a corrupt repo, a bad `.git/config`): a retry cannot + // fix it, so it must NOT taint — a retryable 503 would invite a conformant + // client to retry-storm a fresh `cat-file` per attempt against the broken + // repo. The tail renders it as a terminal 500, and only if nothing served. + Ok(Err(store::ProbeError::Deterministic(e))) => { + tracing::warn!(repo = %repo.name, err = %e, "object-type probe hit a deterministic fault (corrupt repo/config); skipping repo without a verdict"); + walk.deterministic_fault = true; + return GateOutcome::Skip; + } + Err(e) => { + tracing::warn!(repo = %repo.name, err = %e, "object-type probe task panicked; skipping repo without a verdict"); + walk.taint("probe"); + return GateOutcome::Skip; + } + } + }; + + // Per-object gating applies only under a path-scoped rule (KTD4); otherwise the "/" + // gate above is the whole story. A blob is gated on the caller's allowed-blob set, a + // tree on the allowed-tree set (#135), a commit/tag on the repo's reachable + // commit/tag set (#173) — each a full-history walk sharing the per-request cap and + // per-walk IP quota. + let path_scoped = has_path_scoped_rule(rules); + let gated = path_scoped && matches!(obj_type.as_str(), "blob" | "tree" | "commit" | "tag"); + if gated { + let already = match obj_type.as_str() { + "blob" => walk.allowed_blob_memo.contains_key(&repo.id), + "tree" => walk.allowed_tree_memo.contains_key(&repo.id), + "commit" | "tag" => walk.reachable_ct_memo.contains_key(&repo.id), + other => unreachable!("gated admits only blob/tree/commit/tag, got {other}"), + }; + if !already { + // Budget gate for the walk stage (#174 F3): probed-present is not a serve, so + // a walk is never STARTED with no budget left. + if walk + .budget_left( + ctx, + state.config.ipfs_request_budget_secs, + &repo.name, + "visibility walk", + ) + .is_none() + { + return GateOutcome::Skip; + } + // Per-request fan-out ceiling (INV-10): once this many walks have run, skip + // THIS walk-requiring candidate and keep scanning (a later walk-free copy + // must still serve). `walks` is bumped only inside this block, so walk-free + // candidates never consume budget. + // Both parents bound this loop, under different knobs: #173's + // `ipfs_max_history_walks` (an AppState field, seeded from config) and + // #174's `GITLAWB_IPFS_MAX_REPOS_WALKED`. Honor the tighter of the two, so + // neither knob silently stops working after the merge. + let walk_cap = std::cmp::min( + state.ipfs_max_history_walks as usize, + state.config.ipfs_max_repos_walked, + ); + if walk.walks as usize >= walk_cap { + // The walk ceiling truncated the search: a later repo (possibly one that + // authorizes this caller) is left unwalked, so absence is unproven — + // record it so the tail returns 503, not a false 404 (#173, F2). + tracing::warn!( + cap = walk_cap, + repo = %repo.name, + "/ipfs request hit the per-request walk cap; skipping repo without a verdict" + ); + walk.taint("walk-cap"); + return GateOutcome::Skip; + } + // Brake each spawned walk on the source IP (#173, F3, INV-15), BEFORE + // spending walk budget: a throttled candidate neither walks nor consumes + // budget and must not end the request — skip it and keep scanning + // (#173 review, F-C). No key (a test oneshot with no peer/header) skips the + // brake, as the other IP brakes do. On the LEGACY path the probe brake + // above already charged THIS candidate to the source (#173, F3, jatmn), so + // the walk brake must not double-charge it: only the provenance path + // (`legacy_scan == false`, no probe toll) charges here. + if !legacy_scan { + if let Some(key) = + crate::rate_limit::client_key(ctx.headers, ctx.peer, state.push_limiter_trust) + { + if !state.ipfs_work_rate_limiter.check(&key).await { + return GateOutcome::Throttled; + } + } + } + walk.walks += 1; - return Ok((StatusCode::OK, headers, content).into_response()); + let rp = repo_path.clone(); + let r = rules.to_vec(); + let is_public = repo.is_public; + let owner = repo.owner_did.clone(); + let caller_for_walk = ctx.caller_owned.clone(); + let kind = obj_type.clone(); + // Every walk is the DURATION-BOUNDED twin (`run_bounded_git` teardown under + // `git_service_timeout_secs`): the handler holds its /ipfs walk permit + // across this spawn_blocking, and a held permit is only safe if no walk + // child can outlive the deadline (#174 F5). + let git_bin = state.git_bin.clone(); + let git_service_timeout = + std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let walk_deadline = ctx.request_deadline; + let walk_admission = std::sync::Arc::clone(ctx.admission); + let result = tokio::task::spawn_blocking(move || { + // Admission clone (#174 U1): the slot stays taken until this blocking + // work returns, even if the handler future was dropped or this closure + // panics. + let _admission = walk_admission; + // Derive the walk's budget from the request deadline HERE, inside the + // closure, not on the async side before the task is queued. The walk + // starts its own clock when it runs, so a budget computed at queue time + // would hand it the full remainder measured from whenever the blocking + // pool got to it — the queue delay would go uncharged and the walk could + // finish past the request budget. Computing it at task start charges the + // delay against the deadline; a queue delay that eats the whole remainder + // saturates this to zero and the walk fails closed (no verdict, taint), + // which is the safe direction. Same fix as the upload-pack walk in + // `api/repos.rs`, and this route is anonymously reachable. + // TESTING GAP: the queue-delay path is reasoned, not executed. Observing + // it needs a runtime with the blocking pool pinned and parked, and the + // `#[sqlx::test]` harness gives no seam for that. + let walk_timeout = std::cmp::min( + git_service_timeout, + walk_deadline.saturating_duration_since(std::time::Instant::now()), + ); + match kind.as_str() { + "blob" => allowed_blob_set_for_caller_bounded( + &rp, + &git_bin, + walk_timeout, + &r, + is_public, + &owner, + caller_for_walk.as_deref(), + ), + "tree" => allowed_tree_set_for_caller_bounded( + &rp, + &git_bin, + walk_timeout, + &r, + is_public, + &owner, + caller_for_walk.as_deref(), + ), + "commit" | "tag" => { + reachable_commit_tag_oids_bounded(&rp, &git_bin, walk_timeout) + } + other => unreachable!("gated admits only blob/tree/commit/tag, got {other}"), + } + }) + .await; + // Fail closed on a walk error or task panic: we cannot prove readability, so + // skip rather than serve on an unproven gate — and never report absent on one + // either, so the skip taints the scan. + let set = match result { + Ok(Ok(set)) => set, + Ok(Err(e)) => { + tracing::warn!(repo = %repo.name, err = %e, "allowed-set walk failed; skipping repo without a verdict"); + walk.taint("walk-failure"); + return GateOutcome::Skip; + } + Err(e) => { + tracing::warn!(repo = %repo.name, err = %e, "allowed-set walk task panicked; skipping repo without a verdict"); + walk.taint("walk-failure"); + return GateOutcome::Skip; + } + }; + match obj_type.as_str() { + "blob" => walk.allowed_blob_memo.insert(repo.id.clone(), set), + "tree" => walk.allowed_tree_memo.insert(repo.id.clone(), set), + _ => walk.reachable_ct_memo.insert(repo.id.clone(), set), + }; + } + let in_set = match obj_type.as_str() { + "blob" => walk.allowed_blob_memo.get(&repo.id), + "tree" => walk.allowed_tree_memo.get(&repo.id), + _ => walk.reachable_ct_memo.get(&repo.id), + } + .is_some_and(|set| set.contains(sha256_hex)); + if !in_set { + return GateOutcome::Skip; + } } - // Not found in any repo - Err(AppError::RepoNotFound(format!( - "no git object found for CID {cid_str}" - ))) + // Passed the gate — bound the object, read it OFF the async worker, and verify the + // content address, all before any byte egresses. F6 (#173): read_object_content runs a + // blocking `git cat-file` and buffers the whole object; called directly on the Axum + // worker (the type-probe and walk are already off-worker) it blocks a runtime thread, + // and unbounded it can exhaust memory for a large public blob (enumerable from the pins + // index). Precheck the SIZE and run size + read + verify inside spawn_blocking. A + // content-addressed serve cannot verify a STREAMED body (the digest is known only after + // the last byte, by which point the prefix has already egressed), so we never stream: + // buffer-verify-then-serve up to the cap and withhold anything larger. F2's integrity + // check moves in here too, so no unverified bytes are ever assembled into a response. + // + // Budget gate for the read stage (#174 F3): the read never starts past the request + // budget, and its shared deadline clamps to the remainder. + let Some(read_budget) = walk.budget_left( + ctx, + state.config.ipfs_request_budget_secs, + &repo.name, + "content read", + ) else { + return GateOutcome::Skip; + }; + let max_bytes = state.ipfs_max_served_object_bytes; + let read_repo = repo_path.clone(); + let read_sha = sha256_hex.to_string(); + let read_type = obj_type.clone(); + let want_cid = ctx.canonical_cid.to_string(); + // Real `git` for the size and content reads, as the probe above and for the same + // reason: `state.git_bin` is the walk binary. + let git_bin = "git".to_string(); + // Bound the size+read CHILDREN themselves (process-group teardown at + // `git_service_timeout_secs` via the `*_bounded` twins), not an outer tokio timeout + // over an uncancellable `spawn_blocking`: a hung cat-file must be REAPED at the + // deadline rather than left to pin the held /ipfs walk permit (#173 round-10, KTD2). + // No outer timeout, mirroring the bounded walk; a `GitServiceTimeout` from either + // twin surfaces as `ServedRead::ReadErr` -> truncated (retryable 503). + // ONE deadline spans the size and content reads, so a single served candidate + // holds the /ipfs walk permit for at most `git_service_timeout_secs` total, not + // one full timeout per stage (mirrors `build_filtered_pack`'s shared deadline). + let read_deadline = std::time::Instant::now() + + std::cmp::min( + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + read_budget, + ); + let read_admission = std::sync::Arc::clone(ctx.admission); + let read = tokio::task::spawn_blocking(move || -> ServedRead { + // Admission clone (#174 U1): the slot stays taken until this blocking work + // returns, even if the handler future was dropped or this closure panics. + let _admission = read_admission; + let size_budget = read_deadline.saturating_duration_since(std::time::Instant::now()); + match store::object_size_bounded(&git_bin, &read_repo, &read_sha, size_budget) { + Ok(Some(size)) if size > max_bytes => return ServedRead::TooLarge(size), + Ok(Some(_)) => {} + // git ran and reported no such object (or an unparseable size): genuine + // not-found for this candidate. + Ok(None) => return ServedRead::Gone, + // git failed to run OR the bounded read timed out (GitServiceTimeout): an + // infra/timeout failure, not a not-found. + Err(e) => return ServedRead::ReadErr(e.to_string()), + } + let content_budget = read_deadline.saturating_duration_since(std::time::Instant::now()); + let content = match store::read_object_content_bounded( + &git_bin, + &read_repo, + &read_sha, + &read_type, + content_budget, + ) { + Ok(c) => c, + Err(e) => return ServedRead::ReadErr(e.to_string()), + }; + let served = gitlawb_core::cid::Cid::from_git_object_bytes(&content).to_string(); + if served != want_cid { + return ServedRead::Mismatch(served); + } + ServedRead::Ok(content) + }) + .await; + let served_read = match read { + Ok(sr) => sr, + Err(e) => { + tracing::warn!(repo = %repo.name, err = %e, "object read task panicked"); + walk.taint("read"); + return GateOutcome::Skip; + } + }; + let content = match served_read { + ServedRead::Ok(c) => c, + ServedRead::TooLarge(size) => { + tracing::warn!( + repo = %repo.name, size, max = max_bytes, + "withholding object: exceeds the served-object size cap (F6)" + ); + #[cfg(test)] + note_oversize_reject(); + return GateOutcome::Skip; + } + ServedRead::Mismatch(served) => { + tracing::warn!( + repo = %repo.name, requested = %ctx.canonical_cid, served = %served, + "withholding object: served bytes do not hash to the requested CID (legacy provider-CID row?)" + ); + return GateOutcome::Skip; + } + ServedRead::Gone => return GateOutcome::Skip, + ServedRead::ReadErr(e) => { + // Infra failure (git spawn/IO), NOT a not-found: mark the search truncated so + // a wholly-unserved request tails to a retryable 503, never a definitive 404 + // for an authorized caller (INV-25 spirit — logging alone is not surfacing). + tracing::warn!(repo = %repo.name, err = %e, "error reading git object content"); + walk.taint("read"); + return GateOutcome::Skip; + } + }; + let mut resp_headers = HeaderMap::new(); + resp_headers.insert( + HeaderName::from_static("content-type"), + HeaderValue::from_static("application/octet-stream"), + ); + resp_headers.insert( + HeaderName::from_static("x-content-cid"), + HeaderValue::from_str(ctx.cid_str).unwrap_or_else(|_| HeaderValue::from_static("invalid")), + ); + resp_headers.insert( + HeaderName::from_static("x-git-hash"), + HeaderValue::from_str(sha256_hex).unwrap_or_else(|_| HeaderValue::from_static("invalid")), + ); + GateOutcome::Served((StatusCode::OK, resp_headers, content).into_response()) } /// GET /api/v1/ipfs/pins @@ -228,3 +1255,2435 @@ pub async fn list_pins(State(state): State) -> Result = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn reset_preload_queries() { + PRELOAD_QUERIES.with(|c| c.set(0)); +} + +#[cfg(test)] +pub(crate) fn preload_queries() -> usize { + PRELOAD_QUERIES.with(|c| c.get()) +} + +#[cfg(test)] +fn bump_preload_queries() { + PRELOAD_QUERIES.with(|c| c.set(c.get() + 1)); +} + +// Test-only cost counter (F5, #173 round 11): how many times the fallback gate ran the +// `pin_sources_at_cap` / `pin_sources_incomplete` pair. The work-budget peek sits ahead +// of them, so an already-throttled caller leaves this at 0; putting the peek back after +// the pair turns that assertion red. Same thread_local discipline as the preload counter. +#[cfg(test)] +thread_local! { + static MARKER_QUERIES: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn reset_marker_queries() { + MARKER_QUERIES.with(|c| c.set(0)); +} + +#[cfg(test)] +pub(crate) fn marker_queries() -> usize { + MARKER_QUERIES.with(|c| c.get()) +} + +#[cfg(test)] +fn bump_marker_queries() { + MARKER_QUERIES.with(|c| c.set(c.get() + 1)); +} + +// Test-only INV-10 cost counter (F6, U6/U7): how many times the serve path withheld an +// object because it exceeded `ipfs_max_served_object_bytes`. The bounded read must reject +// an oversized object rather than buffer it on the worker; the counter is the both-ways +// guard (a removed size precheck stops incrementing it and serves the oversized object). +// Set from the match arm after `spawn_blocking` resolves, i.e. on the test's runtime +// thread, so the thread-local is read on the same thread it is written. +#[cfg(test)] +thread_local! { + static OVERSIZE_REJECTS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn reset_oversize_rejects() { + OVERSIZE_REJECTS.with(|c| c.set(0)); +} + +#[cfg(test)] +pub(crate) fn oversize_rejects() -> usize { + OVERSIZE_REJECTS.with(|c| c.get()) +} + +#[cfg(test)] +fn note_oversize_reject() { + OVERSIZE_REJECTS.with(|c| c.set(c.get() + 1)); +} + +#[cfg(test)] +mod tests { + //! #174 P1-3 (U3): the public `GET /ipfs/{cid}` walk carries bounded CONCURRENCY + //! admission (a global pool + per-source sub-cap) held through the `spawn_blocking` + //! walk, plus a per-IP route rate limit. These are handler-layer proofs: mount the + //! real handler/router, drive one request, assert the exact 503 shed, then name the + //! mutation that turns each RED. The per-source key resolves an IP only (`Some(ip)` + //! vs `None`), never a DID — both arms are driven so neither is vacuous. The + //! CID-resolution / visibility-gate behavior of the handler itself is covered by the + //! `#[sqlx::test]` suite in `test_support.rs`. + + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use axum::Router; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + /// A router mounting the real `get_by_cid` on `/ipfs/{cid}` with `optional_signature`, + /// matching production wiring for the extractors (`PeerAddr` reads `ConnectInfo`). + fn ipfs_router(state: crate::state::AppState) -> Router { + Router::new() + .route( + "/ipfs/{cid}", + axum::routing::get(crate::api::ipfs::get_by_cid), + ) + .layer(axum::middleware::from_fn(crate::auth::optional_signature)) + .with_state(state) + } + + /// A syntactically valid CIDv1(raw, sha2-256) string the handler decodes past its + /// CID/hash-code validation, so the request reaches the walk admission (not a 400). + fn valid_cid() -> String { + gitlawb_core::cid::Cid::from_git_object_bytes(b"blob 5\0hello") + .as_str() + .to_string() + } + + fn get_cid(cid: &str, peer: Option) -> Request { + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + if let Some(p) = peer { + req.extensions_mut().insert(ConnectInfo(p)); + } + req + } + + /// Run real git, asserting success. Shared by the F2 scan-verdict tests. + fn run_git(args: &[&str], cwd: &std::path::Path) { + let out = std::process::Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + } + + /// Seed a repo row plus a REAL sha256 bare repo at its acquired path holding one + /// committed blob (`src/secret.txt` = `content`). Returns `(repo_id, blob_oid)`. + /// Same recipe as `get_by_cid_walk_permit_held_through_blocking_walk`: the CID + /// digest IS the sha256 object id under `--object-format=sha256`, so the real + /// `cat-file` probe finds the blob. + async fn seed_repo_with_blob( + state: &crate::state::AppState, + tmp: &std::path::Path, + owner: &str, + name: &str, + content: &[u8], + ) -> (String, String) { + state + .db + .upsert_mirror_repo(owner, name, &format!("/unused-{name}"), None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + let _ = std::fs::remove_dir_all(&bare); + std::fs::create_dir_all(&bare).unwrap(); + let work = tmp.join(format!("work-{owner}-{name}")); + std::fs::create_dir_all(work.join("src")).unwrap(); + std::fs::write(work.join("src/secret.txt"), content).unwrap(); + run_git( + &["init", "-q", "--object-format=sha256", "-b", "main"], + &work, + ); + run_git(&["config", "user.email", "t@t"], &work); + run_git(&["config", "user.name", "t"], &work); + run_git(&["add", "src/secret.txt"], &work); + run_git(&["commit", "-q", "-m", "seed"], &work); + run_git( + &[ + "clone", + "--bare", + "-q", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + tmp, + ); + let out = std::process::Command::new("git") + .args(["rev-parse", "HEAD:src/secret.txt"]) + .current_dir(&work) + .output() + .expect("git rev-parse runs"); + assert!(out.status.success(), "rev-parse failed"); + let oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); + // Register the CID index entry the resolver needs to map a requested CID back + // to this oid, keyed on the object's raw CONTENT (what the serve path + // recomputes and verifies against) and with NULL provenance, which is what + // routes a request to the bounded legacy scan (#173). + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(content) + .as_str() + .to_string(); + state + .db + .record_pinned_cid(&oid, &cid, None) + .await + .expect("register the seeded blob in the CID index"); + (rec.id, oid) + } + + /// A 64-hex object id that exists in no repo, for the scan-verdict tests whose + /// whole point is that nothing serves. + fn absent_oid() -> String { + "f2".repeat(32) + } + + /// Register a LEGACY (NULL-provenance) `pinned_cids` row and return its CID. + /// + /// The scan-verdict tests below predate the CID index (#173): they drove a bare + /// CID and relied on the handler treating the CID's own digest as the git oid. + /// The index-backed resolver does not do that (a pin CID digests raw object + /// content, not the framed git object), so without a row `oids_for_cid` comes back + /// empty and the handler 404s before any repo is visited. NULL provenance is what + /// routes the request to the bounded legacy scan, which is the loop these tests + /// are about. + async fn seed_legacy_pin(state: &crate::state::AppState, oid: &str) -> String { + let cid = cid_for_oid(oid); + state + .db + .record_pinned_cid(oid, &cid, None) + .await + .expect("seed a legacy NULL-provenance pin row"); + cid + } + + /// The CID to request for an object a test seeded through `seed_repo_with_blob`. + /// + /// That helper registers the index entry keyed on the object's raw CONTENT, which + /// is what the serve path recomputes and compares the requested CID against + /// (#173 F2). Deriving the key from the oid instead yields a CID the gate passes + /// and the integrity check then rejects as a legacy provider-CID row, so the + /// candidate is withheld and a serving test sees a skip rather than its 200. + async fn seed_legacy_pin_for_oid(state: &crate::state::AppState, oid: &str) -> String { + if let Some(cid) = state.db.cid_for_oid(oid).await.expect("read the CID index") { + return cid; + } + // A test that built its object by hand (to corrupt it, say) has no entry yet. + // Its object never serves, so the content check never runs and any stable key + // will do; derive one from the oid. + seed_legacy_pin(state, oid).await + } + + /// CIDv1(raw, sha2-256) for a sha256 object id, as the handler resolves it. + fn cid_for_oid(oid: &str) -> String { + let oid_bytes = gitlawb_core::cid::sha256_hex_to_bytes(oid).unwrap(); + gitlawb_core::cid::Cid::from_sha256_bytes(&oid_bytes) + .as_str() + .to_string() + } + + /// Fake git for the WALK only (`state.git_bin`): empty refs, `rev-parse` + /// resolves, and each `rev-list` appends one line to `log` and prints nothing — + /// every walked repo yields an EMPTY allowed-set (path-gate deny verdict) and + /// the log's line count == the number of expensive walks run. The probe and the + /// content read shell to the real `git`, so seeded objects must genuinely exist. + #[cfg(unix)] + fn walk_logging_fake_git(dir: &std::path::Path, log: &std::path::Path) -> String { + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + for-each-ref) : ;;\n\ + rev-parse) echo deadbeef ;;\n\ + rev-list) echo walk >> \"{}\" ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + log.display() + ); + let git_path = dir.join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + git_path.to_str().unwrap().to_string() + } + + /// F2 buried-row repro: with more readable repos than `ipfs_max_repos_walked`, + /// existing PUBLIC content past the cap must still serve. The cap counts + /// EXPENSIVE walks only — this request has no path-scoped rules anywhere, so it + /// runs ZERO walks (the fake-git walk log stays empty) and the cap can never cut + /// the scan: the blob buried in the OLDER-updated repo (iterated last under + /// `list_all_repos`' updated_at DESC) serves its 200. Before F2 the cap counted + /// visibility-passing VISITS and broke the loop into the opaque 404 — existing + /// content misreported absent because of unrelated repos. MUTATION (RED): count + /// visits against the cap again (re-add the check+increment at the visibility + /// gate) and the buried row 503s instead of serving. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_buried_public_row_past_walk_cap_still_serves(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let walk_log = tmp.path().join("walks.log"); + state.git_bin = walk_logging_fake_git(tmp.path(), &walk_log); + // Tighter than the repo count: the old visit-counting cap cut the scan here. + let mut cfg = (*state.config).clone(); + cfg.ipfs_max_repos_walked = 1; + state.config = Arc::new(cfg); + + // Seed the blob-carrying repo FIRST so its updated_at is OLDER: the empty + // repo is iterated first and the blob row sits past the old visit budget. + let (_, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6f2buried", + "buried", + b"buried row proof\n", + ) + .await; + seed_repo_with_blob( + &state, + tmp.path(), + "z6f2buried", + "fresh", + b"unrelated content\n", + ) + .await; + + let peer: SocketAddr = "203.0.113.60:5000".parse().unwrap(); + let cid = seed_legacy_pin_for_oid(&state, &oid).await; + let resp = ipfs_router(state) + .oneshot(get_cid(&cid, Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::OK, + "a public blob in a repo past the walk cap must still serve — the cap \ + counts expensive walks and this scan needs none" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + assert_eq!(&body[..], b"buried row proof\n"); + let walks = std::fs::read_to_string(&walk_log) + .map(|s| s.lines().count()) + .unwrap_or(0); + assert_eq!( + walks, 0, + "a request with no path-scoped rules anywhere must run zero expensive walks" + ); + } + + /// F2 walk-cap skip-and-continue: exhausting `ipfs_max_repos_walked` skips the + /// walk-NEEDING repo without a verdict but keeps the scan alive. Three public + /// repos carry the same blob, newest first: the first (path-scoped) consumes the + /// cap-of-1 walk and denies (empty allowed-set — a verdict); the second + /// (path-scoped) needs a walk the cap forbids and is skipped WITHOUT one (taint); + /// the third is plain public and serves the 200 from a cheap probe — found beats + /// taint, and exactly one expensive walk ran. Before F2 the cap broke the loop at + /// the second repo and the request 404'd despite the public copy. MUTATION (RED): + /// turn the walk-cap skip back into a `break` and the public copy never serves. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_walk_cap_skip_continues_to_later_public_copy(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let walk_log = tmp.path().join("walks.log"); + state.git_bin = walk_logging_fake_git(tmp.path(), &walk_log); + let mut cfg = (*state.config).clone(); + cfg.ipfs_max_repos_walked = 1; + state.config = Arc::new(cfg); + + // Insert order = oldest first, so iteration (updated_at DESC) is reversed: + // gatedwalk, then gatedskip, then pubcopy. Identical content -> one CID. + let content = b"skip and continue proof\n"; + let (_, oid) = + seed_repo_with_blob(&state, tmp.path(), "z6f2skip", "pubcopy", content).await; + let (skip_id, _) = + seed_repo_with_blob(&state, tmp.path(), "z6f2skip", "gatedskip", content).await; + let (walk_id, _) = + seed_repo_with_blob(&state, tmp.path(), "z6f2skip", "gatedwalk", content).await; + for id in [&walk_id, &skip_id] { + state + .db + .set_visibility_rule( + id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderCCCCCCCCCCCCCCCCCCCCCCCC".to_string()], + "z6f2skip", + ) + .await + .unwrap(); + } + + let peer: SocketAddr = "203.0.113.61:5000".parse().unwrap(); + let cid = seed_legacy_pin_for_oid(&state, &oid).await; + let resp = ipfs_router(state) + .oneshot(get_cid(&cid, Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::OK, + "the walk-cap skip must continue the scan so the plain public copy serves" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + assert_eq!(&body[..], content.as_slice()); + let walks = std::fs::read_to_string(&walk_log) + .map(|s| s.lines().count()) + .unwrap_or(0); + assert_eq!( + walks, 1, + "cap honored exactly: the first path-scoped repo walks, the second is cut" + ); + } + + /// F2 visit ceiling: `ipfs_max_repo_visits` bounds the acquire+probe cost class + /// (each visit can be a full Tigris archive fetch on a cache miss). Unlike the + /// walk cap there is no cheap way to keep scanning, so exhaustion STOPS the scan + /// — and the stop is a truncation, not an absence: with ceiling 1 the newer + /// empty repo consumes the only visit and the blob-carrying older repo is never + /// probed, so the request sheds a retryable 503 + Retry-After, never a false + /// 404. MUTATION (RED): drop the ceiling check and the blob serves (200); drop + /// only the taint on the break and the 503 decays to a 404. + #[sqlx::test] + async fn get_by_cid_visit_ceiling_stops_scan_with_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let mut cfg = (*state.config).clone(); + cfg.ipfs_max_repo_visits = 1; + state.config = Arc::new(cfg); + + // Blob repo first (older, iterated second); empty repo second (newer, + // consumes the single visit). + let (_, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6f2visit", + "buried", + b"visit ceiling proof\n", + ) + .await; + seed_repo_with_blob(&state, tmp.path(), "z6f2visit", "fresh", b"unrelated\n").await; + + let peer: SocketAddr = "203.0.113.62:5000".parse().unwrap(); + let cid = seed_legacy_pin_for_oid(&state, &oid).await; + let resp = ipfs_router(state) + .oneshot(get_cid(&cid, Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a visit-ceiling truncation must shed a retryable 503, not report absent" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + } + + /// F2 negative arm: a COMPLETE scan that finds nothing keeps its definitive 404 + /// — the truncation 503 must never fire when every candidate reached a verdict. + /// Two public repos both probe clean (the requested CID is nowhere), no rules, + /// no cap or ceiling hit: 404 with no Retry-After. MUTATION (RED): taint the + /// scan unconditionally and this decays into a 503. + #[sqlx::test] + async fn get_by_cid_complete_scan_keeps_definitive_404(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + seed_repo_with_blob(&state, tmp.path(), "z6f2clean", "one", b"content one\n").await; + seed_repo_with_blob(&state, tmp.path(), "z6f2clean", "two", b"content two\n").await; + + // valid_cid() is the "hello" blob — present in neither repo. + let peer: SocketAddr = "203.0.113.63:5000".parse().unwrap(); + let cid = seed_legacy_pin(&state, &absent_oid()).await; + let resp = ipfs_router(state) + .oneshot(get_cid(&cid, Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::NOT_FOUND, + "a complete clean scan is a definitive absence — 404, never the 503 shed" + ); + assert!( + resp.headers().get("retry-after").is_none(), + "a definitive 404 must not advertise a retry" + ); + } + + /// F2 acquire taint: a repo row with NO local copy over a Tigris backend that + /// stalls (a silent local endpoint — accepted, never answered) hits the 1s + /// acquire timeout at the read-acquire site. The skip carries no verdict, so the + /// scan is truncated: retryable 503 + Retry-After, never the old silent-skip 404. + /// MUTATION (RED): drop the taint on the acquire-timeout arm and this decays to + /// a 404. + #[sqlx::test] + async fn get_by_cid_acquire_timeout_taints_scan_to_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + // Endpoint-pinned test client (no AWS_* env reads — env is racy under a + // parallel test run); the silent local endpoint stalls the HEAD + // deterministically. + let endpoint = crate::test_support::silent_http_endpoint().await; + let tigris = + crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) + .await; + state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let mut cfg = (*state.config).clone(); + cfg.git_acquire_timeout_secs = 1; + state.config = Arc::new(cfg); + + // Row exists in the DB but has no local copy, so the read acquire must + // consult Tigris (local-miss path) and stall until the timeout. + state + .db + .upsert_mirror_repo("z6f2acq", "ghost", "/unused-ghost", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.64:5000".parse().unwrap(); + let cid = seed_legacy_pin(&state, &absent_oid()).await; + let resp = ipfs_router(state) + .oneshot(get_cid(&cid, Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an acquire timeout leaves the repo unproven — the scan must shed 503, not 404" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + } + + /// F2 found-beats-taint on the acquire arm: an acquire timeout taints the + /// scan but must NOT stop it — the loop `continue`s, and a later repo that + /// genuinely carries the object still serves. The NEWER row (visited first + /// under `list_all_repos`' updated_at DESC) is a Tigris-backed ghost whose + /// acquire stalls against the silent endpoint and times out at 1s; the + /// OLDER row is a plain public repo carrying the blob, reached next and + /// served from a cheap probe — found beats taint: 200 with the blob bytes, + /// never the truncation 503. MUTATION (RED): turn the acquire-timeout arm's + /// `continue` into a `break` and the public copy never serves (503). + #[sqlx::test] + async fn get_by_cid_acquire_taint_does_not_block_later_public_copy(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Seed the blob repo through a LOCAL-ONLY store first, so seeding never + // consults the (deliberately unreachable) Tigris endpoint. + state.repo_store = + crate::git::repo_store::RepoStore::for_testing(repos_dir.clone(), pool.clone()); + let content = b"acquire taint continue proof\n"; + let (_, oid) = + seed_repo_with_blob(&state, tmp.path(), "z6f2acqcont", "pubcopy", content).await; + // Swap in a Tigris-backed store over the SAME repos_dir (the seeded bare + // repo stays a fast local hit) and add a NEWER ghost row with no local + // copy: its acquire consults the silent local endpoint and stalls to the + // 1s timeout (endpoint-pinned test client, no AWS_* env reads). + let endpoint = crate::test_support::silent_http_endpoint().await; + let tigris = + crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) + .await; + state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); + state + .db + .upsert_mirror_repo("z6f2acqcont", "ghost", "/unused-ghost", None, false) + .await + .unwrap(); + let mut cfg = (*state.config).clone(); + cfg.git_acquire_timeout_secs = 1; + state.config = Arc::new(cfg); + + // Ordering precondition: the ghost must be iterated FIRST (updated_at + // DESC — it was upserted after the blob repo), otherwise the pubcopy + // would serve before the taint ever fires and the continue-vs-break + // distinction would go untested. + let order: Vec = state + .db + .list_all_repos() + .await + .unwrap() + .into_iter() + .map(|r| r.name) + .collect(); + let ghost_pos = order.iter().position(|n| n == "ghost").unwrap(); + let pub_pos = order.iter().position(|n| n == "pubcopy").unwrap(); + assert!( + ghost_pos < pub_pos, + "precondition: the stalling ghost must be iterated before the blob repo; got {order:?}" + ); + + let peer: SocketAddr = "203.0.113.73:5000".parse().unwrap(); + let started = std::time::Instant::now(); + let cid = seed_legacy_pin_for_oid(&state, &oid).await; + let resp = ipfs_router(state) + .oneshot(get_cid(&cid, Some(peer))) + .await + .unwrap(); + // The taint arm demonstrably FIRED on this run: the response can only + // arrive after the ghost's stalled acquire burned its full 1s timeout + // (a cheap skip or a deny verdict would answer near-instantly). + assert!( + started.elapsed() >= std::time::Duration::from_millis(900), + "the ghost's acquire must stall to its timeout before the scan continues; \ + got {:?}", + started.elapsed() + ); + assert_eq!( + resp.status(), + StatusCode::OK, + "an acquire taint must not stop the scan: the later public copy serves" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + assert_eq!( + &body[..], + content.as_slice(), + "the served body must be the blob content from the later public copy" + ); + } + + /// F2 probe taint: a repo row whose local dir does not exist (no Tigris) — + /// `RepoStore::acquire` returns the path anyway (local passthrough), and the + /// `cat-file -t` probe cannot even spawn (missing working dir), so + /// `object_type` is Err. That is not an absence verdict, so the scan is + /// truncated: 503, never 404. A second, real repo probes clean (absent verdict) + /// — the one bad row is what taints. NOTE: the probe shells to the real `git` + /// (not `state.git_bin`), and a clean missing/invalid-object nonzero exit is + /// still `Ok(None)` (an absent verdict) — this arm needs a probe that could + /// not RUN, hence the missing-dir spawn failure here; the corrupt-repo test + /// below drives the stderr-discriminated Err. MUTATION (RED): drop the + /// taint on the probe-error arm and this decays to a 404. + #[sqlx::test] + async fn get_by_cid_probe_error_taints_scan_to_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // Older row: a real repo that probes clean. Newer row: no dir on disk. + seed_repo_with_blob(&state, tmp.path(), "z6f2probe", "real", b"probe clean\n").await; + state + .db + .upsert_mirror_repo("z6f2probe", "ghost", "/unused-ghost", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.65:5000".parse().unwrap(); + let cid = seed_legacy_pin(&state, &absent_oid()).await; + let resp = ipfs_router(state) + .oneshot(get_cid(&cid, Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a failed probe leaves the repo unproven — the scan must shed 503, not 404" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + } + + /// F2 probe taint, corrupt-repo arm: a repo whose git dir EXISTS but is broken + /// (objects/ removed, HEAD garbage) makes the real `cat-file -t` die with the + /// repo-level `fatal: not a git repository` — a probe that could not examine + /// the object store, not an absence verdict, so `object_type` must map it to + /// Err and the scan must shed the probe-tainted 503, never the silent-absence + /// 404. A second, real repo probes clean (absent verdict) — the corrupt row is + /// what taints. MUTATION (RED): map every nonzero cat-file exit back to + /// `Ok(None)` in `object_type` (drop the stderr discrimination) and this + /// decays to a 404. + #[sqlx::test] + async fn get_by_cid_corrupt_repo_dir_probe_error_taints_scan_to_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // Older row: a real repo that probes clean. Newer row: a bare repo whose + // git dir exists on disk but is corrupt at the repo level. + seed_repo_with_blob(&state, tmp.path(), "z6f2corrupt", "real", b"probe clean\n").await; + state + .db + .upsert_mirror_repo("z6f2corrupt", "broken", "/unused-broken", None, false) + .await + .unwrap(); + let rec = state + .db + .get_repo("z6f2corrupt", "broken") + .await + .unwrap() + .unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + std::fs::create_dir_all(&bare).unwrap(); + run_git(&["init", "-q", "--bare", "--object-format=sha256"], &bare); + std::fs::remove_dir_all(bare.join("objects")).unwrap(); + std::fs::write(bare.join("HEAD"), b"junk\n").unwrap(); + + let peer: SocketAddr = "203.0.113.68:5000".parse().unwrap(); + let cid = seed_legacy_pin(&state, &absent_oid()).await; + let resp = ipfs_router(state) + .oneshot(get_cid(&cid, Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a repo-level cat-file fatal leaves the repo unproven — the scan must \ + shed 503, not report the object absent" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("probe"), + "the shed must name the probe taint; got: {body}" + ); + } + + /// #174 F5/U4 (RED-before/GREEN-after): a candidate repo with a corrupt + /// `.git/config` makes `git cat-file` die with `fatal: bad config line N` while + /// `objects/` stays readable. That is a DETERMINISTIC fault, not an absence, and a + /// retry cannot fix it — so the scan must shed a TERMINAL, non-retryable 500, never + /// the old false 404 (`Ok(None)` fell through) and never the retryable 503 (which + /// would invite a conformant client to retry-storm a fresh `cat-file` per attempt). + /// A second, healthy repo probes clean (absent verdict); the corrupt row is what + /// forces the 500. The body must be OPAQUE — no raw git stderr, no filesystem path. + /// MUTATION (RED): route the deterministic fault back to `Ok(None)` in + /// `object_type_bounded` and this decays to a 404; classify it Transient and it + /// decays to a retryable 503. + #[sqlx::test] + async fn get_by_cid_bad_config_repo_is_terminal_500_not_404_or_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // A healthy repo that probes clean (would give a definitive 404 on its own) plus + // a repo whose bare git dir has a corrupt config (objects/ intact). + seed_repo_with_blob(&state, tmp.path(), "z6f5clean", "real", b"probe clean\n").await; + state + .db + .upsert_mirror_repo("z6f5badcfg", "broken", "/unused-badcfg", None, false) + .await + .unwrap(); + let rec = state + .db + .get_repo("z6f5badcfg", "broken") + .await + .unwrap() + .unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + std::fs::create_dir_all(&bare).unwrap(); + run_git(&["init", "-q", "--bare", "--object-format=sha256"], &bare); + // Corrupt the config; leave objects/ readable (the readable-store + git-fails + // combination is exactly what makes this deterministic, not transient). + { + use std::io::Write; + let mut cfg = std::fs::OpenOptions::new() + .append(true) + .open(bare.join("config")) + .unwrap(); + cfg.write_all(b"\n[broken section\nnot a valid = = = line\n") + .unwrap(); + } + + let peer: SocketAddr = "203.0.113.69:5000".parse().unwrap(); + let cid = seed_legacy_pin(&state, &absent_oid()).await; + let resp = ipfs_router(state) + .oneshot(get_cid(&cid, Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::INTERNAL_SERVER_ERROR, + "a bad-config (deterministic) repo fault must shed a terminal 500, never a \ + 404 (false absence) or a retryable 503" + ); + assert!( + resp.headers().get("retry-after").is_none(), + "a terminal 500 must NOT advertise a retry (that is the whole point vs 503)" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + !body.contains("bad config") + && !body.contains(tmp.path().to_str().unwrap()) + && !body.contains(".git") + && !body.contains("fatal"), + "the 500 body must be opaque — no raw git stderr / config text / filesystem \ + path; got: {body}" + ); + } + + /// #174 F5 co-occurrence (RED-before/GREEN-after): a deterministic fault on ONE + /// repo and a TRANSIENT taint on a DIFFERENT repo occur in the same scan, and the + /// requested CID is served by neither. The transiently-skipped repo could hold the + /// object, so a retry can surface it — the correct shed is the RETRYABLE 503, not + /// the terminal 500. Two broken repos drive it, both local so the outcome is + /// deterministic: a bad-`config` repo whose `objects/` stays readable is a + /// DETERMINISTIC probe fault (`deterministic_fault = true`), while a repo whose + /// `objects/` dir is removed is a TRANSIENT probe fault (taints "probe"). A third + /// healthy repo probes clean (absent verdict) so nothing serves. Before the fix the + /// terminal `if deterministic_fault` arm fired first and shed 500 unconditionally, + /// hiding the transiently-skipped repo behind a non-retryable status. MUTATION + /// (RED): drop the `&& truncated_by.is_empty()` gate and this shes 500 again. + #[sqlx::test] + async fn get_by_cid_deterministic_fault_with_cooccurring_transient_taint_is_503_not_500( + pool: sqlx::PgPool, + ) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // Healthy repo that probes clean (definitive absence on its own). + seed_repo_with_blob(&state, tmp.path(), "z6f5coclean", "real", b"probe clean\n").await; + + // Bad-config repo: objects/ readable, config corrupt -> DETERMINISTIC fault. + state + .db + .upsert_mirror_repo("z6f5cobadcfg", "broken", "/unused-badcfg", None, false) + .await + .unwrap(); + let rec = state + .db + .get_repo("z6f5cobadcfg", "broken") + .await + .unwrap() + .unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + std::fs::create_dir_all(&bare).unwrap(); + run_git(&["init", "-q", "--bare", "--object-format=sha256"], &bare); + { + use std::io::Write; + let mut cfg = std::fs::OpenOptions::new() + .append(true) + .open(bare.join("config")) + .unwrap(); + cfg.write_all(b"\n[broken section\nnot a valid = = = line\n") + .unwrap(); + } + + // Corrupt-dir repo: objects/ removed -> TRANSIENT probe fault (taints "probe"), + // a DIFFERENT repo than the deterministic one above. + state + .db + .upsert_mirror_repo("z6f5cocorrupt", "broken", "/unused-corrupt", None, false) + .await + .unwrap(); + let rec2 = state + .db + .get_repo("z6f5cocorrupt", "broken") + .await + .unwrap() + .unwrap(); + let bare2 = state + .repo_store + .acquire(&rec2.owner_did, &rec2.name) + .await + .unwrap(); + std::fs::create_dir_all(&bare2).unwrap(); + run_git(&["init", "-q", "--bare", "--object-format=sha256"], &bare2); + std::fs::remove_dir_all(bare2.join("objects")).unwrap(); + std::fs::write(bare2.join("HEAD"), b"junk\n").unwrap(); + + let peer: SocketAddr = "203.0.113.71:5000".parse().unwrap(); + let cid = seed_legacy_pin(&state, &absent_oid()).await; + let resp = ipfs_router(state) + .oneshot(get_cid(&cid, Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a deterministic fault co-occurring with a transient taint on a DIFFERENT \ + repo must shed the retryable 503 (a retry can surface the object in the \ + transiently-skipped repo), never the terminal 500" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the co-occurrence 503 must carry Retry-After" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("probe"), + "the shed must name the transient probe taint; got: {body}" + ); + } + + /// F2 read taint: the gate passes (the probe reads the truncated loose object's + /// intact "blob 64" header) but the content read fails (`cat-file blob` dies on + /// the deflate stream cut mid-content) — the probe just said the object EXISTS + /// here, so the failed read is no absence verdict: 503, never 404. The loose + /// object is hand-rolled: zlib header + one stored deflate block declaring 72 + /// bytes ("blob 64\0" + 64), truncated after the header NUL + 4 content bytes, + /// no adler trailer. MUTATION (RED): drop the taint on the read-error arm and + /// this decays to a 404. + #[sqlx::test] + async fn get_by_cid_read_error_taints_scan_to_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + state + .db + .upsert_mirror_repo("z6f2read", "corrupt", "/unused-corrupt", None, false) + .await + .unwrap(); + let rec = state + .db + .get_repo("z6f2read", "corrupt") + .await + .unwrap() + .unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + std::fs::create_dir_all(&bare).unwrap(); + run_git(&["init", "-q", "--bare", "--object-format=sha256"], &bare); + // Hand-rolled truncated loose object (dangling is fine: no path-scoped rules, + // so the "/" gate is the whole story and the read follows the probe). + let oid = "6bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459c"; + let mut corrupt: Vec = vec![0x78, 0x01, 0x01, 0x48, 0x00, 0xb7, 0xff]; + corrupt.extend_from_slice(b"blob 64\0AAAA"); + let obj_dir = bare.join("objects").join(&oid[..2]); + std::fs::create_dir_all(&obj_dir).unwrap(); + std::fs::write(obj_dir.join(&oid[2..]), &corrupt).unwrap(); + // Preconditions: the probe classifies it as a blob, the full read fails — + // otherwise the test would pass vacuously via some other arm. + assert_eq!( + crate::git::store::object_type(&bare, oid) + .unwrap() + .as_deref(), + Some("blob"), + "the truncated loose object's header must still probe as a blob" + ); + assert!( + crate::git::store::read_object_content(&bare, oid, "blob").is_err(), + "the truncated loose object's content read must fail" + ); + + let peer: SocketAddr = "203.0.113.66:5000".parse().unwrap(); + let cid = seed_legacy_pin_for_oid(&state, oid).await; + let resp = ipfs_router(state) + .oneshot(get_cid(&cid, Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a failed read after a passed gate leaves the repo unproven — 503, not 404" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + } + + /// F2 denied-is-a-verdict: repos that DENY the caller at the visibility gate + /// are settled, not skipped — an all-denied scan is COMPLETE: 404, zero visits. + /// The private rows deliberately have no local dirs: if the deny didn't + /// short-circuit before the visit, the missing-dir probe would taint the scan + /// into a 503, which the 404 assertion rules out — so the 404 also proves zero + /// acquires, probes, or walks ran for denied rows. + #[sqlx::test] + async fn get_by_cid_all_denied_is_complete_scan_404(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + for name in ["priv-a", "priv-b"] { + let now = chrono::Utc::now(); + state + .db + .create_repo(&crate::db::RepoRecord { + id: uuid::Uuid::new_v4().to_string(), + name: name.to_string(), + owner_did: "did:key:z6MkF2DenyOwnerAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + description: None, + is_public: false, + default_branch: "main".to_string(), + created_at: now, + updated_at: now, + disk_path: format!("/nonexistent/{name}"), + forked_from: None, + machine_id: None, + }) + .await + .unwrap(); + } + + let peer: SocketAddr = "203.0.113.67:5000".parse().unwrap(); + let cid = seed_legacy_pin(&state, &absent_oid()).await; + let resp = ipfs_router(state) + .oneshot(get_cid(&cid, Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::NOT_FOUND, + "an anonymous caller denied by every repo gets a complete-scan 404 — a deny \ + is a verdict and must not visit, taint, or 503" + ); + } + + /// F3 budget expiry mid-loop: one absolute request budget + /// (`ipfs_request_budget_secs`) bounds the whole admitted scan; per-repo + /// stages may not each draw a fresh timeout past it. Budget 1s, per-iteration + /// acquire timeout 2s; the NEWER row is a Tigris-backed ghost (no local copy, + /// silent local endpoint) whose acquire stalls, the OLDER row is a plain + /// public repo carrying the blob. The ghost's acquire runs clamped to the ~1s + /// remainder and times out; at the next repo the budget gate sees zero + /// remaining, taints "budget", and STOPS the scan, so the blob repo is never + /// visited (a visit would probe the healthy public copy and serve 200, which + /// the 503 assertion rules out) and the shed names the budget. Without the + /// budget the acquire would time out at its own 2s, the scan would continue, + /// and the buried blob would serve 200 (the recorded RED). MUTATION (RED): + /// remove the `request_deadline` capture (or make the remaining budget + /// infinite) and this serves 200 again. + #[sqlx::test] + async fn get_by_cid_request_budget_expiry_stops_scan_with_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Seed the blob repo through a LOCAL-ONLY store first, so seeding never + // consults the (deliberately unreachable) Tigris endpoint. + state.repo_store = + crate::git::repo_store::RepoStore::for_testing(repos_dir.clone(), pool.clone()); + let (_, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6f3budget", + "buried", + b"budget expiry proof\n", + ) + .await; + // Swap in a Tigris-backed store over the SAME repos_dir (the seeded bare + // repo stays a fast local hit) and add a NEWER ghost row with no local + // copy: its acquire consults the silent local endpoint and stalls past + // the budget (endpoint-pinned test client, no AWS_* env reads). + let endpoint = crate::test_support::silent_http_endpoint().await; + let tigris = + crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) + .await; + state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); + state + .db + .upsert_mirror_repo("z6f3budget", "ghost", "/unused-ghost", None, false) + .await + .unwrap(); + let mut cfg = (*state.config).clone(); + cfg.ipfs_request_budget_secs = 1; + cfg.git_acquire_timeout_secs = 2; + state.config = Arc::new(cfg); + + let peer: SocketAddr = "203.0.113.70:5000".parse().unwrap(); + let cid = seed_legacy_pin_for_oid(&state, &oid).await; + let resp = ipfs_router(state) + .oneshot(get_cid(&cid, Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted request budget must stop the scan with a retryable 503; \ + scanning on into the later public blob repo would have served 200" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the budget-truncation 503 must carry Retry-After" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("budget"), + "the truncation body must name the budget taint so the operator can \ + map the shed to GITLAWB_IPFS_REQUEST_BUDGET_SECS; got: {body}" + ); + } + + /// F3 clamped walk at expiry: a walk that starts with little budget left runs + /// its git children under `min(git_service_timeout_secs, remaining)`, so the + /// clamp (not any tokio-level abort) is what ends it and a walk can never + /// complete past the budget. Budget 2s, service timeout at its 600s default, + /// fake walk git that sleeps 8s: the walk STARTS (pid file), the walk permit + /// stays held while the blocking walk runs (`available_permits == 0`), the + /// clamped deadline SIGTERM/SIGKILLs the child group at ~2s remaining (the + /// response lands after the ~1s watchdog grace, far before the 8s sleep, and + /// the recorded pid is already dead: a tokio abort would have left it + /// running), the log shows the walk started but never completed, and the + /// request sheds the terminal budget-truncated 503 without ever reaching the + /// OLDER public copy of the same blob (which would have served 200). After + /// the response the permit is free: the spawn_blocking closure genuinely + /// returned. MUTATION (RED): drop the `min` clamp on `walk_timeout` and the + /// walk runs its full 8s sleep (elapsed and log-completion assertions fail). + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_budget_clamps_walk_deadline_and_holds_permit(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let walk_log = tmp.path().join("walks.log"); + let revlist_pid = tmp.path().join("revlist.pid"); + // Fake git for the WALK only: `rev-list` records its pid and a start + // marker, sleeps far past the budget, then records a done marker. Under + // the clamped walk deadline the whole process group is torn down mid + // sleep, so "done" never appears. The 8s sleep also bounds a RED run. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + for-each-ref) : ;;\n\ + rev-parse) echo deadbeef ;;\n\ + rev-list) echo $$ > \"{pid}\"; echo start >> \"{log}\"; sleep 8; echo done >> \"{log}\" ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + pid = revlist_pid.display(), + log = walk_log.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Global walk pool of 1 so the held permit is observable; per-source cap + // permissive so only the global pool matters. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + let mut cfg = (*state.config).clone(); + // The budget is the ONLY thing that can end this walk early: the service + // timeout stays at its generous 600s default. + cfg.ipfs_request_budget_secs = 2; + state.config = Arc::new(cfg); + + // Older row: a plain public copy of the same blob, which must never be + // reached. Newer row: path-scoped, so its blob costs the clamped walk. + let content = b"budget walk clamp proof\n"; + let (_, oid) = + seed_repo_with_blob(&state, tmp.path(), "z6f3clamp", "pubcopy", content).await; + let (walk_id, _) = + seed_repo_with_blob(&state, tmp.path(), "z6f3clamp", "gated", content).await; + state + .db + .set_visibility_rule( + &walk_id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderDDDDDDDDDDDDDDDDDDDDDDDD".to_string()], + "z6f3clamp", + ) + .await + .unwrap(); + + let sem = state.git_ipfs_walk_semaphore.clone(); + let cid = seed_legacy_pin_for_oid(&state, &oid).await; + let router = ipfs_router(state); + let started = std::time::Instant::now(); + let peer: SocketAddr = "203.0.113.71:5000".parse().unwrap(); + let mut fut = Box::pin(router.oneshot(get_cid(&cid, Some(peer)))); + + // Drive until the fake git's rev-list records its pid: the walk is now in + // the blocking pool and the request future is `.await`ing its join. Stop + // polling the instant the future completes (re-polling would panic). + let mut walk_pid: Option = None; + let mut early = None; + for _ in 0..500 { + let done = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&revlist_pid) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + walk_pid = Some(p); + break; + } + if let Ok(resp) = done { + early = Some(resp.map(|r| r.status())); + break; + } + } + let pid = walk_pid.unwrap_or_else(|| { + panic!( + "the budget-clamped walk must have STARTED (nonzero remaining); early: {early:?}" + ) + }); + // Reap the sleeping child on drop so a RED run leaks no orphan. + struct ReapOnDrop(i32); + impl Drop for ReapOnDrop { + fn drop(&mut self) { + unsafe { + libc::kill(self.0, libc::SIGKILL); + } + } + } + let _cleanup = ReapOnDrop(pid); + + // While the blocking walk runs the permit is HELD: the budget never frees + // a slot whose blocking thread is still burning. + assert_eq!( + sem.available_permits(), + 0, + "the walk permit must stay held while the budget-clamped walk runs" + ); + + let resp = tokio::time::timeout(std::time::Duration::from_secs(20), &mut fut) + .await + .expect("the clamped walk deadline must end the request; it never hung") + .unwrap(); + let elapsed = started.elapsed(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a budget-clamped walk that could not finish leaves no verdict: 503, not 404/200" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("budget"), + "the terminal shed must name the budget taint; got: {body}" + ); + // Deadline-killed at ~remaining, not run to completion: the response + // lands at ~budget + the watchdog's kill/reap slack, well before the 8s + // sleep could have finished. + assert!( + elapsed < std::time::Duration::from_secs(7), + "the clamped git deadline must end the walk at ~remaining; got {elapsed:?}" + ); + // The child group is already dead AT response time: the clamp killed it. + // (A tokio-level abort of the walk future would have answered while the + // blocking thread and its child still ran.) + assert_eq!( + unsafe { libc::kill(pid, 0) }, + -1, + "the walk's git child must be reaped by the clamped deadline before the response" + ); + let log = std::fs::read_to_string(&walk_log).unwrap_or_default(); + assert!( + log.contains("start"), + "the walk must have started (the budget gate passed with remaining > 0)" + ); + assert!( + !log.contains("done"), + "the walk must never complete past the budget; the clamp kills it mid-run" + ); + // The spawn_blocking closure returned and the handler finished: the + // permit is free again (held through the blocking run, no longer). + assert_eq!( + sem.available_permits(), + 1, + "the walk permit must free once the blocking walk genuinely returns" + ); + } + + /// #174 F3 hung-probe reap (RED-before/GREEN-after): the `git cat-file -t` + /// probe runs OFF the async worker under the reaped bounded runner, so a hung or + /// corrupt object store cannot pin a runtime worker or the held IPFS permits. + /// `objects/info/alternates` is a FIFO with no writer, so real `git cat-file -t` + /// blocks at odb setup forever. With the probe bounded to + /// `min(git_service_timeout, remaining budget)` (~1s here), the watchdog tears the + /// git process group down at the deadline and the probe returns Err — a taint, not + /// a verdict — so the scan sheds a retryable 503 naming the probe, no walk ever + /// starts, and the whole request returns in bounded time. + /// + /// Load-bearing: with the probe on the bare async worker (pre-fix) this FIFO blocks + /// the handler forever (no feeder frees it) and the request hangs — the wrapping + /// timeout fires (RED). With the reaped bounded probe it returns 503 promptly. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_hung_probe_is_reaped_and_sheds_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let walk_log = tmp.path().join("walks.log"); + state.git_bin = walk_logging_fake_git(tmp.path(), &walk_log); + let mut cfg = (*state.config).clone(); + cfg.ipfs_request_budget_secs = 1; + state.config = Arc::new(cfg); + + let (repo_id, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6f3probe", + "gated", + b"probe-then-expire proof\n", + ) + .await; + state + .db + .set_visibility_rule( + &repo_id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderEEEEEEEEEEEEEEEEEEEEEEEE".to_string()], + "z6f3probe", + ) + .await + .unwrap(); + + // Hang the REAL-git probe indefinitely: `objects/info/alternates` as a FIFO + // with no writer blocks `git cat-file -t` at odb setup forever. There is no + // feeder — the reaped bounded runner must tear the git process group down at + // the deadline; a bare unbounded probe would block the handler here. + let rec = state + .db + .get_repo("z6f3probe", "gated") + .await + .unwrap() + .unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + let fifo = bare.join("objects").join("info").join("alternates"); + let c_path = std::ffi::CString::new(fifo.to_str().unwrap()).unwrap(); + assert_eq!( + unsafe { libc::mkfifo(c_path.as_ptr(), 0o644) }, + 0, + "mkfifo(objects/info/alternates) must succeed" + ); + let peer: SocketAddr = "203.0.113.72:5000".parse().unwrap(); + // The request must return in bounded time: the reaped probe sheds a 503; a + // bare unbounded probe would block on the FIFO forever (no feeder frees it). + let cid = seed_legacy_pin_for_oid(&state, &oid).await; + let resp = tokio::time::timeout( + std::time::Duration::from_secs(15), + ipfs_router(state).oneshot(get_cid(&cid, Some(peer))), + ) + .await + .expect("the hung probe must be reaped, not block the handler") + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "probed-present with the budget gone must shed the truncation 503: \ + never the walked 404, never a serve" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("probe"), + "the shed must name the reaped-probe taint; got: {body}" + ); + let walks = std::fs::read_to_string(&walk_log) + .map(|s| s.lines().count()) + .unwrap_or(0); + assert_eq!( + walks, 0, + "no walk may START once the budget is exhausted, even for a probed-present object" + ); + } + + /// Shed at capacity: an exhausted `git_ipfs_walk_semaphore` sheds a `/ipfs/{cid}` + /// request with 503 BEFORE any DB/git walk (the acquire is the first thing after CID + /// validation), so a lazy DB-free state suffices — exactly like the served-git shed + /// tests. MUTATION (RED): delete the `git_ipfs_walk_semaphore` acquire in + /// `get_by_cid` and the request no longer sheds here (it falls through to the DB / + /// walk and returns something other than 503). + #[tokio::test] + async fn get_by_cid_sheds_with_503_when_walk_pool_exhausted() { + let mut state = crate::test_support::test_state_lazy(); + // Global /ipfs walk pool exhausted; per-source cap permissive so only the global + // pool can shed. Route rate limit is applied as a layer in production, not here. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(0)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let peer: SocketAddr = "203.0.113.9:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted /ipfs walk pool must shed the request with 503" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + + /// Per-source sub-cap, the `Some(ip)` arm: with per-source = 1 and the source pinned + /// at its single slot, a request from THAT source sheds 503 (global pool has room), + /// while a request from a DIFFERENT source is NOT shed by the cap (it proceeds past + /// admission). Pinning proves the `PeerAddr`/`HeaderMap` extractors resolved the key + /// — an inert `None` key would never shed on the per-source cap. MUTATION (RED): + /// delete the `git_ipfs_walk_per_caller` acquire and the capped source no longer + /// sheds. + #[tokio::test] + async fn get_by_cid_per_source_cap_sheds_same_source_admits_other() { + let mut state = crate::test_support::test_state_lazy(); + // Global pool has room; the per-source cap is 1. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(8)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let capped: SocketAddr = "203.0.113.20:5000".parse().unwrap(); + let other: SocketAddr = "203.0.113.21:5000".parse().unwrap(); + + // Pin the capped source at its single walk slot. + let _slot = state + .git_ipfs_walk_per_caller + .try_acquire(&capped.ip().to_string()) + .expect("first walk slot for the capped source IP"); + + let cid = valid_cid(); + // The capped source sheds on the per-source cap even with global capacity free. + let resp = ipfs_router(state.clone()) + .oneshot(get_cid(&cid, Some(capped))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its per-source /ipfs walk cap must shed 503 with global capacity free" + ); + + // A DIFFERENT source is NOT shed by the per-source cap: it clears admission and + // proceeds (then errors on the lazy DB, which is not a 503). + let resp = ipfs_router(state) + .oneshot(get_cid(&cid, Some(other))) + .await + .unwrap(); + assert_ne!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a different source must not be shed by the per-source cap" + ); + } + + /// The `None`-key arm: a request with no resolvable source key (no trusted-proxy + /// header, no `ConnectInfo`) is bounded by the GLOBAL pool only, never the per-source + /// sub-cap. With the global pool exhausted it still sheds 503 (the counterpart to the + /// `Some(ip)` arm above, so neither arm is vacuous). + #[tokio::test] + async fn get_by_cid_none_key_arm_sheds_on_global_pool() { + let mut state = crate::test_support::test_state_lazy(); + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(0)); + // Per-source cap permissive so only the global pool can shed. + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // No ConnectInfo + no trusted header -> client_key resolves None. + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), None)) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a None-key request must still shed 503 on the exhausted GLOBAL /ipfs walk pool" + ); + } + + /// Map self-bound (INV-15): the `/ipfs` per-source map is a `PerCallerConcurrency` + /// built via `with_default_max_keys`, so a distinct-source-key flood cannot grow it + /// past the cap and a rejected key never allocates (reject-before-insert). Mirrors + /// `per_caller_concurrency_map_is_self_bounding_and_reject_before_insert` for the + /// pool U3 adds. + #[tokio::test] + async fn ipfs_walk_per_caller_map_is_self_bounding_and_reject_before_insert() { + let lim = crate::rate_limit::PerCallerConcurrency::new(4, 3); + // Acquire+drop a flood of distinct keys — the map self-empties (a key is removed + // the instant its in-flight count hits zero). + for i in 0..50 { + let _p = lim.try_acquire(&format!("src{i}")); + } + assert_eq!( + lim.tracked_keys(), + 0, + "an acquire+drop flood of distinct sources leaves the /ipfs map empty" + ); + // Reject-before-insert: hold max_keys distinct sources, then a new one sheds + // without growing the map. + let held: Vec<_> = (0..3) + .map(|i| lim.try_acquire(&format!("h{i}")).unwrap()) + .collect(); + assert_eq!( + lim.tracked_keys(), + 3, + "three distinct sources held concurrently" + ); + assert!( + lim.try_acquire("h3").is_none(), + "a new source key at max_keys is rejected" + ); + assert_eq!( + lim.tracked_keys(), + 3, + "the rejected key did not allocate an entry (reject-before-insert)" + ); + drop(held); + } + + /// Build the shared `/ipfs` TREE-walk fixture. A fake `git` whose `rev-list` records + /// its pid then sleeps ~6s (so the tree walk blocks deterministically inside + /// `run_bounded_git`) and whose `cat-file -t` answers "tree" (so the bounded + /// object-type probe, `object_type_bounded` on `state.git_bin`, routes into the + /// tree-gate arm); a real SHA-256 bare repo with a committed `src/` tree pinned WITH + /// provenance; and a path-scoped rule so the gate takes the tree-walk branch. Returns + /// the tempdir (keep it alive for the whole test), the state (the caller sets the walk + /// semaphores), the requested CID, and the rev-list pidfile path. + #[cfg(unix)] + async fn seed_tree_walk_fixture( + pool: sqlx::PgPool, + ) -> ( + tempfile::TempDir, + crate::state::AppState, + String, + std::path::PathBuf, + ) { + use std::process::Command; + + let tmp = tempfile::TempDir::new().unwrap(); + let revlist_pid = tmp.path().join("revlist.pid"); + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + for-each-ref) : ;;\n\ + rev-parse) echo deadbeef ;;\n\ + cat-file) if [ \"$2\" = \"-t\" ]; then echo tree; fi ;;\n\ + rev-list) echo $$ > \"{}\"; sleep 6 ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + revlist_pid.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let owner = "z6ipfstree"; + let name = "iptree"; + state + .db + .upsert_mirror_repo(owner, name, "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + let _ = std::fs::remove_dir_all(&bare); + std::fs::create_dir_all(&bare).unwrap(); + let run = |args: &[&str], cwd: &std::path::Path| { + let out = Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + }; + let work = tmp.path().join("work"); + std::fs::create_dir_all(work.join("src")).unwrap(); + std::fs::write( + work.join("src/secret.txt"), + b"ipfs tree walk retain proof\n", + ) + .unwrap(); + run( + &["init", "-q", "--object-format=sha256", "-b", "main"], + &work, + ); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "src/secret.txt"], &work); + run(&["commit", "-q", "-m", "seed"], &work); + run( + &[ + "clone", + "--bare", + "-q", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + tmp.path(), + ); + let tree_oid = { + let out = Command::new("git") + .args(["rev-parse", "HEAD:src"]) + .current_dir(&work) + .output() + .expect("git rev-parse runs"); + assert!(out.status.success(), "rev-parse failed"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + assert_eq!( + crate::git::store::object_type(&bare, &tree_oid) + .unwrap() + .as_deref(), + Some("tree"), + "the seeded sha256 tree must exist so the handler reaches the tree walk" + ); + let (_ty, raw) = crate::git::store::read_object(&bare, &tree_oid) + .unwrap() + .expect("tree object readable"); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(&raw).to_string(); + state + .db + .record_pinned_cid(&tree_oid, &cid, Some(&rec.id)) + .await + .unwrap(); + state + .db + .set_visibility_rule( + &rec.id, + "/src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkF5IpfsTreeReaderAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + + (tmp, state, cid, revlist_pid) + } + + /// Retain-through-blocking (#174 F5, the load-bearing async property, on the + /// NEWLY-BOUNDED TREE path): the walk admission is held until the `spawn_blocking` + /// walk actually RETURNS, not when a tokio timeout fires. The requested CID + /// resolves to a TREE object under a path-scoped rule, so the gate runs + /// `allowed_tree_set_for_caller_bounded` — the walk this integration converts to + /// `run_bounded_git` — rather than the blob walk #174 already proved. With the + /// global pool at size 1, drive a request until its walk (a fake git that hangs on + /// `rev-list`) is in flight; the slot must stay held (`available_permits() == 0`) + /// and a replacement from a DIFFERENT source must shed 503 for as long as the + /// blocking walk runs — even though the request future is only `.await`ing the + /// blocking join. When the blocking walk ends the permit frees and a replacement + /// is admitted. The permit lives INSIDE the handler across the blocking `.await`; + /// move it out (drop before the walk) and the replacement would be admitted while + /// the walk still burns a blocking thread (the bug this guards). + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_walk_permit_held_through_bounded_tree_walk(pool: sqlx::PgPool) { + let (tmp, mut state, cid, revlist_pid) = seed_tree_walk_fixture(pool).await; + // Isolate the global walk pool at size 1; per-source cap permissive so only the + // held global permit can shed the replacement. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + // Keep the fixture tempdir alive for the whole test (its Drop removes the repos). + let _tmp = tmp; + + let sem = state.git_ipfs_walk_semaphore.clone(); + assert_eq!( + sem.available_permits(), + 1, + "one walk slot before the request" + ); + + let router = ipfs_router(state); + let make_req = |peer: SocketAddr| { + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req + }; + + let peer: SocketAddr = "203.0.113.81:5000".parse().unwrap(); + let mut fut = Box::pin(router.clone().oneshot(make_req(peer))); + // Drive until the fake git's rev-list records its pid — the TREE walk is now in + // the blocking pool and the request future is `.await`ing its join, holding the + // walk permit. Stop polling the instant the future completes (re-polling a + // completed oneshot panics). + let mut walk_pid: Option = None; + let mut early = None; + for _ in 0..500 { + let done = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&revlist_pid) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + walk_pid = Some(p); + break; + } + if let Ok(resp) = done { + early = Some(resp.map(|r| r.status())); + break; + } + } + let pid = walk_pid + .unwrap_or_else(|| panic!("the fake git rev-list must have spawned; early: {early:?}")); + // Reap the sleeping child on drop so a RED run leaks no orphan. + struct ReapOnDrop(i32); + impl Drop for ReapOnDrop { + fn drop(&mut self) { + unsafe { + libc::kill(self.0, libc::SIGKILL); + } + } + } + let _cleanup = ReapOnDrop(pid); + + // Load-bearing: while the blocking TREE walk runs, the slot is HELD and a + // replacement from a DIFFERENT source sheds 503 — proving the permit is + // retained across the spawn_blocking join, not freed by a tokio timeout. + assert_eq!( + sem.available_permits(), + 0, + "the walk slot must be held while the spawn_blocking tree walk runs" + ); + let peer2: SocketAddr = "203.0.113.82:5000".parse().unwrap(); + let resp = router.clone().oneshot(make_req(peer2)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a replacement must shed 503 while the prior request's blocking tree walk still runs" + ); + + // Drop the in-flight request — a client disconnect. The detached blocking walk + // keeps running (a spawn_blocking cannot be cancelled) and its git child is still + // occupying a blocking thread and a PID, so the slot it was admitted under must + // STAY TAKEN. Admission is released by the blocking work finishing, never by the + // handler future going away (#174 U1). + // + // MUTATION (RED): make the admission a handler local again (drop the Arc clone + // moved into the spawn_blocking closures) and this assertion fails immediately — + // the permit count returns to 1 the moment the future is dropped, while the + // sleeping child is still alive. + drop(fut); + // Give the runtime a chance to actually run the drop and any woken tasks, so + // this is not merely observing a not-yet-processed release. + for _ in 0..10 { + tokio::task::yield_now().await; + } + assert!( + unsafe { libc::kill(pid, 0) } == 0, + "precondition: the blocking walk's git child must still be alive, or this \ + assertion proves nothing" + ); + assert_eq!( + sem.available_permits(), + 0, + "a client disconnect must NOT release the walk slot while the uncancellable \ + blocking walk it admitted is still running" + ); + + // Now end the blocking work; the slot frees when the closure returns. + unsafe { + libc::kill(pid, libc::SIGKILL); + } + let mut freed = false; + for _ in 0..400 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + assert!( + freed, + "once the blocking walk tears down, the last admission clone drops and frees the slot" + ); + assert_eq!( + sem.available_permits(), + 1, + "admission released exactly once — the single slot is back, not double-freed" + ); + } + + /// Amplification negative (#173 round-10, R1): sequential cancel-spam from ONE source + /// cannot hold more than the per-source cap of concurrent walks. An abandoned + /// blocking walk keeps its per-source permit until its bounded work finishes (up + /// to `git_service_timeout_secs`), so with a per-source cap of 1 a second request from + /// the SAME source sheds 503 even though the GLOBAL pool has room — the source cannot + /// amplify its concurrent walk children past the cap by dropping-and-retrying. (The + /// worst case: an abandoned walk can occupy its global/per-source permit for one + /// bound-interval, so distributed cancel-spam can hold the global pool that long — the + /// accepted bounded-admission tradeoff, not a leak.) + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_cancel_spam_bounded_by_per_source_cap(pool: sqlx::PgPool) { + let (tmp, mut state, cid, revlist_pid) = seed_tree_walk_fixture(pool).await; + // Global pool has ample room (4); the per-source cap is 1. So any shed of a + // same-source replacement is the PER-SOURCE cap, never global exhaustion. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(4)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + let _tmp = tmp; + + let sem = state.git_ipfs_walk_semaphore.clone(); + let per_caller = state.git_ipfs_walk_per_caller.clone(); + let router = ipfs_router(state); + let make_req = |peer: SocketAddr| { + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req + }; + + // Source S fires request 1; drive until its tree walk is in flight (the task now + // holds source S's single per-source permit). + let source_s: SocketAddr = "203.0.113.71:5000".parse().unwrap(); + let mut fut = Box::pin(router.clone().oneshot(make_req(source_s))); + let mut walk_pid: Option = None; + for _ in 0..500 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&revlist_pid) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + walk_pid = Some(p); + break; + } + } + let pid = walk_pid.expect("the fake git rev-list must have spawned"); + struct ReapOnDrop(i32); + impl Drop for ReapOnDrop { + fn drop(&mut self) { + unsafe { + libc::kill(self.0, libc::SIGKILL); + } + } + } + let _cleanup = ReapOnDrop(pid); + + // Cancel-spam: drop request 1's future. The uncancellable blocking walk keeps + // running and KEEPS holding source S's single per-source permit. + drop(fut); + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + // A SECOND request from source S sheds 503. The global pool still has room (only 1 + // of 4 taken), so this is the per-source cap, not global exhaustion. + let resp = router.clone().oneshot(make_req(source_s)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a same-source cancel-spam replacement must shed 503 on the per-source cap \ + while the abandoned walk still holds the source's permit" + ); + assert!( + sem.available_permits() >= 3, + "the shed was the per-source cap, not global exhaustion (global pool still has room)" + ); + assert_eq!( + per_caller.tracked_keys(), + 1, + "exactly one per-source permit is outstanding for the one source — no amplification" + ); + + // Tear the walk down; the closure returns and releases source S's permit + // (tracked_keys returns to 0), so the source is no longer over the cap. + unsafe { + libc::kill(pid, libc::SIGKILL); + } + let mut released = false; + for _ in 0..400 { + if per_caller.tracked_keys() == 0 { + released = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + assert!( + released, + "once the blocking walk tears down it releases source S's per-source permit" + ); + } + + /// Loop bound (cap N) + F2 truncation verdict: one `/ipfs/{cid}` request against a + /// CID present in many path-scoped repos must not serialize an unbounded number of + /// full-history walks — and cutting a candidate WITHOUT a verdict must not report + /// the object absent. With `ipfs_max_repos_walked = 1` and TWO public, path-scoped + /// repos both carrying the blob, the first candidate is walked (empty allowed-set → + /// a deny VERDICT) and the second is cut by the cap (no verdict), so the fake git's + /// `rev-list` runs exactly once and the request sheds a retryable 503 + Retry-After + /// — never the old false 404 (the blob genuinely sits in the second repo). + /// This drives the GITLAWB_IPFS_MAX_REPOS_WALKED knob specifically. The merge left + /// two walk caps in play, this one and the branch's own history-walk ceiling, and + /// the gate takes the tighter of the two; setting this knob to 1 is what makes it + /// the binding one here. A sibling case covers the ceiling. + /// + /// MUTATION (RED): drop `config.ipfs_max_repos_walked` from the `min()` in the walk + /// gate and both repos are walked (count 2); drop the truncation taint on the skip + /// and the 503 decays to a 404. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_caps_repos_walked_knob_bounds_the_walks(pool: sqlx::PgPool) { + use std::process::Command; + + let tmp = tempfile::TempDir::new().unwrap(); + let walk_log = tmp.path().join("walks.log"); + // Fake git for the WALK: empty refs, `rev-parse` resolves, and each `rev-list` + // appends one line to a log (so the number of walks == the line count) and exits + // with EMPTY output (the allowed-set is empty, so every repo path-gates to a + // `continue` and the request 404s after walking). object_type uses the REAL git, + // so the seeded blob below must genuinely exist. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + for-each-ref) : ;;\n\ + rev-parse) echo deadbeef ;;\n\ + rev-list) echo walk >> \"{}\" ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + walk_log.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // The bound under test: walk at most one candidate repo per request. + let mut cfg = (*state.config).clone(); + cfg.ipfs_max_repos_walked = 1; + state.config = Arc::new(cfg); + + // Seed TWO public repos, each with the SAME blob (same content -> same sha256 OID + // -> same CID) under a path-scoped rule, so both are walk candidates for one CID. + let run = |args: &[&str], cwd: &std::path::Path| { + let out = Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + }; + let mut oid = String::new(); + for (i, name) in ["ipa", "ipb"].iter().enumerate() { + let owner = "z6ipfsN"; + state + .db + .upsert_mirror_repo(owner, name, &format!("/unused-{name}"), None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + let _ = std::fs::remove_dir_all(&bare); + std::fs::create_dir_all(&bare).unwrap(); + let work = tmp.path().join(format!("work{i}")); + std::fs::create_dir_all(work.join("src")).unwrap(); + // Identical content in both repos -> identical sha256 blob OID -> one CID. + std::fs::write(work.join("src/secret.txt"), b"loop bound proof\n").unwrap(); + run( + &["init", "-q", "--object-format=sha256", "-b", "main"], + &work, + ); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "src/secret.txt"], &work); + run(&["commit", "-q", "-m", "seed"], &work); + run( + &[ + "clone", + "--bare", + "-q", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + tmp.path(), + ); + if oid.is_empty() { + let out = Command::new("git") + .args(["rev-parse", "HEAD:src/secret.txt"]) + .current_dir(&work) + .output() + .expect("git rev-parse runs"); + oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); + } + state + .db + .set_visibility_rule( + &rec.id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderBBBBBBBBBBBBBBBBBBBBBBBB".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + } + // The resolver maps a requested CID back to an oid through the CID index, so a + // bare digest-as-oid CID resolves to nothing and 404s before any repo is + // visited. Register a legacy NULL-provenance row, which is also what routes the + // request to the bounded legacy scan this cap governs. Neither repo serves, so + // the key need not be the content CID. + let cid = seed_legacy_pin(&state, &oid).await; + + let peer: SocketAddr = "203.0.113.90:5000".parse().unwrap(); + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let resp = ipfs_router(state).oneshot(req).await.unwrap(); + // The first repo's walk yields the empty allowed-set (deny verdict); the second + // repo NEEDS a walk the cap forbids, so the scan is truncated without a verdict + // on it: retryable 503, never a false 404 for the blob it genuinely carries. + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a walk-cap truncation must shed a retryable 503, not report the object absent" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + + let walks = std::fs::read_to_string(&walk_log) + .map(|s| s.lines().count()) + .unwrap_or(0); + assert_eq!( + walks, 1, + "with the per-request repo-walk cap at 1, only the first candidate repo is \ + walked (the second is cut by the cap), so exactly one walk runs; got {walks}" + ); + } + + /// Route rate limit is WIRED (not a silent no-op): the production `build_router` + /// attaches an `IpRateLimiter` extension to the `/ipfs/{cid}` route, so a per-IP + /// flood is braked with 429. A bare `rate_limit_by_ip` layer with no extension does + /// nothing, so this proves the extension is attached. Drive it through the real + /// router with a tight limiter (1/hr): the second request from the same IP is 429. + /// MUTATION (RED): drop the `axum::Extension(ipfs_limiter)` layer in `server.rs` and + /// the second request is no longer braked (it reaches the handler, 404, not 429). + #[sqlx::test] + async fn ipfs_route_ip_rate_limit_is_attached(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + // Tight per-IP /ipfs bucket so the second request from one IP trips 429. + state.ipfs_rate_limiter = + crate::rate_limit::RateLimiter::new(1, std::time::Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let router = crate::server::build_router(state); + let cid = valid_cid(); + let make = |peer: SocketAddr| { + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req + }; + let peer: SocketAddr = "203.0.113.99:5000".parse().unwrap(); + + // First request from this IP passes the brake and reaches the handler (404 — no + // such object anywhere), debiting the single-slot bucket. + let resp = router.clone().oneshot(make(peer)).await.unwrap(); + assert_ne!( + resp.status(), + StatusCode::TOO_MANY_REQUESTS, + "the first /ipfs request from an IP must pass the rate brake" + ); + // Second request from the SAME IP is braked with 429 — proving the limiter + // extension is attached (a bare no-op layer would let it through to 404). + let resp = router.clone().oneshot(make(peer)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::TOO_MANY_REQUESTS, + "an exhausted per-IP /ipfs bucket must brake with 429 — the IpRateLimiter \ + extension must be attached to the route" + ); + // A DIFFERENT IP still has its own budget (independent bucket). + let other: SocketAddr = "203.0.113.100:5000".parse().unwrap(); + let resp = router.oneshot(make(other)).await.unwrap(); + assert_ne!( + resp.status(), + StatusCode::TOO_MANY_REQUESTS, + "a different IP must not be braked by another IP's exhausted bucket" + ); + } + + /// F6/KTD-5: the two initial metadata queries (`list_all_repos`, + /// `list_visibility_rules_for_repos`) run AFTER the scarce walk permits are + /// acquired (held RAII for the whole request) but BEFORE the per-repo loop's + /// first budget gate. Pre-fix they were bare awaits with no deadline, so a query + /// blocked in Postgres pinned the walk slot for the whole stall, past the request + /// budget. Here we hold an ACCESS EXCLUSIVE lock on `repos` so `list_all_repos` + /// blocks; with the budget clamp the request sheds a retryable budget 503 within + /// ~budget and FREES the walk permit, and a follow-up (lock released) is served. + /// + /// Load-bearing: pre-fix the bare await blocks on the lock until the 10s wrapping + /// timeout fires (RED — "never returned within budget"). After the fix it returns + /// the 503 at ~1s and the permit is free again. MUTATION (RED): drop the + /// `tokio::time::timeout` around `list_all_repos` and this hangs past the wrap. + #[sqlx::test] + async fn get_by_cid_stalled_metadata_query_frees_walk_permit(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Global walk pool of 1 so the held/freed permit is directly observable; + // per-source cap permissive so only the global pool matters. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + let mut cfg = (*state.config).clone(); + cfg.ipfs_request_budget_secs = 1; + state.config = Arc::new(cfg); + + let sem = state.git_ipfs_walk_semaphore.clone(); + let cid = seed_legacy_pin(&state, &absent_oid()).await; + let router = ipfs_router(state); + + // Hold an ACCESS EXCLUSIVE lock on `repos` on a dedicated pooled connection: + // `list_all_repos`' SELECT needs ACCESS SHARE, which conflicts, so it blocks + // at lock acquisition regardless of row count. + let mut lock_conn = pool.acquire().await.unwrap(); + sqlx::raw_sql("BEGIN; LOCK TABLE repos IN ACCESS EXCLUSIVE MODE;") + .execute(&mut *lock_conn) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.80:5000".parse().unwrap(); + let started = std::time::Instant::now(); + let resp = tokio::time::timeout( + std::time::Duration::from_secs(10), + router.clone().oneshot(get_cid(&cid, Some(peer))), + ) + .await + .expect("the budget clamp must return within budget; a bare await hangs on the lock") + .unwrap(); + let elapsed = started.elapsed(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a metadata query blocked past the request budget must shed a retryable 503" + ); + assert!( + elapsed < std::time::Duration::from_secs(3), + "the clamp must end the request at ~budget (1s); got {elapsed:?} \ + (pre-fix the bare await blocks on the lock for the whole stall)" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("budget"), + "the shed must name the budget taint so it maps to \ + GITLAWB_IPFS_REQUEST_BUDGET_SECS; got: {body}" + ); + // The scarce walk permit was RAII-dropped on the early return, not pinned for + // the stall: the slot is free again the instant the request returns. + assert_eq!( + sem.available_permits(), + 1, + "the walk permit must be freed on the budget-shed path, not held for the stall" + ); + + // Release the lock; a follow-up request is now SERVED (404 — empty DB), never + // capacity-503'd, proving the slot was not left pinned. + sqlx::raw_sql("ROLLBACK") + .execute(&mut *lock_conn) + .await + .unwrap(); + drop(lock_conn); + let resp2 = router.oneshot(get_cid(&cid, Some(peer))).await.unwrap(); + assert_eq!( + resp2.status(), + StatusCode::NOT_FOUND, + "with the permit freed and the lock released, a follow-up is served (404), \ + not capacity-503'd" + ); + } + + /// F6/KTD-5 FAIL CLOSED (security-critical): `list_visibility_rules_for_repos` is + /// the access-control query. If its timeout let the handler fall through with an + /// empty rule map, the loop would apply no visibility rules and serve an unfiltered + /// listing — exposing a public repo's path-restricted blob. Here a PUBLIC repo + /// carries the blob under a path-scoped rule that denies anon; `visibility_rules` + /// is locked ACCESS EXCLUSIVE so the rule query blocks. The fix returns the budget + /// 503 BEFORE the loop, so the handler NEVER serves (never 200). + /// + /// Load-bearing: pre-fix the bare await blocks on the lock until the 10s wrap fires + /// (RED). After the fix it sheds the 503 at ~1s. The `assert_ne!(200)` is the + /// fail-closed guard: a naive fix that `unwrap_or_default()`s the rules on timeout + /// and falls through would serve the blob 200 and trip it. + #[sqlx::test] + async fn get_by_cid_visibility_rule_timeout_fails_closed(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool.clone()); + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + + // A PUBLIC repo (upsert_mirror_repo sets is_public=true) carrying the blob, + // with a path-scoped rule restricting src/** to a reader that is NOT the anon + // caller. Rules applied => the blob is denied; rules skipped (fall-through) => + // the public repo serves it (the exposure this guard forbids). + let (repo_id, oid) = + seed_repo_with_blob(&state, tmp.path(), "z6f3failclosed", "gated", b"private\n").await; + state + .db + .set_visibility_rule( + &repo_id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderDDDDDDDDDDDDDDDDDDDDDDDD".to_string()], + "z6f3failclosed", + ) + .await + .unwrap(); + + let mut cfg = (*state.config).clone(); + cfg.ipfs_request_budget_secs = 1; + state.config = Arc::new(cfg); + let sem = state.git_ipfs_walk_semaphore.clone(); + let cid = seed_legacy_pin_for_oid(&state, &oid).await; + let router = ipfs_router(state); + + // Lock `visibility_rules` ACCESS EXCLUSIVE: list_all_repos (on `repos`) still + // succeeds, but list_visibility_rules_for_repos blocks on the rule query. + let mut lock_conn = pool.acquire().await.unwrap(); + sqlx::raw_sql("BEGIN; LOCK TABLE visibility_rules IN ACCESS EXCLUSIVE MODE;") + .execute(&mut *lock_conn) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.81:5000".parse().unwrap(); + let started = std::time::Instant::now(); + let resp = tokio::time::timeout( + std::time::Duration::from_secs(10), + router.oneshot(get_cid(&cid, Some(peer))), + ) + .await + .expect("the budget clamp must return within budget; a bare await hangs on the lock") + .unwrap(); + let elapsed = started.elapsed(); + + let status = resp.status(); + // Fail closed: the handler must NEVER emit the listing with no rules applied. + assert_ne!( + status, + StatusCode::OK, + "a visibility-rule query timeout must DENY, never serve the path-restricted \ + blob from the public repo (that would expose private content)" + ); + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "a visibility-rule query blocked past the budget must shed the retryable budget 503" + ); + assert!( + elapsed < std::time::Duration::from_secs(3), + "the clamp must end the request at ~budget (1s); got {elapsed:?}" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("budget"), + "the fail-closed shed must name the budget taint; got: {body}" + ); + assert_eq!( + sem.available_permits(), + 1, + "the walk permit must be freed on the fail-closed budget-shed path" + ); + + sqlx::raw_sql("ROLLBACK") + .execute(&mut *lock_conn) + .await + .unwrap(); + drop(lock_conn); + } +} diff --git a/crates/gitlawb-node/src/api/issues.rs b/crates/gitlawb-node/src/api/issues.rs index 17acf9af..0eacfa72 100644 --- a/crates/gitlawb-node/src/api/issues.rs +++ b/crates/gitlawb-node/src/api/issues.rs @@ -61,11 +61,14 @@ pub async fn create_issue( let json_str = serde_json::to_string(&issue) .map_err(|e| AppError::BadRequest(format!("serialization error: {e}")))?; + // Shed 503 + Retry-After on an exhausted write-lock POOL instead of a generic + // git 500 (#173 F1). This path holds no admission permit, so it reaches the pool + // unthrottled; reuse the push handler's mapping so the two cannot drift. let guard = state .repo_store .acquire_write(&record.owner_did, &record.name) .await - .map_err(|e| AppError::Git(e.to_string()))?; + .map_err(|e| crate::api::repos::acquire_write_app_error(&e, &repo))?; let disk_path = guard.path().to_path_buf(); let create_result = git_issues::create_issue(&disk_path, &issue_id, &json_str); @@ -229,11 +232,13 @@ pub async fn close_issue( .await? .ok_or_else(|| AppError::RepoNotFound(format!("{owner}/{repo}")))?; + // Same capacity shed as create_issue above (#173 F1): an exhausted write-lock + // pool is a 503 + Retry-After, not a 500 git error. let guard = state .repo_store .acquire_write(&record.owner_did, &record.name) .await - .map_err(|e| AppError::Git(e.to_string()))?; + .map_err(|e| crate::api::repos::acquire_write_app_error(&e, &repo))?; let disk_path = guard.path().to_path_buf(); // Owner OR issue author may close. The author lives in the issue's git-JSON @@ -279,3 +284,168 @@ pub async fn close_issue( Ok(Json(issue)) } + +/// #173 F1 follow-up: the two issue write paths reach `acquire_write` holding NO +/// admission permit (unlike the push handler, which is capped by the git-push +/// semaphore), so they are the callers most likely to meet an exhausted write-lock +/// POOL under load. An exhausted pool is a capacity signal, so both must shed +/// 503 + Retry-After (`AppError::Overloaded`) the way the push handler does, not +/// report the generic 500 git error that says nothing about retrying. +#[cfg(test)] +mod lock_pool_shed_tests { + use super::*; + use axum::response::IntoResponse; + use sqlx::PgPool; + + fn seed_repo(owner_did: &str, name: &str) -> crate::db::RepoRecord { + let now = Utc::now(); + crate::db::RepoRecord { + id: Uuid::new_v4().to_string(), + name: name.to_string(), + owner_did: owner_did.to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + created_at: now, + updated_at: now, + disk_path: format!("/tmp/{name}"), + forked_from: None, + machine_id: None, + } + } + + /// State whose repo store draws write locks from a ONE-connection pool with a + /// short checkout timeout, so a single held guard exhausts it promptly rather + /// than at the pool default. + async fn one_connection_lock_pool_state(pool: &PgPool) -> AppState { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.repo_store = crate::git::repo_store::RepoStore::new( + std::path::PathBuf::from("/tmp/gitlawb-issues-lockpool"), + None, + crate::git::repo_store::build_lock_pool(pool, 1, std::time::Duration::from_secs(1)), + ); + state + } + + /// The shed must be a real 503 carrying Retry-After, not just an internal enum + /// variant: assert on the rendered response so a remapping of `Overloaded` is + /// caught here too. + fn assert_sheds_503_with_retry_after(err: AppError, what: &str) { + let resp = err.into_response(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "{what}: an exhausted write-lock pool must shed 503, not a 500 git error" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .map(|v| v.to_str().unwrap()), + Some("1"), + "{what}: a capacity shed must tell the client when to retry" + ); + } + + /// RED-before/GREEN-after for `create_issue`. Both directions: the shed while the + /// only lock-pool connection is held by a guard on a DIFFERENT repo (so this is + /// pool capacity, not advisory-lock contention on this repo), and the must-not + /// case once that connection is back. + #[sqlx::test] + async fn create_issue_lock_pool_exhaustion_sheds_503_not_500(pool: PgPool) { + let owner = "did:key:zISSUECREATELOCKPOOLAAAAAAAAAAAAAAAAAAAA"; + let state = one_connection_lock_pool_state(&pool).await; + state + .db + .create_repo(&seed_repo(owner, "lp-create")) + .await + .expect("seed repo"); + + let held = state + .repo_store + .acquire_write(owner, "other-repo") + .await + .expect("the first write takes the only lock-pool connection"); + + let shed = create_issue( + State(state.clone()), + Extension(AuthenticatedDid(owner.to_string())), + Path((owner.to_string(), "lp-create".to_string())), + Json(CreateIssueRequest { + title: "t".to_string(), + body: None, + signed_payload: None, + }), + ) + .await; + let err = shed.expect_err("an exhausted lock pool must fail the call"); + assert_sheds_503_with_retry_after(err, "create_issue"); + + // MUST-NOT: with the pool free again the call is not shed as capacity (it + // fails later on the nonexistent on-disk repo, which is a git 500). + held.release(false).await; + let admitted = create_issue( + State(state.clone()), + Extension(AuthenticatedDid(owner.to_string())), + Path((owner.to_string(), "lp-create".to_string())), + Json(CreateIssueRequest { + title: "t".to_string(), + body: None, + signed_payload: None, + }), + ) + .await; + assert!( + !matches!(admitted, Err(AppError::Overloaded(_))), + "with the lock pool free, create_issue must not be shed as capacity; got {:?}", + admitted.err() + ); + } + + /// RED-before/GREEN-after for `close_issue`, same two directions. + #[sqlx::test] + async fn close_issue_lock_pool_exhaustion_sheds_503_not_500(pool: PgPool) { + let owner = "did:key:zISSUECLOSELOCKPOOLBBBBBBBBBBBBBBBBBBBBB"; + let state = one_connection_lock_pool_state(&pool).await; + state + .db + .create_repo(&seed_repo(owner, "lp-close")) + .await + .expect("seed repo"); + + let held = state + .repo_store + .acquire_write(owner, "other-repo") + .await + .expect("the first write takes the only lock-pool connection"); + + let shed = close_issue( + State(state.clone()), + Extension(AuthenticatedDid(owner.to_string())), + Path(( + owner.to_string(), + "lp-close".to_string(), + "deadbeef".to_string(), + )), + ) + .await; + let err = shed.expect_err("an exhausted lock pool must fail the call"); + assert_sheds_503_with_retry_after(err, "close_issue"); + + held.release(false).await; + let admitted = close_issue( + State(state.clone()), + Extension(AuthenticatedDid(owner.to_string())), + Path(( + owner.to_string(), + "lp-close".to_string(), + "deadbeef".to_string(), + )), + ) + .await; + assert!( + !matches!(admitted, Err(AppError::Overloaded(_))), + "with the lock pool free, close_issue must not be shed as capacity; got {:?}", + admitted.err() + ); + } +} diff --git a/crates/gitlawb-node/src/api/mod.rs b/crates/gitlawb-node/src/api/mod.rs index 71bfa43c..df10175a 100644 --- a/crates/gitlawb-node/src/api/mod.rs +++ b/crates/gitlawb-node/src/api/mod.rs @@ -208,11 +208,15 @@ mod authz_guard { (issues, "create_issue", "authorize_repo_read("), (bounties, "create_bounty", "authorize_repo_read("), (repos, "fork_repo", "authorize_repo_read("), - // get_by_cid gates each iterated repo row directly via visibility_check - // (KTD2a: it must NOT route through authorize_repo_read's fuzzy re-resolve). - (ipfs, "get_by_cid", "visibility_check("), - // #94 sibling read surfaces: gate private-repo metadata on read - // visibility (public repos stay anonymous; private repos 404). + // get_by_cid resolves each candidate (provenance path + legacy scan) through + // the shared `gate_and_serve`; the gate markers themselves are asserted + // below. This row proves the delegation is real, so the gate is actually + // reached rather than dead code. The delegated gate still calls + // `visibility_check` directly and never `authorize_repo_read`, so it keeps + // the property the pre-merge marker enforced: no fuzzy re-resolve. + (ipfs, "get_by_cid", "gate_and_serve("), + // Sibling read surfaces: gate private-repo metadata on read visibility + // (public repos stay anonymous; private repos 404). (replicas, "list_replicas", "authorize_repo_read("), (protect, "list_protected_branches", "authorize_repo_read("), (labels, "list_labels", "authorize_repo_read("), @@ -250,6 +254,22 @@ mod authz_guard { "visibility::require_owner must use did_matches for DID-safe owner matching" ); + // The CID read surface (#173) enforces its gate inside the shared + // `gate_and_serve`, which BOTH the provenance path and the legacy scan call, so + // the markers must live there (the get_by_cid row above only proves delegation). + // The repo's own "/" visibility check (KTD2a — never authorize_repo_read's fuzzy + // re-resolve) and the quarantine hard-drop BEFORE visibility (INV-11) are both + // load-bearing: removing either re-opens a leak on the provenance path. + let gate_body = fn_body(ipfs, "gate_and_serve"); + assert!( + gate_body.contains("visibility_check("), + "gate_and_serve must gate the CID read surface via visibility_check (KTD2a)" + ); + assert!( + gate_body.contains("if quarantined"), + "gate_and_serve must hard-drop a quarantined repo before the visibility gate (INV-11)" + ); + for (src, func, marker) in rows { let body = fn_body(src, func); assert!( diff --git a/crates/gitlawb-node/src/api/pulls.rs b/crates/gitlawb-node/src/api/pulls.rs index 26be6109..6255ef24 100644 --- a/crates/gitlawb-node/src/api/pulls.rs +++ b/crates/gitlawb-node/src/api/pulls.rs @@ -209,11 +209,14 @@ pub async fn merge_pr( return Err(AppError::BadRequest(format!("PR is already {}", pr.status))); } + // Shed 503 + Retry-After on an exhausted write-lock POOL instead of a generic + // git 500 (#173 F1). Merging holds no admission permit, so it reaches the pool + // unthrottled; reuse the push handler's mapping so the two cannot drift. let guard = state .repo_store .acquire_write(&record.owner_did, &record.name) .await - .map_err(|e| AppError::Git(e.to_string()))?; + .map_err(|e| crate::api::repos::acquire_write_app_error(&e, &name))?; let disk_path = guard.path().to_path_buf(); let merger_did = auth.0; let merge_result = store::merge_branch( @@ -424,3 +427,116 @@ pub async fn list_comments( let comments = state.db.list_pr_comments(&pr.id).await?; Ok(Json(serde_json::json!({ "comments": comments }))) } + +/// #173 F1 follow-up: `merge_pr` reaches `acquire_write` holding NO admission permit +/// (unlike the push handler, which is capped by the git-push semaphore), so it is one +/// of the callers most likely to meet an exhausted write-lock POOL under load. An +/// exhausted pool is a capacity signal, so the merge must shed 503 + Retry-After +/// (`AppError::Overloaded`) the way the push handler does, not report the generic +/// 500 git error that says nothing about retrying. +#[cfg(test)] +mod lock_pool_shed_tests { + use super::*; + use axum::response::IntoResponse; + use sqlx::PgPool; + + fn seed_repo(owner_did: &str, name: &str) -> crate::db::RepoRecord { + let now = Utc::now(); + crate::db::RepoRecord { + id: Uuid::new_v4().to_string(), + name: name.to_string(), + owner_did: owner_did.to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + created_at: now, + updated_at: now, + disk_path: format!("/tmp/{name}"), + forked_from: None, + machine_id: None, + } + } + + /// RED-before/GREEN-after for `merge_pr`. Both directions: the shed while the only + /// lock-pool connection is held by a guard on a DIFFERENT repo (so this is pool + /// capacity, not advisory-lock contention on this repo), and the must-not case + /// once that connection is back. + #[sqlx::test] + async fn merge_pr_lock_pool_exhaustion_sheds_503_not_500(pool: PgPool) { + let owner = "did:key:zMERGELOCKPOOLOWNERAAAAAAAAAAAAAAAAAAAAA"; + let mut state = crate::test_support::test_state(pool.clone()).await; + // One lock-pool connection with a short checkout timeout, so a single held + // guard exhausts it promptly rather than at the pool default. + state.repo_store = crate::git::repo_store::RepoStore::new( + std::path::PathBuf::from("/tmp/gitlawb-pulls-lockpool"), + None, + crate::git::repo_store::build_lock_pool(&pool, 1, std::time::Duration::from_secs(1)), + ); + + let repo = seed_repo(owner, "lp-merge"); + let repo_id = repo.id.clone(); + state.db.create_repo(&repo).await.expect("seed repo"); + let now = Utc::now().to_rfc3339(); + state + .db + .create_pr(&PullRequest { + id: Uuid::new_v4().to_string(), + repo_id: repo_id.clone(), + number: 1, + title: "lp".to_string(), + body: None, + author_did: owner.to_string(), + source_branch: "feature".to_string(), + target_branch: "main".to_string(), + status: "open".to_string(), + merged_by_did: None, + merged_at: None, + created_at: now.clone(), + updated_at: now, + }) + .await + .expect("seed open PR"); + + let held = state + .repo_store + .acquire_write(owner, "other-repo") + .await + .expect("the first write takes the only lock-pool connection"); + + let shed = merge_pr( + State(state.clone()), + Extension(AuthenticatedDid(owner.to_string())), + Path((owner.to_string(), "lp-merge".to_string(), 1)), + ) + .await; + let err = shed.expect_err("an exhausted lock pool must fail the call"); + let resp = err.into_response(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "merge_pr: an exhausted write-lock pool must shed 503, not a 500 git error" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .map(|v| v.to_str().unwrap()), + Some("1"), + "merge_pr: a capacity shed must tell the client when to retry" + ); + + // MUST-NOT: with the pool free again the merge is not shed as capacity (it + // fails later on the nonexistent on-disk repo, which is a git 500). + held.release(false).await; + let admitted = merge_pr( + State(state.clone()), + Extension(AuthenticatedDid(owner.to_string())), + Path((owner.to_string(), "lp-merge".to_string(), 1)), + ) + .await; + assert!( + !matches!(admitted, Err(AppError::Overloaded(_))), + "with the lock pool free, merge_pr must not be shed as capacity; got {:?}", + admitted.err() + ); + } +} diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index b38b177b..fa812069 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -39,11 +39,21 @@ const ZERO_SHA: &str = "0000000000000000000000000000000000000000"; /// `withheld` is `None`, so an unvetted push neither replicates blobs nor /// announces. Returning both keeps the gate's announce decision a single /// source rather than recomputing it at each call site. +/// +/// The walk arm runs under a `git_encrypt_semaphore` admission permit (#174 F4): +/// by the time the receive-pack tail calls this, the handler's write permit has +/// already been released (receive_pack's AdmissionGuard drops when the git group +/// is reaped), so without the gate a burst of completed pushes accumulates +/// unbounded concurrent full-history walks. `encrypt_sem` is threaded in so the +/// no-walk fast paths (not announceable; no path-scoped rule) never touch it. async fn replication_withheld_set( + encrypt_sem: std::sync::Arc, rules: Option>, owner_did: &str, is_public: bool, disk_path: std::path::PathBuf, + git_bin: String, + timeout: std::time::Duration, ) -> (bool, Option>) { let announce = match &rules { Some(rules) => crate::visibility::listable_at_root(rules, is_public, owner_did, None), @@ -64,9 +74,17 @@ async fn replication_withheld_set( // that off the async worker thread. Some(rules) => { let owner_did = owner_did.to_string(); + // Scan admission (#174 F4): DEFER, never shed — dropping the walk + // would skip the vetting and fail the push's replication closed for + // no reason. Residuals at `acquire_scan_permit`. + let permit = + crate::state::acquire_scan_permit(encrypt_sem, &disk_path, "withheld walk").await; tokio::task::spawn_blocking(move || { - crate::git::visibility_pack::withheld_blob_oids( - &disk_path, &rules, is_public, &owner_did, None, + // The permit lives inside the blocking closure: a started walk + // always completes holding it. + let _permit = permit; + crate::git::visibility_pack::withheld_blob_oids_bounded( + &disk_path, &git_bin, timeout, &rules, is_public, &owner_did, None, ) }) .await @@ -100,18 +118,46 @@ async fn replication_withheld_set( /// non-blobs plus allowed blobs. Any error in either walk (or a task panic) /// pins nothing this push, mirroring the degraded-path shape of /// `replication_withheld_set`. +/// +/// Always walks (there is no no-git arm), so the whole blocking scan runs under +/// one `git_encrypt_semaphore` admission permit (#174 F4) — see +/// `acquire_scan_permit` for the defer rationale and the honest residuals. +#[allow(clippy::too_many_arguments)] async fn fail_closed_full_scan_objects( + encrypt_sem: std::sync::Arc, disk_path: std::path::PathBuf, rules: Vec, is_public: bool, owner_did: String, candidates: Vec, + git_bin: String, + timeout: std::time::Duration, ) -> Vec { + // Scan admission (#174 F4): DEFER, never shed; the permit moves into the + // closure so a started scan always completes holding it. + let permit = + crate::state::acquire_scan_permit(encrypt_sem, &disk_path, "fail-closed full scan").await; tokio::task::spawn_blocking(move || -> anyhow::Result> { - let allowed = crate::git::visibility_pack::replicable_blob_set( - &disk_path, &rules, is_public, &owner_did, + let _permit = permit; + // One whole-scan deadline shared across both phases (#174 F4). A fresh + // `Instant::now() + timeout` for phase 2 let a large-but-successful phase 1 plus + // a full phase 2 hold the scan permit ~2x the configured budget. Sharing the + // deadline caps total occupancy at ~1x: phase 1 runs against the remaining + // budget, and if it consumes the budget phase 2 gets what is left and fails + // closed (pins nothing) rather than over-holding — the safe direction. The cost + // is honest: a genuinely large repo whose phase 1 nears the budget under-pins + // this push rather than the previous silent ~2x hold; size the budget so both + // phases normally fit. + let deadline = std::time::Instant::now() + timeout; + let allowed = crate::git::visibility_pack::replicable_blob_set_bounded( + &disk_path, + &git_bin, + deadline.saturating_duration_since(std::time::Instant::now()), + &rules, + is_public, + &owner_did, )?; - let all_blobs = crate::git::push_delta::all_blob_oids(&disk_path)?; + let all_blobs = crate::git::push_delta::all_blob_oids(&disk_path, &git_bin, deadline)?; Ok(crate::git::visibility_pack::replicable_objects_fail_closed( candidates, &allowed, &all_blobs, )) @@ -508,6 +554,47 @@ pub async fn git_info_refs( auth: Option>, ) -> Result { let name = smart_http_repo_name(&repo)?; + let service = query + .service + .ok_or_else(|| AppError::BadRequest("missing ?service= parameter".into()))?; + // Reject an unsupported service BEFORE taking a read slot or doing any DB/Tigris + // work (#174 P2-1). git_info_refs otherwise treats everything that is not + // git-receive-pack as a read op, so an unauthenticated `?service=anything` to a + // public repo would consume a read permit and the visibility/Tigris work before + // validate_service rejected it downstream in smart_http. + if service != "git-upload-pack" && service != "git-receive-pack" { + return Err(AppError::BadRequest(format!( + "unsupported git service: {service}" + ))); + } + // #62 cheap load shed: if the pool this service draws from is ALREADY saturated, + // shed this request with a 503 before it does any DB/disk work. Best-effort and + // permit-less, so it is a snapshot, not admission: it spares THIS request's DB + // work once the pool has filled, and nothing more. It is NOT a bound on the DB + // window. Permits are only held from `git_permit` below, after the visibility and + // rate gates, so a burst arriving while permits are free all proceeds into the DB + // and none of it sheds here. That ordering is deliberate (a denied or rate-limited + // request must consume no slot, and one source must not hold global slots through + // the DB/visibility window); bounding the DB window itself would need an admission + // mechanism this peek is not. + { + // The receive-pack advertisement peeks its DEDICATED advert pool, not the + // write pool the authenticated POST uses (#174) — matching the held acquire + // below, so the pre-DB shed and the authoritative hold agree on the pool. + let pool = if service == "git-receive-pack" { + &state.git_push_advert_semaphore + } else { + &state.git_read_semaphore + }; + if pool.available_permits() == 0 { + tracing::warn!( + "served-git concurrency cap reached; shedding request with 503 (pre-DB)" + ); + return Err(AppError::Overloaded( + "git service at capacity, retry shortly".into(), + )); + } + } tracing::info!(owner = %owner, repo = %name, "info/refs request"); let record = state .db @@ -521,9 +608,6 @@ pub async fn git_info_refs( return Err(AppError::RepoNotFound(format!("{owner}/{name}"))); } - let service = query - .service - .ok_or_else(|| AppError::BadRequest("missing ?service= parameter".into()))?; tracing::debug!(service = %service, repo = %name, "info/refs service"); // Enforce read visibility on the ref advertisement, for BOTH services. The @@ -563,32 +647,880 @@ pub async fn git_info_refs( } } - // For receive-pack (push), download the latest from Tigris so the client - // sees the same refs that acquire_write() will operate on. - let disk_path = if service == "git-receive-pack" { - state - .repo_store - .acquire_fresh(&record.owner_did, &record.name) - .await + // Per-source concurrency sub-cap (#174), keyed on the resolved source IP and + // acquired AFTER the visibility + push-rate gates (KTD7) so a denied or + // rate-limited request never consumes a slot; held for the whole op. The + // upload-pack advertisement is bounded on the read pool (git_read_per_caller). + // The receive-pack advertisement draws from its own dedicated advert pool + // (git_push_advert_semaphore, see the _permit block below), so it is bounded per + // source by git_push_advert_per_caller instead: without this, an anonymous + // multi-source flood of push-handshake advertisements could hold every advert-pool + // slot across acquire_fresh and shed other sources' advertisements, since the + // per-IP push rate limiter caps rate, not concurrency (#174 review fix). + let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); + let _caller_permit = if service == "git-receive-pack" { + acquire_read_caller_permit( + &state.git_push_advert_per_caller, + caller_key.as_deref(), + name, + "receive-pack advert", + )? } else { - state - .repo_store - .acquire(&record.owner_did, &record.name) - .await - } - .map_err(|e| { - tracing::error!(repo = %name, service = %service, err = %e, "repo acquire failed"); - AppError::Git(e.to_string()) - })?; + acquire_read_caller_permit( + &state.git_read_per_caller, + caller_key.as_deref(), + name, + "info/refs", + )? + }; + + // Shed with a 503 before spawning git when the concurrency cap is saturated; + // held for the whole op (incl. the smart_http call), released on return. Taken + // AFTER the per-source cap above so one source cannot occupy global slots it + // would be sub-cap-denied for during the DB/visibility window and starve other + // sources; still before acquire_fresh/git so it bounds the fresh Tigris acquire + // and git exec (INV-10). The receive-pack advertisement is phase one of a push, + // but it is ANON-reachable, so it draws from the dedicated advert pool + // (`git_push_advert_semaphore`), NOT the write pool the authenticated POST uses: + // an advert flood can at worst exhaust the advert pool, never a permit a push + // POST needs at admission (#174 U2). A clone flood on the read pool likewise + // can't touch either. The upload-pack advertisement stays on the read pool with + // its per-caller sub-cap. + let _permit = if service == "git-receive-pack" { + git_permit(&state.git_push_advert_semaphore)? + } else { + git_permit(&state.git_read_semaphore)? + }; - smart_http::info_refs(&disk_path, &service) + // For receive-pack (push), download the latest from Tigris so the client + // sees the same refs that acquire_write() will operate on. + // + // Bound the acquire under `git_acquire_timeout_secs`: the concurrency permit is + // already held above, and `git_service_timeout_secs` only starts once git spawns, + // so an un-deadlined acquire (a hung Tigris HEAD/GET here) pins the permit until + // the pool drains (#174 P1-2). On expiry the handler-local `_permit`/`_caller_permit` + // drop on the early return (the AdmissionGuard is not built until after acquire), + // so the shed frees the slot; return a bounded 503. + let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); + let acquire_fut = async { + if service == "git-receive-pack" { + state + .repo_store + .acquire_fresh(&record.owner_did, &record.name) + .await + } else { + state + .repo_store + .acquire(&record.owner_did, &record.name) + .await + } + }; + let disk_path = tokio::time::timeout(acquire_deadline, acquire_fut) .await + .map_err(|_elapsed| { + tracing::warn!(repo = %name, service = %service, "repo acquire timed out; shedding with 503"); + AppError::Overloaded("git service acquisition timed out, retry shortly".into()) + })? .map_err(|e| { - tracing::error!(repo = %name, service = %service, err = %e, "info_refs git failed"); + tracing::error!(repo = %name, service = %service, err = %e, "repo acquire failed"); AppError::Git(e.to_string()) + })?; + + // Move the admission permits into the guard so they release only after the spawned + // git process group is confirmed reaped, on complete/timeout/disconnect — not the + // instant a disconnect drops this future while the detached reaper is still tearing + // the group down (#174 P1-a). The handler keeps no copy: `_permit`/`_caller_permit` + // are moved in, so admission tracks the real process lifetime. + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + smart_http::info_refs( + &state.git_bin, + &service, + &disk_path, + git_timeout, + Some(admission), + ) + .await + .map_err(|e| { + let app = git_service_app_error(&e); + match &app { + AppError::Timeout(_) => { + tracing::warn!(repo = %name, service = %service, "info/refs advertisement timed out") + } + _ => { + tracing::error!(repo = %name, service = %service, err = %e, "info_refs git failed") + } + } + app }) } +/// Acquire a permit from the served-git concurrency semaphore, or shed the +/// request with a 503 + Retry-After when every slot is in use. Bind the returned +/// permit to a named local so it is held for the whole git op (it releases on +/// drop); a bare `_` would release it immediately. +fn git_permit( + sem: &std::sync::Arc, +) -> Result { + sem.clone().try_acquire_owned().map_err(|_| { + // Surface the shed so operators can see the cap engaging, mirroring the + // receive-pack rate-limit warn above. A silent 503 makes a saturated or + // misconfigured cap look like a client problem instead of a capacity one. + tracing::warn!("served-git concurrency cap reached; shedding request with 503"); + AppError::Overloaded("git service at capacity, retry shortly".into()) + }) +} + +/// Resolve the per-caller key for the read sub-cap (#174): always the resolved +/// source IP (`client_key`), never the signed DID. Public read routes accept any +/// valid `did:key` via `optional_signature` with no admission step, so keying on +/// the DID would let one host mint disposable DIDs to multiply its per-source +/// budget; the push path already throttles on the resolved source IP for exactly +/// this DID-farm reason (`rate_limit.rs`, `IpRateLimiter`). `None` when no key +/// resolves (no trusted header and no peer): such a request is bounded by the +/// global read pool only, never a 500. The per-source-IP key is only as granular +/// as `trust`; see the `max_concurrent_reads_per_caller` config doc. +fn read_caller_key( + headers: &axum::http::HeaderMap, + peer: Option, + trust: crate::rate_limit::TrustedProxy, +) -> Option { + crate::rate_limit::client_key(headers, peer, trust) +} + +/// Acquire the per-caller read sub-cap permit (#174), or shed with a 503. `key` is +/// `None` when no caller key resolves — that request is bounded by the global read +/// pool only and is never shed here (returns `Ok(None)`). `handler` labels the shed +/// log line. Shared by both read handlers so the two acquire sites cannot drift. +fn acquire_read_caller_permit( + limiter: &crate::rate_limit::PerCallerConcurrency, + key: Option<&str>, + repo: &str, + handler: &str, +) -> Result> { + match key { + Some(k) => match limiter.try_acquire(k) { + Some(p) => Ok(Some(p)), + None => { + tracing::warn!(repo = %repo, caller = %k, handler, "per-caller cap reached; shedding with 503"); + Err(AppError::Overloaded( + "git service at capacity for this caller, retry shortly".into(), + )) + } + }, + None => Ok(None), + } +} + +/// Acquire an encryption-walk admission permit, then run the bounded withheld-blob +/// recipients walk. Blocks (defers) when `git_encrypt_semaphore` is full rather than +/// shedding — the walk is background so added latency is fine, and dropping it would +/// lose the withheld-blob recovery copy (#174 P1-e). Bounds the number of concurrent +/// post-push encryption walks so N fast completed pushes cannot spawn N concurrent +/// full-history git walks. Mirrors the original `spawn_blocking(...).await` return +/// shape so the caller's `Ok(Ok(recipients))` match is unchanged. +async fn withheld_recipients_gated( + encrypt_sem: std::sync::Arc, + repo_path: std::path::PathBuf, + git_bin: String, + timeout: std::time::Duration, + rules: Vec, + is_public: bool, + owner_did: String, +) -> std::result::Result< + anyhow::Result>>, + tokio::task::JoinError, +> { + let permit = encrypt_sem + .acquire_owned() + .await + .expect("git_encrypt_semaphore is never closed"); + tokio::task::spawn_blocking(move || { + // The permit lives inside the blocking closure (#174 U4, the F4 contract): a + // started walk always completes holding it, so a dropped future cannot free + // the slot while this uncancellable walk still occupies a thread and a PID. + let _permit = permit; + crate::git::visibility_pack::withheld_blob_recipients_bounded( + &repo_path, &git_bin, timeout, &rules, is_public, &owner_did, + ) + }) + .await +} + +/// Everything the detached post-push pin/encrypt task needs, cloned once at spawn +/// and shared by the snapshot iteration and every coalesced-drain iteration +/// (#174 F5). +struct EncryptTaskCtx { + ipfs_api: String, + repo_path: std::path::PathBuf, + db: Arc, + repo_id: String, + owner_did: String, + repo_name: String, + irys_url: String, + http_client: Arc, + node_did: String, + node_keypair: Arc, + git_bin: String, + git_timeout: std::time::Duration, + encrypt_sem: Arc, + pin_sem: Arc, +} + +/// The detached post-push pin/encrypt task (#174 P2-2 + F5): run this push's own +/// pre-resolved snapshot through the pipeline, then loop-drain every push that +/// coalesced against the in-flight key until a finish attempt finds nothing +/// pending (which releases the key in the same critical section). +/// +/// The loop holds NO encrypt-pool permit at the task level: each helper it calls +/// (`replication_withheld_set`, `resolve_candidates_for_push`, +/// `fail_closed_full_scan_objects`, `withheld_recipients_gated`) acquires and +/// releases its own walk permit, so the drain makes progress at pool size 1 — a +/// task-level permit would nest over those same-semaphore acquires and deadlock. +async fn run_encrypt_pin_task( + ctx: EncryptTaskCtx, + guard: crate::state::EncryptInflightGuard, + snapshot_objects: Vec, + snapshot_rules: Option>, + snapshot_is_public: bool, +) { + // The snapshot is this push's own work, resolved before spawn, so it belongs + // to the id captured at spawn. + pin_and_encrypt_objects( + &ctx, + &ctx.repo_id, + snapshot_objects, + snapshot_rules, + snapshot_is_public, + ) + .await; + let mut guard = guard; + loop { + match guard.finish_or_take_pending() { + crate::state::FinishOutcome::Finished(_) => break, + crate::state::FinishOutcome::Pending(g, pending) => { + guard = g; + // `drain_repo_id`, not ctx.repo_id: see resolve_drain_object_list. + if let Some((drain_repo_id, object_list, rules, is_public)) = + resolve_drain_object_list(&ctx, pending).await + { + pin_and_encrypt_objects(&ctx, &drain_repo_id, object_list, rules, is_public) + .await; + } + } + } + } +} + +/// Test-only entry point: build an [`EncryptTaskCtx`] from a test `AppState` (with +/// an overridable `ipfs_api` for a mock Kubo server and an explicit `disk_path` for +/// the fixture repo) and run the real drain task. Keeps `EncryptTaskCtx` and +/// `run_encrypt_pin_task` private to this module. +/// +/// `owner_did` and `repo_name` must name the real seeded row: the drain re-fetches +/// the record by owner/name every lap, so a blank name resolves `Gone` and every +/// lap would pin nothing. +#[cfg(test)] +#[allow(clippy::too_many_arguments)] +pub(crate) async fn run_encrypt_pin_task_for_test( + state: &AppState, + guard: crate::state::EncryptInflightGuard, + disk_path: std::path::PathBuf, + repo_id: String, + owner_did: String, + repo_name: String, + ipfs_api: String, + snapshot_objects: Vec, + snapshot_rules: Option>, + snapshot_is_public: bool, +) { + let ctx = EncryptTaskCtx { + ipfs_api, + repo_path: disk_path, + db: state.db.clone(), + repo_id, + owner_did, + repo_name, + irys_url: String::new(), + http_client: std::sync::Arc::clone(&state.http_client), + node_did: state.node_did.to_string(), + node_keypair: std::sync::Arc::clone(&state.node_keypair), + git_bin: state.git_bin.clone(), + git_timeout: std::time::Duration::from_secs(state.config.git_service_timeout_secs), + encrypt_sem: state.git_encrypt_semaphore.clone(), + pin_sem: state.pin_semaphore.clone(), + }; + run_encrypt_pin_task( + ctx, + guard, + snapshot_objects, + snapshot_rules, + snapshot_is_public, + ) + .await; +} + +/// Test-only fault-injection seam for the drain re-reads in +/// `resolve_drain_object_list`. The behavior worth testing lives on the `Err` arm of +/// the two re-reads, which a real Postgres pool will not produce on demand, so the two +/// reads go through the wrappers below and consult this table first. Keyed by `repo_id` +/// (a fresh uuid per test) so tests running in parallel in one process cannot see each +/// other's injections, and it also records the ATTEMPT counts the retry-bound +/// assertions key on. +#[cfg(test)] +pub(crate) mod drain_faults { + use std::collections::HashMap; + use std::sync::{Mutex, OnceLock}; + + #[derive(Default, Clone, Copy, Debug)] + pub(crate) struct Counters { + pub(crate) repo_read_failures_left: usize, + pub(crate) rules_read_failures_left: usize, + pub(crate) repo_read_attempts: usize, + pub(crate) rules_read_attempts: usize, + } + + fn table() -> &'static Mutex> { + static TABLE: OnceLock>> = OnceLock::new(); + TABLE.get_or_init(|| Mutex::new(HashMap::new())) + } + + /// Make the next `repo_read_failures` repo re-reads and the next + /// `rules_read_failures` rule re-reads for `repo_id` return `Err`, then succeed. + pub(crate) fn inject(repo_id: &str, repo_read_failures: usize, rules_read_failures: usize) { + table().lock().unwrap().insert( + repo_id.to_string(), + Counters { + repo_read_failures_left: repo_read_failures, + rules_read_failures_left: rules_read_failures, + ..Default::default() + }, + ); + } + + /// Observed attempt counts (and remaining injections) for `repo_id`. + pub(crate) fn counters(repo_id: &str) -> Counters { + table() + .lock() + .unwrap() + .get(repo_id) + .copied() + .unwrap_or_default() + } + + /// Production-path hook: count one repo re-read attempt, return whether it must fail. + pub(crate) fn take_repo_read(repo_id: &str) -> bool { + let mut map = table().lock().unwrap(); + let c = map.entry(repo_id.to_string()).or_default(); + c.repo_read_attempts += 1; + if c.repo_read_failures_left > 0 { + c.repo_read_failures_left -= 1; + return true; + } + false + } + + /// Production-path hook: count one rules re-read attempt, return whether it must fail. + pub(crate) fn take_rules_read(repo_id: &str) -> bool { + let mut map = table().lock().unwrap(); + let c = map.entry(repo_id.to_string()).or_default(); + c.rules_read_attempts += 1; + if c.rules_read_failures_left > 0 { + c.rules_read_failures_left -= 1; + return true; + } + false + } +} + +/// The drain's repo re-read, behind the test-only fault seam above. +async fn drain_get_repo(ctx: &EncryptTaskCtx) -> anyhow::Result> { + #[cfg(test)] + if drain_faults::take_repo_read(&ctx.repo_id) { + return Err(anyhow::anyhow!("injected repo re-read failure")); + } + ctx.db.get_repo(&ctx.owner_did, &ctx.repo_name).await +} + +/// The drain's visibility-rule re-read, behind the test-only fault seam above. +async fn drain_list_rules( + ctx: &EncryptTaskCtx, + record_id: &str, +) -> anyhow::Result> { + #[cfg(test)] + if drain_faults::take_rules_read(&ctx.repo_id) { + return Err(anyhow::anyhow!("injected visibility-rule re-read failure")); + } + ctx.db.list_visibility_rules(record_id).await +} + +/// Attempts allowed for the drain re-read before the lap gives up. The coalesced +/// push's work is already out of the pending slot (`finish_or_take_pending` took it +/// in the same critical section that kept the key), and there is no reconciliation +/// sweep to re-derive it: a transient read error must be RETRIED here or that push's +/// pin/encrypt pass is gone. The bound keeps a sustained outage from spinning +/// forever; on exhaustion the work is still lost (the pre-existing residual), but +/// the give-up is logged at ERROR so it is observable instead of silent. +const DRAIN_REREAD_MAX_ATTEMPTS: usize = 3; + +/// Backoff before the next re-read attempt. Doubles per attempt. +const DRAIN_REREAD_BACKOFF: std::time::Duration = std::time::Duration::from_millis(50); + +/// The outcome of the drain's fresh state re-read, keeping the three cases distinct +/// that a single `Err => None` collapses into one: a usable refresh, a repo that +/// genuinely no longer exists (terminal, and NOT a retry), and a transient read +/// failure (retryable). +enum DrainRefresh { + State { + /// Boxed only to keep the enum small: `RepoRecord` dwarfs the other two + /// variants, which carry nothing (clippy::large_enum_variant). + record: Box, + rules: Vec, + }, + Gone, + Failed, +} + +/// Re-read repo state for a drain lap, retrying transient read errors. +/// +/// Both reads are retryable and neither may be read as an absence: an `Err` from the +/// repo row is not "the repo is gone", and an `Err` from the rule list is not "this +/// repo has no rules" (`.ok()` made those indistinguishable, and a `None` rule set +/// makes `replication_withheld_set` return `None`, which skips the entire lap). Only +/// `Ok(None)` on the repo row is a terminal absence, and it consumes no retry budget. +/// +/// The whole `RepoRecord` comes back, not just its rules and flags: the caller writes +/// against `record.id` from this FRESH re-fetch, never `ctx.repo_id` frozen at spawn. +async fn drain_refresh_state(ctx: &EncryptTaskCtx) -> DrainRefresh { + let mut backoff = DRAIN_REREAD_BACKOFF; + for attempt in 1..=DRAIN_REREAD_MAX_ATTEMPTS { + let record = match drain_get_repo(ctx).await { + Ok(Some(rec)) => Box::new(rec), + Ok(None) => return DrainRefresh::Gone, + Err(e) => { + tracing::warn!( + repo = %ctx.repo_id, err = %e, attempt, + "coalesced drain: repo re-read failed; retrying" + ); + tokio::time::sleep(backoff).await; + backoff *= 2; + continue; + } + }; + // record.id, never the spawn-time ctx.repo_id: the record above is re-fetched + // fresh by owner/name, and a delete+re-create between spawn and drain gives + // the row a NEW id - rules read against the stale id come back empty and + // would fail open for the new row. + match drain_list_rules(ctx, &record.id).await { + Ok(rules) => return DrainRefresh::State { record, rules }, + Err(e) => { + tracing::warn!( + repo = %ctx.repo_id, err = %e, attempt, + "coalesced drain: visibility-rule re-read failed; retrying" + ); + tokio::time::sleep(backoff).await; + backoff *= 2; + } + } + } + tracing::error!( + repo = %ctx.repo_id, + attempts = DRAIN_REREAD_MAX_ATTEMPTS, + "coalesced drain: re-read failed on every attempt; the coalesced push's \ + pin/encrypt pass is dropped (no reconciliation sweep re-derives it)" + ); + DrainRefresh::Failed +} + +/// Resolve a coalesced-drain iteration's replicable object list. Re-fetches the +/// repo record and visibility rules FRESH — rules tightened between the coalesced +/// push and its drain must be honored, fail closed: a newly-withheld blob is not +/// pinned, and a repo that is no longer announceable (or whose record cannot be +/// re-read) pins nothing at all (`None`). Returns the filtered object list plus +/// the fresh rules/is_public snapshot for the encrypt stage — the same +/// resolution → withheld-filter pipeline the receive-pack tail runs. +/// The returned `String` is the repo id the drain's encrypt/anchor writes must +/// use: the id from the FRESH re-fetch, never `ctx.repo_id` frozen at task spawn. +/// A delete+recreate under the same slug gives the row a new id, and metadata +/// written against the dead id is invisible to readers on the live row (#174 U3). +async fn resolve_drain_object_list( + ctx: &EncryptTaskCtx, + pending: crate::state::PendingWork, +) -> Option<( + String, + Vec, + Option>, + bool, +)> { + // Both re-reads are bounded-retried: a transient blip must not discard the + // coalesced push's work (`finish_or_take_pending` already took it out of the + // pending slot and no sweep re-derives it). The rules come back from the same + // refresh, read against the FRESH record.id, never the spawn-time ctx.repo_id. + let (record, rules_opt) = match drain_refresh_state(ctx).await { + DrainRefresh::State { record, rules } => (*record, Some(rules)), + DrainRefresh::Gone => { + tracing::warn!( + repo = %ctx.repo_id, + "coalesced drain: repo record is gone; dropping the pending work" + ); + return None; + } + DrainRefresh::Failed => { + tracing::warn!( + repo = %ctx.repo_id, + "coalesced drain: repo re-fetch failed; pinning nothing (fail closed)" + ); + return None; + } + }; + let (_announce, withheld) = replication_withheld_set( + ctx.encrypt_sem.clone(), + rules_opt.clone(), + &record.owner_did, + record.is_public, + ctx.repo_path.clone(), + ctx.git_bin.clone(), + ctx.git_timeout, + ) + .await; + let withheld_set = match withheld { + Some(w) => w, + None => { + tracing::info!( + repo = %ctx.repo_id, + "coalesced drain: repo is not announceable under current rules; \ + pinning nothing (fail closed)" + ); + return None; + } + }; + let (new_tips, old_tips, force_full_scan) = match pending { + crate::state::PendingWork::Tips(pairs) => { + let new_tips: Vec = pairs + .iter() + .map(|(_, n)| n.clone()) + .filter(|s| s != ZERO_SHA) + .collect(); + let old_tips: Vec = pairs + .into_iter() + .map(|(o, _)| o) + .filter(|s| s != ZERO_SHA) + .collect(); + (new_tips, old_tips, false) + } + // The overflow marker forces the full scan via the explicit flag. It must + // never be encoded as a plain empty-tips call: empty tips resolve to an + // empty delta and would pin nothing (the F5 silent loss again). + crate::state::PendingWork::FullScan => (Vec::new(), Vec::new(), true), + }; + let pin_set = crate::git::push_delta::resolve_candidates_for_push( + ctx.encrypt_sem.clone(), + ctx.repo_path.clone(), + new_tips, + old_tips, + ctx.git_bin.clone(), + ctx.git_timeout, + force_full_scan, + ) + .await; + let object_list = if pin_set.full_scan { + fail_closed_full_scan_objects( + ctx.encrypt_sem.clone(), + ctx.repo_path.clone(), + rules_opt.clone().unwrap_or_default(), + record.is_public, + record.owner_did.clone(), + pin_set.candidates, + ctx.git_bin.clone(), + ctx.git_timeout, + ) + .await + } else { + crate::git::visibility_pack::replicable_objects(pin_set.candidates, &withheld_set) + }; + // record.id, never ctx.repo_id: the record above is re-fetched fresh by + // owner/name, and a delete+re-create between spawn and drain gives the row a + // NEW id. This is the same rule the rules read a few lines up already follows. + Some((record.id, object_list, rules_opt, record.is_public)) +} + +/// Re-derive the Pinata replication object set for a push from its ref-update +/// tuples (#174 F2 / KTD-3). +/// +/// The detached Pinata task used to MOVE the push's full pre-resolved object list +/// into its closure and hold it across the `pin_semaphore` await. Under a slow +/// Pinata backend every later push then parked a fresh task each still retaining +/// an MB-scale OID list, so outstanding memory grew O(pushes x object-list) — +/// unbounded. The task now captures only the small `(ref, old, new)` tuples and +/// calls this once a pin slot frees, re-deriving the SAME OID set via +/// `git rev-list` (the delta scan) filtered by the current withheld set. Retained +/// memory is O(ref tuples); the object list is materialized only inside the +/// pin-bounded section, so at most `pin_semaphore` permits' worth exist at once. +/// +/// Coalescing / shedding were rejected because the task's per-ref work is +/// non-idempotent (branch->CID upsert, gossip, GraphQL broadcast, Arweave anchor, +/// peer-notify); dropping a later push's task drops its announcements. Only the +/// retained object list is dropped here, not the task, so every push's effects +/// still fire exactly once. +/// +/// Exactly like `resolve_drain_object_list`: the withheld and candidate sets are +/// recomputed from the rules snapshot captured at post-receive tail start (NOT a +/// fresh read at pin-worker time), so a rule tightened up to that point is honored +/// and the filter always fails closed (a newly-withheld blob is not pinned; a +/// no-longer-announceable repo pins nothing). A tightening AFTER tail-start — before +/// a slow re-derivation runs — is not reflected, matching the old retained-list +/// behavior. Every git child runs through the same INV-22 bounded, +/// process-group-reaped helpers the sibling post-receive scans use +/// (`replication_withheld_set`, `resolve_candidates_for_push`, +/// `fail_closed_full_scan_objects`). +#[allow(clippy::too_many_arguments)] +async fn pinata_object_list_for_refs( + encrypt_sem: Arc, + disk_path: std::path::PathBuf, + ref_updates: &[(String, String, String)], + rules_opt: Option>, + is_public: bool, + owner_did: String, + git_bin: String, + timeout: std::time::Duration, +) -> (bool, Vec) { + let (announce, withheld) = replication_withheld_set( + encrypt_sem.clone(), + rules_opt.clone(), + &owner_did, + is_public, + disk_path.clone(), + git_bin.clone(), + timeout, + ) + .await; + // Not announceable, or the withheld walk failed: replicate nothing (fail + // closed), mirroring the receive-pack tail's `withheld == None` handling. The + // announce decision is returned with the list because this recomputation is the + // tail's only walk once coalescing runs ahead of it (#174 F2a): a repo whose + // walk is failing must still suppress gossip, the GraphQL broadcast, Arweave and + // peer-notify, and `announce` is false on exactly those arms. + let withheld_set = match withheld { + Some(w) => w, + None => return (announce, Vec::new()), + }; + let new_tips: Vec = ref_updates + .iter() + .map(|(_, _, new)| new.clone()) + .filter(|s| s != ZERO_SHA) + .collect(); + let old_tips: Vec = ref_updates + .iter() + .map(|(_, old, _)| old.clone()) + .filter(|s| s != ZERO_SHA) + .collect(); + let pin_set = crate::git::push_delta::resolve_candidates_for_push( + encrypt_sem.clone(), + disk_path.clone(), + new_tips, + old_tips, + git_bin.clone(), + timeout, + false, + ) + .await; + let object_list = if pin_set.full_scan { + fail_closed_full_scan_objects( + encrypt_sem, + disk_path, + rules_opt.unwrap_or_default(), + is_public, + owner_did, + pin_set.candidates, + git_bin, + timeout, + ) + .await + } else { + crate::git::visibility_pack::replicable_objects(pin_set.candidates, &withheld_set) + }; + (announce, object_list) +} + +/// The pin/encrypt pipeline shared by the snapshot iteration and the +/// coalesced-drain iterations: local IPFS pin, then (path-scoped rules only) the +/// admission-gated recipients walk → encrypt-then-pin → Arweave manifest anchor. +/// Pin `object_list` to the local IPFS node under the global pin-admission permit +/// (#174 F6). `EncryptInflight` bounds the pin-task COUNT to one per repo, but each +/// pin loop holds a full per-push object-id list while walking it; this permit bounds +/// how many pin loops RUN CONCURRENTLY, and therefore how many such MB-scale lists are +/// held WHILE BEING PINNED. DEFERS (waits) when the pool is full and never drops, since +/// a dropped pin loses the replication copy. +/// +/// What it does NOT bound, stated plainly rather than implied closed: on this path the +/// caller materializes `object_list` BEFORE this function acquires, so a task that is +/// parked here waiting for a permit is still holding its full list. The parked-task +/// count is capped only per repo by `EncryptInflight`, so across distinct repos the +/// retained list memory is not bounded by this pool. Bounding that is a real change to +/// the capture shape and is deliberately not attempted here; the Pinata twin below +/// avoids it by acquiring BEFORE it derives its list. +// Eight because the merge of #173 and #174 landed both sets of arguments on one +// signature: #174's pin-admission permit plus #173's git seam and pin provenance. +// Each is a distinct value the pin loop needs and none is derivable from another, so +// a wrapper struct here would only rename the same eight fields. +#[allow(clippy::too_many_arguments)] +async fn pin_new_objects_gated( + pin_sem: &Arc, + ipfs_api: &str, + repo_path: &std::path::Path, + git_bin: &str, + git_timeout: std::time::Duration, + object_list: Vec, + db: &Arc, + repo_id: &str, +) -> Vec<(String, String)> { + // Nothing to pin: answer before taking a permit (#174 F2b). The permit bounds how + // many pin loops run concurrently, and an empty list does no pinning, so parking + // here would spend a global pin slot on no work. The pool DEFERS rather + // than sheds, so those calls stall pins for every other repo. Empty is the normal + // shape for a push whose walk failed or that may replicate nothing. + if object_list.is_empty() { + return Vec::new(); + } + let _permit = pin_sem + .clone() + .acquire_owned() + .await + .expect("pin_semaphore is never closed"); + crate::ipfs_pin::pin_new_objects( + ipfs_api, + repo_path, + git_bin, + git_timeout, + object_list, + db, + repo_id, + crate::ipfs_pin::PIN_BATCH_BUDGET, + ) + .await +} + +/// `repo_id` is passed explicitly rather than read from `ctx` so the two callers +/// stay honest about which id they mean: the snapshot iteration passes +/// `ctx.repo_id` (its own push's row), while a coalesced drain passes the id from +/// its fresh re-fetch, which differs after a delete+recreate (#174 U3). +async fn pin_and_encrypt_objects( + ctx: &EncryptTaskCtx, + repo_id: &str, + object_list: Vec, + rules: Option>, + is_public: bool, +) { + let pinned = pin_new_objects_gated( + &ctx.pin_sem, + &ctx.ipfs_api, + &ctx.repo_path, + &ctx.git_bin, + ctx.git_timeout, + object_list, + &ctx.db, + // The drain's repo id, never `ctx.repo_id` frozen at spawn (#174 U3): pin + // provenance must name the row the reader will resolve against. + repo_id, + ) + .await; + if !pinned.is_empty() { + tracing::info!(count = pinned.len(), "pinned git objects to IPFS"); + for (sha, cid) in &pinned { + tracing::info!(sha = %sha, %cid, "pinned"); + } + } + + // Option B1: encrypt-then-pin the withheld blobs so authorized readers can + // recover them when the origin cannot serve them. No path-scoped rule can + // withhold a blob, so withheld_blob_recipients would return an empty map + // after a full per-ref walk; skip it. Mirrors the has_path_scoped_rule gate + // on the other two withheld-walk sites. + if let Some(rules) = rules.filter(|r| visibility_pack::has_path_scoped_rule(r)) { + // Bound the number of concurrent post-push encryption walks (#174 P1-e): + // acquire an admission permit before the full-history walk, deferring + // when the pool is full rather than shedding the recovery pin. + let recip = withheld_recipients_gated( + ctx.encrypt_sem.clone(), + ctx.repo_path.clone(), + ctx.git_bin.clone(), + ctx.git_timeout, + rules, + is_public, + ctx.owner_did.clone(), + ) + .await; + if let Ok(Ok(recipients)) = recip { + let node_seed = ctx.node_keypair.to_seed(); + let delta = crate::encrypted_pin::encrypt_and_pin( + &ctx.ipfs_api, + &ctx.repo_path, + &ctx.db, + repo_id, + &node_seed, + &recipients, + ) + .await; + + // Option B3: anchor a per-push manifest of the blobs sealed this + // push to Arweave, so the oid->cid index survives total node loss. + // Best-effort; never fails the push. + if !delta.is_empty() && !ctx.irys_url.is_empty() { + let owner_short = crate::db::normalize_owner_key(&ctx.owner_did); + let repo_slug = format!("{owner_short}/{}", ctx.repo_name); + let ts = chrono::Utc::now().to_rfc3339(); + let manifest = crate::arweave::EncryptedManifest { + repo: &repo_slug, + owner_did: &ctx.owner_did, + node_did: &ctx.node_did, + timestamp: &ts, + blobs: &delta, + }; + match crate::arweave::anchor_encrypted_manifest( + &ctx.http_client, + &ctx.irys_url, + &manifest, + ) + .await + { + Ok(tx) if !tx.is_empty() => tracing::info!( + repo = %repo_slug, + tx_id = %tx, + "anchored encrypted manifest to Arweave" + ), + Ok(_) => {} + Err(e) => tracing::warn!( + repo = %repo_slug, + err = %e, + "encrypted manifest anchor failed" + ), + } + } + } + } +} + +/// Map an `acquire_write` failure to the right `AppError`. An exhausted repo write-lock +/// POOL is a capacity signal, not a broken repo, so it sheds 503 + Retry-After the same +/// way the admission caps around it do; it used to fall into the generic git 500, which +/// tells the client nothing about retrying (#173 F1). Anything else stays a git error. +/// +/// Shared with the non-push `acquire_write` callers (`api/issues.rs`, `api/pulls.rs`) +/// rather than copied: those hold no admission permit, so they meet an exhausted pool +/// first, and a second copy of this mapping would be free to drift from the push path. +pub(crate) fn acquire_write_app_error(err: &anyhow::Error, repo: &str) -> AppError { + if err + .downcast_ref::() + .is_some() + { + tracing::warn!(repo = %repo, err = %err, "write-lock pool exhausted; shedding with 503"); + AppError::Overloaded("git write locks at capacity, retry shortly".into()) + } else { + tracing::error!(repo = %repo, err = %err, "acquire_write failed"); + AppError::Git(err.to_string()) + } +} + /// Map an error from a `smart_http` git service call to the right `AppError`: /// [`smart_http::GitServiceTimeout`] to 504, a malformed client request to 400, /// anything else to a 500 git error. Pure (no logging) so it is unit-testable; @@ -614,8 +1546,19 @@ pub async fn git_upload_pack( State(state): State, Path((owner, repo)): Path<(String, String)>, auth: Option>, + crate::rate_limit::PeerAddr(peer): crate::rate_limit::PeerAddr, + headers: axum::http::HeaderMap, body: Bytes, ) -> Result { + // #62 cheap load shed. Permit-less snapshot, not admission; see git_info_refs for + // what it does and does not bound. The authoritative hold is `git_permit` below, + // after the per-source cap. + if state.git_read_semaphore.available_permits() == 0 { + tracing::warn!("served-git concurrency cap reached; shedding request with 503 (pre-DB)"); + return Err(AppError::Overloaded( + "git service at capacity, retry shortly".into(), + )); + } let name = smart_http_repo_name(&repo)?; let record = state .db @@ -636,11 +1579,41 @@ pub async fn git_upload_pack( return Err(AppError::RepoNotFound(format!("{owner}/{name}"))); } - let disk_path = state - .repo_store - .acquire(&record.owner_did, &record.name) - .await - .map_err(|e| AppError::Git(e.to_string()))?; + // Per-caller read sub-cap (#174): after the visibility gate (KTD7) so a + // visibility-denied caller never consumes a scarce read slot. Keyed on the + // resolved source IP (never the signed DID, #174 U1); no resolvable key -> + // global read pool only. + let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); + let _caller_permit = acquire_read_caller_permit( + &state.git_read_per_caller, + caller_key.as_deref(), + name, + "upload-pack", + )?; + + // Shed with a 503 before spawning git when the concurrency cap is saturated; + // held for the whole op (incl. the smart_http call), released on return. Taken + // AFTER the per-source cap above so one source cannot occupy global slots it + // would be sub-cap-denied for during the DB/visibility window and starve other + // sources; still before acquire/git so it bounds the Tigris acquire and git + // exec (INV-10). + let _permit = git_permit(&state.git_read_semaphore)?; + + // Bound the acquire under `git_acquire_timeout_secs` so a hung Tigris HEAD/GET + // cannot pin the read permit indefinitely (#174 P1-2). The permit is a handler + // local here (moved into the AdmissionGuard only below, once git is spawned), so + // the early return on timeout drops it and frees the slot; shed a bounded 503. + let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); + let disk_path = tokio::time::timeout( + acquire_deadline, + state.repo_store.acquire(&record.owner_did, &record.name), + ) + .await + .map_err(|_elapsed| { + tracing::warn!(repo = %name, "repo acquire timed out; shedding with 503"); + AppError::Overloaded("git service acquisition timed out, retry shortly".into()) + })? + .map_err(|e| AppError::Git(e.to_string()))?; let body_len = body.len(); // No path-scoped rule can withhold an individual blob, and the whole-repo @@ -648,35 +1621,103 @@ pub async fn git_upload_pack( // withheld walk and serve the pack directly. let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); let resp = if !visibility_pack::has_path_scoped_rule(&rules) { - smart_http::upload_pack(&disk_path, body, git_timeout).await + // Plain (non-path-scoped) serve: move both admission permits into the guard so + // they release only after the spawned git group is reaped, on + // complete/timeout/disconnect — not the instant a disconnect drops this future + // (#174 P1-a). The handler keeps no copy. + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + smart_http::upload_pack(&state.git_bin, &disk_path, body, git_timeout, Some(admission)).await } else { - // withheld_blob_oids walks every ref with blocking `git ls-tree`; keep - // that off the async worker thread. - let withheld = { + // withheld_blob_oids walks every ref with blocking `git ls-tree`; keep that + // off the async worker thread. Move BOTH admission permits INTO the blocking + // task so they are held for the walk's real duration: spawn_blocking cannot be + // cancelled, so on a client disconnect the handler future drops but the walk + // keeps running — and now so do its permits, released only when the walk + // finishes rather than the instant the future drops (#174 P1-b). On success the + // task hands the permits back so the serve phase below keeps them; on a + // dropped future the returned tuple (with the permits) is discarded only when + // the blocking task completes, so admission tracks the real git work. + // ONE deadline spans the walk AND the serve below (#174 U1 follow-up). A fresh + // `git_timeout` for the serve let a slow-but-successful walk plus a full serve + // hold this read permit ~2x the configured budget. Sharing the deadline caps + // that at ~1x: the walk runs against the remaining budget, and if it consumes + // the budget the serve gets what is left and is reaped rather than over-holding + // — the safe direction, same tradeoff as `fail_closed_full_scan_objects` and + // `build_filtered_pack`. The cost is honest: a genuinely slow walk on a large + // repo 504s this clone instead of silently holding the pool for ~2x, so size + // `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` so both phases normally fit. + let deadline = std::time::Instant::now() + git_timeout; + let (withheld, _permit, _caller_permit) = { let path = disk_path.clone(); let rules = rules.clone(); let owner_did = record.owner_did.clone(); let caller_owned = caller.map(str::to_string); let is_public = record.is_public; + let git_bin = state.git_bin.clone(); tokio::task::spawn_blocking(move || { - visibility_pack::withheld_blob_oids( + // Derive the walk's budget from the shared deadline HERE, inside the + // closure, not on the async side before the task is queued. The walk + // starts its own clock when it runs, so a budget computed at queue time + // would hand it a full budget measured from whenever the blocking pool + // got to it, leaving the permit held for queue-delay PLUS the budget. + // Computing it at task start charges that queue delay against the + // shared deadline, which is what makes the ~1x bound true rather than + // approximate. + let walk_budget = deadline.saturating_duration_since(std::time::Instant::now()); + let withheld = visibility_pack::withheld_blob_oids_bounded( &path, + &git_bin, + walk_budget, &rules, is_public, &owner_did, caller_owned.as_deref(), - ) + ); + (withheld, _permit, _caller_permit) }) .await .map_err(|e| AppError::Git(e.to_string()))? - .map_err(|e| AppError::Git(e.to_string()))? }; - + // A walk that hit its deadline carries GitServiceTimeout; map it to 504 like + // the smart_http paths, not a generic 500 (#174 U3). + let withheld = withheld.map_err(|e| git_service_app_error(&e))?; + + // Move the permits returned by the walk into the guard, ONE construction + // site for both serve arms below, so admission tracks the served git group's + // reap (complete/timeout/disconnect) whether the pack is plain or filtered. + // The handler keeps no copy (F1: handler-local permits would drop the + // instant a disconnect drops this future, mid-reap). + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + // Computed AFTER the walk's await, so the serve gets what the walk left, not a + // second full budget. A walk that consumed the whole budget saturates this to + // zero, which the serve surfaces as GitServiceTimeout -> 504 rather than + // running unbounded. + let serve_budget = deadline.saturating_duration_since(std::time::Instant::now()); if withheld.is_empty() { - smart_http::upload_pack(&disk_path, body, git_timeout).await + // No blobs to withhold: serve the plain pack (the walk already held the + // permits per be0cdd6; the guard hands them to the serve). + smart_http::upload_pack( + &state.git_bin, + &disk_path, + body, + serve_budget, + Some(admission), + ) + .await } else { tracing::info!(repo = %name, caller = ?caller, withheld = withheld.len(), "serving filtered pack"); - smart_http::upload_pack_excluding(&disk_path, body, &withheld).await + // The guard threads through both filtered-pack stages (rev-list, then + // pack-objects), so a disconnect mid-stage keeps the permits held until + // that stage's process group is reaped (F1). + smart_http::upload_pack_excluding( + &state.git_bin, + &disk_path, + body, + &withheld, + serve_budget, + Some(admission), + ) + .await } } .map_err(|e| { @@ -855,9 +1896,23 @@ pub async fn git_receive_pack( State(state): State, Path((owner, repo)): Path<(String, String)>, Extension(auth): Extension, + crate::rate_limit::PeerAddr(peer): crate::rate_limit::PeerAddr, + headers: axum::http::HeaderMap, body: Bytes, ) -> Result { let name = smart_http_repo_name(&repo)?; + // Fast-path shed before the DB lookup when the write pool is ALREADY saturated, so a + // push flood against a full pool does not hit Postgres per request. Best-effort + // (racy) and NON-holding: a snapshot, not admission. It spares this request's DB + // work once the pool has filled; pushes arriving while permits are free all proceed + // into the DB, so it does not bound that window. The authoritative, held permit is + // taken after the per-repo lease below, so a lease-blocked waiter pins no write slot + // (#174 F3 review). + if state.git_write_semaphore.available_permits() == 0 { + return Err(AppError::Overloaded( + "git service at capacity, retry shortly".into(), + )); + } tracing::info!(owner = %owner, repo = %name, "receive-pack request"); let record = state .db @@ -927,25 +1982,235 @@ pub async fn git_receive_pack( } } - tracing::debug!(repo = %name, "acquiring write lock"); - let guard = state - .repo_store - .acquire_write(&record.owner_did, &record.name) + // Per-repo in-process write lease (#174 U2/F3): SUPPLEMENTS the cluster-wide pg + // advisory lock. Acquire it BEFORE acquire_write (one consistent order everywhere, + // so the two serializers can never invert into a self-hang) so a second SAME-NODE + // push to this repo blocks here rather than racing a disconnected first push's git + // group while its detached reaper is still tearing it down over the shared local + // objects/ dir. Taking it before the pg lock also means a blocked second writer pins + // no pooled pg connection while it waits. The lease rides the write-path + // AdmissionGuard into the reaper (clone (a)) and spans the clean-path Tigris upload + // in guard.release (clone (b)); it frees only when the LAST clone drops. steal_after + // is sized above ONE legitimate hold (a full receive-pack under git_service_timeout + + // the ~4s reaper cap + the Tigris upload). It is NOT a guarantee that only a leaked + // lease is reclaimed: a waiter's timeout starts at acquire(), not at the head of the + // FIFO queue, so a same-repo backlog whose CUMULATIVE wait exceeds steal_after can + // steal while an earlier waiter is still writing. Correctness does not rest on the + // bound — on the non-disconnect path the retained pg advisory lock still serializes + // the stealer at acquire_write (a spurious 503, not a race); the only corruption-capable + // overlap is the ~4s disconnect/reap window, which the reaper-carried clone (a) covers. + // Saturating, not unchecked. `GIT_SERVICE_TIMEOUT_SECS_MAX` now keeps every parsed + // value inside this arithmetic, so on the configured path this cannot overflow; it is + // deliberate defense in depth for the construction paths clap does not cover (tests + // build `Config` by mutation, and nothing stops a future caller doing the same). The + // failure it holds off is not cosmetic: unchecked, `* 2 + 60` panics the push in debug + // and in release WRAPS to a bound short enough for a waiter to steal a live push's + // lease. Saturating also states the intent — a timeout that large means no steal, which + // is what an effectively-disabled service bound implies. + let lease_steal_after = std::time::Duration::from_secs( + state + .config + .git_service_timeout_secs + .saturating_mul(2) + .saturating_add(60), + ); + + // Parked waiters are bounded INSIDE the lease, not by an admission permit taken above + // it. `body: Bytes` means axum has already buffered the whole pack before this handler + // runs, and the park runs to steal_after (1260s at defaults), so an unbounded waiter + // set would let same-repo pushes stack buffered bodies. `acquire` therefore counts its + // LIVE WAITERS against GITLAWB_REPO_LEASE_MAX_WAITERS and returns None past the cap, + // which sheds here as a 503 + Retry-After like the other admission paths. The cap is + // per repo and counts only handlers actually parked, so it denies same-repo + // concurrency on the contended repo alone: a push to any other repo, from any source, + // is untouched. Taking a per-source or global admission permit above the park instead + // is what the F1 review rejected, since GITLAWB_TRUSTED_PROXY defaults to unset and + // every pusher behind a proxy/NAT then resolves to ONE key, turning one contended repo + // into a node-wide denial. + // The stable disk identity, never record.id: the row id rotates on a + // delete+recreate under the same slug while the bare repo on disk is reused, + // and an id-keyed lease stops serializing exactly across that rotation. + let repo_key = crate::state::repo_identity_key(&record.owner_did, &record.name); + let lease = state + .repo_write_leases + .acquire(&repo_key, lease_steal_after) .await - .map_err(|e| { - tracing::error!(repo = %name, err = %e, "acquire_write failed"); - AppError::Git(e.to_string()) + .ok_or_else(|| { + tracing::warn!( + repo = %name, + "repo write-lease waiter cap reached; shedding with 503" + ); + AppError::Overloaded("repo is busy with another push, retry shortly".into()) })?; - let disk_path = guard.path().to_path_buf(); + + // Admission permits are taken HERE, AFTER the per-repo lease and BEFORE acquire_write. + // Ordering is the fix (#174 P2 DoS): the lease is a block-and-wait serializer, so a + // second same-repo push can park on `acquire` above for up to steal_after. Taking the + // scarce write permits only once we own the lease means a lease-blocked waiter pins NO + // write-pool slot while it waits. Otherwise a few hostile sources could stack same-repo + // pushes, hold every global slot on zero-byte lease-waiters, and shed 503 on every push + // to every OTHER repo node-wide. The per-source sub-cap belongs below the park for the + // same reason: its key is the resolved source IP, which collapses to one key for every + // pusher when GITLAWB_TRUSTED_PROXY is unset (the default), so above the park it sheds + // cross-tenant too. Still before acquire_write, so the git op stays admission-gated + // (INV-10) and a saturated pool sheds 503 before spawning git. + // + // Per-source sub-cap first (#174 P1-d): one source IP cannot occupy the whole write + // pool via many slow pushes. Owner enforcement defaults off, so any valid did:key is + // accepted (auth != authz) and the push rate limiter bounds arrival RATE, not in-flight + // concurrency. Keyed on the resolved source IP, NEVER the signed DID (a DID farm defeats + // a DID key); no resolvable key -> global write pool only. Then the global write permit: + // pushes draw from the dedicated WRITE pool, separate from reads, and it is held for the + // whole op (moved into the AdmissionGuard below). + let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); + let _caller_permit = acquire_read_caller_permit( + &state.git_write_per_caller, + caller_key.as_deref(), + name, + "receive-pack", + )?; + let _permit = git_permit(&state.git_write_semaphore)?; + + tracing::debug!(repo = %name, "acquiring write lock"); + // Bound the write acquire under `git_acquire_timeout_secs`. acquire_write's + // advisory-lock loop already caps at ~60s, but its per-iteration + // `pg_try_advisory_lock().fetch_one(&pool)` can block indefinitely on a hung / + // exhausted Postgres pool (so the 60-count never advances) — and the write permit + // is held the whole time, draining the pool (#174 P1-2). The outer + // `tokio::time::timeout` cancels a mid-sleep/mid-`fetch_one` future, so it bounds + // both the loop and a hung iteration. Cancelling here is only safe because + // acquire_write holds its advisory lock on a connection from a pool whose + // `after_release` hook unlocks (#173): the dropped future used to leave the lock + // held with no guard alive to release it, wedging later pushes to that repo. The + // permit is a handler local here (moved into the AdmissionGuard only after this), + // so the early return on timeout drops it and frees the slot; shed a bounded 503. + let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); + let guard = tokio::time::timeout( + acquire_deadline, + state + .repo_store + .acquire_write(&record.owner_did, &record.name), + ) + .await + .map_err(|_elapsed| { + tracing::warn!(repo = %name, "acquire_write timed out; shedding with 503"); + AppError::Overloaded("git service acquisition timed out, retry shortly".into()) + })? + .map_err(|e| acquire_write_app_error(&e, name))?; + let disk_path = guard.path().to_path_buf(); tracing::debug!(repo = %name, path = %disk_path.display(), "running git receive-pack"); let body_len = body.len(); let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); - let receive_result = smart_http::receive_pack(&disk_path, body, git_timeout).await; + // Move both admission permits into the guard so they release only after the spawned + // receive-pack process group is reaped, on complete/timeout/disconnect — not the + // instant a disconnect drops this future while the detached reaper runs (#174 P1-a). + // The handler keeps no copy. This is independent of the write-lock `guard.release` + // below: admission tracks the git process lifetime, the write lock tracks the repo. + // + // The WRITE LOCK rides the same seam (#173 F2). `guard.release(..)` below is only + // reached if `receive_pack` returns, so on a client disconnect the guard would drop + // with the future and the lock pool's `after_release` hook would free the advisory + // lock immediately, while `KillGroupOnDrop`'s detached reaper is still giving the + // group its ~2s SIGTERM grace. A second push admitted in that window puts two + // `git receive-pack` groups on one repo, which is exactly what the timeout path + // reaps to prevent ("a caller releasing a write lock can't race them"). Sharing the + // guard rather than moving it outright is what lets the SUCCESS path still reclaim + // it for the Tigris upload: the copy retained here can only DELAY release, never + // perform it early, because the handler reaches the take below only after + // `receive_pack` has returned (group reaped or disarmed). On the disconnect path + // this copy dies with the future and the reaper's copy is last, so the lock frees + // after the reap with no upload, which is the release(success = false) semantics an + // interrupted push must have. + let guard = std::sync::Arc::new(std::sync::Mutex::new(Some(guard))); + // Clone (a) of the write lease rides this AdmissionGuard: on a client disconnect the + // guard moves into KillGroupOnDrop's detached reaper, so the lease frees only after + // the receive-pack group is reaped — NOT at the disconnect instant (which is exactly + // when RepoWriteGuard::Drop frees the pg lock). Tying the lease to RepoWriteGuard + // instead would drop it at disconnect and reopen the F3 race. + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit) + .with_hold(std::sync::Arc::clone(&guard)) + .with_lease(lease.clone()); + let receive_result = smart_http::receive_pack( + &state.git_bin, + &disk_path, + body, + git_timeout, + Some(admission), + ) + .await; + + // #174 F2/U5: the post-receive replication tail runs in an independently owned + // task. It parks on `git_encrypt_semaphore` (withheld / candidate / full-scan + // resolution), so leaving it in the request future means a client/proxy disconnect + // while parked silently drops this push's pins, recovery copy, and announcements. + // + // Spawned HERE, ABOVE `guard.release()`, because `release` is itself cancellable: + // on success it awaits the Tigris upload and then the advisory unlock, both while + // this future is still tied to the client connection, and the pack has ALREADY + // landed on disk by then. Spawning below `release` left exactly that window open, + // where a disconnect meant a durable push with no tail (the same class F2 closed, + // one step earlier). + // + // The success gate is explicit rather than the `?` below, because that `?` now + // sits under this spawn: `release` runs on failure too, so anchoring the tail on + // it would pin and announce a half-applied repo, the state `release(false)` + // deliberately refuses to upload and one a hostile pusher can produce on demand + // by aborting a pack mid-transfer. + // + // The tail is read-only on `disk_path` (walk plus plumbing) and takes neither the + // write lease nor the advisory lock, so running it concurrently with the upload + // below waits on nothing this handler still holds. Everything after (touch_repo, + // metrics, trust score, certificates, webhooks) stays in the cancellable handler. + // + // The tail also runs CONCURRENTLY with certificate issuance rather than after it, + // so a ref can be announced before its signed certificate exists. That window is + // accepted: cert issuance already fails open (errors are logged and skipped) and + // the gossip event carries `cert_id: None` regardless, so no announce consumer + // reads a certificate out of it. Each push owns its own tail, including its own + // always-spawned announce, so per-push announcements are never coalesced away. + // + // ACCEPTED RESIDUAL, and it is the cost of this ordering: the tail also runs + // concurrently with the Tigris upload inside `release` below, where before it ran + // after. So a ref can be announced while the shared durable copy is still the old + // one, and on a disconnect here the upload is cancelled outright while the + // detached tail still pins and announces. What makes that acceptable is that + // upload-then-announce was never actually guaranteed: `release` tolerates a failed + // upload by design (it warns and continues to the unlock), so an announce over a + // stale Tigris copy was already reachable before this reorder, and it self-heals, + // since `acquire_fresh` falls back to the local copy and the next push re-uploads. + // The alternative, detaching `release` and the tail together to keep the ordering, + // would return 200 to the pusher before the durable copy lands, which is a larger + // change to the client contract than the window it closes. + let push_succeeded = receive_result.is_ok(); + if push_succeeded { + tokio::spawn(post_receive_replication_tail( + state.clone(), + record.clone(), + ref_updates.clone(), + disk_path.clone(), + auth.0.to_string(), + )); + } // Always release the advisory lock — even on error — to prevent stale locks // from blocking subsequent pushes. Only upload to Tigris when the push // succeeded; uploading a half-applied repo would propagate corruption. - guard.release(receive_result.is_ok()).await; + // Reclaim the write lock from the shared cell (#173 F2). This is only reachable + // once `receive_pack` has returned, so the admission guard's copy can only ever + // DELAY release, never perform it early; on the disconnect path this line is not + // reached at all and the reaper's copy is last. + let reclaimed = guard + .lock() + .expect("repo write-lock mutex poisoned") + .take() + .expect("the write lock is only taken here, and only once"); + reclaimed.release(push_succeeded).await; + // Clean path: clone (a) already dropped inside run_git_service when the receive-pack + // group was reaped; clone (b) held here spanned the success-only Tigris upload that + // ran inside release() above. Drop it now so a second same-repo push proceeds the + // moment this write is durable, rather than at end of the (longer) handler tail. On + // the disconnect path this line is never reached: clone (a) rides the reaper (F3). + drop(lease); let result = receive_result.map_err(|e| { let app = git_service_app_error(&e); @@ -1048,21 +2313,96 @@ pub async fn git_receive_pack( } } + Ok(result) +} + +/// The detached post-receive replication tail (#174 F2): everything a landed push +/// still owes after its git response has been returned: the replication decision, +/// the per-repo-coalesced pin/encrypt task, and this push's own Pinata + announce +/// task. Split out of `git_receive_pack` so the ordering the coalescing gate depends +/// on is directly testable; the handler spawns it and returns. +async fn post_receive_replication_tail( + state: AppState, + record: RepoRecord, + ref_updates: Vec, + disk_path: std::path::PathBuf, + did: String, +) { // Replication enforcement (Phase 2): decide once per push whether the public // may read this repo at all and, if so, which blob OIDs must not leave the - // node. `withheld == None` means replicate nothing (private / mode A / - // undetermined): skip every pin so even commit and tree objects (which - // withheld_blob_oids never lists) stay local. `announce` gates the - // network-facing announcements. Fail closed: a private or undetermined repo - // never leaks. + // node. `withheld == None` means this push pins nothing (private / mode A / + // undetermined, or a walk that failed): skip every pin so even commit and tree + // objects (which withheld_blob_oids never lists) stay local. Fail closed: a + // private or undetermined repo never leaks. The announce decision that gates + // the network-facing sends is taken separately, below. let rules_opt = state.db.list_visibility_rules(&record.id).await.ok(); - let (announce, withheld) = replication_withheld_set( - rules_opt.clone(), - &record.owner_did, - record.is_public, - disk_path.clone(), - ) - .await; + + // #174 F2a: take the per-repo coalescing key BEFORE the walk, not after it. + // `replication_withheld_set` decides announceability from the rules snapshot + // alone and returns `(false, None)` before it touches the scan pool or spawns + // any git, so the same predicate can be evaluated here and used to gate + // `try_begin`. With the gate below the walk, rapid pushes to one repo each + // parked on `git_encrypt_semaphore` and re-ran the walk plus the object-list + // materialization before finding out they were going to coalesce; now a push + // that will coalesce does none of that. Not announceable is the same as + // before: nothing replicates, so no key is taken and no walk runs. + let announce_at_root = match &rules_opt { + Some(rules) => { + crate::visibility::listable_at_root(rules, record.is_public, &record.owner_did, None) + } + None => false, + }; + let mut coalesced = false; + let mut inflight = None; + if announce_at_root { + let tip_pairs: Vec<(String, String)> = ref_updates + .iter() + .map(|u| (u.old_sha.clone(), u.new_sha.clone())) + .collect(); + // Same stable disk identity as the lease above (#174 U2): keyed on + // record.id, a post-recreate push would take a fresh key and run a second + // encrypt task against the same on-disk repo instead of coalescing. + let coalesce_key = crate::state::repo_identity_key(&record.owner_did, &record.name); + match state.encrypt_inflight.try_begin(&coalesce_key, tip_pairs) { + crate::state::BeginOutcome::Coalesced => { + coalesced = true; + tracing::debug!( + repo = %record.id, + "post-push encryption task already in flight for this repo; coalesced \ + — this push's tip pairs are queued for that task's drain" + ); + } + crate::state::BeginOutcome::Admitted(guard) => inflight = Some(guard), + } + } + + // The walk feeds this push's own pin/encrypt snapshot, so it is skipped both + // when nothing may replicate and when this push coalesced (the in-flight + // task drains its tips). The walk's own announce decision is deliberately + // not kept here: it does not exist on the coalesced path, and the Pinata / + // announce tail below re-derives it (see `do_pinata_replication`). + let withheld = if coalesced || !announce_at_root { + None + } else { + replication_withheld_set( + state.git_encrypt_semaphore.clone(), + rules_opt.clone(), + &record.owner_did, + record.is_public, + disk_path.clone(), + state.git_bin.clone(), + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + ) + .await + .1 + }; + + // #174 F2b: did THIS push run its own walk and have it fail? An admitted push that + // could not be vetted must not go on to take a pin permit and re-run the same + // failing walk in the Pinata worker below. Only the COALESCED path needs the + // rules-only predicate there: it has no walk of its own, so the worker's + // re-derivation is its only fail-closed source. + let own_walk_failed = announce_at_root && !coalesced && withheld.is_none(); // Resolve the per-push pin candidate set once, off the async worker, then // filter to what may actually replicate. Delta path: the reachable-only @@ -1071,7 +2411,7 @@ pub async fn git_receive_pack( // so fail closed — replicate a blob only if it is reachable AND // visibility-allowed (#99). Only computed when something will actually // replicate; every degraded path logs rather than failing silently. - let object_list: Vec = if let Some(withheld_set) = withheld.clone() { + let object_list: Vec = if let Some(withheld_set) = withheld { let new_tips: Vec = ref_updates .iter() .map(|u| u.new_sha.clone()) @@ -1083,18 +2423,25 @@ pub async fn git_receive_pack( .filter(|s| s != ZERO_SHA) .collect(); let pin_set = crate::git::push_delta::resolve_candidates_for_push( + state.git_encrypt_semaphore.clone(), disk_path.clone(), new_tips, old_tips, + state.git_bin.clone(), + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + false, ) .await; if pin_set.full_scan { fail_closed_full_scan_objects( + state.git_encrypt_semaphore.clone(), disk_path.clone(), rules_opt.clone().unwrap_or_default(), record.is_public, record.owner_did.clone(), pin_set.candidates, + state.git_bin.clone(), + std::time::Duration::from_secs(state.config.git_service_timeout_secs), ) .await } else { @@ -1105,107 +2452,77 @@ pub async fn git_receive_pack( }; // Pin new git objects to the local IPFS node (no-op if ipfs_api is empty). - // Skipped entirely when the public cannot read the repo (withheld == None). - if withheld.is_some() { - let object_list_ipfs = object_list.clone(); - let ipfs_api = state.config.ipfs_api.clone(); - let repo_path_clone = disk_path.clone(); - let db_clone = state.db.clone(); - let rules_for_enc = rules_opt.clone(); - let repo_id = record.id.clone(); - let owner_did = record.owner_did.clone(); - let is_public = record.is_public; - let irys_url = state.config.irys_url.clone(); - let http_client = std::sync::Arc::clone(&state.http_client); - let node_did_str = state.node_did.to_string(); - let node_seed = state.node_keypair.to_seed(); - let repo_name = record.name.clone(); - tokio::spawn(async move { - let pinned = crate::ipfs_pin::pin_new_objects( - &ipfs_api, - &repo_path_clone, - object_list_ipfs, - &db_clone, - ) - .await; - if !pinned.is_empty() { - tracing::info!(count = pinned.len(), "pinned git objects to IPFS"); - for (sha, cid) in &pinned { - tracing::info!(sha = %sha, %cid, "pinned"); - } - } - - // Option B1: encrypt-then-pin the withheld blobs so authorized - // readers can recover them when the origin cannot serve them. - // No path-scoped rule can withhold a blob, so withheld_blob_recipients - // would return an empty map after a full per-ref walk; skip it. Mirrors - // the has_path_scoped_rule gate on the other two withheld-walk sites. - if let Some(rules) = rules_for_enc.filter(|r| visibility_pack::has_path_scoped_rule(r)) - { - let p = repo_path_clone.clone(); - let owner = owner_did.clone(); - let recip = tokio::task::spawn_blocking(move || { - crate::git::visibility_pack::withheld_blob_recipients( - &p, &rules, is_public, &owner, - ) - }) - .await; - if let Ok(Ok(recipients)) = recip { - let delta = crate::encrypted_pin::encrypt_and_pin( - &ipfs_api, - &repo_path_clone, - &db_clone, - &repo_id, - &node_seed, - &recipients, - ) - .await; - - // Option B3: anchor a per-push manifest of the blobs sealed - // this push to Arweave, so the oid->cid index survives total - // node loss. Best-effort; never fails the push. - if !delta.is_empty() && !irys_url.is_empty() { - let owner_short = crate::db::normalize_owner_key(&owner_did); - let repo_slug = format!("{owner_short}/{repo_name}"); - let ts = chrono::Utc::now().to_rfc3339(); - let manifest = crate::arweave::EncryptedManifest { - repo: &repo_slug, - owner_did: &owner_did, - node_did: &node_did_str, - timestamp: &ts, - blobs: &delta, - }; - match crate::arweave::anchor_encrypted_manifest( - &http_client, - &irys_url, - &manifest, - ) - .await - { - Ok(tx) if !tx.is_empty() => tracing::info!( - repo = %repo_slug, - tx_id = %tx, - "anchored encrypted manifest to Arweave" - ), - Ok(_) => {} - Err(e) => tracing::warn!( - repo = %repo_slug, - err = %e, - "encrypted manifest anchor failed" - ), - } - } - } - } - }); + // Skipped entirely when the public cannot read the repo (no key was taken). + // + // Coalesce-and-requeue per repo (#174 P2-2 + F5): the spawned task's walks park + // on `git_encrypt_semaphore` (which DEFERS when the pool is full rather than + // dropping the recovery copy). To bound the OUTSTANDING task set, at most one + // task per repo is in flight; a push arriving while one is in flight does NOT + // spawn a duplicate — and is NOT dropped either. The in-flight task pins only + // its own pre-spawn object-list snapshot, so this push's (old, new) tip pairs + // are merged into the in-flight key's pending slot in the same critical section + // as the presence check, and the task loop-drains them (fresh rules, fail + // closed) before releasing the key. Without the requeue a coalesced push's pins + // and recovery copies would be silently absent until an unrelated later push + // (the F5 loss). The guard still releases the key on panic (Drop on unwind), so + // a crashed walk never permanently locks the repo out. + // + // #174 F2a: the key was taken above, so this is only the spawn. An admitted + // push ALWAYS spawns, including when its own walk failed and `object_list` is + // therefore empty: pushes can have coalesced into the pending slot while that + // walk ran, and the task's drain loop is what consumes them. Releasing or + // dropping the guard instead would discard that work with a warn (the F5 loss + // class again), and the two are indistinguishable from outside (Drop removes + // the key whenever the guard is still armed). + if let Some(inflight_guard) = inflight { + let ctx = EncryptTaskCtx { + ipfs_api: state.config.ipfs_api.clone(), + repo_path: disk_path.clone(), + db: state.db.clone(), + repo_id: record.id.clone(), + owner_did: record.owner_did.clone(), + repo_name: record.name.clone(), + irys_url: state.config.irys_url.clone(), + http_client: std::sync::Arc::clone(&state.http_client), + node_did: state.node_did.to_string(), + node_keypair: std::sync::Arc::clone(&state.node_keypair), + git_bin: state.git_bin.clone(), + git_timeout: std::time::Duration::from_secs(state.config.git_service_timeout_secs), + encrypt_sem: state.git_encrypt_semaphore.clone(), + pin_sem: state.pin_semaphore.clone(), + }; + tokio::spawn(run_encrypt_pin_task( + ctx, + inflight_guard, + object_list, + rules_opt.clone(), + record.is_public, + )); } - // Pin new git objects to Pinata, then record branch→CID and gossip + // Pin new git objects to Pinata, then record branch→CID and gossip. + // + // #174 P2-2 scope note: this SECOND detached spawn is deliberately NOT brought + // under the per-repo encryption coalescing above, because unlike the idempotent + // recovery-copy walk it does PER-PUSH, PER-REF work — branch→CID upserts, gossip + // publish, GraphQL subscription broadcast, Arweave anchoring, and peer notify, each + // keyed to THIS push's ref_updates. Coalescing (or shedding) it against an in-flight + // task for the same repo would DROP a later push's ref-update announcements (a + // correctness regression), not merely delay a duplicate. So the task stays one per + // push and every push's effects fire exactly once. + // + // #174 F2 / KTD-3: {bounded memory, no dropped effects, no handler latency} are + // jointly unsatisfiable by coalesce/shed/block, so instead of retaining the full + // object list we bound the thing that actually accumulates. The task captures only + // the small ref tuples and RE-DERIVES the object set inside the worker once a pin + // slot frees (see `pinata_object_list_for_refs`); the MB-scale OID list is never + // held by a parked task. { let pinata_jwt = state.config.pinata_jwt.clone(); let pinata_upload_url = state.config.pinata_upload_url.clone(); let repo_path_clone = disk_path.clone(); let db_clone = state.db.clone(); + let repo_id = record.id.clone(); let http_client = Arc::clone(&state.http_client); let node_did_str = state.node_did.to_string(); let repo_slug = format!( @@ -1225,21 +2542,80 @@ pub async fn git_receive_pack( let owner_did_for_arweave = record.owner_did.clone(); let self_public_url = state.config.public_url.clone(); let node_keypair = Arc::clone(&state.node_keypair); - let object_list_pinata = object_list; - let do_pinata_replication = withheld.is_some(); + // #174 F2a: gated on the cheap announce predicate, not on `withheld`. + // `withheld` is None for a push that coalesced (it never walked), and + // this task's work is per-push and non-idempotent, so keying it on the + // walk result would silently stop pinning and stop recording a branch to + // CID mapping for every coalesced push. The fail-closed source for this + // path is now `pinata_object_list_for_refs`'s own recomputation of + // `replication_withheld_set` inside the pin permit: it returns an empty + // list AND announce=false when the walk fails or the repo may not + // replicate, so neither blobs nor announcements escape an unvetted push. + // + // #174 F2b: except when THIS push already ran that walk and it failed. The + // re-derivation would fail the same way, so it buys nothing, and it would buy + // it at the price of a global pin permit plus a second round of git children. + // The pin pool DEFERS rather than sheds, so enough such pushes stall pins + // node-wide. A coalesced push never walked, so it is unaffected. + let do_pinata_replication = announce_at_root && !own_walk_failed; + // #174 F2 / KTD-3: capture only the small inputs the re-derivation needs; the + // MB-scale object list is NOT moved in. `pinata_object_list_for_refs` recomputes + // it from these once a pin slot frees. rules/owner/is_public drive the fresh + // fail-closed withheld filter; encrypt_sem + git_bin + timeout keep the re-derive + // git children under the same INV-22 bounded, group-reaped scan admission. + let pinata_rules_opt = rules_opt.clone(); + let pinata_owner_did = record.owner_did.clone(); + let pinata_is_public = record.is_public; + let pinata_git_bin = state.git_bin.clone(); + let pinata_git_timeout = + std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let pinata_encrypt_sem = state.git_encrypt_semaphore.clone(); + // Same global pin-admission bound as the IPFS loop (#174 F6): the Pinata pin + // loop holds a re-derived object-id list while pinning it, so it shares the cap. + // It DEFERS on a full pool rather than dropping the pin. + let pin_sem_pinata = state.pin_semaphore.clone(); tokio::spawn(async move { - let pinned = if do_pinata_replication { - crate::pinata::pin_new_objects( - &http_client, - &pinata_upload_url, - &pinata_jwt, - &repo_path_clone, - object_list_pinata, - &db_clone, + // `announce` comes back from the re-derivation below rather than from + // the tail's own walk (#174 F2a): a coalesced push has no walk of its + // own, and this is the recomputation that fails closed for it. When + // the repo is not announceable at root there is no re-derivation and + // no announcement either, which is the same answer the walk gave. + let (announce, pinned) = if do_pinata_replication { + let _pin_permit = pin_sem_pinata + .acquire_owned() + .await + .expect("pin_semaphore is never closed"); + // Re-derive the object set now that a pin slot is free (#174 F2 / + // KTD-3). A parked task retained only `ref_updates_clone` (O(ref + // tuples)), never this list, so a slow Pinata backend cannot grow + // outstanding memory O(pushes x object-list). Fresh + fail-closed; + // each git child is INV-22 bounded and process-group reaped. + let (announce, object_list) = pinata_object_list_for_refs( + pinata_encrypt_sem, + repo_path_clone.clone(), + &ref_updates_clone, + pinata_rules_opt, + pinata_is_public, + pinata_owner_did, + pinata_git_bin, + pinata_git_timeout, + ) + .await; + ( + announce, + crate::pinata::pin_new_objects( + &http_client, + &pinata_upload_url, + &pinata_jwt, + &repo_path_clone, + object_list, + &db_clone, + &repo_id, + ) + .await, ) - .await } else { - Vec::new() + (false, Vec::new()) }; if !pinned.is_empty() { @@ -1336,7 +2712,9 @@ pub async fn git_receive_pack( .await; } Ok(_) => {} - Err(e) => tracing::warn!(repo=%repo_slug, err=%e, "Arweave anchor failed"), + Err(e) => { + tracing::warn!(repo=%repo_slug, err=%e, "Arweave anchor failed") + } } } } @@ -1376,8 +2754,6 @@ pub async fn git_receive_pack( } }); } - - Ok(result) } /// GET /api/v1/repos/{owner}/{repo}/refs @@ -1647,6 +3023,10 @@ pub async fn get_icaptcha_proof( // ── Pkt-line parsing ────────────────────────────────────────────────────── +/// `Clone` so `git_receive_pack` can hand the parsed updates to the detached +/// replication tail at the durability boundary while the certificate and webhook +/// loops below still iterate their own copy (#174 U5). +#[derive(Clone)] struct RefUpdate { old_sha: String, new_sha: String, @@ -1869,6 +3249,17 @@ mod tests { assert!(matches!(git_service_app_error(&other), AppError::Git(_))); } + #[test] + fn git_permit_sheds_at_capacity_and_releases() { + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + let p1 = git_permit(&sem).expect("first acquire succeeds"); + // At capacity the next request is shed with Overloaded (-> 503), not queued. + assert!(matches!(git_permit(&sem), Err(AppError::Overloaded(_)))); + // Releasing the permit frees the slot for the next request. + drop(p1); + assert!(git_permit(&sem).is_ok()); + } + fn repo_owned_by(owner_did: &str) -> crate::db::RepoRecord { let now = chrono::Utc::now(); crate::db::RepoRecord { @@ -1899,16 +3290,42 @@ mod tests { let dummy = std::path::PathBuf::from("/nonexistent"); // Private: no rules at all. - let (announce, _) = replication_withheld_set(None, OWNER_DID, false, dummy.clone()).await; + let (announce, _) = replication_withheld_set( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), + None, + OWNER_DID, + false, + dummy.clone(), + "git".into(), + std::time::Duration::from_secs(600), + ) + .await; assert!(!announce, "private repo (no rules) must not announce"); // Private: empty rule set, is_public=false → still not listable at root. - let (announce, _) = - replication_withheld_set(Some(vec![]), OWNER_DID, false, dummy.clone()).await; + let (announce, _) = replication_withheld_set( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), + Some(vec![]), + OWNER_DID, + false, + dummy.clone(), + "git".into(), + std::time::Duration::from_secs(600), + ) + .await; assert!(!announce, "private repo (empty rules) must not announce"); // Public: empty rule set, is_public=true → listable at root, announces. - let (announce, _) = replication_withheld_set(Some(vec![]), OWNER_DID, true, dummy).await; + let (announce, _) = replication_withheld_set( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), + Some(vec![]), + OWNER_DID, + true, + dummy, + "git".into(), + std::time::Duration::from_secs(600), + ) + .await; assert!(announce, "public repo must announce"); } @@ -2000,6 +3417,463 @@ mod tests { } } + #[cfg(unix)] + fn write_fake_git(dir: &std::path::Path, body: &str) -> String { + use std::os::unix::fs::PermissionsExt; + let p = dir.join("fakegit"); + std::fs::write(&p, body).unwrap(); + let mut perm = std::fs::metadata(&p).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&p, perm).unwrap(); + p.to_str().unwrap().to_string() + } + + /// #174 U6: the `info/refs` advertisement must run the CONFIGURED git binary, + /// like upload-pack and receive-pack already do. It passed a literal "git", so a + /// fake-git harness could drive the pack paths but never the advertisement. + /// + /// MUTATION (RED): restore the `"git"` literal at the `smart_http::info_refs` + /// call and the fake's marker never reaches the response body. + #[cfg(unix)] + #[sqlx::test] + async fn u6_info_refs_runs_the_configured_git_binary(pool: sqlx::PgPool) { + use axum::extract::{Path, Query, State}; + use http_body_util::BodyExt; + + let tmp = tempfile::TempDir::new().unwrap(); + // Distinctive advertisement so the assertion cannot pass on real git's output. + let body = "#!/bin/sh\n\ + case \"$1\" in\n\ + upload-pack) printf 'U6-FAKE-ADVERTISEMENT' ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + let state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6u6adv", "a1", false).await; + + let resp = git_info_refs( + State(state), + Path(("z6u6adv".to_string(), "a1".to_string())), + Query(InfoRefsQuery { + service: Some("git-upload-pack".to_string()), + }), + crate::rate_limit::PeerAddr(Some("203.0.113.95:5000".parse().unwrap())), + axum::http::HeaderMap::new(), + None, + ) + .await + .expect("the advertisement must succeed"); + + let bytes = resp.into_body().collect().await.unwrap().to_bytes(); + let text = String::from_utf8_lossy(&bytes); + assert!( + text.contains("U6-FAKE-ADVERTISEMENT"), + "info/refs must run state.git_bin, not a hardcoded \"git\"; body was: {text:?}" + ); + } + + /// #174 U4: `withheld_recipients_gated` must hold its encrypt-scan permit INSIDE + /// the blocking closure, matching every other post-receive scan helper (see + /// `replication_withheld_set`, where the permit is moved in with "a started walk + /// always completes holding it"). + /// + /// Holding it in the async frame instead means dropping the future releases the + /// permit while the uncancellable `spawn_blocking` walk keeps running, so the + /// encrypt pool admits a replacement scan against a slot still occupied by a live + /// git child. + /// + /// MUTATION (RED): hoist `_permit` back out of the closure and the + /// still-held-after-drop assertion fails — the count returns to 1 immediately. + #[cfg(unix)] + #[tokio::test] + async fn u4_encrypt_scan_permit_is_held_through_the_blocking_walk() { + use std::time::Duration; + let tmp = tempfile::TempDir::new().unwrap(); + let pidfile = tmp.path().join("u4_walk.pid"); + // Hang on the first real walk command, whatever it is; only rev-parse answers. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + rev-parse) echo deadbeef ;;\n\ + *) echo $$ > \"{pid}\"; while true; do sleep 1; done ;;\n\ + esac\n\ + exit 0\n", + pid = pidfile.display(), + ); + let git_bin = write_fake_git(tmp.path(), &body); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + + let mut fut = Box::pin(withheld_recipients_gated( + sem.clone(), + tmp.path().to_path_buf(), + git_bin, + Duration::from_secs(600), + vec![vis_rule("/secret/**", &[])], + true, + "did:key:z6MkU4OwnerAAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + )); + + // Drive until the blocking walk's git child records its pid. + let mut walk_pid: Option = None; + for _ in 0..500 { + let _ = tokio::time::timeout(Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&pidfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + walk_pid = Some(p); + break; + } + } + let pid = walk_pid.expect("the fake git walk command must have spawned"); + struct ReapOnDrop(i32); + impl Drop for ReapOnDrop { + fn drop(&mut self) { + unsafe { + libc::kill(self.0, libc::SIGKILL); + } + } + } + let _cleanup = ReapOnDrop(pid); + + assert_eq!( + sem.available_permits(), + 0, + "the scan permit must be held while the blocking walk runs" + ); + + drop(fut); + for _ in 0..10 { + tokio::task::yield_now().await; + } + assert!( + unsafe { libc::kill(pid, 0) } == 0, + "precondition: the walk's git child must still be alive, or this \ + assertion proves nothing" + ); + assert_eq!( + sem.available_permits(), + 0, + "dropping the future must NOT release the encrypt-scan permit while the \ + uncancellable blocking walk it admitted is still running" + ); + + unsafe { + libc::kill(pid, libc::SIGKILL); + } + let mut freed = false; + for _ in 0..400 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + assert!( + freed, + "once the blocking walk ends the scan permit must return to the pool" + ); + } + + /// #174 (write-pool twin, vetted by execution not reasoning): the receive-pack + /// post-push replication walk is bounded. Drive `replication_withheld_set` with an + /// injected fake git that hangs on `rev-list` and a short budget: it must RETURN + /// within the budget (so `git_receive_pack` releases the write permit it holds + /// across this await, rather than pinning it for the hang) AND fail closed + /// (announce suppressed) because the walk could not be vetted. Proves this path + /// funnels through the bounded `blob_paths`, on the write-permit-holding side. + #[cfg(unix)] + #[tokio::test] + async fn replication_walk_is_bounded_and_fails_closed_on_a_hung_git() { + use std::time::Duration; + let tmp = tempfile::TempDir::new().unwrap(); + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) sleep 30 ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + // Public root (announceable) + a path-scoped rule, so the walk actually runs + // rather than taking the has_path_scoped_rule short-circuit. + let rules = Some(vec![vis_rule("/secret/**", &[])]); + + let result = tokio::time::timeout( + Duration::from_secs(10), + replication_withheld_set( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), + rules, + OWNER_DID, + true, + tmp.path().to_path_buf(), + git_bin, + Duration::from_millis(200), + ), + ) + .await + .expect( + "replication_withheld_set must return within the budget; a hung walk must \ + not pin the write permit git_receive_pack holds across it", + ); + assert_eq!( + result, + (false, None), + "a walk that could not be vetted must suppress the announce (fail closed)" + ); + } + + /// #174 (serve-path 504, vetted by execution): a hung withheld-blob walk on the + /// upload-pack POST maps to 504, not a generic 500. Real repo dir on disk (so + /// acquire's fast path returns it) + a path-scoped rule (so the walk runs) + + /// an injected fake git that hangs on rev-list. The handler must return 504, + /// proving git_upload_pack routes the walk's GitServiceTimeout through + /// git_service_app_error end to end. + #[cfg(unix)] + #[sqlx::test] + async fn upload_pack_hung_withheld_walk_returns_504(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) sleep 30 ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n"; + let fake = write_fake_git(tmp.path(), body); + + let mut state = crate::test_support::test_state(pool).await; + state.git_bin = fake; + let mut cfg = (*state.config).clone(); + cfg.git_service_timeout_secs = 1; + state.config = std::sync::Arc::new(cfg); + state + .db + .upsert_mirror_repo("z6srv504", "sv", "/tmp/z6srv504-sv", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6srv504", "sv").await.unwrap().unwrap(); + // Path-scoped rule so has_path_scoped_rule() is true and the walk runs; the + // public root still lets an anonymous caller past the "/" gate. + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &[], + OWNER_DID, + ) + .await + .unwrap(); + // acquire()'s fast path returns the local path when it exists on disk. + let disk = std::path::Path::new("/tmp/z6srv504/sv.git"); + std::fs::create_dir_all(disk).unwrap(); + + let peer: SocketAddr = "203.0.113.91:7000".parse().unwrap(); + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::POST) + .uri("/z6srv504/sv/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let status = router.oneshot(req).await.unwrap().status(); + let _ = std::fs::remove_dir_all("/tmp/z6srv504"); + assert_eq!( + status, + StatusCode::GATEWAY_TIMEOUT, + "a hung withheld-blob walk must surface as 504, not a generic 500" + ); + } + + /// #174 U1 follow-up (RED-before/GREEN-after): the path-scoped upload-pack branch + /// shares ONE deadline across the withheld-blob walk and the pack serve, so one + /// clone cannot hold a read permit for ~2x `git_service_timeout_secs`. A walk that + /// consumes most of the budget must leave the serve only the REMAINDER, so the + /// serve is reaped and the request is a 504. + /// + /// Load-bearing: give the serve a fresh `git_timeout` instead of the remainder and + /// the fake `upload-pack` (1.2s) fits inside a fresh 2s budget, completes, and the + /// status is no longer 504 (RED). This is the ~2x-budget hold the unit removes. + /// + /// Plain-serve arm: the fake git lists no refs and fails `rev-parse`, so the walk + /// yields an empty withheld set and the branch takes `upload_pack`. `rev-list` + /// carries the walk's cost. + #[cfg(unix)] + #[sqlx::test] + async fn upload_pack_shares_one_deadline_across_walk_and_plain_serve(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + // Walk: no refs (for-each-ref empty), HEAD does not resolve (rev-parse exit 1), + // rev-list burns 1.2s of the 2s budget and lists no commits -> empty withheld. + // Serve: upload-pack needs 1.2s, which does NOT fit the ~0.8s remainder but + // WOULD fit a fresh 2s budget. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-parse) exit 1 ;;\n rev-list) sleep 1.2 ;;\n upload-pack) sleep 1.2 ;;\n *) : ;;\nesac\nexit 0\n"; + let fake = write_fake_git(tmp.path(), body); + + let mut state = crate::test_support::test_state(pool).await; + state.git_bin = fake; + let mut cfg = (*state.config).clone(); + cfg.git_service_timeout_secs = 2; + state.config = std::sync::Arc::new(cfg); + state + .db + .upsert_mirror_repo("z6shared1", "sv", "/tmp/z6shared1-sv", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6shared1", "sv").await.unwrap().unwrap(); + // Path-scoped rule so has_path_scoped_rule() is true and the walk runs. + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &[], + OWNER_DID, + ) + .await + .unwrap(); + let disk = std::path::Path::new("/tmp/z6shared1/sv.git"); + std::fs::create_dir_all(disk).unwrap(); + + let peer: SocketAddr = "203.0.113.92:7000".parse().unwrap(); + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::POST) + .uri("/z6shared1/sv/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let status = router.oneshot(req).await.unwrap().status(); + let _ = std::fs::remove_dir_all("/tmp/z6shared1"); + assert_eq!( + status, + StatusCode::GATEWAY_TIMEOUT, + "the walk and the serve must share ONE deadline, so a walk that burns most \ + of the budget leaves the serve only the remainder and the serve is reaped; \ + a non-504 here means the serve got a fresh full budget (the ~2x hold)" + ); + } + + /// #174 U1 follow-up, FILTERED arm (RED-before/GREEN-after): the shared deadline + /// must reach `upload_pack_excluding` too, not just the plain `upload_pack`. Same + /// property as the plain-arm test, different serve function, because the branch + /// threads the remainder into both arms and a fix that missed one would leave the + /// ~2x hold reachable by any clone of a repo that actually withholds something. + /// + /// The fake git yields a NON-EMPTY withheld set: one ref peeling to a commit, a + /// resolvable HEAD, one commit, and an `ls-tree -rz` record placing a blob under + /// `/secret/`, which the path-scoped rule denies to an anonymous caller. `ls-tree` + /// carries the walk's cost (walk-only), and `pack-objects` carries the serve's, so + /// the two phases are independently attributable. + /// + /// Load-bearing: hand the serve a fresh `git_timeout` and `pack-objects` (1.2s) + /// fits a fresh 2s budget, completes, and the status is no longer 504 (RED). + #[cfg(unix)] + #[sqlx::test] + async fn upload_pack_shares_one_deadline_across_walk_and_filtered_serve(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + let commit = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + let blob = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + // for-each-ref lists one ref; cat-file peels it to a commit (the fail-closed + // ref check); rev-parse resolves HEAD; rev-list lists the one commit; ls-tree + // emits " blob \t" (NUL-delimited) under secret/ and burns + // 1.2s of the 2s budget; pack-objects is the serve's 1.2s cost. + let body = format!( + "#!/bin/sh\ncase \"$1\" in\n \ + for-each-ref) echo refs/heads/main ;;\n \ + cat-file) echo commit ;;\n \ + rev-parse) echo {commit} ;;\n \ + rev-list) echo {commit} ;;\n \ + ls-tree) printf '100644 blob {blob}\\tsecret/f.txt' ; sleep 1.2 ;;\n \ + pack-objects) sleep 1.2 ;;\n \ + *) : ;;\nesac\nexit 0\n" + ); + let fake = write_fake_git(tmp.path(), &body); + + let mut state = crate::test_support::test_state(pool).await; + state.git_bin = fake; + let mut cfg = (*state.config).clone(); + cfg.git_service_timeout_secs = 2; + state.config = std::sync::Arc::new(cfg); + state + .db + .upsert_mirror_repo("z6shared2", "sv", "/tmp/z6shared2-sv", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6shared2", "sv").await.unwrap().unwrap(); + // Denies /secret/** to an anonymous caller, so the blob above is withheld and + // the branch takes upload_pack_excluding rather than the plain serve. + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &[], + OWNER_DID, + ) + .await + .unwrap(); + let disk = std::path::Path::new("/tmp/z6shared2/sv.git"); + std::fs::create_dir_all(disk).unwrap(); + + let peer: SocketAddr = "203.0.113.93:7000".parse().unwrap(); + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::POST) + .uri("/z6shared2/sv/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let status = router.oneshot(req).await.unwrap().status(); + let _ = std::fs::remove_dir_all("/tmp/z6shared2"); + assert_eq!( + status, + StatusCode::GATEWAY_TIMEOUT, + "the FILTERED serve must also take the shared deadline's remainder; a \ + non-504 here means upload_pack_excluding got a fresh full budget and the \ + ~2x hold is still reachable whenever a repo withholds a blob" + ); + } + + /// #174 (F2 sizing edge, vetted by execution): the receive-pack advertisement + /// per-source cap comes from `rate_limit::per_source_push_cap`, the same helper + /// both main.rs derivation sites call, so it is never 0 even at the minimum + /// write-pool size (1). A 0 cap would make PerCallerConcurrency shed EVERY + /// receive-pack advertisement and break all pushes. + #[test] + fn advert_per_caller_cap_sizing_is_never_zero() { + let cap = crate::rate_limit::per_source_push_cap; + for pushes in [1usize, 4, 8, 32, 256] { + assert!( + cap(pushes) >= 1, + "advert cap must be >= 1 for pushes={pushes}" + ); + } + assert_eq!(cap(1), 1, "minimum write pool must derive cap 1, not 0"); + assert_eq!( + cap(32), + 4, + "default write pool 32 derives cap 4 (~8 source IPs to fill)" + ); + // A cap of 1 admits one and sheds the second from the same source. + let lim = crate::rate_limit::PerCallerConcurrency::new(cap(1), 100); + let _held = lim.try_acquire("src").expect("first advert admitted"); + assert!( + lim.try_acquire("src").is_none(), + "second advert from the same source is shed" + ); + } + #[test] fn fork_owner_full_did_with_path_rule_allowed() { // Owner reads everything (implicit reader), so nothing is withheld. @@ -2597,50 +4471,5619 @@ mod tests { ); } - /// Repo creation must be throttled by the per-IP creation limiter BEFORE - /// signature verification — otherwise a DID farm (one throwaway did:key per - /// repo, each carrying a valid but machine-solved iCaptcha proof) walks past - /// the per-DID limiter and floods the network, as in the recurring spam-repo - /// incidents. A 429 (not a 401) on an unsigned request from an exhausted IP - /// proves the IP brake runs outermost, ahead of auth. + /// #174 P2-1: an unsupported `?service=` must be rejected with 400 BEFORE taking a + /// read slot or doing DB/Tigris work. Isolate it: exhaust the read pool so a read + /// op WOULD shed 503 at the pre-DB check — a garbage service must still return 400 + /// (validation runs first), proving `?service=anything` cannot consume the read + /// pool. Removing the validation makes this 503 (RED). #[sqlx::test] - async fn repo_creation_is_rate_limited_by_ip(pool: sqlx::PgPool) { + async fn info_refs_rejects_unsupported_service_before_the_read_slot(pool: sqlx::PgPool) { use axum::body::Body; use axum::extract::ConnectInfo; use axum::http::{Method, Request, StatusCode}; use std::net::SocketAddr; - use std::time::Duration; + use std::sync::Arc; + use tokio::sync::Semaphore; use tower::ServiceExt; let mut state = crate::test_support::test_state(pool).await; - // Tiny limit, keyed on the socket peer (no trusted proxy). - state.create_ip_rate_limiter = - crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); - state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6svcowner", "svc", "/tmp/svc", None, false) + .await + .unwrap(); + // Exhaust the read pool: a read op would shed 503 at the pre-DB check. + state.git_read_semaphore = Arc::new(Semaphore::new(0)); - let peer: SocketAddr = "203.0.113.77:7000".parse().unwrap(); - // Exhaust this peer's single-request budget up front. - assert!( + let router = crate::server::build_router(state); + let peer: SocketAddr = "203.0.113.90:7000".parse().unwrap(); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6svcowner/svc/info/refs?service=git-explode") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + + let status = router.oneshot(req).await.unwrap().status(); + assert_eq!( + status, + StatusCode::BAD_REQUEST, + "an unsupported ?service= must be 400 before the read-pool shed, not 503" + ); + } + + /// #174 (jatmn P1): the anon-reachable receive-pack advertisement + /// (`GET info/refs?service=git-receive-pack`) draws from a DEDICATED advert pool + /// (`git_push_advert_semaphore`), NOT the write pool the authenticated POST uses. + /// Proven at the handler by saturating each pool to zero and checking who shares + /// it (INV-10, across the auth boundary). The load-bearing pair: + /// * advert pool at 0 -> the advert SHEDS 503 (it is bound to that pool); + /// * write pool at 0 -> the advert SURVIVES (it can NOT consume a permit the + /// authenticated POST needs — the reservation jatmn asked for). + /// Revert the branch to `git_write_semaphore` and BOTH flip: the advert-pool-0 + /// case stops shedding and the write-pool-0 case starts shedding (the exact + /// anon-sheds-authed-push starvation). + #[sqlx::test] + async fn receive_pack_advertisement_draws_from_dedicated_advert_pool(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + // Build a fresh state with the three pools sized independently, then drive one + // info/refs advertisement for `service` and return its handler status. + async fn advert_status( + pool: &sqlx::PgPool, + read_permits: usize, + write_permits: usize, + advert_permits: usize, + service: &str, + ) -> StatusCode { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.git_read_semaphore = Arc::new(Semaphore::new(read_permits)); + state.git_write_semaphore = Arc::new(Semaphore::new(write_permits)); + state.git_push_advert_semaphore = Arc::new(Semaphore::new(advert_permits)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; state - .create_ip_rate_limiter - .check(&peer.ip().to_string()) + .db + .upsert_mirror_repo("z6wpadv", "wp", "/tmp/wp-nonexistent", None, false) .await + .unwrap(); + let peer: SocketAddr = "203.0.113.61:6000".parse().unwrap(); + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/z6wpadv/wp/info/refs?service={service}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + router.oneshot(req).await.unwrap().status() + } + + // Advert pool saturated (read + write free): the receive-pack advert SHEDS, + // proving it is bound to the dedicated advert pool. + assert_eq!( + advert_status(&pool, 8, 8, 0, "git-receive-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "receive-pack advertisement draws from the dedicated advert pool: a saturated advert pool sheds it 503" + ); + // WRITE pool saturated (advert + read free): the advert SURVIVES. This is the + // reservation — an advert flood can never occupy a permit the authenticated + // push POST relies on at admission. + assert_ne!( + advert_status(&pool, 8, 0, 8, "git-receive-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "receive-pack advertisement must NOT draw from the write pool: a saturated write pool must not shed it" + ); + // Read pool saturated (advert + write free): the advert SURVIVES (never on the read pool). + assert_ne!( + advert_status(&pool, 0, 8, 8, "git-receive-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "receive-pack advertisement must not draw from the read pool" + ); + // Read pool saturated: the upload-pack advertisement still SHEDS (unchanged). + assert_eq!( + advert_status(&pool, 0, 8, 8, "git-upload-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "upload-pack advertisement stays on the read pool: a saturated read pool sheds it 503" ); + // Write + advert pools saturated, read free: the upload-pack advertisement is + // UNAFFECTED, proving reads never touch either write-side pool. + assert_ne!( + advert_status(&pool, 8, 0, 0, "git-upload-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "upload-pack advertisement never touches the write or advert pool" + ); + } + + /// #174 U2: the receive-pack advertisement is a write-path op, so it must not be + /// shed by the READ per-caller sub-cap even when the caller's source IP has + /// exhausted its read budget (e.g. concurrent clones from the same host). Fill + /// the IP's read per-caller slot, then the receive-pack advertisement from that + /// same IP must still get through. Restore the unconditional read-cap acquire on + /// the receive-pack branch and this goes 503. + #[sqlx::test] + async fn receive_pack_advertisement_ignores_read_per_caller_cap(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6wpc", "wp", "/tmp/wp-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.71:6000".parse().unwrap(); + // Exhaust the source IP's single READ per-caller slot, as concurrent clones + // from the same host would. + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("fill the IP's read per-caller slot"); let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6wpc/wp/info/refs?service=git-receive-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + assert_ne!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "receive-pack advertisement must not be shed by the read per-caller cap: it is a write-path op" + ); + } + + /// #174 (review fix): the anon-reachable receive-pack advertisement draws from its + /// own dedicated advert pool, so it is bounded per source by + /// `git_push_advert_per_caller` to stop one source from monopolizing that pool and + /// shedding other sources' advertisements. Fill one source IP's advert slot; its next receive-pack advertisement + /// sheds 503, while a different source and the upload-pack advertisement are + /// unaffected. Remove the advert-cap acquisition and the same-source assertion + /// goes green-not-503. + #[sqlx::test] + async fn receive_pack_advertisement_capped_per_source(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_push_advert_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6advcap", "ac", "/tmp/ac-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.81:6000".parse().unwrap(); + // Fill this source IP's single receive-pack-advertisement slot. + let _slot = state + .git_push_advert_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first advert slot for this source IP"); + + // Same source: the receive-pack advertisement sheds 503 (advert cap full). + let router = crate::server::build_router(state.clone()); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6advcap/ac/info/refs?service=git-receive-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + assert_eq!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its receive-pack advertisement cap must shed 503, so it cannot monopolize the advert pool" + ); + + // A DIFFERENT source keeps its own advert budget -> not shed. + let other: SocketAddr = "203.0.113.82:6000".parse().unwrap(); + let router2 = crate::server::build_router(state.clone()); + let mut req2 = Request::builder() + .method(Method::GET) + .uri("/z6advcap/ac/info/refs?service=git-receive-pack") + .body(Body::empty()) + .unwrap(); + req2.extensions_mut().insert(ConnectInfo(other)); + assert_ne!( + router2.oneshot(req2).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a different source must keep its own receive-pack advertisement budget" + ); + + // The upload-pack advertisement is NOT bounded by the receive-pack advert cap. + let router3 = crate::server::build_router(state); + let mut req3 = Request::builder() + .method(Method::GET) + .uri("/z6advcap/ac/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req3.extensions_mut().insert(ConnectInfo(peer)); + assert_ne!( + router3.oneshot(req3).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "the upload-pack advertisement must not be shed by the receive-pack advert cap" + ); + } + + /// #174 SC2 (info_refs probe): the per-caller read sub-cap sheds a caller that + /// is already at its concurrency budget on the upload-pack advertisement, while + /// a DIFFERENT caller still enters. Remove the sub-cap from `git_info_refs` and + /// the same-caller assertion goes green-not-503 — this is the info_refs half of + /// the two-handler mutation probe. + #[sqlx::test] + async fn info_refs_per_caller_cap_sheds_one_caller_not_others(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6pcadv", "pc", "/tmp/pc-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.31:5000".parse().unwrap(); + // Fill this caller's single read slot (a clone shares the Arc-backed map). + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first slot for this caller"); + + // Same caller (IP) at its cap -> shed 503 before the git/Tigris work. + let router = crate::server::build_router(state.clone()); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6pcadv/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + assert_eq!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a caller already at its per-caller read cap must shed the advertisement with 503" + ); + + // A DIFFERENT caller (IP) has its own budget -> not shed by the per-caller cap. + let other: SocketAddr = "203.0.113.32:5000".parse().unwrap(); + let router2 = crate::server::build_router(state.clone()); + let mut req2 = Request::builder() + .method(Method::GET) + .uri("/z6pcadv/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req2.extensions_mut().insert(ConnectInfo(other)); + assert_ne!( + router2.oneshot(req2).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a different caller must not be shed by another caller's saturated budget" + ); + } + + /// #174 SC2 (upload_pack probe): the same per-caller shed on the POST + /// upload-pack path. Remove the sub-cap from `git_upload_pack` and this goes + /// green-not-503 — the upload_pack half of the two-handler mutation probe. + #[sqlx::test] + async fn upload_pack_per_caller_cap_sheds_one_caller_not_others(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6pcupl", "pc", "/tmp/pc-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.41:5000".parse().unwrap(); + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first slot for this caller"); + + let router = crate::server::build_router(state.clone()); let mut req = Request::builder() .method(Method::POST) - .uri("/api/v1/repos") - .header("content-type", "application/json") - .body(Body::from(r#"{"name":"flood","is_public":true}"#)) + .uri("/z6pcupl/pc/git-upload-pack") + .body(Body::from(&b"0000"[..])) .unwrap(); req.extensions_mut().insert(ConnectInfo(peer)); + assert_eq!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a caller already at its per-caller read cap must shed upload-pack with 503" + ); - let status = router.oneshot(req).await.unwrap().status(); + let other: SocketAddr = "203.0.113.42:5000".parse().unwrap(); + let router2 = crate::server::build_router(state.clone()); + let mut req2 = Request::builder() + .method(Method::POST) + .uri("/z6pcupl/pc/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + req2.extensions_mut().insert(ConnectInfo(other)); + assert_ne!( + router2.oneshot(req2).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a different caller must not be shed by another caller's saturated budget" + ); + } + + /// #174 (review fix): the per-source caller cap is an independent brake that + /// sheds a capped source even when the global pool has free capacity — the + /// sub-cap is not a mere pre-filter for pool exhaustion. Proven by leaving the + /// global read pool with capacity (so the pre-DB early shed passes) AND + /// pre-holding the source's upload-pack read sub-cap: the request reaches the + /// caller cap and sheds there, so its 503 body reads "for this caller". Remove + /// the `acquire_read_caller_permit` call and the capped source falls through to + /// the git op instead of shedding with "for this caller" — this is the + /// caller-cap acquire probe for the info/refs upload-pack branch. + #[sqlx::test] + async fn info_refs_upload_pack_per_source_cap_sheds_with_global_capacity(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + // Global read pool has free capacity (early shed passes); source pre-held at + // its per-caller cap so it sheds on the caller cap, not the global pool. + state.git_read_semaphore = Arc::new(Semaphore::new(4)); + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6ordir", "oi", "/tmp/oi-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.91:5000".parse().unwrap(); + // Pin this source at its single upload-pack read slot. + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first read slot for this source IP"); + + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6ordir/oi/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let resp = router.oneshot(req).await.unwrap(); assert_eq!( - status, - StatusCode::TOO_MANY_REQUESTS, - "repo creation must be IP-throttled before signature verification" + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its read sub-cap must shed 503 even with global pool capacity" + ); + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .expect("body bytes"); + let body = String::from_utf8_lossy(&bytes); + assert!( + body.contains("for this caller"), + "the per-source cap is an independent brake: with global capacity free, the capped source must still shed with the caller-cap body, got {body}" + ); + } + + /// #174 (review fix): same independent-brake guarantee for the receive-pack + /// advertisement branch of info/refs — its per-source cap + /// (`git_push_advert_per_caller`) sheds a capped source even when the global + /// write pool has capacity. Leave the global write pool with capacity (so the + /// pre-DB early shed passes) and pre-hold the source's advert slot: the request + /// reaches the caller cap, so the 503 body reads "for this caller". Remove the + /// caller-cap acquire and the capped source falls through instead of shedding + /// with "for this caller". The push rate limiter is left permissive so the + /// request reaches the caller cap. + #[sqlx::test] + async fn info_refs_receive_pack_per_source_cap_sheds_with_global_capacity(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + // Global write pool has free capacity (early shed passes); source pre-held at + // its advert sub-cap so it sheds on the caller cap, not the global pool. + state.git_write_semaphore = Arc::new(Semaphore::new(4)); + state.git_push_advert_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + // Permissive push rate limiter so the advertisement passes the rate gate and + // reaches the per-source concurrency cap. + state.push_rate_limiter = crate::rate_limit::RateLimiter::new(100, Duration::from_secs(60)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6ordrp", "or", "/tmp/or-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.92:5000".parse().unwrap(); + // Pin this source at its single receive-pack advertisement slot. + let _slot = state + .git_push_advert_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first advert slot for this source IP"); + + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6ordrp/or/info/refs?service=git-receive-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its advert sub-cap must shed 503 even with global write pool capacity" + ); + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .expect("body bytes"); + let body = String::from_utf8_lossy(&bytes); + assert!( + body.contains("for this caller"), + "the per-source advert cap is an independent brake: with global write capacity free, the capped source must still shed with the caller-cap body, got {body}" + ); + } + + /// #174 (review fix): same independent-brake guarantee for the POST upload-pack + /// handler — its per-source read cap sheds a capped source even when the global + /// read pool has capacity. Leave the global read pool with capacity (so the + /// pre-DB early shed passes) and pre-hold the source's read slot: the request + /// reaches the caller cap, so the 503 body reads "for this caller". Remove the + /// caller-cap acquire and the capped source falls through instead of shedding + /// with "for this caller". + #[sqlx::test] + async fn upload_pack_per_source_cap_sheds_with_global_capacity(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + // Global read pool has free capacity (early shed passes); source pre-held at + // its per-caller cap so it sheds on the caller cap, not the global pool. + state.git_read_semaphore = Arc::new(Semaphore::new(4)); + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6ordup", "ou", "/tmp/ou-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.93:5000".parse().unwrap(); + // Pin this source at its single read slot. + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first read slot for this source IP"); + + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::POST) + .uri("/z6ordup/ou/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its read sub-cap must shed 503 even with global pool capacity" + ); + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .expect("body bytes"); + let body = String::from_utf8_lossy(&bytes); + assert!( + body.contains("for this caller"), + "the per-source cap is an independent brake: with global capacity free, the capped source must still shed with the caller-cap body, got {body}" + ); + } + + /// #174 U3 (P1-b, RED-before/GREEN-after): a client disconnect during the + /// path-scoped withheld-blob walk must NOT release the read admission while the + /// uncancellable `spawn_blocking` walk is still running. The handler takes the + /// global read permit, enters the walk (a fake git hangs on rev-list), then the + /// request future is dropped mid-walk. With both permits moved into the blocking + /// task the global slot stays occupied until the walk finishes; on the pre-fix code + /// the handler-local permits drop on future-drop and the slot frees instantly (RED), + /// letting disconnect-spam exceed the cap while real git work keeps running. + #[sqlx::test] + async fn upload_pack_permit_held_through_walk_after_disconnect(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + let revlist_pid = tmp.path().join("revlist.pid"); + // Fake git: resolve refs fast, hang on rev-list (recording its pid first). The + // ~6s sleep bounds the walk so a broken fix cannot wedge the suite. + let body = format!( + "#!/bin/sh\ncase \"$1\" in\n rev-list) echo $$ > \"{}\" ; sleep 6 ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n", + revlist_pid.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + // Root the repo store at this test's TempDir so the bare repo is isolated per + // run (the default for_testing store uses a fixed /tmp path that would collide + // across runs). + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.git_read_semaphore = Arc::new(Semaphore::new(1)); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let owner = "z6up3rd"; + let name = "up3"; + state + .db + .upsert_mirror_repo(owner, name, "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + // Real bare repo at the path acquire() computes, so the handler reaches the walk. + state + .repo_store + .init(&rec.owner_did, &rec.name) + .await + .unwrap(); + // A path-scoped rule so has_path_scoped_rule() is true (the walk path) without + // denying the "/" gate for the public repo. + state + .db + .set_visibility_rule( + &rec.id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3ReaderAAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + + let sem = state.git_read_semaphore.clone(); + assert_eq!( + sem.available_permits(), + 1, + "one read slot before the request" + ); + + let router = crate::server::build_router(state); + let peer: SocketAddr = "203.0.113.77:5000".parse().unwrap(); + let mut req = Request::builder() + .method(Method::POST) + .uri(format!("/{owner}/{name}/git-upload-pack")) + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + + let mut fut = Box::pin(router.oneshot(req)); + // Drive until the walk's rev-list starts (its pidfile appears) — i.e. the + // request is inside the spawn_blocking walk, holding the global read permit. + let mut in_walk = false; + for _ in 0..500 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if revlist_pid.exists() { + in_walk = true; + break; + } + } + assert!( + in_walk, + "the walk's rev-list must start (request reached the spawn_blocking walk)" + ); + assert_eq!( + sem.available_permits(), + 0, + "the read slot is held while the walk runs" + ); + + // Client disconnect: drop the request future mid-walk. + drop(fut); + + // Load-bearing: the slot must STAY held while the uncancellable walk runs. On + // the pre-fix code the handler-local permits drop here and the slot frees at + // once (RED). + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + assert_eq!( + sem.available_permits(), + 0, + "on disconnect the read admission must be held until the spawn_blocking walk \ + finishes, not released the instant the future drops (P1-b)" + ); + + // Cleanup: let the walk finish so the slot releases and no blocking task leaks. + for _ in 0..400 { + if sem.available_permits() == 1 { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + if let Some(p) = std::fs::read_to_string(&revlist_pid) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + unsafe { + libc::kill(p, libc::SIGKILL); + } + } + } + + /// #174 U1 (P1-a, plain-spawn residual, RED-before/GREEN-after): on the PLAIN + /// (non-path-scoped) upload-pack path a client disconnect must NOT release the + /// global read admission while the detached process-group reaper is still tearing + /// down a git group that ignores SIGTERM. The `be0cdd6` fix moved permits into the + /// path-scoped `spawn_blocking` walk; this closes the residual plain path, where the + /// permits were handler-locals that dropped the instant the future was dropped. + /// + /// Isolate the GLOBAL pool: read pool = 1, per-source cap + rate limiter permissive, + /// so the only thing that can shed a replacement is the leaked global permit. Drive + /// the handler until git spawns, disconnect, then assert the global slot stays held + /// (`available_permits() == 0`) AND a replacement sheds 503 while the group is alive; + /// after the reaper SIGKILLs+reaps the group the slot frees and a replacement is no + /// longer shed by the global cap. On the pre-fix code the handler-local permit drops + /// on future-drop and the slot frees at once (RED). + #[cfg(unix)] + #[sqlx::test] + async fn upload_pack_plain_permit_held_through_group_reap_after_disconnect(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + let descfile = tmp.path().join("desc.pid"); + // Fake git for the plain upload-pack path (invoked as `git upload-pack + // --stateless-rpc `). It forks a descendant that TRAPS SIGTERM, records its + // pid, and loops ~20s, then `wait`s — so on disconnect the group leader dies on + // the reaper's SIGTERM but the descendant survives until the reaper escalates to + // SIGKILL, keeping the group alive (ESRCH not reached) across the observation + // window. Bounded so a broken fix leaks no permanent orphan. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + upload-pack)\n\ + sh -c 'trap \"\" TERM; echo $$ > \"{}\"; i=0; while [ $i -lt 20 ]; do sleep 1; i=$((i+1)); done' &\n\ + wait ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + descfile.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + // Isolate the global read pool: size 1; per-source cap + rate limiter permissive + // so only the leaked global permit can shed the replacement. + state.git_read_semaphore = Arc::new(Semaphore::new(1)); + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let owner = "z6up1st"; + let name = "up1"; + state + .db + .upsert_mirror_repo(owner, name, "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + // Real bare repo at the path acquire() computes, so the handler reaches the + // spawn. No path-scoped rule -> the PLAIN serve branch (this test's target). + state + .repo_store + .init(&rec.owner_did, &rec.name) + .await + .unwrap(); + + let sem = state.git_read_semaphore.clone(); + assert_eq!( + sem.available_permits(), + 1, + "one read slot before the request" + ); + + let router = crate::server::build_router(state); + let make_req = |peer: SocketAddr| { + let mut req = Request::builder() + .method(Method::POST) + .uri(format!("/{owner}/{name}/git-upload-pack")) + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req + }; + + let peer: SocketAddr = "203.0.113.71:5000".parse().unwrap(); + let mut fut = Box::pin(router.clone().oneshot(make_req(peer))); + // Drive until git spawns (the descendant records its pid) — the request is + // inside the plain serve, holding the global read permit. Stop polling the + // instant the future completes (re-polling a completed oneshot panics); read the + // descfile first so a spawn that recorded its pid then returned is still caught. + let mut spawned: Option = None; + let mut early = None; + for _ in 0..500 { + let done = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&descfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + spawned = Some(p); + break; + } + if let Ok(resp) = done { + early = Some(resp.map(|r| r.status())); + break; + } + } + let desc = spawned + .unwrap_or_else(|| panic!("the fake git must have spawned; early finish: {early:?}")); + // Kill the descendant regardless of outcome so a RED run leaks no orphan. + struct ReapOnDrop(i32); + impl Drop for ReapOnDrop { + fn drop(&mut self) { + unsafe { + libc::kill(self.0, libc::SIGKILL); + } + } + } + let _cleanup = ReapOnDrop(desc); + assert!( + unsafe { libc::kill(desc, 0) == 0 }, + "descendant should be running before the disconnect" + ); + assert_eq!( + sem.available_permits(), + 0, + "the read slot is held while the git op runs" + ); + + // Client disconnect: drop the request future. The detached reaper now owns the + // AdmissionGuard and will not drop it until the group is ESRCH-confirmed reaped. + drop(fut); + + // Load-bearing: the slot must STAY held while the SIGTERM-ignoring group is still + // alive. On the pre-fix code the handler-local permit drops here and the slot + // frees at once (RED). Check quickly (before the reaper's ~2s SIGKILL escalation). + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + assert!( + unsafe { libc::kill(desc, 0) == 0 }, + "the SIGTERM-ignoring descendant must still be alive during the hold window" + ); + assert_eq!( + sem.available_permits(), + 0, + "on disconnect the read admission must be HELD until the process group is \ + reaped, not released the instant the future drops (P1-a)" + ); + // A replacement request from a DIFFERENT source must shed 503 — the only pool + // that can shed it is the leaked global permit (per-source cap is permissive). + let peer2: SocketAddr = "203.0.113.72:5000".parse().unwrap(); + let resp = router.clone().oneshot(make_req(peer2)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "while the prior group is still alive the held global permit must shed a \ + replacement with 503" + ); + + // After the reaper SIGKILLs + reaps the group the AdmissionGuard drops and the + // slot frees. Poll for recovery. + let mut freed = false; + for _ in 0..400 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + assert!( + freed, + "once the reaper confirms the group gone the admission guard must drop and \ + free the global slot" + ); + // A replacement is now no longer shed by the global cap (it proceeds past + // admission; it then fails downstream on the fake git, which is not a 503). + let peer3: SocketAddr = "203.0.113.73:5000".parse().unwrap(); + let resp = router.oneshot(make_req(peer3)).await.unwrap(); + assert_ne!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "after the group is reaped the freed slot must admit a replacement" + ); + } + + /// F1 (filtered-serve residual of #174 P1-a, RED-before/GREEN-after): on the + /// FILTERED (path-scoped, non-empty withheld) upload-pack branch a client + /// disconnect mid-pack-objects must NOT release the read admission while the + /// detached reaper is still tearing down the git group. Pre-fix the handler held + /// the permits as locals (`_hold`) across `upload_pack_excluding`, so dropping + /// the future released them instantly and disconnect-spam could exceed the read + /// caps during each reap window (RED). The fix threads the AdmissionGuard through + /// both filtered-pack stages so it rides `KillGroupOnDrop` into the reaper. + /// + /// Same isolation as the plain-path test above: read pool = 1, per-source cap + /// permissive, so only the global permit can shed a replacement. The fake git + /// serves the withheld walk (for-each-ref/rev-parse/rev-list/ls-tree) with a blob + /// under the denied `/src/**` subtree so the filtered branch is taken, answers + /// the pack build's rev-list fast, and hangs pack-objects in a SIGTERM-trapping + /// descendant. The descendant hang is first-invocation-only (keyed on the + /// pidfile's existence) so the post-reap replacement request completes fast. + #[cfg(unix)] + #[sqlx::test] + async fn upload_pack_filtered_permit_held_through_group_reap_after_disconnect( + pool: sqlx::PgPool, + ) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + let descfile = tmp.path().join("desc.pid"); + let commit = "1111111111111111111111111111111111111111"; + let blob = "2222222222222222222222222222222222222222"; + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + rev-parse) echo {commit} ;;\n\ + rev-list) echo {commit} ;;\n\ + ls-tree) printf '100644 blob {blob}\\tsrc/x.txt' ;;\n\ + pack-objects)\n\ + if [ ! -e \"{desc}\" ]; then\n\ + sh -c 'trap \"\" TERM; echo $$ > \"{desc}\"; i=0; while [ $i -lt 20 ]; do sleep 1; i=$((i+1)); done' &\n\ + wait\n\ + fi ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + desc = descfile.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + // Isolate the global read pool: size 1; per-source cap + rate limiter permissive + // so only the leaked global permit can shed the replacement. + state.git_read_semaphore = Arc::new(Semaphore::new(1)); + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let owner = "z6upf1"; + let name = "upf"; + state + .db + .upsert_mirror_repo(owner, name, "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + // Real bare repo at the path acquire() computes, so the handler reaches the + // walk and the filtered serve. + state + .repo_store + .init(&rec.owner_did, &rec.name) + .await + .unwrap(); + // Path-scoped rule denying the anonymous caller under /src, matching the + // fake ls-tree's blob path, so the withheld set is NON-EMPTY and the + // filtered (upload_pack_excluding) branch is taken. + state + .db + .set_visibility_rule( + &rec.id, + "/src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkUF1ReaderAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + + let sem = state.git_read_semaphore.clone(); + assert_eq!( + sem.available_permits(), + 1, + "one read slot before the request" + ); + + let router = crate::server::build_router(state); + let make_req = |peer: SocketAddr| { + let mut req = Request::builder() + .method(Method::POST) + .uri(format!("/{owner}/{name}/git-upload-pack")) + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req + }; + + let peer: SocketAddr = "203.0.113.81:5000".parse().unwrap(); + let mut fut = Box::pin(router.clone().oneshot(make_req(peer))); + // Drive until the pack-objects descendant records its pid: the request is + // past the walk, inside the filtered serve's stage 2, holding the read permit. + // Stop polling the instant the future completes (re-polling a completed + // oneshot panics); read the descfile first so a spawn that recorded its pid + // then returned is still caught. + let mut spawned: Option = None; + let mut early = None; + for _ in 0..500 { + let done = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&descfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + spawned = Some(p); + break; + } + if let Ok(resp) = done { + early = Some(resp.map(|r| r.status())); + break; + } + } + let desc = spawned.unwrap_or_else(|| { + panic!("the fake pack-objects must have spawned; early finish: {early:?}") + }); + // Kill the descendant regardless of outcome so a RED run leaks no orphan. + struct ReapOnDrop(i32); + impl Drop for ReapOnDrop { + fn drop(&mut self) { + unsafe { + libc::kill(self.0, libc::SIGKILL); + } + } + } + let _cleanup = ReapOnDrop(desc); + assert!( + unsafe { libc::kill(desc, 0) == 0 }, + "descendant should be running before the disconnect" + ); + assert_eq!( + sem.available_permits(), + 0, + "the read slot is held while the filtered serve runs" + ); + + // Client disconnect: drop the request future mid-pack-objects. The detached + // reaper must now own the AdmissionGuard and hold it until ESRCH. + drop(fut); + + // Load-bearing: the slot must STAY held while the SIGTERM-ignoring group is + // still alive. On the pre-fix code the handler-local `_hold` drops here and + // the slot frees at once (RED). Check quickly (before the reaper's ~2s + // SIGKILL escalation). + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + assert!( + unsafe { libc::kill(desc, 0) == 0 }, + "the SIGTERM-ignoring descendant must still be alive during the hold window" + ); + assert_eq!( + sem.available_permits(), + 0, + "on disconnect the read admission must be HELD until the filtered serve's \ + process group is reaped, not released the instant the future drops (F1)" + ); + // A replacement request from a DIFFERENT source must shed 503: the only pool + // that can shed it is the held global permit (per-source cap is permissive). + let peer2: SocketAddr = "203.0.113.82:5000".parse().unwrap(); + let resp = router.clone().oneshot(make_req(peer2)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "while the prior group is still alive the held global permit must shed a \ + replacement with 503" + ); + + // After the reaper SIGKILLs + reaps the group the AdmissionGuard drops and + // the slot frees. Poll for recovery. + let mut freed = false; + for _ in 0..400 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + assert!( + freed, + "once the reaper confirms the group gone the admission guard must drop and \ + free the global slot" + ); + // A replacement is now admitted and completes: the fake pack-objects takes + // its fast path (the descfile exists), so the filtered serve returns instead + // of hanging. + let peer3: SocketAddr = "203.0.113.83:5000".parse().unwrap(); + let resp = router.oneshot(make_req(peer3)).await.unwrap(); + assert_ne!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "after the group is reaped the freed slot must admit a replacement" + ); + } + + /// #174 U1 (P1-a): the `None`-key arm — a request with no resolvable source key + /// (no trusted-proxy header, no peer) is bounded by the GLOBAL read pool only, never + /// a per-source cap. With the global read pool exhausted such a request still sheds + /// 503, proving the plain path admits/sheds on the global pool for the `None` arm + /// (the counterpart to the `Some(ip)` arm above). Complements the resolver-arm rule: + /// neither arm is vacuous. + #[tokio::test] + async fn upload_pack_plain_none_key_arm_sheds_on_global_pool() { + use axum::body::Body; + use axum::http::{Method, Request, StatusCode}; + use axum::Router; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state_lazy(); + // Global read pool exhausted; per-source cap permissive so only the global pool + // can shed. No ConnectInfo + no trusted header -> read_caller_key resolves None. + state.git_read_semaphore = Arc::new(Semaphore::new(0)); + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let router = Router::new() + .route( + "/{owner}/{repo}/git-upload-pack", + axum::routing::post(crate::api::repos::git_upload_pack), + ) + .with_state(state); + // No ConnectInfo extension and no XFF header: the caller key is None. + let req = Request::builder() + .method(Method::POST) + .uri("/alice/repo.git/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a None-key request must still shed 503 on the exhausted GLOBAL read pool" + ); + } + + /// #174 U4 (P1-d, RED-before/GREEN-after): the authenticated receive-pack POST + /// carries a per-source WRITE sub-cap so one source IP cannot monopolize the write + /// pool with many slow pushes (owner enforcement defaults off, so disposable DIDs + /// are free). Global write pool has capacity; the source is pre-held at its single + /// write slot. A push from THAT source sheds (Overloaded/503) — which also proves + /// the PeerAddr+HeaderMap extractors resolve a key (without them the key is None and + /// the cap is inert, never shedding). A push from a DIFFERENT source is NOT shed by + /// the cap. Called directly so the test needs no signed request; the handler is + /// where the cap lives. Remove the `git_write_per_caller` acquire and the capped + /// source no longer sheds (RED). + #[sqlx::test] + async fn receive_pack_per_source_write_cap_sheds_capped_source_not_others(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + let mut state = crate::test_support::test_state(pool).await; + // Global write pool has capacity; the per-source cap is 1. + state.git_write_semaphore = Arc::new(Semaphore::new(4)); + state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6rp4wr", "rp4", "/tmp/rp4-nonexistent", None, false) + .await + .unwrap(); + + let did = "did:key:z6MkReceivePackWriteCapProofDidAAAAAAAAAA"; + let capped: SocketAddr = "203.0.113.44:5000".parse().unwrap(); + let other: SocketAddr = "203.0.113.45:5000".parse().unwrap(); + + // Pin the capped source at its single write slot. + let _slot = state + .git_write_per_caller + .try_acquire(&capped.ip().to_string()) + .expect("first write slot for the capped source IP"); + + // A push from the capped source must shed on the per-source write cap even with + // global write capacity free. The shed also proves the source-IP key resolved + // via the extractors (an inert None key would fall through to Ok(None)). + let capped_result = git_receive_pack( + State(state.clone()), + Path(("z6rp4wr".to_string(), "rp4".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(capped)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await; + assert!( + matches!(capped_result, Err(AppError::Overloaded(_))), + "a source at its per-source write cap must shed (Overloaded/503) with global \ + pool capacity free; got {capped_result:?}" + ); + + // A push from a DIFFERENT source must NOT be shed by the per-source cap — it + // proceeds past admission (and fails later on the nonexistent repo, which is not + // an Overloaded error). + let other_result = git_receive_pack( + State(state.clone()), + Path(("z6rp4wr".to_string(), "rp4".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(other)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await; + assert!( + !matches!(other_result, Err(AppError::Overloaded(_))), + "a different source must not be shed by the per-source write cap while the \ + capped source holds its slot; got {other_result:?}" + ); + } + + /// #174 U2 (P1-2, RED-before/GREEN-after): the storage-acquisition phase is bounded + /// by `git_acquire_timeout_secs`, so a stalled backend releases the admission permit + /// and sheds a 503 instead of pinning the pool. The permit is taken BEFORE + /// `acquire_write`, whose advisory-lock loop can spin ~60s (and whose per-iteration + /// `pg_try_advisory_lock` can block indefinitely on a hung pool), so without the + /// `tokio::time::timeout` wrapper the permit is held far past the deadline. + /// + /// Real stall (no `RepoStore` trait to fake): hold the SAME session-level advisory + /// lock `acquire_write` derives (`advisory_lock_key(owner_slug, repo_name)`, where + /// `owner_slug = owner_did.replace([':','/'], "_")`) on a second pooled connection, + /// so the handler's `pg_try_advisory_lock` returns false every iteration and the loop + /// must retry against the deadline. `git_acquire_timeout_secs = 2`; the request must + /// return 503 (Overloaded) at ~2s (NOT ~59s), and the write permit must be released + /// (`available_permits()` recovers to full once the shed returns). Covers R2. + /// + /// Load-bearing / mutation: remove the `tokio::time::timeout` wrapper on + /// `acquire_write` and the loop runs to ~59s with the permit held the whole time — + /// the `< DEADLINE_CEILING` timing assertion goes RED (observed ~59s) and the permit + /// stays pinned past the deadline. Restore to return GREEN. + #[sqlx::test] + async fn receive_pack_acquire_deadline_sheds_and_releases_permit(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + // Reproduce acquire_write's session-level advisory-lock key exactly so the + // second-connection lock collides with the handler's pg_try_advisory_lock + // (repo_store.rs: advisory_lock_key over owner_slug then repo_name). + fn advisory_lock_key(owner_slug: &str, repo_name: &str) -> i64 { + use std::hash::{Hash, Hasher}; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + owner_slug.hash(&mut hasher); + repo_name.hash(&mut hasher); + hasher.finish() as i64 + } + + let owner = "z6acqdead"; + let name = "acq1"; + // owner_slug as local_path() computes it from the record's owner_did. The + // mirror row stores the short owner as owner_did, so slug == owner (no ':'/'/'). + let owner_slug = owner.replace([':', '/'], "_"); + let lock_key = advisory_lock_key(&owner_slug, name); + + let mut state = crate::test_support::test_state(pool.clone()).await; + // Isolate the write pool at size 1 so available_permits() cleanly reports + // held (0) vs released (1). Per-source cap + trust permissive so only the + // write pool / acquire path can gate. + state.git_write_semaphore = Arc::new(Semaphore::new(1)); + state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Short acquire deadline: the fix must shed here, well before acquire_write's + // ~59s advisory-lock loop would bail on its own. + const ACQUIRE_TIMEOUT_SECS: u64 = 2; + let mut cfg = (*state.config).clone(); + cfg.git_acquire_timeout_secs = ACQUIRE_TIMEOUT_SECS; + // Keep the git-service timeout large so the deadline under test is the acquire + // one, not git execution (which is never reached on the stalled path anyway). + cfg.git_service_timeout_secs = 600; + state.config = std::sync::Arc::new(cfg); + + state + .db + .upsert_mirror_repo(owner, name, "/tmp/z6acqdead-acq1", None, false) + .await + .unwrap(); + + // Hold the advisory lock on a dedicated pooled connection (a distinct session), + // so the handler's pg_try_advisory_lock($lock_key) returns false every iteration + // and acquire_write's real loop must retry against the deadline. Released when + // this connection drops at end of test. + let mut lock_conn = pool + .acquire() + .await + .expect("second connection for the lock"); + sqlx::query("SELECT pg_advisory_lock($1)") + .bind(lock_key) + .execute(&mut *lock_conn) + .await + .expect("hold the advisory lock on the second connection"); + + let did = "did:key:z6MkAcquireDeadlineProofDidAAAAAAAAAAAAAAAA"; + let peer: SocketAddr = "203.0.113.61:5000".parse().unwrap(); + + let sem = state.git_write_semaphore.clone(); + assert_eq!( + sem.available_permits(), + 1, + "one write slot before the request" + ); + + // Drive the authenticated push in the background so we can observe the permit is + // held while acquire_write stalls, then that it is released on the shed. + let state_for_task = state.clone(); + let start = std::time::Instant::now(); + let handle = tokio::spawn(async move { + git_receive_pack( + State(state_for_task), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(peer)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await + }); + + // The handler takes the write permit BEFORE acquire_write, so once it is stalled + // in the advisory-lock loop the pool reports 0 available. Wait for that to prove + // the permit is genuinely held during the stall (and the request really reached + // acquire_write, not an earlier reject). + let mut held = false; + for _ in 0..200 { + if sem.available_permits() == 0 { + held = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert!( + held, + "the write permit must be held while acquire_write stalls on the advisory lock" + ); + + // The bounded acquire deadline must shed with 503 (Overloaded), NOT wait out the + // ~59s advisory-lock loop. Ceiling is comfortably above the 2s deadline + task + // scheduling but far below 59s, so a RED run (no wrapper -> ~59s) fails here. + const DEADLINE_CEILING: std::time::Duration = std::time::Duration::from_secs(20); + let result = tokio::time::timeout( + DEADLINE_CEILING + std::time::Duration::from_secs(10), + handle, + ) + .await + .expect("the handler must return within the ceiling — a hang means the acquire deadline is missing (RED)") + .expect("the receive-pack task must not panic"); + let elapsed = start.elapsed(); + + assert!( + matches!(result, Err(AppError::Overloaded(_))), + "a stalled acquire_write must shed with Overloaded/503 at the acquire deadline; \ + got {result:?}" + ); + assert!( + elapsed < DEADLINE_CEILING, + "the shed must land at ~{ACQUIRE_TIMEOUT_SECS}s (the acquire deadline), not ~59s \ + (the advisory-lock loop). Observed {elapsed:?}; without the timeout wrapper this \ + is ~59s (RED)" + ); + + // Permit release on expiry: the Overloaded return drops the handler-local permit, + // so the isolated write pool must recover to full. A leaked permit here means the + // pool drains under a stalled backend (the #174 P1-2 bug). + let mut freed = false; + for _ in 0..200 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert!( + freed, + "on the acquire-deadline shed the write permit must be released; the pool did \ + not recover to full (permit leaked)" + ); + + // Follow-up admits once the contended lock is released: release the second-conn + // lock, then a fresh push proceeds PAST admission (it fails later on the + // nonexistent on-disk repo, which is NOT an Overloaded/503). Proves the freed + // slot is usable, not merely counted. + sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(lock_key) + .execute(&mut *lock_conn) + .await + .expect("release the advisory lock"); + // The follow-up asserts an ADMISSION property (the freed write slot is usable), + // so it must not inherit the 2s acquire deadline the shed above is testing. That + // budget covers a real advisory-lock round trip, and on a loaded machine it + // expires and returns the same `Overloaded` this assertion reads as a drained + // pool: a correct run going red for a reason the test is not about. + let mut followup_cfg = (*state.config).clone(); + followup_cfg.git_acquire_timeout_secs = 120; + state.config = std::sync::Arc::new(followup_cfg); + let followup = git_receive_pack( + State(state.clone()), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some("203.0.113.62:5000".parse().unwrap())), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await; + assert!( + !matches!(followup, Err(AppError::Overloaded(_))), + "once the lock frees, a follow-up push must admit past the (recovered) write \ + pool and acquire; got {followup:?}" + ); + } + + /// Reproduce `repo_store::advisory_lock_key` (private there) so a test can probe the + /// exact key `acquire_write` derives. + #[cfg(unix)] + fn write_lock_key(owner_slug: &str, repo_name: &str) -> i64 { + use std::hash::{Hash, Hasher}; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + owner_slug.hash(&mut hasher); + repo_name.hash(&mut hasher); + hasher.finish() as i64 + } + + #[cfg(unix)] + fn pid_alive(pid: i32) -> bool { + // SAFETY: kill(2) with signal 0 only probes; it takes integers and borrows no + // Rust memory. + unsafe { libc::kill(pid, 0) == 0 } + } + + /// SIGKILL the recorded pids if the test unwinds, so a RED run leaks no orphan. + #[cfg(unix)] + struct KillOnPanic(Vec); + #[cfg(unix)] + impl Drop for KillOnPanic { + fn drop(&mut self) { + for pid in &self.0 { + // SAFETY: as above. + unsafe { + libc::kill(*pid, libc::SIGKILL); + } + } + } + } + + /// Is the repo write lock takeable from an INDEPENDENT session right now? Session + /// advisory locks are re-entrant within their own session, so this must not run on + /// any connection the code under test might be using. + #[cfg(unix)] + async fn write_lock_is_takeable(pool: &sqlx::PgPool, key: i64) -> bool { + let mut probe = pool.acquire().await.expect("probe connection"); + let taken: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *probe) + .await + .expect("probe try-lock"); + if taken.0 { + sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *probe) + .await + .expect("probe unlock"); + } + taken.0 + } + + /// #173 F2 (RED-before/GREEN-after): on a CLIENT DISCONNECT the repo write lock must + /// stay held until the receive-pack process group is confirmed reaped. + /// + /// The handler's `guard.release(..)` line is only reached if `receive_pack` returns. + /// When the request future is dropped mid-push the guard drops instead, and (since + /// #173 U1 gave the lock pool an `after_release` hook) that FREES the advisory lock + /// immediately, while `KillGroupOnDrop`'s detached reaper is still giving the group + /// its ~2s SIGTERM grace. A second `acquire_write` admitted inside that window puts + /// two `git receive-pack` groups on one repo. `smart_http.rs` states the invariant + /// the other way round on the timeout path: "a caller releasing a write lock can't + /// race them". + /// + /// Real seam, not a stand-in: the production `git_receive_pack` handler, a fake git + /// whose descendant IGNORES SIGTERM (so the group genuinely survives the grace and + /// the window is ~2s wide, not a scheduling artifact), and the lock probed from an + /// independent session. RED before the fix: the lock is takeable while the group is + /// still alive. GREEN after: takeable only once the group is gone. + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_disconnect_holds_the_write_lock_until_the_group_is_reaped( + pool: sqlx::PgPool, + ) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let owner = "z6disc"; + let name = "dc1"; + let repos_dir = tempfile::TempDir::new().unwrap(); + let tmp = tempfile::TempDir::new().unwrap(); + let descfile = tmp.path().join("desc.pid"); + // The leader dies on the group SIGTERM; its descendant traps SIGTERM and loops + // (bounded at ~30s so a RED run leaks nothing permanent), so the group is only + // gone once the reaper escalates to SIGKILL. The descendant inherits the stdout + // pipe, which keeps drive_git_child's read_to_end pending until we drop. + let body = format!( + "#!/bin/sh\n\ + sh -c 'trap \"\" TERM; echo $$ > \"{}\"; i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done' &\n\ + wait\n", + descfile.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + + let mut state = crate::test_support::test_state(pool.clone()).await; + state.git_bin = git_bin; + state.repo_store = crate::git::repo_store::RepoStore::new( + repos_dir.path().to_path_buf(), + None, + crate::git::repo_store::build_lock_pool(&pool, 4, std::time::Duration::from_secs(5)), + ); + let mut cfg = (*state.config).clone(); + // Long enough that the git-service timeout is never what ends this push; the + // disconnect is. + cfg.git_service_timeout_secs = 600; + state.config = std::sync::Arc::new(cfg); + state + .db + .upsert_mirror_repo(owner, name, "/tmp/z6disc-dc1", None, false) + .await + .unwrap(); + + // The mirror row stores the short owner as owner_did, so the slug is the owner. + let key = write_lock_key(&owner.replace([':', '/'], "_"), name); + // Probe from a pool that is NOT the store's lock pool and NOT the harness pool. + let probe = sqlx::postgres::PgPoolOptions::new() + .max_connections(2) + .connect_lazy_with((*pool.connect_options()).clone()); + + assert!( + write_lock_is_takeable(&probe, key).await, + "the write lock must be free before the push" + ); + + // Drive the handler a slice at a time until the fake git's SIGTERM-ignoring + // descendant records its pid, i.e. receive-pack is genuinely running under the + // write lock. `Ok(_)` means the handler returned early; stop polling then, since + // re-polling a completed future panics. + // + // Retried on a miss for the same reason `smart_http`'s disconnect tests retry: + // under `cargo test` fork-storm load a freshly written fake `git` can transiently + // fail to exec (ETXTBSY, a concurrent worker forked while its write fd was open), + // which leaves no pid. A losing attempt's future is dropped, which reaps whatever + // spawned and releases its write lock, so retries do not leak. The winning + // attempt's future is kept PENDING: dropping it below is the disconnect under test. + const SPAWN_ATTEMPTS: u64 = 12; + let (fut, desc) = { + let mut attempt = 0u64; + loop { + attempt += 1; + let _ = std::fs::remove_file(&descfile); + let mut fut = Box::pin(git_receive_pack( + State(state.clone()), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid( + "did:key:z6MkDisconnectWriteLockProofDidAAAAAAAA".to_string(), + )), + crate::rate_limit::PeerAddr(Some( + "203.0.113.81:5000".parse::().unwrap(), + )), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + )); + let mut found: Option = None; + for _ in 0..500 { + let finished = + tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut) + .await + .is_ok(); + if let Some(p) = std::fs::read_to_string(&descfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + found = Some(p); + break; + } + if finished { + break; + } + } + match found { + Some(p) => break (fut, p), + None => { + drop(fut); + assert!( + attempt < SPAWN_ATTEMPTS, + "the push never reached receive-pack after {SPAWN_ATTEMPTS} \ + attempts (persistent failure, not a transient runner miss)" + ); + tokio::time::sleep(std::time::Duration::from_millis(100 * attempt)).await; + } + } + } + }; + let _cleanup = KillOnPanic(vec![desc]); + assert!( + pid_alive(desc), + "the receive-pack group must be running before the disconnect" + ); + assert!( + !write_lock_is_takeable(&probe, key).await, + "the write lock must be held while receive-pack runs" + ); + + // Client disconnect: drop the request future mid-receive-pack. + drop(fut); + + let mut takeable_while_group_alive = false; + let mut freed_after_reap = false; + for _ in 0..800 { + let takeable = write_lock_is_takeable(&probe, key).await; + let group_alive = pid_alive(desc); + if takeable && group_alive { + takeable_while_group_alive = true; + } + if takeable && !group_alive { + freed_after_reap = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + // Clean up regardless so a RED run leaves no orphan behind. + // SAFETY: kill(2) takes integers only. + unsafe { + libc::kill(desc, libc::SIGKILL); + } + assert!( + !takeable_while_group_alive, + "the repo write lock was takeable while a receive-pack group was still alive \ + on that repo: a second push can enter and two git receive-pack groups run \ + against one repo (#173 F2)" + ); + assert!( + freed_after_reap, + "the write lock must be released once the disconnected push's group is reaped" + ); + // The other half of the disconnect invariant, and the reason the guard rides the + // reaper rather than being released there: an interrupted push must not publish a + // half-applied repo. The guard is gone by now (the lock above only frees when it + // is), so the upload site has had its whole chance to be reached. The positive + // control is `receive_pack_success_reclaims_and_releases_the_write_lock`, which + // observes the same counter at 1: without it, a zero here would pass on any build + // where an upload is simply impossible. + assert_eq!( + state.repo_store.tigris_upload_site_reached(), + 0, + "a push interrupted by a client disconnect must not reach the Tigris upload \ + site: publishing a half-applied repo propagates it to every node that later \ + downloads it (#173 F2)" + ); + } + + /// #173 F2, the other half: carrying the write lock through the admission seam must + /// NOT cost the success path its `release(true)`. A push that completes normally has + /// to reclaim the lock and release it explicitly (that is what performs the Tigris + /// upload), synchronously, not leave it to the pool's `after_release` net. The lock + /// is probed immediately after the handler returns, with no polling, so a fix that + /// only ever dropped the guard would fail here. + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_success_reclaims_and_releases_the_write_lock(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let owner = "z6succ"; + let name = "sc1"; + let repos_dir = tempfile::TempDir::new().unwrap(); + let tmp = tempfile::TempDir::new().unwrap(); + // A receive-pack that succeeds. It DRAINS stdin first: exiting while the handler + // is still writing the request body would EPIPE that write, which + // `drive_git_child` surfaces as an error after a successful exit status, making + // the push fail for a reason that has nothing to do with the lock under test. + let git_bin = write_fake_git(tmp.path(), "#!/bin/sh\ncat >/dev/null\nexit 0\n"); + + let mut state = crate::test_support::test_state(pool.clone()).await; + state.git_bin = git_bin; + state.repo_store = crate::git::repo_store::RepoStore::new( + repos_dir.path().to_path_buf(), + None, + crate::git::repo_store::build_lock_pool(&pool, 4, std::time::Duration::from_secs(5)), + ); + state + .db + .upsert_mirror_repo(owner, name, "/tmp/z6succ-sc1", None, false) + .await + .unwrap(); + + let key = write_lock_key(&owner.replace([':', '/'], "_"), name); + let probe = sqlx::postgres::PgPoolOptions::new() + .max_connections(2) + .connect_lazy_with((*pool.connect_options()).clone()); + + // Retried ONLY on the ETXTBSY exec race a freshly written fake `git` hits under + // fork-storm load (a concurrent test worker forked while its write fd was open). + // Narrow on purpose: any other failure still fails the assertion below loudly. + const SPAWN_ATTEMPTS: u64 = 12; + let mut result = None; + for attempt in 1..=SPAWN_ATTEMPTS { + let outcome = tokio::time::timeout( + std::time::Duration::from_secs(30), + git_receive_pack( + State(state.clone()), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid( + "did:key:z6MkPushSuccessReleaseProofDidAAAAAAAA".to_string(), + )), + crate::rate_limit::PeerAddr(Some( + "203.0.113.83:5000".parse::().unwrap(), + )), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ), + ) + .await + .expect("the push must return"); + let exec_race = + matches!(&outcome, Err(AppError::Git(m)) if m.contains("Text file busy")); + if exec_race && attempt < SPAWN_ATTEMPTS { + tokio::time::sleep(std::time::Duration::from_millis(100 * attempt)).await; + continue; + } + result = Some(outcome); + break; + } + let result = result.expect("one attempt must have produced an outcome"); + assert!( + result.is_ok(), + "the fake receive-pack succeeds, so the handler must too; got {result:?}" + ); + + // No polling: `release` unlocks on the connection that took the lock, so the + // lock is free the instant the handler returns. Falling back to the async + // `after_release` net would not satisfy this. + assert!( + write_lock_is_takeable(&probe, key).await, + "a completed push must reclaim its write lock and release it synchronously" + ); + // POSITIVE CONTROL for the disconnect case's "no upload" assertion. A push that + // completed does reach the Tigris upload site, exactly once, so the zero the + // disconnect test observes is a real difference between the two paths rather than + // an artifact of tests running with no Tigris client configured. Exactly once, + // not at least once: a retried exec race releases with success = false and must + // not count. + assert_eq!( + state.repo_store.tigris_upload_site_reached(), + 1, + "a completed push must reach the Tigris upload site once" + ); + } + + /// #173 F1 (RED-before/GREEN-after): an exhausted repo write-lock POOL is a capacity + /// signal, so the push must shed 503 + Retry-After (Overloaded) like every other + /// admission path here, not report a 500 git error. Both directions: the shed with + /// the single lock-pool connection occupied by a guard on a DIFFERENT repo (so this + /// is pool capacity, not advisory-lock contention), and the must-not case once that + /// connection is back. Before the fix `acquire_write`'s checkout failure fell into + /// the generic `AppError::Git` arm (500, no Retry-After). + #[sqlx::test] + async fn receive_pack_lock_pool_exhaustion_sheds_503_not_500(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let owner = "z6lockpool"; + let name = "lp1"; + let mut state = crate::test_support::test_state(pool.clone()).await; + // One lock-pool connection, short checkout timeout so the exhaustion surfaces + // promptly rather than at the handler's own acquire deadline. + state.repo_store = crate::git::repo_store::RepoStore::new( + std::path::PathBuf::from("/tmp/gitlawb-lockpool-shed"), + None, + crate::git::repo_store::build_lock_pool(&pool, 1, std::time::Duration::from_secs(1)), + ); + state + .db + .upsert_mirror_repo(owner, name, "/tmp/z6lockpool-lp1", None, false) + .await + .unwrap(); + + let did = "did:key:z6MkLockPoolShedProofDidAAAAAAAAAAAAAAAAAA"; + let peer: SocketAddr = "203.0.113.71:5000".parse().unwrap(); + + // Occupy the only lock-pool connection with a write on an UNRELATED repo. + let held = state + .repo_store + .acquire_write(owner, "other-repo") + .await + .expect("the first write takes the only lock-pool connection"); + + let shed = git_receive_pack( + State(state.clone()), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(peer)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await; + assert!( + matches!(shed, Err(AppError::Overloaded(_))), + "an exhausted lock pool must shed 503 + Retry-After, not a 500 git error; \ + got {shed:?}" + ); + + // MUST-NOT: with the pool free again, the push is not shed as capacity (it fails + // later on the nonexistent on-disk repo, which is a git error, not Overloaded). + held.release(false).await; + let admitted = git_receive_pack( + State(state.clone()), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some("203.0.113.72:5000".parse().unwrap())), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await; + assert!( + !matches!(admitted, Err(AppError::Overloaded(_))), + "with the lock pool free, a push must not be shed as capacity; got {admitted:?}" + ); + } + + /// #174 U5 (P1-e, RED-before/GREEN-after): the post-push encryption walk acquires a + /// `git_encrypt_semaphore` permit before running, so completed pushes cannot spawn + /// unbounded concurrent full-history walks. With the pool exhausted the gated walk + /// must DEFER (block on admission) and NOT run its rev-list; on the pre-fix code + /// (no acquire) the walk runs regardless of the pool (RED). It defers rather than + /// sheds — releasing the permit lets the SAME walk run and pin (durability stays + /// fail-closed). Exercises the gating seam directly; the detached push task calls + /// this exact helper. + #[tokio::test] + async fn encrypt_walk_defers_when_pool_exhausted() { + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let marker = tmp.path().join("revlist.ran"); + // Fake git records when rev-list runs (the walk's first git call). + let body = format!( + "#!/bin/sh\ncase \"$1\" in\n rev-list) echo ran > \"{}\" ;;\n *) : ;;\nesac\nexit 0\n", + marker.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + let git_bin = git_path.to_str().unwrap().to_string(); + let owner = "did:key:z6MkEncWalkOwnerAAAAAAAAAAAAAAAAAAAAAAAA".to_string(); + + // Exhaust the pool: hold its only permit so a gated walk must defer. + let sem = Arc::new(Semaphore::new(1)); + let held = sem.clone().acquire_owned().await.unwrap(); + + // Blocked: the gated walk must NOT complete or run rev-list while exhausted. + let blocked = tokio::time::timeout( + Duration::from_millis(500), + withheld_recipients_gated( + sem.clone(), + tmp.path().to_path_buf(), + git_bin.clone(), + Duration::from_secs(5), + Vec::new(), + true, + owner.clone(), + ), + ) + .await; + assert!( + blocked.is_err(), + "the encryption walk must defer (block on admission) when the pool is exhausted" + ); + assert!( + !marker.exists(), + "the walk's rev-list must not run while its admission permit is unavailable (P1-e)" + ); + + // Release admission: the SAME walk now runs (defer, not shed) — rev-list fires. + drop(held); + let ran = withheld_recipients_gated( + sem, + tmp.path().to_path_buf(), + git_bin, + Duration::from_secs(5), + Vec::new(), + true, + owner, + ) + .await; + assert!( + ran.is_ok(), + "with a permit the walk runs and joins: {ran:?}" + ); + assert!( + marker.exists(), + "once admission is available the deferred walk runs its rev-list" + ); + } + + /// F4 defer proof 1: `replication_withheld_set`'s WALK arm acquires a + /// `git_encrypt_semaphore` permit before its spawn_blocking git walk, deferring + /// (never shedding) when the pool is exhausted — while its no-walk fast paths + /// (no path-scoped rule; not announceable) complete WITHOUT touching the pool. + /// On ungated code the walk runs regardless of a zero-permit pool (RED). + #[cfg(unix)] + #[tokio::test] + async fn replication_walk_defers_when_scan_pool_exhausted() { + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let marker = tmp.path().join("git.ran"); + // Fake git records ANY invocation (the walk's first call is rev-parse), then + // behaves well enough for a successful empty walk: HEAD probe succeeds, + // rev-list lists no commits. + let body = format!( + "#!/bin/sh\necho ran >> \"{}\"\ncase \"$1\" in\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n", + marker.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + let scoped_rules = || Some(vec![vis_rule("/secret/**", &[])]); + + // Zero-permit pool: every gated walk must park forever. + let sem: Arc = Arc::new(Semaphore::new(0)); + + // Fast path A (negative arm): announceable, NO path-scoped rule -> zero git + // work, must complete immediately without acquiring from the empty pool. + let fast = tokio::time::timeout( + Duration::from_millis(500), + replication_withheld_set( + sem.clone(), + Some(vec![]), + OWNER_DID, + true, + tmp.path().to_path_buf(), + git_bin.clone(), + Duration::from_secs(5), + ), + ) + .await + .expect("the no-path-scoped-rule fast path must not park on the scan pool"); + assert_eq!(fast, (true, Some(std::collections::HashSet::new()))); + + // Fast path B (negative arm): not announceable (no rules) -> zero git work, + // must complete immediately without acquiring. + let fast = tokio::time::timeout( + Duration::from_millis(500), + replication_withheld_set( + sem.clone(), + None, + OWNER_DID, + false, + tmp.path().to_path_buf(), + git_bin.clone(), + Duration::from_secs(5), + ), + ) + .await + .expect("the not-announceable fast path must not park on the scan pool"); + assert_eq!(fast, (false, None)); + assert!(!marker.exists(), "the fast paths must spawn no git at all"); + + // Walk arm with the pool exhausted: must DEFER (park), spawning no git. + let blocked = tokio::time::timeout( + Duration::from_millis(500), + replication_withheld_set( + sem.clone(), + scoped_rules(), + OWNER_DID, + true, + tmp.path().to_path_buf(), + git_bin.clone(), + Duration::from_secs(5), + ), + ) + .await; + assert!( + blocked.is_err(), + "the withheld walk must defer (park on admission) when the pool is exhausted" + ); + assert!( + !marker.exists(), + "the withheld walk's git must not spawn while its admission permit is unavailable (F4)" + ); + + // Release admission: the SAME walk now runs (defer, not shed) and succeeds. + sem.add_permits(1); + let ran = replication_withheld_set( + sem, + scoped_rules(), + OWNER_DID, + true, + tmp.path().to_path_buf(), + git_bin, + Duration::from_secs(5), + ) + .await; + assert!( + marker.exists(), + "once admission is available the deferred withheld walk runs its git" + ); + assert_eq!( + ran, + (true, Some(std::collections::HashSet::new())), + "the released walk completes and vets the (empty) withheld set" + ); + } + + /// F4 defer proof 3: `fail_closed_full_scan_objects` ALWAYS walks, so its + /// spawn_blocking is always admission-gated: with the pool exhausted it defers + /// and spawns no git; with a permit the same call runs. Ungated it runs + /// regardless (RED). + #[cfg(unix)] + #[tokio::test] + async fn full_scan_pin_walk_defers_when_scan_pool_exhausted() { + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let marker = tmp.path().join("git.ran"); + let body = format!( + "#!/bin/sh\necho ran >> \"{}\"\ncase \"$1\" in\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n", + marker.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + let candidates = vec!["3333333333333333333333333333333333333333".to_string()]; + + let sem: Arc = Arc::new(Semaphore::new(0)); + let blocked = tokio::time::timeout( + Duration::from_millis(500), + fail_closed_full_scan_objects( + sem.clone(), + tmp.path().to_path_buf(), + vec![vis_rule("/secret/**", &[])], + true, + OWNER_DID.to_string(), + candidates.clone(), + git_bin.clone(), + Duration::from_secs(5), + ), + ) + .await; + assert!( + blocked.is_err(), + "the fail-closed full scan must defer (park on admission) when the pool is exhausted" + ); + assert!( + !marker.exists(), + "the full scan's git must not spawn while its admission permit is unavailable (F4)" + ); + + // Release admission: the SAME scan now runs (defer, not shed). + sem.add_permits(1); + let _objs = fail_closed_full_scan_objects( + sem, + tmp.path().to_path_buf(), + vec![vis_rule("/secret/**", &[])], + true, + OWNER_DID.to_string(), + candidates, + git_bin, + Duration::from_secs(5), + ) + .await; + assert!( + marker.exists(), + "once admission is available the deferred full scan runs its git" + ); + } + + /// #174 F4 (RED-before/GREEN-after): the two full-scan phases share ONE whole-scan + /// deadline. Phase 1 (`replicable_blob_set_bounded`) succeeds but consumes almost + /// the whole budget; phase 2 (`all_blob_oids`) then gets only the remainder. With a + /// shared deadline phase 2 is reaped and the scan fails closed (pins nothing) — the + /// safe direction. With a FRESH `Instant::now() + timeout` for phase 2 (pre-fix) it + /// gets a full second budget, completes with an empty blob set, and the non-blob + /// candidate is kept — so the result is NON-empty (RED) and the permit is held ~2x. + #[cfg(unix)] + #[tokio::test] + async fn full_scan_shares_one_deadline_across_both_phases() { + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + // Phase 1 (ls-tree) sleeps 1.5s and succeeds (empty tree); phase 2 + // (cat-file --batch-all-objects) sleeps 1.5s. With a 2s whole-scan budget the + // shared deadline leaves phase 2 only ~0.5s, so it is reaped; a fresh 2s budget + // would let it finish. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-parse) echo deadbeef ;;\n rev-list) echo aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ;;\n ls-tree) sleep 1.5 ;;\n cat-file) case \"$*\" in *--batch-all-objects*) sleep 1.5 ;; *) : ;; esac ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + // A candidate that is NOT a blob (never appears in all_blob_oids): kept by + // replicable_objects_fail_closed only if phase 2 actually ran to completion. + let candidates = vec!["cccccccccccccccccccccccccccccccccccccccc".to_string()]; + + let sem: Arc = Arc::new(Semaphore::new(1)); + let objs = fail_closed_full_scan_objects( + sem, + tmp.path().to_path_buf(), + vec![vis_rule("/secret/**", &[])], + true, + OWNER_DID.to_string(), + candidates, + git_bin, + Duration::from_secs(2), + ) + .await; + assert!( + objs.is_empty(), + "a large-but-successful phase 1 must leave phase 2 only the SHARED whole-scan \ + remainder, so it reaps and the scan fails closed; got {objs:?} (a fresh phase-2 \ + budget completed the scan and kept the candidate — the ~2x-budget bug)" + ); + } + + /// #174 F6 (RED-before/GREEN-after): a post-push pin loop holds this push's full + /// object-id list while walking it, so concurrent pin loops across many repos must + /// be bounded by a global permit, not just the per-repo task count. `pin_new_objects_gated` + /// DEFERS (waits) when the pin pool is exhausted rather than running unbounded. + /// + /// Load-bearing: without the permit acquire the pin loop runs immediately even with + /// the pool held (RED — the deferral assertion fails). With it, it parks. + #[sqlx::test] + async fn pin_new_objects_gated_defers_when_pin_pool_exhausted(pool: sqlx::PgPool) { + use std::sync::Arc; + use tokio::sync::Semaphore; + + let state = crate::test_support::test_state(pool).await; + let db = state.db.clone(); + let tmp = tempfile::TempDir::new().unwrap(); + let pin_sem = Arc::new(Semaphore::new(1)); + // Hold the only pin permit. + let held = pin_sem.clone().acquire_owned().await.unwrap(); + + // Empty ipfs_api makes the pin itself a no-op, but the loop must still DEFER on + // the exhausted pin pool rather than run. The object list is non-empty because + // an empty one takes no permit at all by design (#174 F2b). + let objects = vec!["0123456789abcdef0123456789abcdef01234567".to_string()]; + let blocked = tokio::time::timeout( + std::time::Duration::from_millis(500), + pin_new_objects_gated( + &pin_sem, + "", + tmp.path(), + "git", + std::time::Duration::from_secs(30), + objects.clone(), + &db, + "repo-gated-a", + ), + ) + .await; + assert!( + blocked.is_err(), + "a pin loop must defer while the pin pool is exhausted (#174 F6)" + ); + + // Release admission: the SAME call now completes. + drop(held); + let out = tokio::time::timeout( + std::time::Duration::from_secs(5), + pin_new_objects_gated( + &pin_sem, + "", + tmp.path(), + "git", + std::time::Duration::from_secs(30), + objects, + &db, + "repo-gated-b", + ), + ) + .await + .expect("the pin loop completes once admission frees"); + assert!(out.is_empty(), "an empty ipfs_api pins nothing"); + } + + /// #174 F2b: the pin permit bounds how many pin loops run concurrently, so a call + /// with NOTHING to pin must not take one. It otherwise spends a global pin slot on no + /// work, and the pool DEFERS rather than sheds, so those calls stall pins for every + /// other repo. The empty case is the normal shape for a push whose walk failed or + /// that may replicate nothing. + /// + /// Load-bearing: without the guard this call parks on the exhausted pool exactly like + /// the non-empty one above, and the completion assertion fails. + #[sqlx::test] + async fn pin_new_objects_gated_takes_no_permit_for_an_empty_object_list(pool: sqlx::PgPool) { + use std::sync::Arc; + use tokio::sync::Semaphore; + + let state = crate::test_support::test_state(pool).await; + let db = state.db.clone(); + let tmp = tempfile::TempDir::new().unwrap(); + let pin_sem = Arc::new(Semaphore::new(1)); + // Hold the only pin permit for the whole call. + let _held = pin_sem.clone().acquire_owned().await.unwrap(); + + let out = tokio::time::timeout( + std::time::Duration::from_millis(500), + pin_new_objects_gated( + &pin_sem, + "", + tmp.path(), + "git", + std::time::Duration::from_secs(30), + vec![], + &db, + "repo-gated-empty", + ), + ) + .await + .expect("an empty object list must not wait on pin admission (#174 F2b)"); + assert!(out.is_empty(), "and it pins nothing"); + assert_eq!( + pin_sem.available_permits(), + 0, + "the test still holds the only permit, so the call never took one" + ); + } + + /// Shared fixture for the F4 handler-layer tests: a state whose repo_store and + /// git_bin point at the given tempdir/fake-git, plus a seeded on-disk repo, + /// optionally with a path-scoped rule (so the post-receive walks actually run). + #[cfg(unix)] + async fn f4_state_with_repo( + pool: sqlx::PgPool, + tmp: &std::path::Path, + git_bin: &str, + owner: &str, + name: &str, + path_scoped: bool, + ) -> AppState { + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.git_bin = git_bin.to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo(owner, name, &format!("/unused-{owner}-{name}"), None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + state + .repo_store + .init(&rec.owner_did, &rec.name) + .await + .unwrap(); + if path_scoped { + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkF4ReaderAAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + } + state + } + + /// A pkt-line receive-pack body carrying one branch-create ref update, so the + /// handler's post-receive tail resolves a non-empty new-tip set (the delta + /// scan's git stages run). + fn ref_update_body(new_sha: &str) -> axum::body::Bytes { + let line = format!("{ZERO_SHA} {new_sha} refs/heads/main"); + axum::body::Bytes::from(format!("{:04x}{}0000", line.len() + 4, line)) + } + + /// F4 scenario 2 — push-burst bound at the handler layer: with a scan pool of + /// ONE, two concurrent pushes to two path-scoped repos never have more than one + /// scan's git alive at a time (an atomic mkdir lock in the fake git detects any + /// overlap), and BOTH pushes still succeed 200 — defer, not shed. Two distinct + /// repos on purpose: the per-repo advisory write lock must not be what + /// serializes the scans. + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_burst_scans_serialized_and_both_pushes_succeed(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let lockdir = tmp.path().join("scan.lock"); + let ranfile = tmp.path().join("scan.ran"); + let overlap = tmp.path().join("scan.overlap"); + // receive-pack succeeds instantly; every candidate-scan git op (cat-file / + // rev-list / ls-tree) holds an atomic mkdir lock for 150ms — a second scan + // process alive at the same instant records an overlap. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + receive-pack) cat > /dev/null 2>/dev/null ;;\n\ + rev-parse) echo deadbeef ;;\n\ + cat-file|rev-list|ls-tree)\n\ + if mkdir \"{lock}\" 2>/dev/null; then\n\ + echo 1 >> \"{ran}\"\n\ + sleep 0.15\n\ + rmdir \"{lock}\"\n\ + else\n\ + echo 1 >> \"{over}\"\n\ + fi\n\ + if [ \"$1\" = cat-file ]; then echo commit; fi ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + lock = lockdir.display(), + ran = ranfile.display(), + over = overlap.display(), + ); + let git_bin = write_fake_git(tmp.path(), &body); + + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f4burst1", "b1", true).await; + // Second path-scoped repo on the same state/store. + state + .db + .upsert_mirror_repo("z6f4burst2", "b2", "/unused-z6f4burst2-b2", None, false) + .await + .unwrap(); + let rec2 = state + .db + .get_repo("z6f4burst2", "b2") + .await + .unwrap() + .unwrap(); + state + .repo_store + .init(&rec2.owner_did, &rec2.name) + .await + .unwrap(); + state + .db + .set_visibility_rule( + &rec2.id, + "/secret/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkF4ReaderAAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec2.owner_did, + ) + .await + .unwrap(); + // Scan pool of ONE: at most one post-receive walk may run at a time. + state.git_encrypt_semaphore = Arc::new(Semaphore::new(1)); + + let did = "did:key:z6MkF4BurstPusherAAAAAAAAAAAAAAAAAAAAAAAA"; + let new_sha = "1111111111111111111111111111111111111111"; + let push = |owner: &'static str, name: &'static str, peer: &'static str| { + let state = state.clone(); + tokio::spawn(async move { + git_receive_pack( + State(state), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(peer.parse::().unwrap())), + axum::http::HeaderMap::new(), + ref_update_body(new_sha), + ) + .await + }) + }; + + let (a, b) = ( + push("z6f4burst1", "b1", "203.0.113.71:5000"), + push("z6f4burst2", "b2", "203.0.113.72:5000"), + ); + let a = tokio::time::timeout(std::time::Duration::from_secs(60), a) + .await + .expect("push A must complete — a scan gate must defer, never wedge") + .expect("push A task must not panic"); + let b = tokio::time::timeout(std::time::Duration::from_secs(60), b) + .await + .expect("push B must complete — a scan gate must defer, never wedge") + .expect("push B task must not panic"); + let a = a.expect("push A must succeed"); + let b = b.expect("push B must succeed"); + assert_eq!(a.status(), 200, "push A lands 200 despite scan contention"); + assert_eq!(b.status(), 200, "push B lands 200 despite scan contention"); + + // Wait for both pushes' detached scan tails to drain through the pool of 1 + // before reading the detector files. The WHOLE tail (withheld walk included) now + // runs detached (#174 F2), so poll until every expected scan has run rather than a + // fixed sleep, which is load-sensitive under a parallel test run. + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); + loop { + let ran = std::fs::read_to_string(&ranfile) + .unwrap_or_default() + .lines() + .count(); + if ran >= 6 || std::time::Instant::now() >= deadline { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + // Small settle so the last scan's rmdir has landed before the overlap check. + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + assert!( + !overlap.exists(), + "with a scan pool of 1, no two scans' git may ever be alive at once \ + (found overlap records: {:?})", + std::fs::read_to_string(&overlap).unwrap_or_default() + ); + let ran = std::fs::read_to_string(&ranfile).unwrap_or_default(); + assert!( + ran.lines().count() >= 6, + "both pushes' scans must actually have run (withheld walk + delta probe + \ + delta rev-list each); got {} runs", + ran.lines().count() + ); + } + + /// F4 scenario 3 — fast-path non-acquisition at the handler layer: a push to a + /// public repo with NO path-scoped rules does zero post-receive git scanning + /// (the withheld short-circuit; a deletion-free flush-only body resolves no new + /// tips), so it must complete 200 even with the scan pool at ZERO permits. + /// A gate that wrongly captured a no-walk path would park this push forever. + /// Note: `resolve_candidates_for_push` spawns git for ANY non-empty new-tip set + /// (the per-tip cat-file probe), so the genuinely git-free negative arm is the + /// no-ref-update body, not a branch-create push. + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_no_scan_fast_path_completes_with_zero_scan_permits(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let marker = tmp.path().join("scan.ran"); + let body = format!( + "#!/bin/sh\ncase \"$1\" in\n receive-pack) cat > /dev/null 2>/dev/null ;;\n cat-file|rev-list|ls-tree) echo 1 >> \"{}\" ;;\n *) : ;;\nesac\nexit 0\n", + marker.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f4fast", "f1", false).await; + state.git_encrypt_semaphore = Arc::new(Semaphore::new(0)); + + let peer: SocketAddr = "203.0.113.73:5000".parse().unwrap(); + let resp = tokio::time::timeout( + std::time::Duration::from_secs(30), + git_receive_pack( + State(state), + Path(("z6f4fast".to_string(), "f1".to_string())), + Extension(crate::auth::AuthenticatedDid( + "did:key:z6MkF4FastPusherAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + )), + crate::rate_limit::PeerAddr(Some(peer)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ), + ) + .await + .expect("a no-scan push must not park on the (empty) scan pool") + .expect("the push must succeed"); + assert_eq!(resp.status(), 200); + assert!( + !marker.exists(), + "the no-walk fast paths must spawn no scan git at all" + ); + } + + /// F4 scenario 4 — landed-push-never-fails: a push whose post-receive walk must + /// park (pool held elsewhere) DEFERS and then returns the receive-pack success + /// once admission frees; contention never converts the landed push into a 5xx. + /// #174 F2 (RED-before/GREEN-after): the post-receive replication tail parks on + /// `git_encrypt_semaphore` (withheld/candidate/full-scan resolution). Leaving it in the + /// request future means a client/proxy disconnect while parked silently loses this + /// push's pins, recovery copy, and announcements (state.rs documented this residual). + /// The fix moves the whole tail into an independently owned task, so the handler + /// returns its receive-pack 200 WITHOUT waiting on the scan pool and a disconnect can + /// no longer drop the work. + /// + /// Load-bearing: with the tail inline (pre-fix) the handler parks while the pool is + /// held and does NOT return within the bound (RED — the timeout fires). With the + /// detached tail it returns 200 promptly (GREEN). + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_landed_push_returns_without_parking_on_scan_pool(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let body = "#!/bin/sh\ncase \"$1\" in\n receive-pack) cat > /dev/null 2>/dev/null ;;\n rev-parse) echo deadbeef ;;\n cat-file) echo commit ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f4park", "p1", true).await; + let sem = Arc::new(Semaphore::new(1)); + state.git_encrypt_semaphore = sem.clone(); + // Hold the pool's only permit: the post-receive scan would park if it ran in the + // request future. + let held = sem.clone().acquire_owned().await.unwrap(); + + let peer: SocketAddr = "203.0.113.74:5000".parse().unwrap(); + // The handler must return its receive-pack 200 WITHOUT waiting on the held scan + // pool — the tail is owned by a detached task. Pre-fix this times out. + let resp = tokio::time::timeout( + std::time::Duration::from_secs(5), + git_receive_pack( + State(state), + Path(("z6f4park".to_string(), "p1".to_string())), + Extension(crate::auth::AuthenticatedDid( + "did:key:z6MkF4ParkPusherAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + )), + crate::rate_limit::PeerAddr(Some(peer)), + axum::http::HeaderMap::new(), + ref_update_body("2222222222222222222222222222222222222222"), + ), + ) + .await + .expect("the handler must return without parking on the held scan pool") + .expect("contention must never convert a landed push into an error"); + assert_eq!( + resp.status(), + 200, + "the response is the receive-pack success, returned before the detached tail runs" + ); + + // The detached tail is still owned: release admission and let it drain cleanly. + drop(held); + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + } + + // ---- #174 U4 (P2-2): post-push encryption task set bounded by per-repo coalescing ---- + // + // The residual jatmn found is not the WALK (bounded by `git_encrypt_semaphore`, + // proven by `encrypt_walk_defers_when_pool_exhausted` above) but the OUTER + // `tokio::spawn` + its parked `acquire_owned().await` waiters: N rapid pushes to a + // repo spawn N tasks that each park holding cloned object lists/rules/keys — an + // unbounded outstanding set. U4 bounds it by coalescing per repo: before spawning, + // if a task for the repo is in flight, skip the duplicate. Crucially this DEFERS a + // duplicate walk (the newer push's objects are covered by the pending one) and does + // NOT shed — there is no reconciliation sweep, so a dropped job would permanently + // lose the withheld-blob recovery copy (`2a54c15`'s fail-closed durability stance). + // + // These drive the coalescing seam (`EncryptInflight`) that the detached spawn at + // `repos.rs` consults directly (the try_begin gate on the in-flight set, guarded by + // `withheld.is_some()`). Observing `encrypt_and_pin`'s IPFS effect end-to-end needs a live IPFS node + // (`pin_git_object` hits the API), so the durability property is proven at this + // layer: a coalesced repo's key is released when its task ends, so a later push for + // that repo is processed once — NOT permanently skipped, which is exactly what a + // coalesce->shed mutation would break by dropping the job with no sweep to recover it. + + /// Bounded outstanding set under saturation (R4). Simulate K rapid path-scoped + /// pushes to the SAME repo while the encrypt pool is saturated (every spawned task + /// would park, so none has finished and removed its key): the first `try_begin` + /// admits (spawns), the rest coalesce (skip). The in-flight set holds at 1, not K. + /// + /// MUTATION (RED): removing the coalescing check makes every push spawn — modeled by + /// `simulate_without_coalescing`, which reaches K. If the coalesced count equaled the + /// un-coalesced one the gate would be a no-op; the strict inequality proves it bites. + #[test] + fn u4_outstanding_encrypt_set_is_bounded_to_one_per_repo_under_saturation() { + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkRepoOwnerAAAAAAAAAAAAAAAAAAAAAAAAAAAA/proj"; + const K: usize = 32; + + // Hold every admitted guard so the tasks are "still in flight" (the saturated + // case: all parked on acquire_owned().await, none finished, none removed a key). + let mut admitted = Vec::new(); + let mut coalesced = 0usize; + for _ in 0..K { + match inflight.try_begin(repo, vec![]) { + crate::state::BeginOutcome::Admitted(g) => admitted.push(g), + crate::state::BeginOutcome::Coalesced => coalesced += 1, + } + } + + assert_eq!( + admitted.len(), + 1, + "exactly ONE detached task may spawn per repo while one is in flight — the \ + outstanding set is bounded to 1, not K parked waiters" + ); + assert_eq!( + coalesced, + K - 1, + "the other K-1 rapid pushes to the same repo coalesce (skip spawning)" + ); + assert_eq!( + inflight.len(), + 1, + "the in-flight set holds at most one entry per repo under saturation" + ); + + let no_coalesce = simulate_without_coalescing(K); + assert_eq!( + no_coalesce, K, + "sanity: without the coalescing check all K pushes spawn (the unbounded set \ + the fix prevents) — proves the bound above is not vacuously 1" + ); + assert!( + admitted.len() < no_coalesce, + "coalesced set ({}) must be strictly smaller than the un-coalesced one ({})", + admitted.len(), + no_coalesce + ); + } + + /// Coalescing is PER-REPO: distinct repos are never coalesced against each other, so + /// one repo in flight cannot starve a second repo's recovery copy. + #[test] + fn u4_distinct_repos_each_admit_one_encrypt_task() { + use crate::state::BeginOutcome; + let inflight = crate::state::EncryptInflight::new(); + let a = inflight.try_begin("owner/repo-a", vec![]); + let b = inflight.try_begin("owner/repo-b", vec![]); + let c = inflight.try_begin("owner/repo-c", vec![]); + assert!( + matches!(&a, BeginOutcome::Admitted(_)) + && matches!(&b, BeginOutcome::Admitted(_)) + && matches!(&c, BeginOutcome::Admitted(_)), + "three distinct repos each admit their own encryption task" + ); + assert_eq!(inflight.len(), 3, "one in-flight entry per distinct repo"); + } + + /// NO LOST RECOVERY COPY — the security guard (R4/R6). Coalescing must DELAY a + /// duplicate walk, never permanently drop a repo's recovery copy. Observable + /// property: once an in-flight task ENDS (its guard drops — completion, error, or + /// panic-unwind) the repo key is released, so the NEXT push for that repo is admitted + /// and processed again. A coalesce->shed mutation would drop the job AND never + /// re-admit — with no reconciliation sweep the copy is lost forever. Here re-admission + /// survives normal completion AND a panic, so no permanent skip / no leaked key. + #[test] + fn u4_coalesced_repo_is_reprocessed_after_task_ends_not_permanently_skipped() { + use crate::state::{BeginOutcome, FinishOutcome}; + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkDurableRepoBBBBBBBBBBBBBBBBBBBBBBBBB/repo"; + + // Push #1 admits and "spawns". A concurrent push #2 (task #1 still in flight) + // coalesces — no duplicate spawn; its (empty) tip set is recorded, not lost. + let guard1 = admit(&inflight, repo); + assert!( + matches!(inflight.try_begin(repo, vec![]), BeginOutcome::Coalesced), + "while task #1 is in flight, push #2 to the same repo coalesces" + ); + + // Task #1 finishes normally: nothing pending (push #2 carried no tips), so + // the empty-pending check removes the key in its critical section. + assert!( + matches!(guard1.finish_or_take_pending(), FinishOutcome::Finished(_)), + "no pending tips — the task exits and releases the key" + ); + assert_eq!( + inflight.len(), + 0, + "when the in-flight task ends its repo key is released — the set does not leak" + ); + + // A LATER push for the SAME repo is admitted again (processed, not skipped + // forever). This is what coalesce->shed breaks: shed drops the job and no sweep + // re-derives the missing copy, so the recovery copy is permanently lost. + let guard2 = admit(&inflight, repo); + // An errored task (guard dropped without finishing) still releases the key. + drop(guard2); + assert_eq!(inflight.len(), 0); + + // Durability across PANIC: a task that panics mid-walk must still release its + // key (the still-armed guard's Drop runs on unwind), so one crashed walk never + // permanently locks a repo out of future recovery copies. Coalesce real tips + // first: the panic loses them (logged), and the loss must not corrupt the set. + let panicked = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _g = admit(&inflight, repo); + assert!(matches!( + inflight.try_begin(repo, vec![("old1".to_string(), "new1".to_string())]), + BeginOutcome::Coalesced + )); + assert_eq!(inflight.len(), 1); + panic!("simulate the detached encryption task panicking mid-walk"); + })); + assert!(panicked.is_err(), "the simulated task panicked"); + assert_eq!( + inflight.len(), + 0, + "a panicked encryption task still releases its repo key (Drop on unwind) — no \ + permanent leak that would block every future recovery copy for the repo" + ); + // The next push is re-admitted, and the panicked task's pending tips did NOT + // survive into it (they are lost-and-logged, recovered only by a later push). + let guard3 = admit(&inflight, repo); + assert!( + matches!(guard3.finish_or_take_pending(), FinishOutcome::Finished(_)), + "the pre-panic pending tips must not leak into the re-admitted task" + ); + } + + /// Degenerate state: the first push on a cold/empty in-flight set always admits + /// (never a false coalesce on an empty set). + #[test] + fn u4_first_push_on_a_cold_set_always_admits() { + let inflight = crate::state::EncryptInflight::new(); + assert!(inflight.is_empty(), "cold set is empty"); + assert!( + matches!( + inflight.try_begin("owner/first", vec![]), + crate::state::BeginOutcome::Admitted(_) + ), + "the first push on a cold in-flight set must admit (never falsely coalesce)" + ); + } + + /// Unwrap an Admitted outcome (panic on Coalesced) — the u4/u5 suites' shorthand. + fn admit( + inflight: &crate::state::EncryptInflight, + repo: &str, + ) -> crate::state::EncryptInflightGuard { + match inflight.try_begin(repo, vec![]) { + crate::state::BeginOutcome::Admitted(g) => g, + crate::state::BeginOutcome::Coalesced => panic!("expected {repo} to admit"), + } + } + + /// Model of the pre-fix / mutated code: no coalescing check, so every push spawns. + /// Returns the count of tasks spawned (== the size of the unbounded outstanding set + /// the fix prevents), used as the RED comparison in the bound test above. + fn simulate_without_coalescing(pushes: usize) -> usize { + (0..pushes).count() + } + + // ---- #174 U5 (F5): a push that loses try_begin is REQUEUED, never dropped ---- + // + // F5: the in-flight task pins only its own pre-spawn object-list snapshot, so a + // push B arriving while task A is in flight used to be SKIPPED outright (the old + // None arm) — B's pins and recovery copies were silently absent until an + // unrelated later push re-walked the repo. U5 records B's (old, new) tip pairs + // into the in-flight key's pending slot in the SAME critical section as the + // presence check, and A's task loop-drains them before releasing the key. + + /// The F5 lost-update repro. Task A is in flight past its snapshot; push B + /// coalesces carrying its tip pair; when A finishes its snapshot iteration the + /// tracker must hand A exactly B's recorded work with the key retained — and only + /// an empty pending check may remove the key. On pre-U5 code this is RED: the + /// coalesce arm records B's work nowhere and there is no drain surface at all. + #[test] + fn u5_coalesced_push_work_is_drained_by_the_inflight_task() { + use crate::state::{BeginOutcome, FinishOutcome, PendingWork}; + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkF5LostUpdateCCCCCCCCCCCCCCCCCCCCCCCC/repo"; + + // Push A admits; its spawned task is "in flight past its snapshot". + let guard_a = match inflight.try_begin(repo, vec![]) { + BeginOutcome::Admitted(g) => g, + BeginOutcome::Coalesced => panic!("first push must admit"), + }; + + // Push B lands while A is in flight: coalesced, tip pair recorded. + let b_pair = ( + "b0ldb0ldb0ldb0ldb0ldb0ldb0ldb0ldb0ldb0ld".to_string(), + "bnewbnewbnewbnewbnewbnewbnewbnewbnewbnew".to_string(), + ); + match inflight.try_begin(repo, vec![b_pair.clone()]) { + BeginOutcome::Coalesced => {} + BeginOutcome::Admitted(_) => panic!("push B must coalesce while A is in flight"), + } + + // A finishes its snapshot iteration: it must be handed B's work (drained), + // not exit — an exit here is exactly the F5 silent loss. + match guard_a.finish_or_take_pending() { + FinishOutcome::Pending(guard_a, work) => { + assert_eq!( + work, + PendingWork::Tips(vec![b_pair]), + "A drains exactly B's recorded tip pair" + ); + assert_eq!(inflight.len(), 1, "the key is retained while A iterates"); + // Nothing further pending: A now exits and releases the key. + match guard_a.finish_or_take_pending() { + FinishOutcome::Finished(_) => {} + FinishOutcome::Pending(..) => panic!("no second batch was recorded"), + } + assert_eq!( + inflight.len(), + 0, + "an empty pending check at task end releases the key" + ); + } + FinishOutcome::Finished(_) => panic!( + "F5: B's coalesced work vanished — the in-flight task exited without draining it" + ), + } + } + + /// Drain-vs-admit race, both orderings driven deterministically through the lock + /// API (the check+merge and check+remove are each ONE critical section, so a push + /// can only land on one side of A's final pending check — never inside it): + /// before it, the push is merged and A drains it; after it, the key is gone and + /// the push is admitted as a fresh task. Neither ordering loses the work. A + /// check-then-record split (merge moved outside try_begin's critical section) + /// turns ordering 1 RED: the work recorded after A's check is never drained. + #[test] + fn u5_drain_vs_admit_race_loses_no_work_in_either_ordering() { + use crate::state::{BeginOutcome, FinishOutcome, PendingWork}; + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkF5RaceOrderDDDDDDDDDDDDDDDDDDDDDDDDD/repo"; + let pair = ("cold".to_string(), "cnew".to_string()); + + // Ordering 1: push C lands BEFORE A's final pending check → merged in + // try_begin's critical section → A must drain it (key retained). + let guard_a = admit(&inflight, repo); + assert!(matches!( + inflight.try_begin(repo, vec![pair.clone()]), + BeginOutcome::Coalesced + )); + match guard_a.finish_or_take_pending() { + FinishOutcome::Pending(g, work) => { + assert_eq!(work, PendingWork::Tips(vec![pair.clone()])); + assert!(matches!( + g.finish_or_take_pending(), + FinishOutcome::Finished(_) + )); + } + FinishOutcome::Finished(_) => { + panic!("a push merged before the final check must be drained, not lost") + } + } + assert!(inflight.is_empty()); + + // Ordering 2: push C lands AFTER A's final pending check removed the key → + // it must be ADMITTED as a fresh task (its own snapshot covers its work). + let guard_a = admit(&inflight, repo); + assert!(matches!( + guard_a.finish_or_take_pending(), + FinishOutcome::Finished(_) + )); + match inflight.try_begin(repo, vec![pair]) { + BeginOutcome::Admitted(g) => drop(g), + BeginOutcome::Coalesced => panic!( + "a push landing after the key was removed must admit a new task — a \ + coalesce here records work no task will ever drain" + ), + } + } + + /// Exit-vs-successor (the double-remove hazard). A's normal exit removes the key + /// and disarms the guard in ONE critical section, and the disarmed guard is + /// handed back — so its eventual Drop lands in the real remove→drop window. A + /// successor task B admitted inside that window must keep ITS key when A's guard + /// finally drops. With the disarm reverted (Drop removing unconditionally) this + /// is RED: dropping A's guard deletes B's key and the third push falsely admits + /// a second task for the repo. + #[test] + fn u5_disarmed_guard_drop_never_removes_a_successor_key() { + use crate::state::{BeginOutcome, FinishOutcome}; + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkF5DisarmEEEEEEEEEEEEEEEEEEEEEEEEEEEE/repo"; + + // A admits and exits normally; HOLD the disarmed guard to keep the window open. + let guard_a = admit(&inflight, repo); + let disarmed = match guard_a.finish_or_take_pending() { + FinishOutcome::Finished(g) => g, + FinishOutcome::Pending(..) => panic!("nothing was pending"), + }; + assert!(inflight.is_empty(), "A's exit released the key"); + + // Successor B is admitted inside the remove→drop window. + let guard_b = admit(&inflight, repo); + assert_eq!(inflight.len(), 1); + + // A's disarmed guard now drops. B's key must SURVIVE: a third push still + // coalesces against B's in-flight task. + drop(disarmed); + assert_eq!( + inflight.len(), + 1, + "dropping A's disarmed guard must not remove successor B's key" + ); + assert!( + matches!(inflight.try_begin(repo, vec![]), BeginOutcome::Coalesced), + "B's task is still the (only) in-flight task — at-most-one-per-repo holds" + ); + drop(guard_b); + assert!(inflight.is_empty()); + } + + /// Pending overflow: past the 1024-pair bound the slot degrades to the FullScan + /// marker (bounded memory under a hostile push burst); at exactly the bound it + /// stays a Tips batch. The marker is an explicit variant, never an empty tip + /// list — an empty-tips encoding would drain to an empty delta and pin nothing. + #[test] + fn u5_pending_overflow_degrades_to_full_scan_marker() { + use crate::state::{BeginOutcome, FinishOutcome, PendingWork}; + let inflight = crate::state::EncryptInflight::new(); + let pair = |i: usize| (format!("old{i}"), format!("new{i}")); + + // At the bound: exactly 1024 pairs stay a Tips batch. + let repo_at = "owner/at-bound"; + let g = admit(&inflight, repo_at); + assert!(matches!( + inflight.try_begin(repo_at, (0..1024).map(pair).collect()), + BeginOutcome::Coalesced + )); + match g.finish_or_take_pending() { + FinishOutcome::Pending(g, PendingWork::Tips(v)) => { + assert_eq!(v.len(), 1024, "at the bound the pairs are kept verbatim"); + assert!(matches!( + g.finish_or_take_pending(), + FinishOutcome::Finished(_) + )); + } + other => panic!( + "expected a Tips batch at the bound, got {:?}", + match other { + FinishOutcome::Pending(_, w) => Some(w), + FinishOutcome::Finished(_) => None, + } + ), + } + + // Past the bound: the accumulated slot degrades to FullScan and later + // merges are absorbed (still one bounded marker, not a growing list). + let repo_over = "owner/over-bound"; + let g = admit(&inflight, repo_over); + assert!(matches!( + inflight.try_begin(repo_over, (0..1024).map(pair).collect()), + BeginOutcome::Coalesced + )); + assert!(matches!( + inflight.try_begin(repo_over, vec![pair(9999)]), + BeginOutcome::Coalesced + )); + assert!(matches!( + inflight.try_begin(repo_over, vec![pair(10000)]), + BeginOutcome::Coalesced + )); + match g.finish_or_take_pending() { + FinishOutcome::Pending(g, work) => { + assert_eq!( + work, + PendingWork::FullScan, + "overflow degrades to the explicit FullScan marker" + ); + assert!(matches!( + g.finish_or_take_pending(), + FinishOutcome::Finished(_) + )); + } + FinishOutcome::Finished(_) => panic!("the overflowed pending work vanished"), + } + } + + // ---- u5 drain-pipeline fixtures: a real git repo + a DB repo row ---- + + fn u5_git(dir: &std::path::Path, args: &[&str]) -> String { + let out = std::process::Command::new("git") + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + + fn u5_init_repo(dir: &std::path::Path) { + u5_git(dir, &["init", "-q", "-b", "main"]); + u5_git(dir, &["config", "user.email", "t@t"]); + u5_git(dir, &["config", "user.name", "t"]); + } + + /// Commit `name` (parent dirs created) with `body`; returns the commit sha. + fn u5_commit_file(dir: &std::path::Path, name: &str, body: &str) -> String { + let path = dir.join(name); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(&path, body).unwrap(); + u5_git(dir, &["add", name]); + u5_git(dir, &["commit", "-qm", &format!("add {name}")]); + u5_git(dir, &["rev-parse", "HEAD"]) + } + + /// A drain-task context over the test state and an on-disk repo. Empty + /// `ipfs_api`/`irys_url` keep the pin/anchor stages inert (no network). + fn u5_ctx( + state: &AppState, + rec: &crate::db::RepoRecord, + repo_path: std::path::PathBuf, + git_bin: &str, + sem: std::sync::Arc, + ) -> EncryptTaskCtx { + EncryptTaskCtx { + ipfs_api: String::new(), + repo_path, + db: state.db.clone(), + repo_id: rec.id.clone(), + owner_did: rec.owner_did.clone(), + repo_name: rec.name.clone(), + irys_url: String::new(), + http_client: std::sync::Arc::clone(&state.http_client), + node_did: state.node_did.to_string(), + node_keypair: std::sync::Arc::clone(&state.node_keypair), + git_bin: git_bin.to_string(), + git_timeout: std::time::Duration::from_secs(600), + encrypt_sem: sem, + pin_sem: std::sync::Arc::new(tokio::sync::Semaphore::new(64)), + } + } + + /// #174 U3: a coalesced drain that runs after the repo row was deleted and + /// recreated under the same owner/name must resolve the LIVE row's id, not the + /// id frozen into the task ctx at spawn. Encrypted-pin metadata written under + /// the dead id is invisible to authorized readers on the live row. + /// + /// `resolve_drain_object_list` already re-fetches by owner/name and uses + /// `record.id` for the visibility-rule read; this binds the same id to the + /// encrypt write, which was still taking `ctx.repo_id`. + #[sqlx::test] + async fn u3_drain_resolves_the_refetched_repo_id_after_an_id_rotation(pool: sqlx::PgPool) { + let raw = pool.clone(); + let state = crate::test_support::test_state(pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + u5_init_repo(tmp.path()); + let c1 = u5_commit_file(tmp.path(), "a.txt", "one\n"); + + state + .db + .upsert_mirror_repo("z6u3rot", "r", "/u3-rotation", None, false) + .await + .unwrap(); + let before = state.db.get_repo("z6u3rot", "r").await.unwrap().unwrap(); + let ctx = u5_ctx( + &state, + &before, + tmp.path().to_path_buf(), + "git", + std::sync::Arc::new(tokio::sync::Semaphore::new(4)), + ); + assert_eq!(ctx.repo_id, before.id, "ctx captures the spawn-time id"); + + // Delete + recreate under the SAME owner/name. This is the rotation: a new + // row id over the same on-disk bare repo. + sqlx::query("DELETE FROM repos WHERE id = $1") + .bind(&before.id) + .execute(&raw) + .await + .unwrap(); + let after = crate::db::RepoRecord { + id: Uuid::new_v4().to_string(), + name: before.name.clone(), + owner_did: before.owner_did.clone(), + description: None, + is_public: true, + default_branch: "main".to_string(), + created_at: Utc::now(), + updated_at: Utc::now(), + disk_path: before.disk_path.clone(), + forked_from: None, + machine_id: None, + }; + state.db.create_repo(&after).await.unwrap(); + assert_ne!(after.id, before.id, "the recreate really did rotate the id"); + + let (drain_id, _list, _rules, _pub) = resolve_drain_object_list( + &ctx, + crate::state::PendingWork::Tips(vec![(ZERO_SHA.to_string(), c1.clone())]), + ) + .await + .expect("a public repo drains to a pin list"); + + assert_eq!( + drain_id, after.id, + "the drain must write its encrypted-pin metadata under the LIVE row id" + ); + assert_ne!( + drain_id, ctx.repo_id, + "the spawn-time id is the deleted row; metadata written there is \ + unreachable from the live repo" + ); + } + + /// The drain resolves a coalesced push's tip pair to exactly that push's + /// introduced objects (delta semantics — the F5 observable: push B's pins are + /// recorded by the drain, and pre-existing objects are not re-listed). + #[sqlx::test] + async fn u5_drain_resolves_coalesced_tips_to_their_objects(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + u5_init_repo(tmp.path()); + let c1 = u5_commit_file(tmp.path(), "a.txt", "one\n"); + let c2 = u5_commit_file(tmp.path(), "b.txt", "two\n"); + state + .db + .upsert_mirror_repo("z6u5delta", "d", "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6u5delta", "d").await.unwrap().unwrap(); + let ctx = u5_ctx( + &state, + &rec, + tmp.path().to_path_buf(), + "git", + std::sync::Arc::new(tokio::sync::Semaphore::new(4)), + ); + + // Push B advanced main c1 -> c2 and lost try_begin; its pair was coalesced. + let (_drain_id, list, _rules, is_public) = resolve_drain_object_list( + &ctx, + crate::state::PendingWork::Tips(vec![(c1.clone(), c2.clone())]), + ) + .await + .expect("a public repo drains to a pin list"); + assert!(is_public, "mirror rows are public"); + let got: std::collections::HashSet = list.into_iter().collect(); + let new_blob = u5_git(tmp.path(), &["rev-parse", "HEAD:b.txt"]); + let old_blob = u5_git(tmp.path(), &["rev-parse", &format!("{c1}:a.txt")]); + assert!( + got.contains(&c2) && got.contains(&new_blob), + "B's commit and blob are in the drained pin list (the F5 fix)" + ); + assert!( + !got.contains(&c1) && !got.contains(&old_blob), + "pre-existing objects are not re-listed (delta, not full scan)" + ); + } + + /// The FullScan marker drains through the FLAGGED full-scan path to a NON-EMPTY + /// candidate set. RED arm of the encoding: were the marker a plain empty-tips + /// call, the deletion-only fast path would return an empty delta and the drain + /// would pin nothing (the F5 silent loss resurfacing). + #[sqlx::test] + async fn u5_drain_full_scan_marker_yields_nonempty_candidates(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + u5_init_repo(tmp.path()); + let c1 = u5_commit_file(tmp.path(), "a.txt", "one\n"); + state + .db + .upsert_mirror_repo("z6u5full", "f", "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6u5full", "f").await.unwrap().unwrap(); + let ctx = u5_ctx( + &state, + &rec, + tmp.path().to_path_buf(), + "git", + std::sync::Arc::new(tokio::sync::Semaphore::new(4)), + ); + + let (_drain_id, list, _rules, _pub) = + resolve_drain_object_list(&ctx, crate::state::PendingWork::FullScan) + .await + .expect("a public repo drains to a pin list"); + let got: std::collections::HashSet = list.into_iter().collect(); + assert!( + !got.is_empty(), + "the FullScan drain must enumerate the repo — an empty list means the \ + marker collapsed into the empty-tips fast path" + ); + let blob = u5_git(tmp.path(), &["rev-parse", "HEAD:a.txt"]); + assert!( + got.contains(&c1) && got.contains(&blob), + "the full-scan drain covers the repo's commit and blob" + ); + } + + /// Rules tightened between the coalesced push and its drain are honored, fail + /// closed: the drain re-fetches rules/is_public fresh, so (1) a newly-withheld + /// blob is NOT pinned, and (2) a repo whose root became unreadable to the + /// anonymous public drains to nothing at all. + #[sqlx::test] + async fn u5_drain_honors_rules_tightened_after_the_push(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + u5_init_repo(tmp.path()); + u5_commit_file(tmp.path(), "pub.txt", "public\n"); + let c2 = u5_commit_file(tmp.path(), "secret/hidden.txt", "sealed\n"); + state + .db + .upsert_mirror_repo("z6u5tight", "t", "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6u5tight", "t").await.unwrap().unwrap(); + let ctx = u5_ctx( + &state, + &rec, + tmp.path().to_path_buf(), + "git", + std::sync::Arc::new(tokio::sync::Semaphore::new(4)), + ); + let pending = || crate::state::PendingWork::Tips(vec![(ZERO_SHA.to_string(), c2.clone())]); + + // At push time the repo had no rules. TIGHTEN before the drain: /secret/** + // becomes reader-gated. The drain must re-fetch and withhold the new blob. + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &[READER_DID.to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + let (_drain_id, list, _rules, _pub) = resolve_drain_object_list(&ctx, pending()) + .await + .expect("still announceable at root"); + let got: std::collections::HashSet = list.into_iter().collect(); + let pub_blob = u5_git(tmp.path(), &["rev-parse", "HEAD:pub.txt"]); + let secret_blob = u5_git(tmp.path(), &["rev-parse", "HEAD:secret/hidden.txt"]); + assert!( + got.contains(&pub_blob), + "the still-public blob is pinned by the drain" + ); + assert!( + !got.contains(&secret_blob), + "a blob withheld by a rule added AFTER the push must NOT be pinned by \ + the drain (fresh rules, fail closed)" + ); + + // Tighten further: root becomes reader-gated → not announceable to the + // anonymous public → the drain pins nothing at all. + state + .db + .set_visibility_rule( + &rec.id, + "/", + crate::db::VisibilityMode::A, + &[READER_DID.to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + assert!( + resolve_drain_object_list(&ctx, pending()).await.is_none(), + "a repo no longer announceable under current rules drains to nothing \ + (fail closed)" + ); + } + + /// #174 F2 / KTD-3 re-derivation equivalence: the object set the Pinata worker + /// re-derives from ONLY the ref tuples (`pinata_object_list_for_refs`, run once a + /// pin slot frees) must equal exactly what the old retained `object_list` would + /// have pinned — the inline-resolved delta, filtered by the withheld set. If the + /// two differ, the memory fix changed what gets pinned; they must not. + #[tokio::test] + async fn f2_pinata_rederivation_equals_retained_object_list() { + let tmp = tempfile::TempDir::new().unwrap(); + u5_init_repo(tmp.path()); + let c1 = u5_commit_file(tmp.path(), "a.txt", "one\n"); + let c2 = u5_commit_file(tmp.path(), "b.txt", "two\n"); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(4)); + let timeout = std::time::Duration::from_secs(600); + + // What the OLD retained-list task WOULD have pinned: the inline pipeline the + // receive-pack tail ran before moving `object_list` into the closure — the + // delta for main c1 -> c2, filtered by the (empty) withheld set. + let candidates = crate::git::push_delta::resolve_candidates_for_push( + sem.clone(), + tmp.path().to_path_buf(), + vec![c2.clone()], + vec![c1.clone()], + "git".to_string(), + timeout, + false, + ) + .await; + assert!( + !candidates.full_scan, + "the c1 -> c2 push is a delta, not a full scan" + ); + let retained: std::collections::HashSet = + crate::git::visibility_pack::replicable_objects( + candidates.candidates, + &std::collections::HashSet::new(), + ) + .into_iter() + .collect(); + + // What the worker re-derives from only the (ref, old, new) tuples. Empty rules + // + is_public => announceable, withheld = {} (the common Pinata case). + let ref_updates = vec![("refs/heads/main".to_string(), c1.clone(), c2.clone())]; + let rederived: std::collections::HashSet = pinata_object_list_for_refs( + sem.clone(), + tmp.path().to_path_buf(), + &ref_updates, + Some(Vec::new()), + true, + "z6MkPinataOwnerAAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + "git".to_string(), + timeout, + ) + .await + .1 + .into_iter() + .collect(); + + let new_blob = u5_git(tmp.path(), &["rev-parse", "HEAD:b.txt"]); + assert!( + retained.contains(&c2) && retained.contains(&new_blob), + "the push introduced the new commit and blob" + ); + assert_eq!( + rederived, retained, + "the worker's git rev-list re-derivation must yield exactly the object set \ + the retained list would have pinned — the memory fix must not change what pins" + ); + } + + /// #174 F2 / KTD-3 reaped + deadline-bounded: the worker's re-derivation git children + /// run through the same INV-22 bounded, process-group-reaped helpers the sibling scans + /// use. On a git that hangs on both `rev-list` and `--batch-all-objects`, + /// `pinata_object_list_for_refs` must RETURN within the watchdog budget (the group is + /// SIGKILLed + reaped at the deadline), not block. A bare `Command::output()` here + /// would hang past the ceiling (RED). + #[cfg(unix)] + #[tokio::test] + async fn f2_pinata_rederivation_is_deadline_bounded_and_reaped() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::TempDir::new().unwrap(); + // Empty rules => replication_withheld_set short-circuits (no git). The tip + // peel (`cat-file -t`) reports a commit so the delta stage proceeds; rev-list + // and the full-scan `cat-file --batch-all-objects` both hang (bounded 30s so a + // broken test cannot leak a permanent orphan). + let fake = dir.path().join("fakegit"); + std::fs::write( + &fake, + "#!/bin/sh\ncase \"$1\" in\n \ + cat-file) case \"$*\" in *--batch-all-objects*) i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;; *) echo commit ;; esac ;;\n \ + rev-list) i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;;\n \ + *) : ;;\nesac\nexit 0\n", + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + let git_bin = fake.to_str().unwrap().to_string(); + + let ref_updates = vec![( + "refs/heads/main".to_string(), + ZERO_SHA.to_string(), + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef".to_string(), + )]; + let got = tokio::time::timeout( + std::time::Duration::from_secs(10), + pinata_object_list_for_refs( + std::sync::Arc::new(tokio::sync::Semaphore::new(4)), + dir.path().to_path_buf(), + &ref_updates, + Some(Vec::new()), + true, + "z6MkPinataOwnerAAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + git_bin, + std::time::Duration::from_millis(400), + ), + ) + .await + .expect( + "pinata_object_list_for_refs must return within the watchdog budget — a hang \ + means the re-derivation git is not deadline-bounded / group-reaped (RED)", + ); + assert!( + got.1.is_empty(), + "a hung git yields nothing this push (the reconciliation sweep backstops)" + ); + } + + /// Hot-repo drain at encrypt-pool size 1: the task loop holds NO task-level + /// permit, so per-iteration helper acquires (withheld walk, candidate scan, + /// recipients walk) each get the pool's single permit in turn and BOTH the + /// snapshot iteration and the coalesced-drain iteration complete. RED if the + /// loop takes a task-level permit: the first helper acquire nests over the + /// same exhausted semaphore and the task parks forever. + #[sqlx::test] + async fn u5_hot_repo_drain_completes_at_pool_size_one(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + u5_init_repo(tmp.path()); + u5_commit_file(tmp.path(), "pub.txt", "public\n"); + let c2 = u5_commit_file(tmp.path(), "secret/hidden.txt", "sealed\n"); + state + .db + .upsert_mirror_repo("z6u5hot", "h", "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6u5hot", "h").await.unwrap().unwrap(); + // A path-scoped rule so every gated walk actually runs (withheld walk on + // the drain, recipients walk on both iterations). READER_DID carries no + // resolvable key, so the encrypt stage plans no seal and stays offline. + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &[READER_DID.to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + let rules = state.db.list_visibility_rules(&rec.id).await.unwrap(); + + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + let ctx = u5_ctx(&state, &rec, tmp.path().to_path_buf(), "git", sem); + + let inflight = crate::state::EncryptInflight::new(); + let guard = admit(&inflight, &rec.id); + assert!(matches!( + inflight.try_begin(&rec.id, vec![(ZERO_SHA.to_string(), c2)]), + crate::state::BeginOutcome::Coalesced + )); + + tokio::time::timeout( + std::time::Duration::from_secs(60), + run_encrypt_pin_task(ctx, guard, Vec::new(), Some(rules), true), + ) + .await + .expect( + "the drain must complete at pool size 1 — a task-level permit would \ + deadlock the helper-internal acquires", + ); + assert!( + inflight.is_empty(), + "the drained task released its repo key on exit" + ); + } + + /// The task LOOP is load-bearing: work coalesced during the snapshot iteration + /// is drained (its candidate scan runs git) before the task exits. RED under + /// the drain-loop revert (task drops its guard after the snapshot without + /// checking pending): the fake git never runs and the marker is absent. + #[cfg(unix)] + #[sqlx::test] + async fn u5_task_drains_coalesced_work_before_exiting(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + let marker = tmp.path().join("git.ran"); + // The drain's candidate scan probes the tip type then walks: report a + // commit tip and an empty rev-list, recording every invocation. + let body = format!( + "#!/bin/sh\necho ran >> \"{}\"\ncase \"$1\" in\n cat-file) echo commit ;;\n *) : ;;\nesac\nexit 0\n", + marker.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + state + .db + .upsert_mirror_repo("z6u5loop", "l", "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6u5loop", "l").await.unwrap().unwrap(); + let ctx = u5_ctx( + &state, + &rec, + tmp.path().to_path_buf(), + &git_bin, + std::sync::Arc::new(tokio::sync::Semaphore::new(4)), + ); + + let inflight = crate::state::EncryptInflight::new(); + let guard = admit(&inflight, &rec.id); + // Push B coalesces mid-flight with a real (created-ref) tip pair. + assert!(matches!( + inflight.try_begin( + &rec.id, + vec![( + ZERO_SHA.to_string(), + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef".to_string(), + )] + ), + crate::state::BeginOutcome::Coalesced + )); + + // Snapshot: empty object list, no rules — the snapshot iteration itself + // spawns no git, so any git invocation below belongs to the DRAIN. + run_encrypt_pin_task(ctx, guard, Vec::new(), None, true).await; + assert!( + marker.exists(), + "the task must drain B's coalesced tips (candidate scan runs git) \ + before exiting — an absent marker is the F5 skip" + ); + assert!( + inflight.is_empty(), + "the empty pending check at task end released the key" + ); + } + + /// #174 SC2 (per-source key, U1): the per-caller read sub-cap keys on the + /// resolved source IP, NOT the signed DID, so a disposable-DID farm cannot + /// multiply its budget. Fill the source IP's single read slot, then drive two + /// requests signed under DIFFERENT DIDs from that SAME IP: both must shed 503 + /// (keyed by the saturated IP, not their own free DID slots). A signed request + /// from a DIFFERENT source IP keeps its own budget. Revert `read_caller_key` to + /// prefer the DID and the same-IP assertions go green-not-503 (each fresh DID + /// gets a free slot) -- the farm-defeat mutation probe. + #[sqlx::test] + async fn info_refs_per_caller_cap_keys_on_ip_not_did(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6pcip", "pc", "/tmp/pc-nonexistent", None, false) + .await + .unwrap(); + + let did_a = "did:key:z6MkPerCallerKeyingProofDidAAAAAAAAAAAAAAAA"; + let did_b = "did:key:z6MkPerCallerKeyingProofDidBBBBBBBBBBBBBBBB"; + let peer: SocketAddr = "203.0.113.51:5000".parse().unwrap(); + + // Fill the SOURCE IP's single read slot; both DIDs' own slots stay free. + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first slot for this source IP"); + + // Signed as DID_A from `peer`: keyed by the saturated source IP -> shed 503. + let router = crate::server::build_router(state.clone()); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6pcip/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req.extensions_mut() + .insert(crate::auth::AuthenticatedDid(did_a.to_string())); + assert_eq!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a signed caller must be keyed by its source IP, not its DID: the saturated IP must shed it 503" + ); + + // Same IP, a DIFFERENT DID: still keyed by the same saturated IP -> also shed. + // The farm defeat: minting a fresh DID buys no fresh per-source budget. + let router2 = crate::server::build_router(state.clone()); + let mut req2 = Request::builder() + .method(Method::GET) + .uri("/z6pcip/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req2.extensions_mut().insert(ConnectInfo(peer)); + req2.extensions_mut() + .insert(crate::auth::AuthenticatedDid(did_b.to_string())); + assert_eq!( + router2.oneshot(req2).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a second DID from the same source IP must also shed 503: a DID farm cannot multiply the per-source budget" + ); + + // A signed caller from a DIFFERENT source IP keeps its own budget -> not shed. + let other: SocketAddr = "203.0.113.52:5000".parse().unwrap(); + let router3 = crate::server::build_router(state.clone()); + let mut req3 = Request::builder() + .method(Method::GET) + .uri("/z6pcip/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req3.extensions_mut().insert(ConnectInfo(other)); + req3.extensions_mut() + .insert(crate::auth::AuthenticatedDid(did_a.to_string())); + assert_ne!( + router3.oneshot(req3).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a signed caller from a different source IP must keep its own per-source budget" + ); + } + + /// #174 SC2 (None-key): a request with no resolvable caller key (no ConnectInfo, + /// no trusted header) must NOT be shed by the per-caller cap even when another + /// caller's budget is full — it is bounded by the global read pool only. A None + /// key never keys into the map, so it never 503s from the per-caller sub-cap. + #[sqlx::test] + async fn info_refs_none_key_bypasses_per_caller_cap(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::http::{Method, Request, StatusCode}; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6pcnone", "pc", "/tmp/pc-nonexistent", None, false) + .await + .unwrap(); + // Saturate an unrelated caller's budget; the None-key request must be + // unaffected because it never keys into the per-caller map. + let _slot = state + .git_read_per_caller + .try_acquire("203.0.113.99") + .expect("hold an unrelated caller's slot"); + + // No ConnectInfo inserted -> PeerAddr is None -> no per-caller key. + let router = crate::server::build_router(state.clone()); + let req = Request::builder() + .method(Method::GET) + .uri("/z6pcnone/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + assert_ne!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a request with no resolvable caller key must not be shed by the per-caller cap" + ); + } + + /// Repo creation must be throttled by the per-IP creation limiter BEFORE + /// signature verification — otherwise a DID farm (one throwaway did:key per + /// repo, each carrying a valid but machine-solved iCaptcha proof) walks past + /// the per-DID limiter and floods the network, as in the recurring spam-repo + /// incidents. A 429 (not a 401) on an unsigned request from an exhausted IP + /// proves the IP brake runs outermost, ahead of auth. + #[sqlx::test] + async fn repo_creation_is_rate_limited_by_ip(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::time::Duration; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + // Tiny limit, keyed on the socket peer (no trusted proxy). + state.create_ip_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let peer: SocketAddr = "203.0.113.77:7000".parse().unwrap(); + // Exhaust this peer's single-request budget up front. + assert!( + state + .create_ip_rate_limiter + .check(&peer.ip().to_string()) + .await + ); + + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::POST) + .uri("/api/v1/repos") + .header("content-type", "application/json") + .body(Body::from(r#"{"name":"flood","is_public":true}"#)) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + + let status = router.oneshot(req).await.unwrap().status(); + assert_eq!( + status, + StatusCode::TOO_MANY_REQUESTS, + "repo creation must be IP-throttled before signature verification" + ); + } + + // ── #174 U2 / F3: second same-repo push serialized until a disconnected first ── + // push's git process GROUP is reaped (RepoWriteLease riding the disconnect reaper). + + /// `kill(pid, 0)` liveness probe (same-uid here, so EPERM never applies). + #[cfg(unix)] + fn f3_alive(pid: i32) -> bool { + unsafe { libc::kill(pid, 0) == 0 } + } + + /// F3 (P1, RED-before/GREEN-after): on a client disconnect DURING receive-pack the + /// disconnected push's git group is torn down by KillGroupOnDrop's detached reaper + /// (~4s TERM/grace/KILL/reap), while RepoWriteGuard::Drop releases the pg advisory + /// lock at the disconnect INSTANT. Without the in-process write lease, a second + /// same-node push then acquires the freed pg lock and mutates the shared local repo + /// WHILE the first group is still writing — a torn snapshot. The lease is held by the + /// write-path AdmissionGuard, which rides that reaper, so the second push must not run + /// its receive-pack (mutate the repo) until the first group is reaped. + /// + /// The fake git labels the pushes by receive-pack arrival order (atomic mkdir): the + /// first (push A) forks a SIGTERM-IGNORING descendant, records its pid, then hangs + /// (so A can be dropped mid-transfer and its group survives the SIGTERM grace); the + /// second (push B, a DIFFERENT source) records that its receive-pack ran — i.e. that + /// B mutated the repo. The load-bearing invariant is strictly ordered, not + /// time-windowed: B's marker must NEVER appear while A's descendant is still alive. + /// + /// Load-bearing: pre-fix (no lease) A's disconnect frees the pg lock, B's + /// acquire_write succeeds within its ~1s retry, and B's receive-pack runs (marker + /// appears) WHILE A's descendant is still alive — RED. With the lease the reaper + /// holds it until the group is ESRCH-gone, so B's marker appears only AFTER — GREEN. + #[cfg(unix)] + #[sqlx::test] + async fn f3_second_push_serialized_until_disconnected_group_reaped(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let tmp = tempfile::TempDir::new().unwrap(); + let seq_a = tmp.path().join("seq_a"); // first receive-pack wins this mkdir = push A + let descfile = tmp.path().join("desc.pid"); // A's SIGTERM-ignoring descendant pid + let b_ran = tmp.path().join("b.ran"); // set when B's receive-pack runs (B mutates) + // receive-pack: first invocation (A) forks a TERM-ignoring descendant (bounded + // loop so a RED run leaks no permanent orphan), records its pid, and hangs in + // `wait`; second (B) records that it ran. rev-parse feeds any tail probe. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + receive-pack)\n\ + cat >/dev/null 2>/dev/null\n\ + if mkdir \"{seq}\" 2>/dev/null; then\n\ + sh -c 'trap \"\" TERM; echo $$ > \"{desc}\"; i=0; while [ $i -lt 60 ]; do sleep 0.1; i=$((i+1)); done' &\n\ + wait\n\ + else\n\ + echo 1 > \"{bran}\"\n\ + fi ;;\n\ + rev-parse) echo deadbeef ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + seq = seq_a.display(), + desc = descfile.display(), + bran = b_ran.display(), + ); + let git_bin = write_fake_git(tmp.path(), &body); + // One repo; A and B push to it (same record.id -> same lease key). Non-path-scoped + // + flush-only body -> no post-receive scans to muddy the observation. + let state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f3repo", "r1", false).await; + let did = "did:key:z6MkF3PusherAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + + // Push A: drive its handler future in slices until it reaches receive-pack and the + // fake records its descendant pid (A now holds the lease and is hung). + let mut fut_a = Box::pin(git_receive_pack( + State(state.clone()), + Path(("z6f3repo".to_string(), "r1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some("203.0.113.81:5000".parse::().unwrap())), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + )); + let mut desc: Option = None; + for _ in 0..1000 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut_a).await; + if let Some(p) = std::fs::read_to_string(&descfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + desc = Some(p); + break; + } + } + let desc = desc.expect("push A must reach receive-pack and record its descendant pid"); + assert!( + f3_alive(desc), + "A's descendant must be alive before the disconnect" + ); + + // Push B: a DIFFERENT source, same repo. It blocks on the lease A holds. + let state_b = state.clone(); + let handle_b = tokio::spawn(async move { + git_receive_pack( + State(state_b), + Path(("z6f3repo".to_string(), "r1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some( + "203.0.113.82:5000".parse::().unwrap(), + )), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await + }); + // Give B time to reach and block on the lease acquire. + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + assert!( + !b_ran.exists(), + "B must not have mutated the repo while A legitimately holds the lease" + ); + + // Client disconnect on A: drop its future. RepoWriteGuard::Drop frees the pg lock + // immediately; the write-path AdmissionGuard (carrying the lease's clone (a)) + // rides KillGroupOnDrop's detached reaper, which now tears down A's group. + drop(fut_a); + + // Load-bearing ordering invariant: while A's descendant is still alive (group not + // yet reaped), B must NOT have run its receive-pack. Poll until the descendant is + // gone; every step it is alive, B's marker must be absent. Pre-fix, B's marker + // appears here (RED); with the lease it can only appear after the reap (GREEN). + let mut reaped = false; + for _ in 0..800 { + if !f3_alive(desc) { + reaped = true; + break; + } + assert!( + !b_ran.exists(), + "F3 RED: push B mutated the repo while push A's disconnected git group \ + was still alive (descendant pid {desc}) — the second writer must be \ + serialized until the first group is reaped" + ); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + // Safety net so a RED run never leaks the orphan. + unsafe { + libc::kill(desc, libc::SIGKILL); + } + assert!( + reaped, + "A's disconnected group must be reaped within the teardown cap" + ); + + // GREEN tail: once the group is reaped the lease frees and B proceeds — its + // receive-pack runs (marker appears) and it returns 200. + let mut b_mutated = false; + for _ in 0..1000 { + if b_ran.exists() { + b_mutated = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert!( + b_mutated, + "push B must proceed and mutate the repo once A's group is reaped" + ); + let resp = tokio::time::timeout(std::time::Duration::from_secs(30), handle_b) + .await + .expect("push B must complete once the lease frees") + .expect("push B task must not panic") + .expect("push B must succeed"); + assert_eq!(resp.status(), 200, "push B lands 200 after serialization"); + } + + /// F3 clean-path no-regression: a clean push (no disconnect) releases the lease after + /// the receive-pack group is reaped and the (success-only) Tigris upload in + /// guard.release runs, so the per-repo lease entry is GC'd and a second same-repo + /// push proceeds immediately. A lease that failed to free on the clean path would + /// wedge every subsequent push to the repo. + #[cfg(unix)] + #[sqlx::test] + async fn f3_clean_push_frees_lease_and_second_push_proceeds(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let tmp = tempfile::TempDir::new().unwrap(); + // Clean receive-pack: drain stdin, exit 0. No hang, no descendant. + let body = "#!/bin/sh\ncase \"$1\" in\n receive-pack) cat >/dev/null 2>/dev/null ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + let state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f3clean", "c1", false).await; + let did = "did:key:z6MkF3CleanPusherAAAAAAAAAAAAAAAAAAAAAAAA"; + + let push = |st: AppState, peer: &'static str| async move { + tokio::time::timeout( + std::time::Duration::from_secs(30), + git_receive_pack( + State(st), + Path(("z6f3clean".to_string(), "c1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(peer.parse::().unwrap())), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ), + ) + .await + .expect("a clean push must not wedge on the lease") + .expect("the push must succeed") + }; + + let a = push(state.clone(), "203.0.113.91:5000").await; + assert_eq!(a.status(), 200, "clean push A lands 200"); + // The clean push freed its lease (both clones dropped) -> the entry GC'd. + assert!( + state.repo_write_leases.is_empty(), + "a clean push must free the per-repo lease (Drop-frees-key) so it never wedges" + ); + + let b = push(state.clone(), "203.0.113.92:5000").await; + assert_eq!( + b.status(), + 200, + "a second same-repo push proceeds after a clean first" + ); + assert!( + state.repo_write_leases.is_empty(), + "the lease entry must be freed again after the second clean push" + ); + } + + /// F3 DoS (P2, RED-before/GREEN-after): a second same-repo push that BLOCKS on the + /// per-repo write lease must hold NO global write permit while it waits. The lease + /// is a block-and-wait serializer, so a lease-blocked waiter can sit for up to + /// steal_after (~a full git_service_timeout window). If it grabs a scarce global + /// write-pool slot BEFORE blocking, a handful of hostile sources can stack same-repo + /// pushes, pin every write slot on lease-waiters sending zero bytes, and shed 503 on + /// every push to every OTHER repo node-wide. The fix acquires the lease BEFORE the + /// two write permits, so a blocked waiter pins no slot. + /// + /// Load-bearing invariant: with the write pool sized to 2, push A holds the lease and + /// is in-flight in receive-pack (1 permit held), and same-repo push B is blocked on + /// the lease, `git_write_semaphore.available_permits()` must stay 1 (only A holds). + /// Pre-fix B takes its permit BEFORE blocking on the lease, draining the pool to 0 + /// (RED). With the reorder B blocks before any permit, so the pool stays at 1 (GREEN). + #[cfg(unix)] + #[sqlx::test] + async fn f3_lease_blocked_waiter_holds_no_write_permit(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let seq_a = tmp.path().join("seq_a"); // first receive-pack wins this mkdir = push A + let a_inpack = tmp.path().join("a.inpack"); // set when A reaches receive-pack (holds lease+permit) + let b_ran = tmp.path().join("b.ran"); // set when B's receive-pack runs (B got past the lease) + // receive-pack: first invocation (A) marks that it reached the pack and hangs in a + // bounded loop (so a RED run leaks no permanent orphan); second (B) marks it ran. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + receive-pack)\n\ + cat >/dev/null 2>/dev/null\n\ + if mkdir \"{seq}\" 2>/dev/null; then\n\ + echo 1 > \"{ainp}\"\n\ + i=0; while [ $i -lt 100 ]; do sleep 0.1; i=$((i+1)); done\n\ + else\n\ + echo 1 > \"{bran}\"\n\ + fi ;;\n\ + rev-parse) echo deadbeef ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + seq = seq_a.display(), + ainp = a_inpack.display(), + bran = b_ran.display(), + ); + let git_bin = write_fake_git(tmp.path(), &body); + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f3dos", "d1", false).await; + // Size the write pool to 2 so one in-flight holder (A) leaves exactly one slot + // free, and a pool-holding waiter (B, pre-fix) would drain it to zero. Sizing to + // 1 would 503 B on the pool before it could block on the lease, hiding the bug. + state.git_write_semaphore = Arc::new(Semaphore::new(2)); + let did = "did:key:z6MkF3DosPusherAAAAAAAAAAAAAAAAAAAAAAAAAA"; + + // Push A: drive its handler future in slices until it reaches receive-pack (it now + // holds the lease and one write permit and is hung). available_permits() drops to 1. + let mut fut_a = Box::pin(git_receive_pack( + State(state.clone()), + Path(("z6f3dos".to_string(), "d1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some("203.0.113.71:5000".parse::().unwrap())), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + )); + for _ in 0..1000 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut_a).await; + if a_inpack.exists() { + break; + } + } + assert!( + a_inpack.exists(), + "push A must reach receive-pack and hold the lease" + ); + assert_eq!( + state.git_write_semaphore.available_permits(), + 1, + "with the pool sized to 2, the single in-flight holder (A) leaves one slot free" + ); + + // Push B: a DIFFERENT source, same repo. It blocks on the lease A holds. + let state_b = state.clone(); + let handle_b = tokio::spawn(async move { + git_receive_pack( + State(state_b), + Path(("z6f3dos".to_string(), "d1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some( + "203.0.113.72:5000".parse::().unwrap(), + )), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await + }); + + // Load-bearing check: while B is a lease-blocked waiter the pool must stay at 1 + // (only A holds a permit). Poll the invariant across a full window; pre-fix B + // grabs the last slot within ms and the pool falls to 0 (RED), post-fix it never + // does (GREEN). A stable state, not a one-shot race: B stays blocked on the lease + // (steal_after is far larger than this window) so once it settles the pool holds. + for _ in 0..100 { + assert_eq!( + state.git_write_semaphore.available_permits(), + 1, + "F3 DoS RED: a lease-blocked same-repo waiter took a global write permit \ + while sending zero bytes, draining the pool — a blocked waiter must pin \ + no write-pool slot" + ); + assert!( + !b_ran.exists(), + "B must not have run receive-pack while A holds the lease" + ); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + // Non-vacuous: B is genuinely parked on the lease, not returned early. + assert!( + !handle_b.is_finished(), + "push B must still be blocked on the lease at this point" + ); + + // Client disconnect on A: drop its future. The write-path AdmissionGuard rides the + // reaper, freeing the lease once A's group is reaped; B then proceeds. + drop(fut_a); + let resp = tokio::time::timeout(std::time::Duration::from_secs(30), handle_b) + .await + .expect("push B must complete once A's group is reaped and the lease frees") + .expect("push B task must not panic") + .expect("push B must succeed"); + assert_eq!(resp.status(), 200, "push B lands 200 after serialization"); + assert!(b_ran.exists(), "push B ran its receive-pack once unblocked"); + } + + /// Backstop for the F1 wait loops below. It is a HANG detector, never the thing an + /// assertion rests on: every F1 conclusion is drawn from a state the loop actually + /// observed (a returned shed, a second reference on the lease entry), so a loaded + /// machine only spends more iterations getting there. It must stay comfortably under + /// `F1_HOLD_SECS` so push A is still holding the lease when the loop gives up. + #[cfg(unix)] + const F1_BACKSTOP: std::time::Duration = std::time::Duration::from_secs(30); + + /// How long `f1_hanging_first_git`'s first receive-pack holds the lease. Bounded so a + /// RED run leaks no permanent orphan, and well above the two sequential `F1_BACKSTOP` + /// windows the discriminator can spend, so the hold never expires mid-observation on + /// a loaded machine. The old bound was 10s, which is under the 19s the pair took on a + /// contended box: push A's git would exit and free the lease mid-test. + #[cfg(unix)] + const F1_HOLD_SECS: usize = 150; + + /// Poll `cond` until it holds, yielding between checks so spawned handlers make + /// progress. Returns false if `cap` elapses first. Callers assert on the state the + /// loop settled into, not on the elapsed time. + #[cfg(unix)] + async fn f1_wait_for(cap: std::time::Duration, mut cond: impl FnMut() -> bool) -> bool { + let deadline = std::time::Instant::now() + cap; + loop { + if cond() { + return true; + } + if std::time::Instant::now() >= deadline { + return false; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + } + + /// Fake git for the U1 lease-park tests. The FIRST receive-pack invocation marks + /// that it reached the pack and hangs in a BOUNDED loop (so a RED run leaks no + /// permanent orphan); every later invocation marks that it got past the lease. + /// Returns `(git_bin, a_inpack marker, later_ran marker)`. + #[cfg(unix)] + fn f1_hanging_first_git( + tmp: &std::path::Path, + ) -> (String, std::path::PathBuf, std::path::PathBuf) { + let seq_a = tmp.join("f1_seq_a"); // first receive-pack wins this mkdir = push A + let a_inpack = tmp.join("f1_a.inpack"); + let later_ran = tmp.join("f1_later.ran"); + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + receive-pack)\n\ + cat >/dev/null 2>/dev/null\n\ + if mkdir \"{seq}\" 2>/dev/null; then\n\ + echo 1 > \"{ainp}\"\n\ + i=0; while [ $i -lt {hold} ]; do sleep 1; i=$((i+1)); done\n\ + else\n\ + echo 1 > \"{later}\"\n\ + fi ;;\n\ + rev-parse) echo deadbeef ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + seq = seq_a.display(), + ainp = a_inpack.display(), + later = later_ran.display(), + hold = F1_HOLD_SECS, + ); + (write_fake_git(tmp, &body), a_inpack, later_ran) + } + + /// Add a second repo to a state built by `f4_state_with_repo`, returning its DB id. + /// The U1 tests need two repos in ONE state to show that shedding on a contended + /// lease is confined to that repo. + #[cfg(unix)] + async fn f1_add_repo(state: &AppState, owner: &str, name: &str) -> String { + state + .db + .upsert_mirror_repo(owner, name, &format!("/unused-{owner}-{name}"), None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + state + .repo_store + .init(&rec.owner_did, &rec.name) + .await + .unwrap(); + rec.id + } + + /// #174 U2: the write lease must register under the STABLE DISK IDENTITY + /// (sanitized owner slug + repo name, what `RepoStore::local_path` and the pg + /// advisory lock key on), not `record.id`. + /// + /// The row id rotates on delete+recreate under the same slug while the bare + /// repo on disk is reused, so an id-keyed lease silently stops serializing + /// across that rotation and lets two writers onto one `objects/` directory. + /// Asserting on the key the holder actually registers under binds the + /// production call site — a helper unit test alone would not. + #[cfg(unix)] + #[sqlx::test] + async fn u2_lease_registers_under_the_disk_identity_not_the_row_id(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let tmp = tempfile::TempDir::new().unwrap(); + let (git_bin, a_inpack, _later_ran) = f1_hanging_first_git(tmp.path()); + let state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6u2key", "k1", false).await; + // Rotate the row to a UUID id first. f4_state_with_repo creates the repo + // through the mirror path, whose id is literally `owner_short/name` and so + // coincides with the identity key — which would leave this test unable to + // tell the two keys apart. A UUID id is also the real shape of the bug: an + // API-created repo, or any repo whose row was recreated under the same slug. + let seeded = state.db.get_repo("z6u2key", "k1").await.unwrap().unwrap(); + let rotated_id = Uuid::new_v4().to_string(); + sqlx::query("UPDATE repos SET id = $1 WHERE id = $2") + .bind(&rotated_id) + .bind(&seeded.id) + .execute(&pool) + .await + .unwrap(); + + let rec = state.db.get_repo("z6u2key", "k1").await.unwrap().unwrap(); + assert_eq!(rec.id, rotated_id, "the row really did take the new id"); + let identity = crate::state::repo_identity_key(&rec.owner_did, &rec.name); + assert_ne!( + identity, rec.id, + "the identity key must differ from the row id, or this test proves nothing" + ); + + let did = "did:key:z6MkU2KeyPusherAAAAAAAAAAAAAAAAAAAAAAAA"; + let peer: SocketAddr = "203.0.113.91:5000".parse().unwrap(); + let handle = tokio::spawn({ + let st = state.clone(); + let did = did.to_string(); + async move { + git_receive_pack( + State(st), + Path(("z6u2key".to_string(), "k1".to_string())), + Extension(crate::auth::AuthenticatedDid(did)), + crate::rate_limit::PeerAddr(Some(peer)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await + } + }); + assert!( + f1_wait_for(F1_BACKSTOP, || a_inpack.exists()).await, + "the push never reached receive-pack within {F1_BACKSTOP:?}" + ); + + assert_eq!( + state.repo_write_leases.refs_for(&identity), + 1, + "the lease holder must be registered under the stable disk identity" + ); + assert_eq!( + state.repo_write_leases.refs_for(&rec.id), + 0, + "and never under the rotating row id" + ); + + handle.abort(); + } + + /// #174 (RED-before/GREEN-after): the lease steal bound derived from + /// `git_service_timeout_secs` must not overflow. Unchecked, `* 2 + 60` panics the push + /// in a debug build and wraps to a short `Duration` in release, and a wrapped bound + /// would let a waiter steal the lease out from under a live push. Drives the handler + /// rather than the arithmetic, so the call-site wiring is what is under test. + /// + /// `GIT_SERVICE_TIMEOUT_SECS_MAX` means clap no longer admits a value this large, and + /// the test keeps `u64::MAX` anyway rather than moving to the ceiling. Two reasons, and + /// the second is the load-bearing one: `Config` is reachable by direct construction, + /// which is how this test and every other one build it; and at the ceiling the + /// arithmetic does not overflow, so a test pinned there would pass with the saturation + /// removed and prove nothing about this line. + #[cfg(unix)] + #[sqlx::test] + async fn push_survives_a_git_service_timeout_that_overflows_the_lease_bound( + pool: sqlx::PgPool, + ) { + use axum::extract::{Path, State}; + use axum::response::IntoResponse; + use axum::Extension; + use std::net::SocketAddr; + + let tmp = tempfile::TempDir::new().unwrap(); + let git_bin = write_fake_git( + tmp.path(), + "#!/bin/sh\n\ + case \"$1\" in\n\ + receive-pack) cat > /dev/null 2>/dev/null ;;\n\ + rev-parse) echo deadbeef ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + ); + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6ovflow", "o1", false).await; + // Every value above (u64::MAX - 60) / 2 overflows the derived bound; u64::MAX is + // the top of that tail, and the value clap accepted before the ceiling landed. + let mut cfg = (*state.config).clone(); + cfg.git_service_timeout_secs = u64::MAX; + state.config = std::sync::Arc::new(cfg); + + let resp = git_receive_pack( + State(state), + Path(("z6ovflow".to_string(), "o1".to_string())), + Extension(crate::auth::AuthenticatedDid( + "did:key:z6MkOverflowPusherAAAAAAAAAAAAAAAAAAAAAA".to_string(), + )), + crate::rate_limit::PeerAddr(Some("203.0.113.90:5000".parse::().unwrap())), + axum::http::HeaderMap::new(), + ref_update_body("1111111111111111111111111111111111111111"), + ) + .await + .expect("push must succeed under a maximal git_service_timeout_secs") + .into_response(); + assert_eq!( + resp.status(), + 200, + "a maximal service timeout must disable the steal bound, not break the push" + ); + } + + /// #174 U1 scenario 1 (RED-before/GREEN-after): parked pushes on one repo's write + /// lease are BOUNDED. `git_receive_pack` takes `body: Bytes`, so axum has already + /// buffered the whole pack (up to `max_pack_bytes`) before the handler runs, and the + /// park runs to `lease_steal_after` = `git_service_timeout_secs * 2 + 60` = 1260s at + /// defaults. An unbounded waiter set is therefore unbounded buffered memory held for + /// 21 minutes. With the cap at K, a holder plus K live waiters means the next push + /// sheds a 503 + Retry-After instead of joining the queue. + /// + /// Read as state, never as elapsed time: the shed is the returned `Overloaded`, and + /// "the queue did not grow" is the waiter count the loop polls to. The bound is on + /// LIVE WAITERS, so the holder is not counted; that is what keeps a leaked lease from + /// wedging the repo (see `steal_on_leaked_lease_still_works_under_the_waiter_cap`). + #[cfg(unix)] + #[sqlx::test] + async fn u1_push_past_the_lease_waiter_cap_sheds_with_503(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::response::IntoResponse; + use axum::Extension; + use std::net::SocketAddr; + + let tmp = tempfile::TempDir::new().unwrap(); + let (git_bin, a_inpack, _later_ran) = f1_hanging_first_git(tmp.path()); + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6u1cap", "c1", false).await; + // Cap of ONE live waiter, so a holder plus one parked push fills the repo's queue. + state.repo_write_leases = crate::state::RepoWriteLeases::new(1); + // The lease keys on the stable disk identity (#174 U2), not the row id. + let repo_id = { + let r = state.db.get_repo("z6u1cap", "c1").await.unwrap().unwrap(); + crate::state::repo_identity_key(&r.owner_did, &r.name) + }; + let did = "did:key:z6MkU1CapPusherAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let push = |peer: SocketAddr| { + let st = state.clone(); + let did = did.to_string(); + async move { + git_receive_pack( + State(st), + Path(("z6u1cap".to_string(), "c1".to_string())), + Extension(crate::auth::AuthenticatedDid(did)), + crate::rate_limit::PeerAddr(Some(peer)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await + } + }; + + // Push A holds the lease (its git hangs) until it is aborted below. + let handle_a = tokio::spawn(push("203.0.113.81:5000".parse().unwrap())); + assert!( + f1_wait_for(F1_BACKSTOP, || a_inpack.exists()).await, + "push A never reached receive-pack within {F1_BACKSTOP:?}" + ); + assert_eq!( + state.repo_write_leases.waiters_for(&repo_id), + 0, + "the uncontended holder must spend no waiter budget" + ); + + // Push B fills the single waiter slot. + let handle_b = tokio::spawn(push("203.0.113.82:5000".parse().unwrap())); + assert!( + f1_wait_for(F1_BACKSTOP, || state + .repo_write_leases + .waiters_for(&repo_id) + == 1) + .await, + "push B never parked on the contended lease within {F1_BACKSTOP:?}" + ); + + // Push C is past the cap: it must be turned away, not queued. + let c = tokio::time::timeout(F1_BACKSTOP, push("203.0.113.83:5000".parse().unwrap())) + .await + .expect("a push past the waiter cap must return, not park"); + assert!( + matches!(c, Err(AppError::Overloaded(_))), + "U1 RED: a push arriving past the repo's live-waiter cap joined the unbounded \ + park queue instead of shedding, holding its fully buffered pack for up to \ + steal_after (1260s at defaults); got {c:?}" + ); + let resp = c.unwrap_err().into_response(); + assert_eq!( + resp.status(), + 503, + "the shed must be a 503, consistent with the other admission paths" + ); + assert_eq!( + resp.headers().get("retry-after").unwrap().to_str().unwrap(), + "1", + "the shed must advertise Retry-After" + ); + assert_eq!( + state.repo_write_leases.waiters_for(&repo_id), + 1, + "the shed must not have joined the queue, and must leave no waiter residue" + ); + assert_eq!( + state.repo_write_leases.refs_for(&repo_id), + 2, + "only the holder and the one real waiter may reference the entry after a shed" + ); + + handle_a.abort(); + let _ = tokio::time::timeout(std::time::Duration::from_secs(60), handle_b).await; + } + + /// #174 U1 scenario 2, THE REGRESSION GUARD for the rejected design. A source parked + /// on repo A's lease must still be served on an UNCONTENDED repo B. The rejected fix + /// bounded parked bodies by taking the per-source write permit ABOVE the lease, which + /// makes a parked push spend that source's node-wide budget: the same pusher is then + /// denied on every other repo, and (since `TrustedProxy` defaults to `None`, so every + /// pusher behind a proxy or NAT resolves to one key) so is everyone else. Moving the + /// per-caller permit back above the lease turns this red. + /// + /// The per-source cap is 2 here: the holder spends one, so a parked push spending the + /// second is what denies repo B. Under the shipped ordering the parked push holds no + /// permit at all and repo B is served. + #[cfg(unix)] + #[sqlx::test] + async fn u1_a_source_parked_on_one_repo_is_still_served_on_another(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let tmp = tempfile::TempDir::new().unwrap(); + let (git_bin, a_inpack, later_ran) = f1_hanging_first_git(tmp.path()); + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6u1two", "r1", false).await; + state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(2, 100); + let repo1 = { + let r = state.db.get_repo("z6u1two", "r1").await.unwrap().unwrap(); + crate::state::repo_identity_key(&r.owner_did, &r.name) + }; + f1_add_repo(&state, "z6u1two", "r2").await; + let did = "did:key:z6MkU1TwoPusherAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let src: SocketAddr = "203.0.113.84:5000".parse().unwrap(); + let push = |repo: &'static str| { + let st = state.clone(); + let did = did.to_string(); + async move { + git_receive_pack( + State(st), + Path(("z6u1two".to_string(), repo.to_string())), + Extension(crate::auth::AuthenticatedDid(did)), + crate::rate_limit::PeerAddr(Some(src)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await + } + }; + + // Push A (this source) holds repo r1's lease; push B (same source) parks on it. + let handle_a = tokio::spawn(push("r1")); + assert!( + f1_wait_for(F1_BACKSTOP, || a_inpack.exists()).await, + "push A never reached receive-pack within {F1_BACKSTOP:?}" + ); + let handle_b = tokio::spawn(push("r1")); + assert!( + f1_wait_for(F1_BACKSTOP, || state.repo_write_leases.waiters_for(&repo1) + == 1) + .await, + "push B never parked on r1's contended lease within {F1_BACKSTOP:?}" + ); + + // Push C: same source, DIFFERENT repo, uncontended. It must be served. + let c = tokio::time::timeout(F1_BACKSTOP, push("r2")) + .await + .expect("a push to an uncontended repo must not park"); + let resp = c.unwrap_or_else(|e| { + panic!( + "U1 scenario 2 RED: a push to an UNCONTENDED repo was denied because the \ + same source had a push parked on a DIFFERENT repo's lease. A parked push \ + must hold no cross-repo admission budget; got {e:?}" + ) + }); + assert_eq!(resp.status(), 200, "the uncontended repo's push lands 200"); + assert!( + later_ran.exists(), + "the uncontended repo's push must have run its receive-pack" + ); + + handle_a.abort(); + let _ = tokio::time::timeout(std::time::Duration::from_secs(60), handle_b).await; + } + + /// #174 U1 scenario 3, THE CROSS-TENANT GUARD. `GITLAWB_TRUSTED_PROXY` is unset by + /// default (`TrustedProxy::None`), so behind an edge proxy, a NAT, or a CI pool every + /// pusher resolves to the SAME source key. A push parked on one repo's lease must not + /// shed a DIFFERENT pusher's push to a DIFFERENT repo. This is the shape that made + /// the rejected design a cross-tenant denial rather than a self-inflicted one: with + /// the per-source permit above the park, four parked pushes deny every push on the + /// node for up to 1260s. + /// + /// Same one source IP for all three pushes (the collapsed-key shape), distinct pusher + /// DIDs, per-source cap 2 so the holder plus a parked push would exhaust it. + #[cfg(unix)] + #[sqlx::test] + async fn u1_parked_push_does_not_shed_another_pusher_behind_the_same_ip(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let tmp = tempfile::TempDir::new().unwrap(); + let (git_bin, a_inpack, later_ran) = f1_hanging_first_git(tmp.path()); + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6u1nat", "n1", false).await; + // The default proxy trust: the resolved key is the peer IP, which is the edge's. + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(2, 100); + let repo1 = { + let r = state.db.get_repo("z6u1nat", "n1").await.unwrap().unwrap(); + crate::state::repo_identity_key(&r.owner_did, &r.name) + }; + f1_add_repo(&state, "z6u1nat", "n2").await; + // Every pusher arrives from the one edge IP, so they share a source key. + let edge: SocketAddr = "203.0.113.85:5000".parse().unwrap(); + let push = |pusher: &'static str, repo: &'static str| { + let st = state.clone(); + async move { + git_receive_pack( + State(st), + Path(("z6u1nat".to_string(), repo.to_string())), + Extension(crate::auth::AuthenticatedDid(pusher.to_string())), + crate::rate_limit::PeerAddr(Some(edge)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await + } + }; + + let handle_a = tokio::spawn(push( + "did:key:z6MkU1NatPusherOneAAAAAAAAAAAAAAAAAAAAAA", + "n1", + )); + assert!( + f1_wait_for(F1_BACKSTOP, || a_inpack.exists()).await, + "pusher one never reached receive-pack within {F1_BACKSTOP:?}" + ); + let handle_b = tokio::spawn(push( + "did:key:z6MkU1NatPusherTwoAAAAAAAAAAAAAAAAAAAAAA", + "n1", + )); + assert!( + f1_wait_for(F1_BACKSTOP, || state.repo_write_leases.waiters_for(&repo1) + == 1) + .await, + "pusher two never parked on n1's contended lease within {F1_BACKSTOP:?}" + ); + + // A third, unrelated pusher behind the same edge IP, on a different repo. + let c = tokio::time::timeout( + F1_BACKSTOP, + push("did:key:z6MkU1NatPusherThreeAAAAAAAAAAAAAAAAAA", "n2"), + ) + .await + .expect("an unrelated pusher's push to an uncontended repo must not park"); + let resp = c.unwrap_or_else(|e| { + panic!( + "U1 scenario 3 RED: one repo's parked push shed an UNRELATED pusher's push \ + to a DIFFERENT repo, because every pusher behind the proxy shares one \ + resolved source key. Contention on one repo must never deny another; \ + got {e:?}" + ) + }); + assert_eq!(resp.status(), 200, "the unrelated pusher lands 200"); + assert!( + later_ran.exists(), + "the unrelated pusher must have run its receive-pack" + ); + + handle_a.abort(); + let _ = tokio::time::timeout(std::time::Duration::from_secs(60), handle_b).await; + } + + /// #174 F1 (U1) key shape: the write sub-cap is keyed on the SOURCE, not the repo. + /// A source at its cap is shed on EVERY repo (otherwise one source could hold four + /// buffered bodies per repo, and the bound would be `repos x cap x max_pack_bytes`), + /// while a different source is shed on none. Uncontended leases here: this pins the + /// key's shape, not the acquisition order. + #[sqlx::test] + async fn f1_write_cap_key_is_per_source_not_per_repo(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + let mut state = crate::test_support::test_state(pool).await; + state.git_write_semaphore = Arc::new(Semaphore::new(4)); + state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + for name in ["k1", "k2"] { + state + .db + .upsert_mirror_repo( + "z6f1key", + name, + &format!("/tmp/f1-key-{name}-nonexistent"), + None, + false, + ) + .await + .unwrap(); + } + + let did = "did:key:z6MkF1KeyPusherAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let capped: SocketAddr = "203.0.113.65:5000".parse().unwrap(); + let other: SocketAddr = "203.0.113.66:5000".parse().unwrap(); + let _slot = state + .git_write_per_caller + .try_acquire(&capped.ip().to_string()) + .expect("pin the capped source at its single write slot"); + + let push = |peer: SocketAddr, repo: &'static str| { + let st = state.clone(); + async move { + git_receive_pack( + State(st), + Path(("z6f1key".to_string(), repo.to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(peer)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await + } + }; + + for repo in ["k1", "k2"] { + let r = push(capped, repo).await; + assert!( + matches!(r, Err(AppError::Overloaded(_))), + "the capped source must shed on repo {repo} too: the sub-cap is per \ + source, not per repo; got {r:?}" + ); + } + let r = push(other, "k2").await; + assert!( + !matches!(r, Err(AppError::Overloaded(_))), + "a different source must not be shed on any repo while the capped source \ + holds its slot; got {r:?}" + ); + } + + /// #174 F1 (U1) fallback: a caller with no resolvable source key (no trusted + /// header, no peer address) takes no per-caller permit and is bounded by the global + /// write pool only, exactly as before the move. `acquire_read_caller_permit` + /// returns `Ok(None)` for a `None` key, so it must never 503 here. + #[sqlx::test] + async fn f1_write_cap_is_inert_without_a_resolvable_source_key(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::sync::Arc; + use tokio::sync::Semaphore; + + let mut state = crate::test_support::test_state(pool).await; + state.git_write_semaphore = Arc::new(Semaphore::new(4)); + state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 1); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6f1none", "n1", "/tmp/f1-none-nonexistent", None, false) + .await + .unwrap(); + // Saturate the limiter's only key slot as well, so a request that DID resolve a + // key would be shed. The keyless request must still pass. + let _slot = state + .git_write_per_caller + .try_acquire("203.0.113.67") + .expect("occupy the limiter's single key slot"); + + let r = git_receive_pack( + State(state.clone()), + Path(("z6f1none".to_string(), "n1".to_string())), + Extension(crate::auth::AuthenticatedDid( + "did:key:z6MkF1NoKeyPusherAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + )), + crate::rate_limit::PeerAddr(None), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await; + assert!( + !matches!(r, Err(AppError::Overloaded(_))), + "a caller with no resolvable source key must fall back to the global write \ + pool only, never shed on the per-caller cap; got {r:?}" + ); + } + + /// #174 F1 (U1) no-regression: moving the per-caller permit above the lease must + /// not leak it. Two SEQUENTIAL clean pushes from the SAME source with the sub-cap + /// set to 1 must both land 200; if the first push's permit outlived its request the + /// second would 503, breaking every repeat pusher. + #[cfg(unix)] + #[sqlx::test] + async fn f1_sequential_pushes_from_one_source_release_the_write_permit(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let tmp = tempfile::TempDir::new().unwrap(); + let body = "#!/bin/sh\ncase \"$1\" in\n receive-pack) cat >/dev/null 2>/dev/null ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f1seq", "s1", false).await; + state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + let did = "did:key:z6MkF1SeqPusherAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let src: SocketAddr = "203.0.113.68:5000".parse().unwrap(); + + for attempt in 1..=2 { + let resp = tokio::time::timeout( + std::time::Duration::from_secs(30), + git_receive_pack( + State(state.clone()), + Path(("z6f1seq".to_string(), "s1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(src)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ), + ) + .await + .expect("a clean push must not wedge") + .unwrap_or_else(|e| panic!("push {attempt} from the same source must succeed: {e:?}")); + assert_eq!(resp.status(), 200, "clean push {attempt} lands 200"); + } + assert_eq!( + state.git_write_per_caller.tracked_keys(), + 0, + "every per-source write permit must be released when its push completes" + ); + } + // ---- #174 F2a: the coalescing gate runs BEFORE the withheld walk ---- + // + // These drive `post_receive_replication_tail` directly (the handler's detached + // tail, extracted so the ordering the gate depends on is observable) over a REAL + // git repo, with a logging git shim in front of the real binary. The shim's log + // is the seam: `ls-tree` lines are withheld-walk children, and a line naming a + // tip sha attributes a scan to the push that pushed it. + + /// A git shim that appends its argv to `log`, then delegates to the real git, so + /// the walks stay real while every child is observable. + #[cfg(unix)] + fn f2a_logging_git(dir: &std::path::Path, log: &std::path::Path) -> String { + write_fake_git( + dir, + &format!( + "#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"{}\"\nexec git \"$@\"\n", + log.display() + ), + ) + } + + fn f2a_log(log: &std::path::Path) -> String { + std::fs::read_to_string(log).unwrap_or_default() + } + + /// Withheld-walk children run so far. `ls-tree` is the walk's signature child + /// (`blob_paths` lists every reachable commit's tree); the delta scan and the + /// full-scan fallback use `rev-list` / `cat-file` instead. + fn f2a_walks(log: &std::path::Path) -> usize { + f2a_log(log) + .lines() + .filter(|l| l.starts_with("ls-tree")) + .count() + } + + /// A state whose git is the shim, plus a repo row (optionally path-scoped, so + /// the withheld walk actually runs rather than taking the no-rule shortcut). + /// The repo's on-disk path is passed to the tail directly, so no repo_store or + /// receive-pack plumbing is involved. + async fn f2a_state( + pool: sqlx::PgPool, + git_bin: &str, + owner: &str, + name: &str, + path_scoped: bool, + ) -> (AppState, crate::db::RepoRecord) { + let mut state = crate::test_support::test_state(pool).await; + state.git_bin = git_bin.to_string(); + state + .db + .upsert_mirror_repo(owner, name, "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + if path_scoped { + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkF2aReaderAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + } + (state, rec) + } + + fn f2a_update(ref_name: &str, new_sha: &str) -> Vec { + vec![RefUpdate { + old_sha: ZERO_SHA.to_string(), + new_sha: new_sha.to_string(), + ref_name: ref_name.to_string(), + }] + } + + const F2A_PUSHER: &str = "did:key:z6MkF2aPusherAAAAAAAAAAAAAAAAAAAAAAAAAA"; + + /// Scenario 1 (the finding). A second rapid push to the same repo coalesces + /// WITHOUT running the withheld walk. Asserted on the walk's git children, not + /// on the `Coalesced` outcome: with `try_begin` below the walk (the pre-fix + /// order) the second push still parks on the scan pool and re-walks, which is + /// exactly the accumulation jatmn found. + /// + /// The pin pool's only permit is held for the whole test, so both pushes' pin + /// tasks park before doing any git of their own: every `ls-tree` in the log is a + /// tail walk. + #[cfg(unix)] + #[sqlx::test] + async fn f2a_coalesced_push_does_not_run_the_withheld_walk(pool: sqlx::PgPool) { + let repo = tempfile::TempDir::new().unwrap(); + let bin = tempfile::TempDir::new().unwrap(); + u5_init_repo(repo.path()); + let c1 = u5_commit_file(repo.path(), "a.txt", "one\n"); + let c2 = u5_commit_file(repo.path(), "secret/s.txt", "two\n"); + let log = bin.path().join("git.log"); + let git_bin = f2a_logging_git(bin.path(), &log); + let (mut state, rec) = f2a_state(pool, &git_bin, "z6f2acoal", "c1", true).await; + state.pin_semaphore = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + let _held = state.pin_semaphore.clone().acquire_owned().await.unwrap(); + + post_receive_replication_tail( + state.clone(), + rec.clone(), + f2a_update("refs/heads/main", &c2), + repo.path().to_path_buf(), + F2A_PUSHER.to_string(), + ) + .await; + let after_first = f2a_walks(&log); + assert!( + after_first >= 1, + "the admitted push must have run the withheld walk; log:\n{}", + f2a_log(&log) + ); + assert_eq!( + state.encrypt_inflight.len(), + 1, + "the admitted push's task holds the repo key while it is parked on the pin pool" + ); + + post_receive_replication_tail( + state.clone(), + rec.clone(), + f2a_update("refs/heads/second", &c1), + repo.path().to_path_buf(), + F2A_PUSHER.to_string(), + ) + .await; + + assert_eq!( + f2a_walks(&log), + after_first, + "a push that coalesces must not run the withheld walk at all; log:\n{}", + f2a_log(&log) + ); + assert_eq!( + state + .encrypt_inflight + .pending_for(&crate::state::repo_identity_key(&rec.owner_did, &rec.name)), + Some(crate::state::PendingWork::Tips(vec![( + ZERO_SHA.to_string(), + c1.clone() + )])), + "the coalesced push's tip pairs are queued for the in-flight task's drain" + ); + } + /// Poll `cond` until it holds, with a bound so a regression fails the test + /// rather than hanging the suite. + async fn f2a_wait_for(mut cond: impl FnMut() -> bool, what: &str) { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); + while !cond() { + assert!( + std::time::Instant::now() < deadline, + "timed out waiting for {what}" + ); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + } + + /// A `rev-list --objects` line names the tips a DELTA scan was asked to resolve, + /// so it attributes that scan to one push's tips. The withheld walk's own + /// `rev-list --all` / `ls-tree` lines never carry a tip as an argument this way. + fn f2a_delta_scanned(log: &std::path::Path, tip: &str) -> bool { + f2a_log(log) + .lines() + .any(|l| l.starts_with("rev-list --objects") && l.contains(tip)) + } + + /// Scenario 2. The tip pairs a coalesced push queues are consumed by the + /// in-flight task's drain, which is what makes coalescing lossless. Asserted on + /// the drained WORK (the delta scan the drain runs for those tips), not on the + /// key going empty: an armed guard's Drop empties the key too, so `is_empty` + /// cannot tell a drain from a discard. + /// + /// The coalesced tips are injected through `try_begin` directly rather than by a + /// second tail. A second tail would spawn its own Pinata worker, which re-derives + /// from the SAME tips, and the two scans are indistinguishable in the git log; the + /// tail-to-`try_begin` half is covered by scenario 1's pending-slot assertion. + #[cfg(unix)] + #[sqlx::test] + async fn f2a_coalesced_tips_are_drained_by_the_inflight_task(pool: sqlx::PgPool) { + let repo = tempfile::TempDir::new().unwrap(); + let bin = tempfile::TempDir::new().unwrap(); + u5_init_repo(repo.path()); + u5_commit_file(repo.path(), "a.txt", "one\n"); + let c2 = u5_commit_file(repo.path(), "secret/s.txt", "two\n"); + let log = bin.path().join("git.log"); + let git_bin = f2a_logging_git(bin.path(), &log); + let (mut state, rec) = f2a_state(pool, &git_bin, "z6f2adrain", "d1", true).await; + state.pin_semaphore = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + let held = state.pin_semaphore.clone().acquire_owned().await.unwrap(); + + // Push A is admitted; its task then parks on the held pin pool, key retained. + post_receive_replication_tail( + state.clone(), + rec.clone(), + f2a_update("refs/heads/main", &c2), + repo.path().to_path_buf(), + F2A_PUSHER.to_string(), + ) + .await; + + // A later push lands while A's task is in flight: it coalesces, and its tip + // advances main to a commit no other push in this test names. + let c3 = u5_commit_file(repo.path(), "b.txt", "three\n"); + assert!( + matches!( + state.encrypt_inflight.try_begin( + &crate::state::repo_identity_key(&rec.owner_did, &rec.name), + vec![(c2.clone(), c3.clone())], + ), + crate::state::BeginOutcome::Coalesced + ), + "a push arriving while a task is in flight must coalesce" + ); + assert!( + !f2a_delta_scanned(&log, &c3), + "nothing has scanned the coalesced tip yet" + ); + + drop(held); + f2a_wait_for( + || f2a_delta_scanned(&log, &c3), + "the in-flight task's drain to resolve the coalesced push's tips", + ) + .await; + } + + /// Scenario 3 (trap 4). A push coalesces WHILE the admitted push is walking, the + /// admitted walk then FAILS, and the coalesced work is still drained. Moving + /// `try_begin` above the walk opens this window, so the failed-walk arm must + /// still spawn the task (with an empty snapshot) rather than let the guard go: + /// dropping it discards the pending tips with a warn. + /// + /// The git shim fails the FIRST `rev-list --all` (the withheld walk's commit + /// enumeration) after signalling that the walk has started and waiting for the + /// test to inject the coalescing push, then behaves normally, so the drain that + /// follows is a real one. + #[cfg(unix)] + #[sqlx::test] + async fn f2a_walk_failure_still_drains_the_coalesced_work(pool: sqlx::PgPool) { + let repo = tempfile::TempDir::new().unwrap(); + let bin = tempfile::TempDir::new().unwrap(); + u5_init_repo(repo.path()); + u5_commit_file(repo.path(), "a.txt", "one\n"); + let c2 = u5_commit_file(repo.path(), "secret/s.txt", "two\n"); + let c3 = u5_commit_file(repo.path(), "b.txt", "three\n"); + let log = bin.path().join("git.log"); + let started = bin.path().join("walk.started"); + let go = bin.path().join("walk.go"); + let once = bin.path().join("walk.once"); + let git_bin = write_fake_git( + bin.path(), + &format!( + "#!/bin/sh\n\ + printf '%s\\n' \"$*\" >> \"{log}\"\n\ + case \"$*\" in\n\ + 'rev-list --all'*)\n\ + if [ ! -f \"{once}\" ]; then\n\ + : > \"{once}\"\n\ + : > \"{started}\"\n\ + while [ ! -f \"{go}\" ]; do sleep 0.05; done\n\ + exit 1\n\ + fi ;;\n\ + esac\n\ + exec git \"$@\"\n", + log = log.display(), + once = once.display(), + started = started.display(), + go = go.display(), + ), + ); + let (state, rec) = f2a_state(pool, &git_bin, "z6f2afail", "f1", true).await; + + let tail = tokio::spawn(post_receive_replication_tail( + state.clone(), + rec.clone(), + f2a_update("refs/heads/main", &c2), + repo.path().to_path_buf(), + F2A_PUSHER.to_string(), + )); + f2a_wait_for(|| started.exists(), "the admitted push's walk to start").await; + + // Mid-walk arrival: the key is already taken, so this push coalesces into the + // slot the walking task owns. (With the gate back below the walk it would be + // ADMITTED here instead, and this assertion is what catches that.) + assert!( + matches!( + state.encrypt_inflight.try_begin( + &crate::state::repo_identity_key(&rec.owner_did, &rec.name), + vec![(c2.clone(), c3.clone())], + ), + crate::state::BeginOutcome::Coalesced + ), + "a push arriving mid-walk must coalesce, not start a second task" + ); + std::fs::write(&go, b"").unwrap(); + tail.await.unwrap(); + + f2a_wait_for( + || f2a_delta_scanned(&log, &c3), + "the failed walk's task to drain the coalesced push's tips", + ) + .await; + } + + /// Mount a Pinata upload endpoint that assigns every object the same CID, and + /// point the state at it. Returns the server (kept alive by the caller) and CID. + async fn f2a_pinata(state: &mut AppState) -> (mockito::ServerGuard, String) { + let cid = "bafyf2acoalescedmapping".to_string(); + let mut server = mockito::Server::new_async().await; + server + .mock("POST", "/") + .with_status(200) + .with_body(format!(r#"{{"data":{{"cid":"{cid}"}}}}"#)) + .expect_at_least(1) + .create_async() + .await; + let mut cfg = (*state.config).clone(); + cfg.pinata_jwt = "f2a-test-jwt".to_string(); + cfg.pinata_upload_url = server.url(); + state.config = std::sync::Arc::new(cfg); + (server, cid) + } + + /// Poll the branch to CID table until the push's mapping lands (the Pinata + /// worker is detached), bounded so a regression fails rather than hangs. + async fn f2a_wait_for_branch_cid( + db: &crate::db::Db, + slug: &str, + what: &str, + ) -> Vec { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); + loop { + let rows = db.list_branch_cids(slug).await.unwrap(); + if !rows.is_empty() { + return rows; + } + assert!( + std::time::Instant::now() < deadline, + "timed out waiting for {what}" + ); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + } + + fn f2a_slug(rec: &crate::db::RepoRecord) -> String { + format!( + "{}/{}", + crate::db::normalize_owner_key(&rec.owner_did), + rec.name + ) + } + + /// Scenario 4 (traps 1 and 2). A coalesced push still does its own per-push work: + /// it records a branch to CID mapping and broadcasts its own ref update. Both + /// would be lost by returning early on `Coalesced`, and the mapping alone would be + /// lost by leaving the Pinata gate on `withheld.is_some()` (a coalesced push never + /// walks, so its `withheld` is None) while a test that only checked "the spawn + /// ran" stayed green. + #[cfg(unix)] + #[sqlx::test] + async fn f2a_coalesced_push_still_pins_and_announces(pool: sqlx::PgPool) { + let repo = tempfile::TempDir::new().unwrap(); + let bin = tempfile::TempDir::new().unwrap(); + u5_init_repo(repo.path()); + let c1 = u5_commit_file(repo.path(), "a.txt", "one\n"); + let c2 = u5_commit_file(repo.path(), "secret/s.txt", "two\n"); + let log = bin.path().join("git.log"); + let git_bin = f2a_logging_git(bin.path(), &log); + let (mut state, rec) = f2a_state(pool, &git_bin, "z6f2apin", "p1", true).await; + let (_server, cid) = f2a_pinata(&mut state).await; + let mut updates = state.ref_update_tx.subscribe(); + + // A task for this repo is already in flight, so the push below coalesces. + let _inflight = match state.encrypt_inflight.try_begin( + &crate::state::repo_identity_key(&rec.owner_did, &rec.name), + Vec::new(), + ) { + crate::state::BeginOutcome::Admitted(g) => g, + crate::state::BeginOutcome::Coalesced => panic!("the first begin must admit"), + }; + + post_receive_replication_tail( + state.clone(), + rec.clone(), + vec![RefUpdate { + old_sha: c1.clone(), + new_sha: c2.clone(), + ref_name: "refs/heads/main".to_string(), + }], + repo.path().to_path_buf(), + F2A_PUSHER.to_string(), + ) + .await; + + let slug = f2a_slug(&rec); + let mapped = f2a_wait_for_branch_cid( + &state.db, + &slug, + "the coalesced push's branch to CID mapping", + ) + .await; + assert_eq!( + mapped.len(), + 1, + "one mapping, for the ref this push advanced" + ); + assert_eq!(mapped[0].ref_name, "refs/heads/main"); + assert_eq!(mapped[0].sha, c2, "mapped to the tip this push landed"); + assert_eq!(mapped[0].cid, cid); + + let broadcast = updates + .try_recv() + .expect("a coalesced push still fires its own announce"); + assert_eq!(broadcast.new_sha, c2); + assert_eq!(broadcast.ref_name, "refs/heads/main"); + } + + /// A push handler whose `release` parks at its pre-unlock point, so a test can + /// drop the future from inside the cancellable post-receive window. Returns the + /// state, the seeded record and the git-invocation log. + /// + /// Path-scoped on purpose: the tail's withheld walk is the observable, and + /// without a path-scoped rule `replication_withheld_set` takes the no-walk + /// shortcut and spawns no git at all. + #[cfg(unix)] + async fn p2_parked_release_state( + pool: sqlx::PgPool, + tmp: &std::path::Path, + owner: &str, + name: &str, + git_body: Option<&str>, + ) -> (AppState, std::path::PathBuf) { + let log = tmp.join("git.log"); + let git_bin = match git_body { + Some(body) => write_fake_git(tmp, body), + None => f2a_logging_git(tmp, &log), + }; + let repos_dir = tmp.join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + + let mut state = crate::test_support::test_state(pool.clone()).await; + state.git_bin = git_bin; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Armed and never notified: every guard this store hands out parks in + // `release` right before the advisory unlock, which is inside the window a + // client disconnect can hit. + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool.clone()) + .with_pre_unlock_gate(std::sync::Arc::new(tokio::sync::Notify::new())); + state + .db + .upsert_mirror_repo(owner, name, &format!("/unused-{owner}-{name}"), None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkP2TailReaderAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + state + .repo_store + .init(&rec.owner_did, &rec.name) + .await + .unwrap(); + (state, log) + } + + const P2_PUSHER: &str = "did:key:z6MkP2TailPusherAAAAAAAAAAAAAAAAAAAAAA"; + + fn p2_push( + state: &AppState, + owner: &str, + name: &str, + ) -> impl std::future::Future> { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + git_receive_pack( + State(state.clone()), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid(P2_PUSHER.to_string())), + crate::rate_limit::PeerAddr(Some("203.0.113.90:5000".parse::().unwrap())), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + } + + fn p2_logged(log: &std::path::Path, prefix: &str) -> bool { + f2a_log(log).lines().any(|l| l.starts_with(prefix)) + } + + /// #174 (jatmn P2, RED-before/GREEN-after): a client disconnect DURING + /// `guard.release()` must not take the replication tail with it. On a + /// successful push `release` awaits the Tigris upload and then the advisory + /// unlock, both cancellation points, while the pack has already landed on disk. + /// Spawning the tail below `release` means a disconnect in that window drops + /// this push's pins, recovery copy and announce: the F2 dropped-tail class, one + /// step earlier in the handler. + /// + /// Load-bearing: with the spawn below `release` the walk's `for-each-ref` never + /// appears after the disconnect (RED). With it above, gated on + /// `receive_result.is_ok()`, it does (GREEN). + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_tail_survives_a_disconnect_during_release(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let (state, log) = p2_parked_release_state(pool, tmp.path(), "z6p2tail", "t1", None).await; + + let mut fut = Box::pin(p2_push(&state, "z6p2tail", "t1")); + + // Drive until receive-pack has run. Nothing between it and `release` awaits, + // so a future that stops completing after that point is parked on the gate. + let mut ran = false; + for _ in 0..1000 { + let step = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + assert!( + step.is_err(), + "the handler must park inside release, not return" + ); + if p2_logged(&log, "receive-pack") { + ran = true; + break; + } + } + assert!(ran, "the push must reach receive-pack"); + // Settle into the parked state. Whether the tail's walk has already started + // by now is immaterial: pre-fix no tail is ever spawned, because `release` + // never returns, so the marker below can only come from a spawn above it. + for _ in 0..5 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + } + + // The disconnect: drop the handler future while `release` is still awaiting. + drop(fut); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while !p2_logged(&log, "for-each-ref") { + assert!( + std::time::Instant::now() < deadline, + "RED: the pack landed but its replication tail never ran. A disconnect \ + during guard.release() took the tail with the handler future — spawn it \ + above release, gated on receive_result.is_ok()" + ); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + + // Not asserted here: that the same disconnect leaves the session advisory + // lock free. A handler-level "the next acquire_write succeeds" probe does not + // discriminate. It still passes with the `Drop` backstop disabled, because the + // guard's `PoolConnection` goes back to the pool when this future is dropped + // and `#[sqlx::test]` builds the pool with `idle_timeout(1s)`, so the session + // ends on its own a beat later and postgres frees the lock with no help from + // the code under test. Measured with the backstop disabled: held at the drop, + // free ~2s later, observed from a session outside the pool. `acquire_write` + // retries for far longer than that, so it waits the release out and reports + // success either way. `write_guard_release_cancelled_mid_unlock_frees_the_lock` + // is the real proof: it probes from a connection held OUT of the pool, 400ms + // after the drop, which is inside that window rather than past it. + } + + /// The must-not direction of the same reorder. Moving the spawn above + /// `release` moves it above the `?` that used to gate it, so the success check + /// has to be explicit: a FAILED receive-pack must still spawn no tail, or a + /// pusher who aborts a pack mid-transfer gets a half-applied repo pinned and + /// announced on demand. + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_failure_spawns_no_tail_even_when_the_client_disconnects( + pool: sqlx::PgPool, + ) { + // receive-pack fails; everything else the handler or a tail might run is + // logged, so any walk child would show up. + let tmp = tempfile::TempDir::new().unwrap(); + let log = tmp.path().join("git.log"); + let body = format!( + "#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"{}\"\n\ + case \"$1\" in receive-pack) cat >/dev/null 2>/dev/null; exit 1 ;; esac\n\ + exec git \"$@\"\n", + log.display() + ); + let (state, log) = + p2_parked_release_state(pool, tmp.path(), "z6p2fail", "t1", Some(&body)).await; + + let mut fut = Box::pin(p2_push(&state, "z6p2fail", "t1")); + let mut ran = false; + for _ in 0..1000 { + let step = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if p2_logged(&log, "receive-pack") { + ran = true; + // A failed push still parks in `release` (the lock is freed on + // failure too); drop it there, the same disconnect as above. + assert!( + step.is_err(), + "the failed push must still park inside release" + ); + break; + } + } + assert!(ran, "the push must reach receive-pack"); + for _ in 0..5 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + } + drop(fut); + + tokio::time::sleep(std::time::Duration::from_millis(750)).await; + assert!( + !p2_logged(&log, "for-each-ref"), + "a failed receive-pack must spawn no replication tail: pinning and \ + announcing a half-applied repo is exactly what release(false) refuses \ + to upload" + ); + } + + /// Scenario 5 (trap 3, fail-closed). On a repo whose withheld walk is failing, a + /// coalesced push must not publish. Before the gate moved, every push on such a + /// repo got `announce = false` from its own walk; a coalesced push has no walk, so + /// the announce decision now comes from the Pinata worker's recomputation, which + /// fails closed the same way. Asserted on the broadcast channel: nothing is sent. + #[cfg(unix)] + #[sqlx::test] + async fn f2a_coalesced_push_on_a_failing_walk_does_not_publish(pool: sqlx::PgPool) { + let repo = tempfile::TempDir::new().unwrap(); + let bin = tempfile::TempDir::new().unwrap(); + u5_init_repo(repo.path()); + let c1 = u5_commit_file(repo.path(), "a.txt", "one\n"); + let c2 = u5_commit_file(repo.path(), "secret/s.txt", "two\n"); + let log = bin.path().join("git.log"); + // Every withheld walk fails: the repo cannot be vetted, so it must neither + // replicate nor announce. + let git_bin = write_fake_git( + bin.path(), + &format!( + "#!/bin/sh\n\ + printf '%s\\n' \"$*\" >> \"{log}\"\n\ + case \"$*\" in 'rev-list --all'*) exit 1 ;; esac\n\ + exec git \"$@\"\n", + log = log.display(), + ), + ); + let (mut state, rec) = f2a_state(pool, &git_bin, "z6f2aclosed", "x1", true).await; + let (_server, _cid) = f2a_pinata(&mut state).await; + let mut updates = state.ref_update_tx.subscribe(); + + let _inflight = match state.encrypt_inflight.try_begin( + &crate::state::repo_identity_key(&rec.owner_did, &rec.name), + Vec::new(), + ) { + crate::state::BeginOutcome::Admitted(g) => g, + crate::state::BeginOutcome::Coalesced => panic!("the first begin must admit"), + }; + + post_receive_replication_tail( + state.clone(), + rec.clone(), + vec![RefUpdate { + old_sha: c1.clone(), + new_sha: c2.clone(), + ref_name: "refs/heads/main".to_string(), + }], + repo.path().to_path_buf(), + F2A_PUSHER.to_string(), + ) + .await; + + // The worker has reached its recomputation (and failed it) once the walk's + // commit enumeration shows up in the log; give the rest of the task a settle. + f2a_wait_for( + || { + f2a_log(&log) + .lines() + .any(|l| l.starts_with("rev-list --all")) + }, + "the Pinata worker's fail-closed recomputation", + ) + .await; + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + + assert!( + matches!( + updates.try_recv(), + Err(tokio::sync::broadcast::error::TryRecvError::Empty) + ), + "a push whose replication could not be vetted must not broadcast" + ); + assert!( + state + .db + .list_branch_cids(&f2a_slug(&rec)) + .await + .unwrap() + .is_empty(), + "and it must pin nothing, so it maps no CID" + ); + } + + /// Scenario 6. A repo the anonymous public cannot read at root takes no key, runs + /// no walk, and spawns no git at all: the cheap predicate answers before anything + /// is acquired, exactly as `replication_withheld_set`'s own early return did. + #[cfg(unix)] + #[sqlx::test] + async fn f2a_private_repo_takes_no_key_and_runs_no_git(pool: sqlx::PgPool) { + let repo = tempfile::TempDir::new().unwrap(); + let bin = tempfile::TempDir::new().unwrap(); + u5_init_repo(repo.path()); + let c1 = u5_commit_file(repo.path(), "a.txt", "one\n"); + let log = bin.path().join("git.log"); + let git_bin = f2a_logging_git(bin.path(), &log); + let (state, mut rec) = f2a_state(pool, &git_bin, "z6f2apriv", "v1", false).await; + rec.is_public = false; + + post_receive_replication_tail( + state.clone(), + rec.clone(), + f2a_update("refs/heads/main", &c1), + repo.path().to_path_buf(), + F2A_PUSHER.to_string(), + ) + .await; + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + + assert!( + state.encrypt_inflight.is_empty(), + "a repo that replicates nothing must not take the coalescing key" + ); + assert_eq!( + f2a_log(&log), + "", + "and it must spawn no git: no walk, no candidate scan, no re-derivation" + ); + } + + /// Scenario 7. The first push is unaffected: it is admitted, it runs the walk, and + /// it still does the full per-push work (pin, mapping, announce). + #[cfg(unix)] + #[sqlx::test] + async fn f2a_first_push_is_admitted_and_does_the_full_work(pool: sqlx::PgPool) { + let repo = tempfile::TempDir::new().unwrap(); + let bin = tempfile::TempDir::new().unwrap(); + u5_init_repo(repo.path()); + let c1 = u5_commit_file(repo.path(), "a.txt", "one\n"); + let c2 = u5_commit_file(repo.path(), "secret/s.txt", "two\n"); + let log = bin.path().join("git.log"); + let git_bin = f2a_logging_git(bin.path(), &log); + let (mut state, rec) = f2a_state(pool, &git_bin, "z6f2afirst", "f1", true).await; + let (_server, cid) = f2a_pinata(&mut state).await; + let mut updates = state.ref_update_tx.subscribe(); + + post_receive_replication_tail( + state.clone(), + rec.clone(), + vec![RefUpdate { + old_sha: c1.clone(), + new_sha: c2.clone(), + ref_name: "refs/heads/main".to_string(), + }], + repo.path().to_path_buf(), + F2A_PUSHER.to_string(), + ) + .await; + + assert!( + f2a_walks(&log) >= 1, + "the admitted push runs the withheld walk itself; log:\n{}", + f2a_log(&log) + ); + let slug = f2a_slug(&rec); + let mapped = f2a_wait_for_branch_cid( + &state.db, + &slug, + "the admitted push's branch to CID mapping", + ) + .await; + assert_eq!(mapped[0].sha, c2); + assert_eq!(mapped[0].cid, cid); + let broadcast = updates + .try_recv() + .expect("the admitted push fires its announce"); + assert_eq!(broadcast.new_sha, c2); + } + + // ---- #174 F2b: a failed OWN walk does not buy a pin permit and a second walk ---- + + /// Withheld-walk commit-enumeration attempts so far. Each `replication_withheld_set` + /// runs exactly one `rev-list --all`, so this counts the walks that were attempted + /// (the `ls-tree` counter above cannot: a walk whose enumeration fails never gets + /// to `ls-tree`). + fn f2b_walk_attempts(log: &std::path::Path) -> usize { + f2a_log(log) + .lines() + .filter(|l| l.starts_with("rev-list --all")) + .count() + } + + /// A NON-coalesced push whose own withheld walk failed must not take a global pin + /// permit and must not re-run the same failing walk in the Pinata worker. + /// + /// The F2a change moved the Pinata gate from `withheld.is_some()` to the rules-only + /// `announce_at_root`, which a coalesced push genuinely needs (it has no walk of its + /// own). But it also let an ADMITTED push whose walk failed acquire `pin_semaphore` + /// and re-derive `replication_withheld_set`, which fails the same way. With + /// `max_concurrent_pin_tasks` defaulting to 8 and the pin pool DEFERRING rather than + /// shedding, eight such pushes stall pins node-wide. + /// + /// Asserted on observable work, twice over, with the pin pool's only permit held for + /// the whole first phase: + /// * the walk attempts while the permit is held. Two: the tail's own (which fails) + /// and the recovery task's recipients walk, which must NOT park on the pin pool + /// for its empty object list. The Pinata worker's is the third and must not exist. + /// * the walk attempts after the permit is released. Still two: nothing was left + /// waiting on the pin pool, which is the permit assertion. + /// + /// Load-bearing both ways. With the gate back on plain `announce_at_root` the Pinata + /// worker parks on the held permit and then walks once it is freed (phase 2 sees 3). + /// Without the empty-list guard on `pin_new_objects_gated` the recovery task parks + /// too, so phase 1 sees 1. + #[cfg(unix)] + #[sqlx::test] + async fn f2b_failed_own_walk_takes_no_pin_permit_and_runs_no_second_walk(pool: sqlx::PgPool) { + let repo = tempfile::TempDir::new().unwrap(); + let bin = tempfile::TempDir::new().unwrap(); + u5_init_repo(repo.path()); + let c1 = u5_commit_file(repo.path(), "a.txt", "one\n"); + let c2 = u5_commit_file(repo.path(), "secret/s.txt", "two\n"); + let log = bin.path().join("git.log"); + // Every withheld walk fails, so this push can never be vetted. + let git_bin = write_fake_git( + bin.path(), + &format!( + "#!/bin/sh\n\ + printf '%s\\n' \"$*\" >> \"{log}\"\n\ + case \"$*\" in 'rev-list --all'*) exit 1 ;; esac\n\ + exec git \"$@\"\n", + log = log.display(), + ), + ); + let (mut state, rec) = f2a_state(pool, &git_bin, "z6f2bfail", "w1", true).await; + let (_server, _cid) = f2a_pinata(&mut state).await; + // One pin permit, held: anything that reaches a pin-admission acquire parks + // instead of running, which is what makes "took no permit" observable. + state.pin_semaphore = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + let held = state.pin_semaphore.clone().acquire_owned().await.unwrap(); + + // Nothing pre-takes the coalescing key, so this push is ADMITTED and runs its + // own walk. + post_receive_replication_tail( + state.clone(), + rec.clone(), + vec![RefUpdate { + old_sha: c1.clone(), + new_sha: c2.clone(), + ref_name: "refs/heads/main".to_string(), + }], + repo.path().to_path_buf(), + F2A_PUSHER.to_string(), + ) + .await; + + f2a_wait_for( + || f2b_walk_attempts(&log) >= 2, + "the tail's own walk and the recovery task's recipients walk", + ) + .await; + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + assert_eq!( + f2b_walk_attempts(&log), + 2, + "a failed own walk must not buy a third walk in the Pinata worker; log:\n{}", + f2a_log(&log) + ); + + // Release pin admission. A task that had parked on it now wakes and walks; + // nothing should have been parked. + drop(held); + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + assert_eq!( + f2b_walk_attempts(&log), + 2, + "nothing may be left waiting on the pin permit for a push that pins \ + nothing; log:\n{}", + f2a_log(&log) + ); + assert!( + state + .db + .list_branch_cids(&f2a_slug(&rec)) + .await + .unwrap() + .is_empty(), + "and the unvetted push still maps no CID" ); } } diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 720fb3ae..e86b5a5f 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -516,10 +516,30 @@ mod tests { rate_limiter: RateLimiter::new(100, Duration::from_secs(60)), create_ip_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), push_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_work_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, + ipfs_max_legacy_probes: crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, + ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, push_limiter_trust: crate::rate_limit::TrustedProxy::None, sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), shutdown_tx: tokio::sync::watch::channel(false).0, + git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + pin_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + encrypt_inflight: crate::state::EncryptInflight::new(), + repo_write_leases: crate::state::RepoWriteLeases::new(8), + git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), + git_push_advert_per_caller: + crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), + git_write_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), + git_ipfs_walk_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_ipfs_walk_per_caller: + crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), + git_bin: "git".to_string(), } } diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index fc2247d9..e88611d6 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -1,6 +1,33 @@ use clap::Parser; use std::path::PathBuf; +/// Upper bound on `git_service_timeout_secs` and `ipfs_request_budget_secs`, in seconds +/// (100 years). +/// +/// Two consumers now, so a future tightening moves both. `ipfs_request_budget_secs` +/// derives only the `Instant` addition in `get_by_cid`, not the lease-steal multiply +/// below, but it shares this ceiling because the defect class and the "set it very large +/// to disable" contract are the same. +/// +/// The knob is not just stored, it is arithmetic input: the write path derives the +/// per-repo lease steal bound from it (`* 2 + 60`), and #174 routed it into +/// `build_filtered_pack` and `blob_paths`, which each build a deadline as +/// `Instant::now() + Duration::from_secs(this)`. That addition panics on overflow in +/// RELEASE as well as debug, so an unbounded `u64` here turns an operator typo into a +/// serve-path crash rather than a very long timeout. Bounding at parse time keeps every +/// derived duration in range at once, instead of hardening each site as it is found. +/// +/// The ceiling is representability, NOT a policy view of a sane timeout, and that +/// distinction is what sets the number. The help text has always told operators to set +/// this very large to disable the bound, so values like `999999999` (~31 years) are +/// working production "off" settings; a tighter, tidier cap would fail those nodes at +/// boot on upgrade over a value that was never the defect. 100 years clears every such +/// setting while staying a factor of about 5.85 under the ~584-year ceiling of a +/// `u64`-nanosecond `Instant`, which is the tightest representation on any platform we +/// build for. Every value that worked before still parses; only the ones that would have +/// panicked are rejected. +pub const GIT_SERVICE_TIMEOUT_SECS_MAX: u64 = 100 * 365 * 24 * 60 * 60; + #[derive(Parser, Debug, Clone)] #[command(name = "gitlawb-node", about = "gitlawb node daemon", version)] pub struct Config { @@ -170,26 +197,54 @@ pub struct Config { /// Maximum wall-clock time a single served git operation (upload-pack / /// receive-pack through `run_git_service`) may run before it is aborted and /// its process group torn down, in seconds. Bounds a git that neither - /// finishes nor disconnects. Must be positive; set it very large to - /// effectively disable the bound. Default: 600s (10 min), generous for large - /// clones. Does not cover the ref advertisement (`info/refs`) or the - /// withheld-blob fetch path (`upload_pack_excluding`, a blocking - /// `spawn_blocking` a tokio timeout cannot cancel); both remain unbounded. + /// finishes nor disconnects. Must be positive and at most + /// [`GIT_SERVICE_TIMEOUT_SECS_MAX`] (100 years, the largest value every derived + /// deadline can represent); setting it very large is still the way to disable the + /// bound. Default: 600s (10 min), generous for large clones. Also bounds the ref + /// advertisement + /// (`info/refs`) and the withheld-blob pack build (`upload_pack_excluding`'s + /// pack-objects stage), which now share the same timeout + process-group + /// teardown (#174). #[arg( long, env = "GITLAWB_GIT_SERVICE_TIMEOUT_SECS", default_value_t = 600, - value_parser = clap::value_parser!(u64).range(1..) + value_parser = clap::value_parser!(u64).range(1..=GIT_SERVICE_TIMEOUT_SECS_MAX) )] pub git_service_timeout_secs: u64, + /// Maximum wall-clock time the storage-acquisition phase of a served git + /// operation may run before the request is shed with a 503, in seconds. This + /// bounds `RepoStore::{acquire,acquire_fresh,acquire_write}` — the Tigris + /// HEAD/GET on a read/advert acquire and the advisory-lock retry loop (incl. a + /// per-iteration `pg_try_advisory_lock` that can block on a hung Postgres pool) + /// on a write acquire. A concurrency permit is taken BEFORE this phase, and + /// `git_service_timeout_secs` only starts once git spawns, so without this the + /// acquire phase is unbounded: a stalled backend pins the permit and drains the + /// pool until every later request 503s. On expiry the permit is released and a + /// bounded 503 + Retry-After is returned (fail-closed). Kept separate from + /// `git_service_timeout_secs` because acquisition and git execution are distinct + /// cost centers — one shared budget would let a slow acquire starve git. Must be + /// positive; set it very large to effectively disable the bound. Default: 30s. + #[arg( + long, + env = "GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS", + default_value_t = 30, + value_parser = clap::value_parser!(u64).range(1..) + )] + pub git_acquire_timeout_secs: u64, + /// Maximum connections in the PostgreSQL pool. This is a cap, not a floor /// (connections open lazily). Size against the database server's - /// max_connections, remembering admin tooling opens its own pool. + /// max_connections, remembering admin tooling opens its own pool. Each + /// concurrent write pins one pooled connection for its whole duration (the + /// advisory lock in `repo_store::acquire_write` is connection-affine), so this + /// must exceed `max_concurrent_git_pushes` by `DB_POOL_APP_HEADROOM` or slow + /// pushes starve every other DB path — enforced by `Config::validate`. #[arg( long, env = "GITLAWB_DB_MAX_CONNECTIONS", - default_value_t = 20, + default_value_t = 48, value_parser = clap::value_parser!(u32).range(1..) )] pub db_max_connections: u32, @@ -234,6 +289,327 @@ pub struct Config { value_parser = clap::value_parser!(u64).range(1..) )] pub db_retry_max_secs: u64, + + /// Maximum number of served git operations (upload-pack / receive-pack / + /// info-refs) allowed to run concurrently. Beyond this the node sheds the + /// request with a clean 503 + Retry-After instead of spawning another git + /// subprocess and risking PID/thread exhaustion. Portable backstop: the + /// compose `pids_limit` is not present on Fly, whose connection-concurrency + /// cap is a different axis (500 connections each fan out to git + + /// pack-objects + threads). Size below the process budget with headroom. + /// + /// This is the READ pool (`git_read_semaphore`): upload-pack and the UPLOAD-PACK + /// `info/refs` advertisement only. The authenticated push POST draws from a + /// separate write pool (`max_concurrent_git_pushes`) that anonymous reads can + /// never reach, and each read caller is additionally bounded by + /// `max_concurrent_reads_per_caller`, so an anonymous flood cannot shed the actual + /// push nor monopolize reads (#174). The anon-reachable RECEIVE-PACK `info/refs` + /// advertisement draws from its OWN dedicated pool (sized like the write pool but + /// disjoint), so an advertisement flood can never occupy a permit the + /// authenticated push POST needs at admission (#174). + /// + /// A permit is held for the whole op. Every git subprocess that STREAMS is + /// duration-bounded and reaps its process group on disconnect: upload-pack, + /// receive-pack, and both info/refs advertisements run under + /// `git_service_timeout_secs` with `process_group(0)` teardown, and the + /// withheld-blob (`upload_pack_excluding`) pack-objects stage plus the push-side + /// candidate-discovery children (`rev-list` / `cat-file`) now run under the same + /// bounded runner with process-group teardown, so a stuck git child no longer + /// holds its slot indefinitely (#174 closed the duration/cancellation gaps this + /// comment previously tracked). + /// + /// Default: 128. Must be between 1 and 1_048_576; the ceiling keeps the value + /// well under tokio's `Semaphore` permit limit so an oversized value is a + /// clean CLI error rather than a boot-time panic. + #[arg( + long, + env = "GITLAWB_MAX_CONCURRENT_GIT_OPS", + default_value_t = 128, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub max_concurrent_git_ops: usize, + + /// Maximum number of concurrent `git-receive-pack` (push) operations. The + /// authenticated push POST draws from this dedicated pool, separate from + /// `max_concurrent_git_ops` (reads), so a flood of anonymous reads cannot shed an + /// authenticated push at admission (#174). The anon-reachable receive-pack + /// `info/refs` advertisement runs in a SEPARATE pool of the same size (derived + /// from this knob), disjoint from this one, so an advertisement flood cannot + /// occupy a POST's slot either (#174). Beyond this a push sheds a clean 503 + + /// Retry-After. + /// + /// Default: 32. Must be between 1 and 1_048_576 (the ceiling keeps the value + /// under tokio's `Semaphore` permit limit so an oversized value is a clean CLI + /// error rather than a boot-time panic). + /// + /// CONNECTION BUDGET. A push holds a Postgres connection from the node's separate + /// advisory-lock pool for the whole receive-pack, and that pool is sized from this + /// knob (this value + 8, clamped to 64 in `main.rs`). The node's total ceiling is + /// therefore `db_max_connections` (default 20) + the lock pool (default 40), i.e. + /// 60 by default, and at most `db_max_connections` + 64. Size BOTH against the + /// database server's `max_connections`: `db_max_connections`' own doc predates the + /// lock pool and no longer covers most of the node's connections. The +8 headroom + /// is shared with the three non-push `acquire_write` callers (`api/issues.rs` x2, + /// `api/pulls.rs`). Raising this knob past the clamp does NOT buy more lock-pool + /// connections; pushes beyond it wait briefly and then shed a 503 + Retry-After. + #[arg( + long, + env = "GITLAWB_MAX_CONCURRENT_GIT_PUSHES", + default_value_t = 32, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub max_concurrent_git_pushes: usize, + + /// Maximum number of pushes that may be PARKED on one repo's in-process write lease + /// at once. Same-repo pushes are serialized (block-and-wait), and a parked push has + /// already had its entire pack buffered by axum, so an unbounded queue on a contended + /// repo is unbounded buffered memory held for up to `git_service_timeout_secs * 2 + + /// 60` (1260s at defaults). Past this cap the newest push sheds a clean 503 + + /// Retry-After instead of joining the queue. + /// + /// The trade: raising it lets more same-repo pushes wait their turn (fewer 503s for a + /// hot repo, more memory pinned by waiters); lowering it sheds sooner. Only pushes to + /// the SAME repo count, and only ones parked right now, so the cap can never deny a + /// push to a different repo. The holder is deliberately not counted: a holder whose + /// cleanup never ran would otherwise pin a slot forever and wedge the repo, which is + /// the failure the `steal_after` reclaim exists to survive. + /// + /// Default: 8, a quarter of the default `max_concurrent_git_pushes` (32). Raising the + /// push pool does not raise this; set it explicitly. Must be between 1 and 1_048_576. + #[arg( + long, + env = "GITLAWB_REPO_LEASE_MAX_WAITERS", + default_value_t = 8, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub repo_lease_max_waiters: usize, + + /// Max concurrent post-push pin loops (`ipfs_pin` and `pinata` + /// `pin_new_objects`) across all repos. `EncryptInflight` bounds the outstanding + /// pin-task COUNT to one per repo, but each pin loop holds a full per-push + /// object-id list (up to `git_max_pack_bytes` worth of OIDs) while it walks it, + /// so N distinct repos could hold N such lists at once. This caps how many run + /// concurrently (#174 F6). Beyond it a pin loop DEFERS (waits) and never drops, + /// since a dropped pin would lose the object's replication copy. + /// + /// It does not cap the memory itself: the local IPFS path builds its list before + /// taking a permit, so tasks parked on this pool still hold theirs, and how many + /// park is capped only per repo. Lowering this knob bounds concurrent pinning, not + /// how much an actor pushing to many repos can retain. + /// + /// Default: 8. Must be between 1 and 1_048_576. + #[arg( + long, + env = "GITLAWB_MAX_CONCURRENT_PIN_TASKS", + default_value_t = 8, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub max_concurrent_pin_tasks: usize, + + /// Maximum concurrent read operations (`upload-pack` and the upload-pack + /// `info/refs` advertisement) a single caller may hold at once, so one caller + /// cannot monopolize the `max_concurrent_git_ops` read pool (#174). Callers are + /// keyed on the RESOLVED SOURCE IP, never the DID — a signature does not move a + /// caller off this cap, so an authenticated client cannot mint DIDs to escape it. + /// IMPORTANT: the source-IP key is only as granular as `GITLAWB_TRUSTED_PROXY`. + /// Left unset (the default), a node behind an edge/NAT keys all callers on the + /// edge IP, so this cap collapses to a single global cap rather than per-client. + /// Set `GITLAWB_TRUSTED_PROXY` to key on the real client; a high-fanout caller (a + /// CI fleet behind one NAT) then needs the operator to raise this. Over-cap for a + /// caller sheds a clean 503 + Retry-After. + /// + /// Default: 16. Must be between 1 and 1_048_576. + #[arg( + long, + env = "GITLAWB_MAX_CONCURRENT_READS_PER_CALLER", + default_value_t = 16, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub max_concurrent_reads_per_caller: usize, + + /// Maximum number of concurrent `GET /ipfs/{cid}` requests that may run their + /// visibility walk at once. The publicly-reachable `/ipfs/{cid}` route runs + /// `allowed_blob_set_for_caller_bounded` in `spawn_blocking` — a full-history + /// git walk (up to `git_service_timeout_secs`) — for each candidate repo. It + /// draws from THIS pool, not any served-git pool: a distinct public cost center + /// on a distinct surface, so sharing a git pool would let anonymous /ipfs + /// traffic shed authenticated git ops (the auth-boundary trap). A permit is + /// held for the whole request (across the repo loop) so it reflects real + /// blocking-thread occupancy, not merely the tokio wait. Beyond this the request + /// sheds a clean 503 + Retry-After. Must be between 1 and 1_048_576; the ceiling + /// keeps the value under tokio's `Semaphore` permit limit so an oversized value + /// is a clean CLI error rather than a boot-time panic. Default: 32. + #[arg( + long, + env = "GITLAWB_MAX_CONCURRENT_IPFS_WALKS", + default_value_t = 32, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub max_concurrent_ipfs_walks: usize, + + /// Maximum concurrent `/ipfs/{cid}` walk requests a single source may hold at + /// once, so one source cannot monopolize `max_concurrent_ipfs_walks` (#174). + /// Callers are keyed on the RESOLVED SOURCE IP (`client_key`/`GITLAWB_TRUSTED_PROXY`), + /// never the DID — `/ipfs` accepts any `did:key` via `optional_signature` with no + /// admission step, so keying on the DID would let one host mint disposable DIDs to + /// multiply its budget. A request with no resolvable key (no trusted header, no + /// peer) is bounded by the global pool only, never this sub-cap. Over-cap sheds a + /// clean 503 + Retry-After. Must be between 1 and 1_048_576. Default: 4. + #[arg( + long, + env = "GITLAWB_IPFS_WALK_PER_SOURCE", + default_value_t = 4, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub ipfs_walk_per_source: usize, + + /// Per-request ceiling on the number of legacy (NULL-provenance) repos the + /// `/ipfs/{cid}` resolver's scan fallback will PROBE (`acquire` + `git cat-file + /// -t`) before giving up. The provenance path targets its recorded sources; the + /// legacy scan, absent this bound, fans one anonymous request out to O(repos) + /// subprocess spawns and cold-cache fetches for a CID enumerable from the public + /// pins index. A truncated scan surfaces as a retryable 503, never a false 404. + /// Wired into `AppState::ipfs_max_legacy_probes` at construction. This knob does + /// not govern the history-walk ceiling; see `ipfs_max_repos_walked` for that. + /// Must be between 1 and 1_048_576. Default: 256. + #[arg( + long, + env = "GITLAWB_IPFS_MAX_LEGACY_PROBES", + default_value_t = crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST as usize, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub ipfs_max_legacy_probes: usize, + + /// Upper bound on the number of EXPENSIVE visibility walks + /// (`allowed_blob_set_for_caller_bounded`, a full-history git walk in a + /// blocking thread) a single `/ipfs/{cid}` request may run. Only a blob in a + /// path-scoped repo costs a walk, so the cap counts exactly those candidates + /// — cheap probe-only visits are bounded by `ipfs_max_repo_visits` instead + /// (counting them here would starve a plain public copy past the cap out of + /// its 200). On exhaustion the walk-needing repo is skipped WITHOUT a verdict + /// and the scan continues; if the request then finds the object nowhere it + /// sheds a retryable 503 + Retry-After rather than misreport existing content + /// absent with a 404. The handler still short-circuits the moment it serves. + /// Must be between 1 and 1_048_576. Default: 64. + /// + /// The effective ceiling is `max(MAX_PIN_SOURCES + 1, this)`. That floor exists + /// so the cap can never truncate a request before its whole bounded provenance + /// source set has been tried, which would falsely 503 a provenanced request, so + /// setting this below the floor widens nothing and is silently raised. + #[arg( + long, + env = "GITLAWB_IPFS_MAX_REPOS_WALKED", + default_value_t = 64, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub ipfs_max_repos_walked: usize, + + /// Ceiling on the number of repos a single `/ipfs/{cid}` request may VISIT — + /// pass the repo-level visibility gate into the acquire + `cat-file` probe. + /// Each visit costs a `RepoStore::acquire` (on a Tigris cache miss that is a + /// full repo-archive download from object storage, so the worst-case + /// object-store fetch count for one request equals this ceiling) plus a git + /// probe subprocess. On exhaustion the scan STOPS — unlike + /// `ipfs_max_repos_walked`, which skips just the walk-needing repo, there is + /// no cheaper way to keep scanning — and the request sheds a retryable 503 + + /// Retry-After rather than a false 404. Must be between 1 and 1_048_576. + /// Default: 1024. + #[arg( + long, + env = "GITLAWB_IPFS_MAX_REPO_VISITS", + default_value_t = 1024, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub ipfs_max_repo_visits: usize, + + /// Absolute wall-clock budget for one admitted `GET /ipfs/{cid}` request's + /// acquire+walk lifetime, in seconds. `max_concurrent_ipfs_walks` bounds how + /// MANY requests hold walk slots; this bounds how LONG one admitted request + /// may keep its slot. Without it, each repo iteration draws a fresh + /// `git_acquire_timeout_secs` and each expensive walk a fresh + /// `git_service_timeout_secs`, so one request scanning many repos could hold + /// a scarce walk slot for hours. Every stage (acquire, `cat-file` probe, + /// visibility walk, content read) starts only while budget remains, and the + /// acquire wait and walk deadline are clamped to `min(their own timeout, + /// remaining budget)`; a stage is never started with zero remaining. On + /// exhaustion the scan stops without a verdict and the request sheds a + /// retryable 503 + Retry-After rather than a false 404. The clamps bound + /// only the acquire and walk stages (overshoot there is the walk watchdog's + /// SIGTERM grace + SIGKILL settle); the `object_type` / + /// `read_object_content` probe subprocesses are budget-checked before they + /// start AND each run under their own deadline (the lesser of + /// `git_service_timeout_secs` and the remaining budget), reaped by + /// process-group teardown, so a hung `cat-file` cannot hold the request's walk + /// slot past it. Still unbounded: the probe's `object_store_readable` check is a + /// synchronous filesystem sweep with nothing to reap, so a wedged filesystem can + /// hold the slot past the deadline. + /// Must be positive, and no larger than `GIT_SERVICE_TIMEOUT_SECS_MAX`. The ceiling is + /// representability, NOT a policy view of a sane budget: `get_by_cid` derives the + /// request deadline as `Instant::now() + Duration::from_secs(this)`, and that addition + /// is an explicit overflow check rather than a debug-only one, so a value near the top + /// of the `u64` range aborts every `/ipfs/{cid}` request in a release build instead of + /// setting a very long budget. The ceiling sits well below where that starts, about a + /// factor of 5.85 (see the constant's own note), so it is a conservative margin rather than the + /// exact overflow point; rejecting at parse time keeps the unrepresentable values out + /// of every reachable configuration. Setting it very large is still the way to + /// effectively disable the budget, and the documented sentinels (`999999999`, + /// `1000000000`) are well inside the range. + /// Default: 600s (10 min), matching `git_service_timeout_secs` so a single full-length + /// walk still fits. + #[arg( + long, + env = "GITLAWB_IPFS_REQUEST_BUDGET_SECS", + default_value_t = 600, + value_parser = clap::value_parser!(u64).range(1..=GIT_SERVICE_TIMEOUT_SECS_MAX) + )] + pub ipfs_request_budget_secs: u64, + + /// Per-client-IP rate limit for `GET /ipfs/{cid}`, in requests per hour. The + /// route is publicly reachable (`optional_signature`) and each request can drive + /// a full-history git walk, so it carries a per-IP flood brake in addition to the + /// concurrency cap above (a rate limit bounds request *rate*, the semaphore + /// bounds concurrent slow holds — different axes). Keyed on the resolved client + /// IP via `GITLAWB_TRUSTED_PROXY`. `0` disables. Default: 600. + /// + /// This is the pure once-per-request ROUTE brake. The resolver's internal + /// per-probe/per-walk WORK budget is a SEPARATE bucket whose capacity is DERIVED + /// from this value (`AppState::ipfs_work_budget`), not a knob of its own; `0` here + /// disables that derived bucket too. + #[arg(long, env = "GITLAWB_IPFS_RATE_LIMIT", default_value_t = 600)] + pub ipfs_rate_limit: usize, + + /// Rows the legacy provider-CID repair sweep reads per batch (U4, #173). + /// + /// The sweep walks every `pinned_cids` row on the node once, repairing rows that + /// releases before this branch keyed on a PROVIDER CID (Kubo dag-pb / Pinata CIDv0) + /// instead of the raw-content resolver key. This bounds one batch, so the sweep can + /// never turn into a single unbounded table scan competing with request traffic. + /// Conservative on purpose: paired with the inter-batch delay below the default is + /// ~64 rows per minute, which finishes a large pin set in hours of idle background + /// work rather than one expensive burst. Must be between 1 and 100_000. + #[arg( + long, + env = "GITLAWB_PIN_REPAIR_SWEEP_BATCH", + default_value_t = 64, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=100_000) + )] + pub pin_repair_sweep_batch: i64, + + /// Seconds the legacy provider-CID repair sweep sleeps between batches (U4, #173). + /// + /// Each batch costs an indexed range scan plus, for the legacy rows in it, a + /// `git cat-file` per row. The delay is what keeps that off the DB's and the disk's + /// critical path: the sweep is repairing rows that have been unresolvable since the + /// upgrade, so finishing slowly is fine and finishing fast at the cost of live + /// traffic is not. `0` disables the pause (test and one-off operational use only). + /// Must be between 0 and 86_400. + #[arg( + long, + env = "GITLAWB_PIN_REPAIR_SWEEP_DELAY_SECS", + default_value_t = 60, + value_parser = clap::builder::RangedU64ValueParser::::new().range(0..=86_400) + )] + pub pin_repair_sweep_delay_secs: u64, } impl Config { @@ -250,6 +626,36 @@ impl Config { } PathBuf::from(&self.key_path) } + + /// DB connections reserved for everything other than held write-locks: auth + /// lookups, visibility-rule reads, the post-receive tail's own DB writes, and + /// admin tooling. A write pins one pooled connection for its whole duration, so + /// the pool must clear the concurrent-write cap by at least this margin. + pub const DB_POOL_APP_HEADROOM: u32 = 8; + + /// Cross-field boot validation. Single-field ranges are enforced by clap; this + /// catches combinations that ship a denial-of-service under otherwise-valid + /// values. Call once at startup and fail fast on `Err`. + pub fn validate(&self) -> Result<(), String> { + // A write pins one pooled connection for its whole duration (the + // connection-affine advisory lock in repo_store::acquire_write), and + // concurrent writes are capped at max_concurrent_git_pushes. If the pool + // does not exceed that cap by DB_POOL_APP_HEADROOM, a burst of slow pushes + // drains every connection and every other DB path 503s. (#174 F1) + let floor = (self.max_concurrent_git_pushes as u64) + (Self::DB_POOL_APP_HEADROOM as u64); + if (self.db_max_connections as u64) < floor { + return Err(format!( + "GITLAWB_DB_MAX_CONNECTIONS ({}) must be at least max_concurrent_git_pushes ({}) \ + + {} headroom = {}: each concurrent write pins one pooled connection for its whole \ + duration, so a smaller pool lets a burst of slow pushes starve every other DB path.", + self.db_max_connections, + self.max_concurrent_git_pushes, + Self::DB_POOL_APP_HEADROOM, + floor + )); + } + Ok(()) + } } #[cfg(test)] @@ -272,4 +678,538 @@ mod tests { Config::try_parse_from(["gitlawb-node", "--git-service-timeout-secs", "0"]).is_err() ); } + + /// #174 (RED-before/GREEN-after): the upper bound is what keeps every duration + /// derived from this knob in range — the lease steal bound's `* 2 + 60` on the write + /// path, and the `Instant::now() + Duration::from_secs(..)` deadlines in + /// `build_filtered_pack` / `blob_paths` on the serve path, which panic on overflow in + /// release builds too. Checked at parse time so no reachable configuration can carry a + /// value those sites cannot represent. + /// + /// The pre-existing "set it very large to disable the bound" settings are asserted + /// alongside the rejections on purpose. The bound exists to exclude unrepresentable + /// values, not to impose a view of a reasonable timeout, so a node that has been + /// running on ~31 years must not start failing at boot on upgrade. + #[test] + fn git_service_timeout_rejects_values_no_derived_duration_can_represent() { + let parse = |secs: u64| { + Config::try_parse_from([ + "gitlawb-node", + "--git-service-timeout-secs", + &secs.to_string(), + ]) + }; + + // Large "disable the bound" values that predate the ceiling still parse. + for disable in [1_000_000_000, 999_999_999] { + assert_eq!( + parse(disable) + .unwrap_or_else(|e| panic!("{disable} was a working setting: {e}")) + .git_service_timeout_secs, + disable + ); + } + + // At the ceiling: accepted, and every derived duration still fits. + let at_max = + parse(GIT_SERVICE_TIMEOUT_SECS_MAX).expect("the documented maximum must parse"); + assert_eq!( + at_max.git_service_timeout_secs, + GIT_SERVICE_TIMEOUT_SECS_MAX + ); + assert!(at_max + .git_service_timeout_secs + .checked_mul(2) + .and_then(|v| v.checked_add(60)) + .is_some()); + assert!(std::time::Instant::now() + .checked_add(std::time::Duration::from_secs( + at_max.git_service_timeout_secs + )) + .is_some()); + + // Past the ceiling, and the top of the u64 range clap used to accept — the value + // that panics `Instant::now() + Duration::from_secs(..)` outright. + for over in [GIT_SERVICE_TIMEOUT_SECS_MAX + 1, u64::MAX] { + assert!( + parse(over).is_err(), + "{over} is past the representable ceiling and must be rejected at parse time" + ); + } + assert!(std::time::Instant::now() + .checked_add(std::time::Duration::from_secs(u64::MAX)) + .is_none()); + } + + #[test] + fn max_concurrent_pin_tasks_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).max_concurrent_pin_tasks, + 8 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-pin-tasks", "2"]) + .max_concurrent_pin_tasks, + 2 + ); + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-pin-tasks", "0"]).is_err() + ); + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-pin-tasks", "1048577"]) + .is_err() + ); + } + + #[test] + fn max_concurrent_git_ops_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).max_concurrent_git_ops, + 128 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-git-ops", "8"]) + .max_concurrent_git_ops, + 8 + ); + // 0 permits would shed every served-git request with a 503; clap must reject it. + assert!(Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-ops", "0"]).is_err()); + // Above the ceiling would panic tokio's Semaphore::new at boot (permits > + // usize::MAX >> 3); clap must reject it as a clean CLI error instead. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-ops", "1048577"]) + .is_err() + ); + // The ceiling itself is accepted. + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-git-ops", "1048576"]) + .max_concurrent_git_ops, + 1_048_576 + ); + } + + #[test] + fn max_concurrent_git_pushes_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).max_concurrent_git_pushes, + 32 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "8"]) + .max_concurrent_git_pushes, + 8 + ); + // 0 permits would shed every push with a 503; clap must reject it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "0"]).is_err() + ); + // Above the ceiling would panic tokio's Semaphore::new at boot; clap rejects it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "1048577"]) + .is_err() + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "1048576"]) + .max_concurrent_git_pushes, + 1_048_576 + ); + } + + #[test] + fn max_concurrent_ipfs_walks_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).max_concurrent_ipfs_walks, + 32 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "4"]) + .max_concurrent_ipfs_walks, + 4 + ); + // 0 permits would shed every /ipfs walk with a 503; clap must reject it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "0"]).is_err() + ); + // Above the ceiling would panic tokio's Semaphore::new at boot; clap rejects it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "1048577"]) + .is_err() + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "1048576"]) + .max_concurrent_ipfs_walks, + 1_048_576 + ); + } + + /// U4 (#173): the repair sweep's bounds are conservative by default and a batch of + /// 0 (a sweep that walks nothing and never terminates) is a CLI error, not a + /// runtime hang. The delay does accept 0, for tests and one-off operational runs. + #[test] + fn pin_repair_sweep_knobs_default_conservatively() { + let c = Config::parse_from(["gitlawb-node"]); + assert_eq!(c.pin_repair_sweep_batch, 64); + assert_eq!(c.pin_repair_sweep_delay_secs, 60); + + assert!(Config::try_parse_from(["gitlawb-node", "--pin-repair-sweep-batch", "0"]).is_err()); + assert!( + Config::try_parse_from(["gitlawb-node", "--pin-repair-sweep-batch", "100001"]).is_err() + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--pin-repair-sweep-batch", "8"]) + .pin_repair_sweep_batch, + 8 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--pin-repair-sweep-delay-secs", "0"]) + .pin_repair_sweep_delay_secs, + 0 + ); + assert!( + Config::try_parse_from(["gitlawb-node", "--pin-repair-sweep-delay-secs", "86401"]) + .is_err() + ); + } + + #[test] + fn ipfs_walk_per_source_defaults_and_rejects_out_of_range() { + assert_eq!(Config::parse_from(["gitlawb-node"]).ipfs_walk_per_source, 4); + assert_eq!( + Config::parse_from(["gitlawb-node", "--ipfs-walk-per-source", "2"]) + .ipfs_walk_per_source, + 2 + ); + // 0 would shed every /ipfs walk from a keyed source; clap must reject it. + assert!(Config::try_parse_from(["gitlawb-node", "--ipfs-walk-per-source", "0"]).is_err()); + assert!( + Config::try_parse_from(["gitlawb-node", "--ipfs-walk-per-source", "1048577"]).is_err() + ); + } + + /// The legacy-probe budget and the expensive-walk cap are SEPARATE knobs with + /// different defaults. They were one field until the probe budget and the walk cap + /// were split apart, so assert both defaults here: a future collapse back into one + /// field silently gives one of the two the other's default. + #[test] + fn ipfs_probe_and_walk_knobs_default_apart_and_reject_out_of_range() { + let default = Config::parse_from(["gitlawb-node"]); + assert_eq!(default.ipfs_max_legacy_probes, 256, "legacy-probe budget"); + assert_eq!(default.ipfs_max_repos_walked, 64, "expensive-walk cap"); + + assert_eq!( + Config::parse_from(["gitlawb-node", "--ipfs-max-legacy-probes", "8"]) + .ipfs_max_legacy_probes, + 8 + ); + // 0 would probe no repos (serve nothing); clap must reject it. + assert!(Config::try_parse_from(["gitlawb-node", "--ipfs-max-legacy-probes", "0"]).is_err()); + assert!( + Config::try_parse_from(["gitlawb-node", "--ipfs-max-legacy-probes", "1048577"]) + .is_err() + ); + assert!(Config::try_parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "0"]).is_err()); + } + + /// The `GITLAWB_IPFS_MAX_LEGACY_PROBES` knob must actually reach the legacy-probe + /// budget it advertises: production seeds `ipfs_max_legacy_probes` from this helper, + /// so the knob is a no-op unless the helper reflects it. RED while the helper returns + /// the hardcoded `MAX_LEGACY_PROBES_PER_REQUEST` (256 regardless of the knob), GREEN + /// once it reads the knob. + #[test] + fn ipfs_max_legacy_probes_wires_the_legacy_probe_budget() { + use crate::state::AppState; + // Knob set to 1 → a one-probe legacy budget. + let one = Config::parse_from(["gitlawb-node", "--ipfs-max-legacy-probes", "1"]); + assert_eq!( + AppState::ipfs_legacy_probe_budget(&one), + 1, + "the knob must control the legacy-probe budget, not be ignored" + ); + // Unset knob preserves the shipped 256-probe behaviour. + let default = Config::parse_from(["gitlawb-node"]); + assert_eq!( + AppState::ipfs_legacy_probe_budget(&default), + 256, + "the default knob keeps the shipped 256-probe budget" + ); + assert_eq!( + AppState::ipfs_legacy_probe_budget(&default), + crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, + "the default budget equals the constant it replaced" + ); + // Ceiling guard: the knob never governs the history-walk ceiling, which must + // stay at MAX_PIN_SOURCES + 1 or a provenanced full source set false-503s. + assert!( + crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST > crate::db::MAX_PIN_SOURCES as u32, + "the history-walk ceiling is independent of the repos-walked knob" + ); + } + + /// The `/ipfs` work-budget capacity is DERIVED from the route limit (R6, KTD6), with + /// a hard floor of one full legacy search per window (the effective + /// `ipfs_max_legacy_probes`). This guards the derived default so a single + /// default-config deep search never self-throttles mid-scan and recreates the F6 + /// admit-then-429 for a legitimate caller. A `RateLimiter` sized to the derived + /// budget must admit the whole probe budget back to back. + #[test] + fn ipfs_work_budget_derives_from_route_limit_and_clears_the_probe_floor() { + use crate::state::AppState; + + // Default config: derived work budget = max(route 600, probe budget 256) = 600, + // comfortably above the 256-probe floor. + let default = Config::parse_from(["gitlawb-node"]); + let budget = AppState::ipfs_work_budget(&default); + assert_eq!(budget, 600, "default derives max(route 600, probe 256)"); + assert!( + budget >= AppState::ipfs_legacy_probe_budget(&default) as usize, + "the work budget must clear one full legacy search per window" + ); + + // Tight route limit (1): the floor lifts the work budget to the probe budget + // (256), NOT down to 1 — a single deep search still completes its full scan. + let tight = Config::parse_from(["gitlawb-node", "--ipfs-rate-limit", "1"]); + assert_eq!( + AppState::ipfs_work_budget(&tight), + 256, + "a tight route limit is floored at the 256-probe budget, not clamped to 1" + ); + + // Raised probe budget lifts the floor with it (the work budget tracks the + // effective probe budget, not the constant). The walk cap is set to a DIFFERENT + // value in the same config on purpose: the two were one field before the split, + // so a floor that silently read the walk cap would return 7 here and still look + // plausible. Only the legacy-probe knob may drive this budget. + let raised = Config::parse_from([ + "gitlawb-node", + "--ipfs-rate-limit", + "10", + "--ipfs-max-legacy-probes", + "1000", + "--ipfs-max-repos-walked", + "7", + ]); + assert_eq!( + AppState::ipfs_work_budget(&raised), + 1000, + "the floor tracks the operator-raised legacy-probe budget, not the walk cap" + ); + + // 0 route limit disables the derived bucket too (a 0-capacity limiter admits all). + let disabled = Config::parse_from(["gitlawb-node", "--ipfs-rate-limit", "0"]); + assert_eq!( + AppState::ipfs_work_budget(&disabled), + 0, + "route limit 0 disables the derived work bucket alongside the route brake" + ); + + // Behavioral floor: a limiter sized to the derived (tight-route) budget admits + // the whole probe budget back to back for one source, then sheds the next. + let budget = AppState::ipfs_work_budget(&tight); + let limiter = + crate::rate_limit::RateLimiter::new(budget, std::time::Duration::from_secs(3600)); + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + rt.block_on(async { + for i in 0..budget { + assert!( + limiter.check("1.2.3.4").await, + "probe {i} of one full default-config scan must be admitted (no mid-scan throttle)" + ); + } + assert!( + !limiter.check("1.2.3.4").await, + "the probe past the derived budget is shed" + ); + }); + } + + #[test] + fn ipfs_max_repos_walked_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).ipfs_max_repos_walked, + 64 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "8"]) + .ipfs_max_repos_walked, + 8 + ); + // 0 would walk no repos (serve nothing); clap must reject it. + assert!(Config::try_parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "0"]).is_err()); + assert!( + Config::try_parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "1048577"]).is_err() + ); + } + + #[test] + fn ipfs_max_repo_visits_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).ipfs_max_repo_visits, + 1024 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--ipfs-max-repo-visits", "8"]) + .ipfs_max_repo_visits, + 8 + ); + // 0 would visit no repos (serve nothing); clap must reject it. + assert!(Config::try_parse_from(["gitlawb-node", "--ipfs-max-repo-visits", "0"]).is_err()); + assert!( + Config::try_parse_from(["gitlawb-node", "--ipfs-max-repo-visits", "1048577"]).is_err() + ); + } + + #[test] + fn ipfs_request_budget_secs_defaults_to_600_and_rejects_zero() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).ipfs_request_budget_secs, + 600 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--ipfs-request-budget-secs", "30"]) + .ipfs_request_budget_secs, + 30 + ); + // 0 would expire every /ipfs request at its first stage (unconditional + // 503); clap must reject it. + assert!( + Config::try_parse_from(["gitlawb-node", "--ipfs-request-budget-secs", "0"]).is_err() + ); + } + + /// #174 (RED-before/GREEN-after): the upper bound is what keeps the deadline derived + /// from this knob in range. `get_by_cid` builds the request budget as + /// `Instant::now() + Duration::from_secs(this)` (api/ipfs.rs), and that addition is an + /// explicit overflow check rather than a debug-only one, so an oversized value aborts + /// every `/ipfs/{cid}` request in a release build instead of setting a very long budget. + /// The route is anon-reachable, so the failure is operator-triggered but publicly felt. + /// Checked at parse time so no reachable configuration can carry a value the deadline + /// cannot represent. + /// + /// The large "disable the bound" settings are asserted alongside the rejections on + /// purpose, the same way the `git_service_timeout_secs` sibling does it. The ceiling + /// exists to exclude unrepresentable values, not to impose a view of a reasonable + /// budget, so a node already running on such a value must not start failing at boot on + /// upgrade. A test that only checked the boundary would pass with a far tighter cap. + #[test] + fn ipfs_request_budget_rejects_values_no_derived_duration_can_represent() { + let parse = |secs: u64| { + Config::try_parse_from([ + "gitlawb-node", + "--ipfs-request-budget-secs", + &secs.to_string(), + ]) + }; + + // Large "disable the bound" values that predate the ceiling still parse. + for disable in [1_000_000_000, 999_999_999] { + assert_eq!( + parse(disable) + .unwrap_or_else(|e| panic!("{disable} was a working setting: {e}")) + .ipfs_request_budget_secs, + disable + ); + } + + // At the ceiling: accepted, and the derived deadline still fits. This knob feeds + // only the `Instant` addition (no multiply derivation like the lease steal bound), + // so there is no `checked_mul` clause to carry over from the sibling test. + let at_max = + parse(GIT_SERVICE_TIMEOUT_SECS_MAX).expect("the documented maximum must parse"); + assert_eq!( + at_max.ipfs_request_budget_secs, + GIT_SERVICE_TIMEOUT_SECS_MAX + ); + assert!(std::time::Instant::now() + .checked_add(std::time::Duration::from_secs( + at_max.ipfs_request_budget_secs + )) + .is_some()); + + // Past the ceiling, and the top of the u64 range clap used to accept. Only the + // latter actually panics `Instant::now() + Duration::from_secs(..)`; the ceiling + // sits well below that, which is the conservative margin the constant documents. + for over in [GIT_SERVICE_TIMEOUT_SECS_MAX + 1, u64::MAX] { + assert!( + parse(over).is_err(), + "{over} is past the representable ceiling and must be rejected at parse time" + ); + } + assert!(std::time::Instant::now() + .checked_add(std::time::Duration::from_secs(u64::MAX)) + .is_none()); + } + + #[test] + fn max_concurrent_reads_per_caller_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).max_concurrent_reads_per_caller, + 16 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-reads-per-caller", "4"]) + .max_concurrent_reads_per_caller, + 4 + ); + // 0 would shed every read from a keyed caller; clap must reject it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-reads-per-caller", "0"]) + .is_err() + ); + assert!(Config::try_parse_from([ + "gitlawb-node", + "--max-concurrent-reads-per-caller", + "1048577" + ]) + .is_err()); + assert_eq!( + Config::parse_from([ + "gitlawb-node", + "--max-concurrent-reads-per-caller", + "1048576" + ]) + .max_concurrent_reads_per_caller, + 1_048_576 + ); + } + + /// #174 F1: a connection-affine write lock pins a pooled connection per + /// concurrent write, so the pool must clear `max_concurrent_git_pushes` by + /// `DB_POOL_APP_HEADROOM` or a push burst starves every other DB path. + /// `validate()` must reject an under-sized pool at boot. + #[test] + fn db_pool_must_clear_the_git_push_cap() { + // Shipped defaults validate (48 >= 32 + 8). + Config::parse_from(["gitlawb-node"]) + .validate() + .expect("default config must validate"); + + // An under-sized pool relative to the push cap is rejected (20 < 32 + 8). + let under = Config::parse_from([ + "gitlawb-node", + "--db-max-connections", + "20", + "--max-concurrent-git-pushes", + "32", + ]); + assert!( + under.validate().is_err(), + "db_max_connections 20 below max_concurrent_git_pushes 32 + headroom must be rejected" + ); + + // Exactly at the floor validates (40 == 32 + 8). + let at_floor = Config::parse_from([ + "gitlawb-node", + "--db-max-connections", + "40", + "--max-concurrent-git-pushes", + "32", + ]); + assert!( + at_floor.validate().is_ok(), + "db_max_connections at the floor (pushes + headroom) must validate" + ); + } } diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index c6ff644b..d2065ad7 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -879,18 +879,17 @@ const MIGRATIONS: &[Migration] = &[ version: 11, name: "ref_update_owner_did", stmts: &[ - // Index deferred — the feed gate (#144) does not read owner_did yet. + // Index deferred: the feed gate (#144) does not read owner_did yet. "ALTER TABLE received_ref_updates ADD COLUMN IF NOT EXISTS owner_did TEXT", ], }, - // Reservation: v17, deliberately not main's current_max + 1 (which is 12). - // The runner keys the applied set on the integer alone, so a version another - // in-flight branch also claims is skipped in full on whichever side merges - // second — no error, no warning, and schema_migrations still reads healthy - // while the column is simply absent. Two open branches already claim into - // this range: #135/#173 holds through 14 (15 once it rebases past v11), and - // #253 took 16. 17 clears both. Gaps are harmless: the runner iterates the - // array and never requires contiguity. + // Reservation: v17 is deliberately not main's current_max + 1. The runner keys the + // applied set on the integer alone, so a version another in-flight branch also + // claims is skipped in full on whichever side merges second: no error, no warning, + // and schema_migrations still reads healthy while the column is simply absent. + // #253 took 16, and the pin-provenance work below took 18-23 when it merged, so 17 + // sits between them. Gaps are harmless: the runner iterates the array and never + // requires contiguity. Migration { version: 17, name: "sync_queue_attempted_at", @@ -901,8 +900,125 @@ const MIGRATIONS: &[Migration] = &[ "ALTER TABLE sync_queue ADD COLUMN IF NOT EXISTS attempted_at TEXT", ], }, + // The six pin-provenance migrations below were numbered 11-16 while this work was + // in flight and moved to 18-23 on merge, because 11 and 16 were claimed elsewhere. + // A database that ran an earlier commit of this branch therefore has schema_migrations + // rows for the old numbers. Those rows are orphans: the runner skips on `version` + // alone and never reads `name`, so nothing detects them, and the DDL below re-runs + // as a no-op against objects that already exist. Recreate any such database rather + // than upgrading it in place. + Migration { + version: 18, + name: "pinned_cids_cid_index", + stmts: &[ + // GET /ipfs/{cid} resolves an incoming CID -> git oid via pinned_cids.cid + // (#173); index it so the per-request lookup is not a table scan. This is + // a NEW versioned migration (not appended to the applied v1 bundle) so a + // node already past v1 actually gets the index. Non-unique on purpose: cid + // is a function of raw content, so a UNIQUE index could reject a legitimate + // record_pinned_cid insert, and colliding rows serve byte-identical content. + "CREATE INDEX IF NOT EXISTS idx_pinned_cids_cid ON pinned_cids(cid)", + ], + }, + Migration { + version: 19, + name: "pinned_cids_repo_provenance", + stmts: &[ + // Record the repository a pin came from so GET /ipfs/{cid} resolves a + // provenanced pin straight to its ONE source repo instead of scanning every + // repo (#173, jatmn round 2 — bounds the anonymous fan-out and removes the + // updated_at-ordering false-404). NEW versioned migration (never appended to + // the applied v1 pinned_cids table) so a node past v1 gets the column. + // Nullable: pins recorded before this migration have no provenance and fall + // back to the legacy repo scan; new pins carry repo_id and resolve to one + // repo. Indexed for the resolver's oid -> repo_id lookup. + "ALTER TABLE pinned_cids ADD COLUMN IF NOT EXISTS repo_id TEXT", + "CREATE INDEX IF NOT EXISTS idx_pinned_cids_repo_id ON pinned_cids(repo_id)", + ], + }, + Migration { + version: 20, + name: "pin_repo_sources", + stmts: &[ + // F1 (#173, jatmn round 8): a shared object (a blob/tree/commit common to + // forks and mirrors) can be pinned from more than one repo. `pinned_cids` + // keeps only the FIRST pinner's `repo_id`, so a shared object first pinned + // from a private/quarantined repo 404s by CID even when a later PUBLIC repo + // also pinned it. Record EVERY pin-path source so `GET /ipfs/{cid}` can try + // each. NEW versioned migration (never appended to an applied block, INV-7). + // Bounded per object at insert time (MAX_PIN_SOURCES) so an adversary pushing + // one object from N repos cannot make resolution O(repos) (R2, INV-10). + "CREATE TABLE IF NOT EXISTS pin_repo_sources ( + sha256_hex TEXT NOT NULL, + repo_id TEXT NOT NULL, + PRIMARY KEY (sha256_hex, repo_id) + )", + "CREATE INDEX IF NOT EXISTS idx_pin_repo_sources_sha ON pin_repo_sources(sha256_hex)", + ], + }, + Migration { + version: 21, + name: "pinned_cids_legacy_provider_cid", + stmts: &[ + // R8 (#173, jatmn round 10): the opportunistic legacy provider-CID repair + // rewrites `pinned_cids.cid` from a stored PROVIDER CID (Kubo dag-pb / + // Pinata CIDv0) to the raw-content resolver key and stashes the OLD value + // here, so the rewrite is auditable and the row's legacy origin survives. + // Distinct from `pinata_cid` on purpose: `has_pinata_cid` gates the Pinata + // pin-skip, so parking a Kubo-legacy CID there would make Pinata forever + // skip re-pinning that object. NEW versioned migration (never appended to an + // applied block, INV-7) so a node past v13 actually gets the column. + // Nullable: only a repaired row sets it. + "ALTER TABLE pinned_cids ADD COLUMN IF NOT EXISTS legacy_provider_cid TEXT", + ], + }, + Migration { + version: 22, + name: "pinned_cids_sources_incomplete", + stmts: &[ + // U3 (#173): `record_pin_source` is BEST EFFORT at every call site, so a + // non-empty, below-cap source set is not proof that every source was + // recorded. An object first pinned from a private repo and later pushed + // from a PUBLIC repo whose record failed keeps a set naming only the + // private source, and the resolver used to call that set complete and 404 + // an object the public repo would serve. Record the miss DURABLY here so + // `GET /ipfs/{cid}` keeps the bounded scan fallback for exactly those + // objects. Not inferable from row counts or timestamps: neither can tell + // "no other source exists" from "a source failed to record", which is the + // whole distinction. NEW versioned migration (never appended to an applied + // block, INV-7). NOT NULL DEFAULT FALSE so every pre-existing row reads as + // complete and ordinary denials stay off the O(repos) path (INV-10). + "ALTER TABLE pinned_cids ADD COLUMN IF NOT EXISTS pin_sources_incomplete BOOLEAN NOT NULL DEFAULT FALSE", + ], + }, + Migration { + version: 23, + name: "pin_repair_sweep_cursor", + stmts: &[ + // U4 (#173): the legacy provider-CID repair sweep walks `pinned_cids` in + // bounded batches over an ordered `sha256_hex` cursor. The cursor has to be + // DURABLE, or a restart rewinds the walk to the start of the table and an + // upgraded node with a large pin set never finishes repairing it. One row + // (`id = 1`, enforced by the CHECK) rather than a key-value table: there is + // exactly one sweep and no second consumer, and a real constraint beats a + // convention nobody can enforce. NEW versioned migration (never appended to + // an applied block, INV-7). No default row is inserted: an absent row is the + // "never swept" state, which the empty-string cursor start already means, so + // there is no first-run special case to get wrong. + "CREATE TABLE IF NOT EXISTS pin_repair_sweep ( + id INTEGER NOT NULL PRIMARY KEY CHECK (id = 1), + cursor TEXT NOT NULL + )", + ], + }, ]; +/// Max distinct source repos recorded per pinned object (F1, #173 jatmn round 8). +/// Bounds both the resolver's per-OID source loop and the `pin_repo_sources` growth, +/// so an adversary re-pushing one object from many repos cannot make resolution +/// O(repos) (R2, INV-10). +pub const MAX_PIN_SOURCES: i64 = 16; + // ── Repos ───────────────────────────────────────────────────────────────────── pub(crate) fn normalize_owner_key(did: &str) -> &str { @@ -1079,6 +1195,22 @@ impl Db { Ok(row.map(row_to_repo)) } + /// Fetch a repo by its stable `id`. Used by the `/ipfs/{cid}` provenance path, + /// which resolves a pin straight to its ONE source repo (#173) instead of + /// scanning `list_all_repos`. `id` is exact, so unlike `get_repo`'s fuzzy + /// owner/name match there is no mirror-vs-canonical disambiguation. + pub async fn get_repo_by_id(&self, id: &str) -> Result> { + let row = sqlx::query( + "SELECT id, name, owner_did, description, is_public, default_branch, + created_at, updated_at, disk_path, forked_from, machine_id + FROM repos WHERE id = $1 LIMIT 1", + ) + .bind(id) + .fetch_optional(&self.pool) + .await?; + Ok(row.map(row_to_repo)) + } + #[allow(dead_code)] pub async fn list_repos(&self, owner_did: &str) -> Result> { let rows = sqlx::query( @@ -2268,20 +2400,387 @@ impl Db { Ok(row.get::("cnt") > 0) } - pub async fn record_pinned_cid(&self, sha256_hex: &str, cid: &str) -> Result<()> { + /// Every git oid a pinned CID maps to (`pinned_cids.cid` -> `sha256_hex`). + /// `GET /ipfs/{cid}` resolves the content-addressed CID a client sends back to + /// the object's git oid this way: a real pin CID digests the raw object + /// content, not the git oid, so the digest cannot be `git cat-file`d directly + /// (#173). The index is unique on the git oid but NON-unique on cid, so two + /// distinct oids can share one content-CID (a tree and a blob whose raw bytes + /// collide, or byte-identical content pinned under two oids). Returning every + /// candidate lets the handler try each rather than pick one arbitrarily and + /// false-404 when the chosen one is withheld or absent while another is + /// readable (#173). Empty when the CID was never pinned on this node. + pub async fn oids_for_cid(&self, cid: &str) -> Result> { + let rows = sqlx::query("SELECT sha256_hex FROM pinned_cids WHERE cid = $1") + .bind(cid) + .fetch_all(&self.pool) + .await?; + Ok(rows + .into_iter() + .map(|r| r.get::("sha256_hex")) + .collect()) + } + + /// Record a pinned object's CID and the repository it was pinned from + /// (`repo_id`, #173). On conflict the `COALESCE` backfills a NULL provenance + /// from a known source while keeping first-pinner-owns: an existing non-NULL + /// `repo_id` is never rewritten by a later push of the same oid, but a legacy + /// pin (or a pin recorded before provenance existed) whose `repo_id` is NULL + /// gets it filled the next time the object is re-pinned with a known source. + /// `cid`/`pinned_at` are left untouched on conflict. `repo_id` is `None` only + /// for a legacy pin with no known source; those fall back to the resolver's scan. + /// + /// The production first-pin path now goes through [`Self::record_pinned_cid_with_source`] + /// (U3, #173) so the pin and its source land atomically; this remains the seam for + /// seeding legacy, source-less rows in tests. + #[cfg_attr(not(test), allow(dead_code))] + pub async fn record_pinned_cid( + &self, + sha256_hex: &str, + cid: &str, + repo_id: Option<&str>, + ) -> Result<()> { sqlx::query( - "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) - VALUES ($1, $2, $3) - ON CONFLICT(sha256_hex) DO NOTHING", + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) + VALUES ($1, $2, $3, $4) + ON CONFLICT(sha256_hex) DO UPDATE SET + repo_id = COALESCE(pinned_cids.repo_id, EXCLUDED.repo_id)", ) .bind(sha256_hex) .bind(cid) .bind(Utc::now().to_rfc3339()) + .bind(repo_id) + .execute(&self.pool) + .await?; + Ok(()) + } + + /// The resolver key currently stored for a pinned object (`pinned_cids.cid`), + /// or `None` for an unpinned oid. The opportunistic legacy-repair path reads + /// it to decide candidacy from the codec of the string alone (no object bytes) + /// before it recomputes anything. + pub async fn cid_for_oid(&self, sha256_hex: &str) -> Result> { + let row = sqlx::query("SELECT cid FROM pinned_cids WHERE sha256_hex = $1") + .bind(sha256_hex) + .fetch_optional(&self.pool) + .await?; + Ok(row.map(|r| r.get::("cid"))) + } + + /// Rewrite a legacy provider-CID row to the raw-content resolver key, stashing + /// the old provider value in `legacy_provider_cid` (#173 R8, KTD8). Before this + /// branch the pin path stored the PROVIDER CID (Kubo dag-pb / Pinata CIDv0) in + /// `cid`; the `/ipfs` resolver recomputes the raw CID and 404s a mismatched key + /// even though `list_pinned_cids` still advertises it. The `WHERE cid = + /// $old_provider_cid` guard makes a concurrent double-repair a no-op (the second + /// writer sees the already-rewritten key and matches nothing) and never touches + /// a row keyed on a different value. Stashed in `legacy_provider_cid`, NOT + /// `pinata_cid`: the latter gates the Pinata pin-skip (`has_pinata_cid`), so a + /// Kubo-legacy CID parked there would make Pinata permanently skip the object. + pub async fn repair_legacy_provider_cid( + &self, + sha256_hex: &str, + raw_cid: &str, + old_provider_cid: &str, + ) -> Result<()> { + sqlx::query( + "UPDATE pinned_cids + SET cid = $2, legacy_provider_cid = $3 + WHERE sha256_hex = $1 AND cid = $3", + ) + .bind(sha256_hex) + .bind(raw_cid) + .bind(old_provider_cid) .execute(&self.pool) .await?; Ok(()) } + /// One ordered batch of `pinned_cids` rows strictly after `cursor`, for the U4 + /// legacy provider-CID repair sweep. Returns `(sha256_hex, cid)` ordered by + /// `sha256_hex` (the table's primary key, so the walk rides the PK index) and + /// capped at `limit` rows, which is what BOUNDS the sweep: one pass can never read + /// more than a batch, however large the pin set is. + /// + /// Deliberately NOT filtered to legacy rows in SQL. "Is this a raw CIDv1" is a + /// multibase+codec decode (`is_raw_cidv1`), which Postgres cannot express, and a + /// prefix-match approximation would silently mis-classify keys under a different + /// multihash. The caller applies the real predicate, so `limit` bounds rows READ + /// (the DB cost), not rows repaired. + pub async fn pinned_cids_after( + &self, + cursor: &str, + limit: i64, + ) -> Result> { + let rows = sqlx::query( + "SELECT sha256_hex, cid FROM pinned_cids + WHERE sha256_hex > $1 + ORDER BY sha256_hex + LIMIT $2", + ) + .bind(cursor) + .bind(limit) + .fetch_all(&self.pool) + .await?; + Ok(rows + .into_iter() + .map(|r| (r.get::("sha256_hex"), r.get::("cid"))) + .collect()) + } + + /// Where the U4 repair sweep's walk left off, or `""` before it has ever run. + /// Empty string sorts below every hex oid, so a first run and a rewound run are + /// the same code path (`sha256_hex > ''` is the whole table). + pub async fn pin_repair_cursor(&self) -> Result { + let row = sqlx::query("SELECT cursor FROM pin_repair_sweep WHERE id = 1") + .fetch_optional(&self.pool) + .await?; + Ok(row + .map(|r| r.get::("cursor")) + .unwrap_or_default()) + } + + /// Persist the sweep's walk position. Written after every batch, so a restart + /// resumes rather than re-walking the table from the beginning. A rewrite is a + /// plain upsert: the sweep is the single writer, and re-repairing an + /// already-repaired row is a no-op anyway (the codec cost gate spares it). + pub async fn set_pin_repair_cursor(&self, cursor: &str) -> Result<()> { + sqlx::query( + "INSERT INTO pin_repair_sweep (id, cursor) VALUES (1, $1) + ON CONFLICT (id) DO UPDATE SET cursor = EXCLUDED.cursor", + ) + .bind(cursor) + .execute(&self.pool) + .await?; + Ok(()) + } + + /// The repository a pinned object was recorded from (`pinned_cids.repo_id`), + /// or `None` for a legacy pin (recorded before provenance existed) or an + /// unpinned oid. `GET /ipfs/{cid}` uses this to gate+serve the ONE source + /// repo instead of scanning every repo (#173). + pub async fn provenance_for_oid(&self, sha256_hex: &str) -> Result> { + let row = sqlx::query("SELECT repo_id FROM pinned_cids WHERE sha256_hex = $1") + .bind(sha256_hex) + .fetch_optional(&self.pool) + .await?; + Ok(row.and_then(|r| r.get::, _>("repo_id"))) + } + + /// Backfill the source repo on an already-pinned object whose provenance is + /// NULL (a legacy pin recorded before provenance existed, #173, jatmn). The + /// `AND repo_id IS NULL` guard keeps first-pinner-owns: an existing non-NULL + /// provenance is left untouched. Touches only `repo_id` and never re-pins the + /// object's bytes, so it is safe to call on the already-pinned skip path. + pub async fn backfill_pin_provenance(&self, sha256_hex: &str, repo_id: &str) -> Result<()> { + sqlx::query( + "UPDATE pinned_cids SET repo_id = $2 WHERE sha256_hex = $1 AND repo_id IS NULL", + ) + .bind(sha256_hex) + .bind(repo_id) + .execute(&self.pool) + .await?; + Ok(()) + } + + /// Record a repository as a source for a pinned object (F1, #173 jatmn round 8), + /// bounded to about `MAX_PIN_SOURCES` distinct repos per object. The count guard + /// lives inside the INSERT (a single statement), which suppresses a re-push of the + /// SAME `(oid, repo)` via `ON CONFLICT DO NOTHING`. It does NOT hard-serialize + /// concurrent inserts of DIFFERENT repos for the same object: under Postgres READ + /// COMMITTED each concurrent writer's count subquery reads a snapshot that omits the + /// others' uncommitted rows, so N concurrent pushers can each see `count < cap` and + /// overshoot by up to N-1 rows. The overshoot is a small constant (bounded by + /// concurrent-pusher count, never O(repos)), and the RESOLVER read side + /// (`pin_sources_for_oid`) caps the ADDITIONAL sources at `MAX_PIN_SOURCES` (always + /// keeping the first-pinner), so the INV-10 bound on serve-time work holds at + /// `O(MAX_PIN_SOURCES + 1)` regardless of a table overshoot. + /// + /// A record that ACTUALLY ADDS a source row also CLEARS the + /// `pin_sources_incomplete` marker for the object, in the SAME transaction as the + /// insert (U3, #173), so the clear cannot drift across the four call sites or land + /// without the row it describes. + /// + /// The clear is gated on `rows_affected() > 0` because the INSERT is a no-op in two + /// ordinary cases: the `(oid, repo)` pair already exists (`ON CONFLICT DO NOTHING`) + /// and the source set is at cap (the count guard). The skip path calls this for + /// EVERY already-pinned object, and on a requeue pass that list is the whole-repo + /// enumeration, so an unconditional clear meant the next coalesced push from a repo + /// already in the set wiped the marker for every object in the repo without + /// recording anything (round 11 regression). The residual, which the gate does not + /// close: the marker is per-object, not per-(object, repo), so a GENUINE record from + /// a third repo C still clears a marker that repo A's failed record set. That is the + /// deliberate cost of a single boolean; closing it needs a per-(oid, repo) marker + /// table, and it fails in the safe direction (the marker only ever ADDS the scan + /// fallback, never removes a source the resolver already tries). + pub async fn record_pin_source(&self, sha256_hex: &str, repo_id: &str) -> Result<()> { + let mut tx = self.pool.begin().await?; + let inserted = sqlx::query( + "INSERT INTO pin_repo_sources (sha256_hex, repo_id) + SELECT $1, $2 + WHERE (SELECT count(*) FROM pin_repo_sources WHERE sha256_hex = $1) < $3 + ON CONFLICT DO NOTHING", + ) + .bind(sha256_hex) + .bind(repo_id) + .bind(MAX_PIN_SOURCES) + .execute(&mut *tx) + .await? + .rows_affected(); + if inserted > 0 { + sqlx::query( + "UPDATE pinned_cids SET pin_sources_incomplete = FALSE + WHERE sha256_hex = $1 AND pin_sources_incomplete", + ) + .bind(sha256_hex) + .execute(&mut *tx) + .await?; + } + tx.commit().await?; + Ok(()) + } + + /// Record a first pin and its source ATOMICALLY (U3, #173). The first-pin path + /// used to run `record_pinned_cid` and `record_pin_source` as two independent + /// best-effort calls, so the pin could land while its source did not, leaving a + /// source set that is silently missing its own first pinner. One transaction + /// removes that window entirely: either both rows land or neither does, and a + /// total failure leaves the object unpinned so the next push retries it. + /// + /// The marker clear carries the same `rows_affected` gate as `record_pin_source`. + /// It is not load-bearing here: this path runs only when `is_pinned` said no row + /// exists, and `mark_pin_sources_incomplete` is a no-op without a `pinned_cids` row, + /// so there is no marker to wrongly clear. The gate is kept for the one window that + /// is not covered by that argument, a concurrent pinner landing the row between the + /// `is_pinned` check and this upsert, and so the two clears cannot drift apart. + pub async fn record_pinned_cid_with_source( + &self, + sha256_hex: &str, + cid: &str, + repo_id: &str, + ) -> Result<()> { + let mut tx = self.pool.begin().await?; + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) + VALUES ($1, $2, $3, $4) + ON CONFLICT(sha256_hex) DO UPDATE SET + repo_id = COALESCE(pinned_cids.repo_id, EXCLUDED.repo_id)", + ) + .bind(sha256_hex) + .bind(cid) + .bind(Utc::now().to_rfc3339()) + .bind(repo_id) + .execute(&mut *tx) + .await?; + let inserted = sqlx::query( + "INSERT INTO pin_repo_sources (sha256_hex, repo_id) + SELECT $1, $2 + WHERE (SELECT count(*) FROM pin_repo_sources WHERE sha256_hex = $1) < $3 + ON CONFLICT DO NOTHING", + ) + .bind(sha256_hex) + .bind(repo_id) + .bind(MAX_PIN_SOURCES) + .execute(&mut *tx) + .await? + .rows_affected(); + if inserted > 0 { + sqlx::query( + "UPDATE pinned_cids SET pin_sources_incomplete = FALSE + WHERE sha256_hex = $1 AND pin_sources_incomplete", + ) + .bind(sha256_hex) + .execute(&mut *tx) + .await?; + } + tx.commit().await?; + Ok(()) + } + + /// Mark this object's pin-source set as KNOWN INCOMPLETE (U3, #173). Called when a + /// `record_pin_source` exhausts its retries, which is the only moment the node + /// knows a source it meant to record is missing. `GET /ipfs/{cid}` reads it to keep + /// the bounded scan fallback for that object, so a public copy that would serve is + /// no longer 404'd. A no-op when no `pinned_cids` row exists (the first-pin path is + /// transactional, so there is no half-recorded pin to describe). + pub async fn mark_pin_sources_incomplete(&self, sha256_hex: &str) -> Result<()> { + sqlx::query("UPDATE pinned_cids SET pin_sources_incomplete = TRUE WHERE sha256_hex = $1") + .bind(sha256_hex) + .execute(&self.pool) + .await?; + Ok(()) + } + + /// Whether this object's pin-source set is KNOWN INCOMPLETE (U3, #173): a + /// `record_pin_source` for it failed outright and no later record has repaired the + /// set. `false` for an unpinned oid and for every row predating the column, so the + /// common path is unchanged and an ordinary denial never fans out (INV-10). + pub async fn pin_sources_incomplete(&self, sha256_hex: &str) -> Result { + let flag: Option = sqlx::query_scalar( + "SELECT pin_sources_incomplete FROM pinned_cids WHERE sha256_hex = $1", + ) + .bind(sha256_hex) + .fetch_optional(&self.pool) + .await?; + Ok(flag.unwrap_or(false)) + } + + /// Every source repository recorded for a pinned object (F1, #173 jatmn round 8): + /// the union of the first-pinner `pinned_cids.repo_id` and the `pin_repo_sources` + /// rows, deduped and ordered for a deterministic resolver walk. + /// + /// The first-pinner (a single row by `pinned_cids`' PK on `sha256_hex`) is ALWAYS + /// included; the `LIMIT MAX_PIN_SOURCES` caps only the ADDITIONAL `pin_repo_sources` + /// rows. This keeps the resolver's per-source work a bounded `O(MAX_PIN_SOURCES + 1)` + /// ceiling (INV-10) while never letting the cap evict the original source. A prior + /// version applied the `LIMIT` to the whole UNION with a lexicographic `ORDER BY`, + /// which let an attacker 404 a legacy public CID (first-pinner in `pinned_cids` but + /// not yet in `pin_repo_sources`) by pushing the same object from `MAX_PIN_SOURCES` + /// repos whose grindable ids sort before it, evicting the public source from the + /// window. Empty for a legacy pin with no known source (it falls back to the repo + /// scan) or an unpinned oid. + pub async fn pin_sources_for_oid(&self, sha256_hex: &str) -> Result> { + let rows = sqlx::query( + "SELECT repo_id FROM pinned_cids + WHERE sha256_hex = $1 AND repo_id IS NOT NULL + UNION + SELECT repo_id FROM ( + SELECT repo_id FROM pin_repo_sources + WHERE sha256_hex = $1 + ORDER BY repo_id + LIMIT $2 + ) capped + ORDER BY repo_id", + ) + .bind(sha256_hex) + .bind(MAX_PIN_SOURCES) + .fetch_all(&self.pool) + .await?; + Ok(rows + .into_iter() + .map(|r| r.get::("repo_id")) + .collect()) + } + + /// Whether `pin_repo_sources` is at the `MAX_PIN_SOURCES` cap for this oid, i.e. + /// the provenance source set returned by [`Self::pin_sources_for_oid`] may be + /// INCOMPLETE. `record_pin_source` stops inserting at exactly `MAX_PIN_SOURCES` + /// rows and drops later sources silently, so a full table is the only observable + /// signal that a servable source (e.g. a later public pinner) may have been + /// dropped. `get_by_cid` uses this to decide whether a provenance miss should fall + /// back to the bounded legacy scan (which gates every repo through the real + /// visibility gate and so finds a dropped public source) rather than 404 — closing + /// the pin-source griefing hole where 16 attacker sources bury a public one. `>=` + /// (not `==`) is defensive against any future overshoot. + pub async fn pin_sources_at_cap(&self, sha256_hex: &str) -> Result { + let count: i64 = + sqlx::query_scalar("SELECT count(*) FROM pin_repo_sources WHERE sha256_hex = $1") + .bind(sha256_hex) + .fetch_one(&self.pool) + .await?; + Ok(count >= MAX_PIN_SOURCES) + } + pub async fn record_encrypted_blob( &self, repo_id: &str, @@ -2352,6 +2851,17 @@ impl Db { Ok(row.map(|r| r.get("recipients_tag"))) } + /// Every pinned object this node ADVERTISES (`GET /api/v1/ipfs/pins`). + /// + /// U4 (#173): rows still keyed on a legacy PROVIDER CID (Kubo dag-pb / Pinata + /// CIDv0, written by releases before this branch) are withheld from the listing. + /// The `/ipfs/{cid}` resolver recomputes the raw-content CID from the object bytes + /// and refuses any row whose stored key does not match, so advertising the legacy + /// key hands a client a CID this node deliberately will not serve. The background + /// repair sweep rewrites those rows to the raw key, and each one reappears here the + /// moment it is repaired. Filtering is done in Rust because the raw-CIDv1 test is a + /// multibase+codec decode (`is_raw_cidv1`), not something SQL can express; it is the + /// SAME predicate the repair path uses as its cost gate, so the two cannot drift. pub async fn list_pinned_cids(&self) -> Result> { let rows = sqlx::query( "SELECT sha256_hex, cid, pinned_at, pinata_cid FROM pinned_cids ORDER BY pinned_at DESC", @@ -2360,6 +2870,7 @@ impl Db { .await?; Ok(rows .into_iter() + .filter(|r| gitlawb_core::cid::is_raw_cidv1(r.get::<&str, _>("cid"))) .map(|r| PinnedCidRecord { sha256_hex: r.get("sha256_hex"), cid: r.get("cid"), @@ -2381,18 +2892,34 @@ impl Db { } /// Record the Pinata CID for a git object. - /// Inserts the row if it doesn't exist (objects pinned directly to Pinata - /// without a prior local IPFS pin get cid = pinata_cid). - pub async fn record_pinata_cid(&self, sha256_hex: &str, pinata_cid: &str) -> Result<()> { + /// + /// `raw_cid` is the locally-computed raw-content CID (`Cid::from_git_object_bytes`, + /// CIDv1/raw/sha2-256), the resolver key `GET /ipfs/{cid}` looks up; `pinata_cid` + /// is the provider CID Pinata returned (a dag-pb/UnixFS CID for gateway retrieval). + /// Inserts the row if it doesn't exist (an object pinned directly to Pinata with + /// no prior local IPFS pin gets `cid = raw_cid`, never the provider CID — a dag-pb + /// provider CID must never become an alias that serves raw bytes that do not hash + /// to it, #173). On conflict `cid` is left untouched: a prior local pin already + /// stored the correct raw CID, and the COALESCE backfills a NULL provenance from a + /// known source while keeping first-pinner-owns. + pub async fn record_pinata_cid( + &self, + sha256_hex: &str, + raw_cid: &str, + pinata_cid: &str, + repo_id: Option<&str>, + ) -> Result<()> { sqlx::query( - "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid) - VALUES ($1, $2, $3, $4) - ON CONFLICT(sha256_hex) DO UPDATE SET pinata_cid = EXCLUDED.pinata_cid", + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid, repo_id) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT(sha256_hex) DO UPDATE SET pinata_cid = EXCLUDED.pinata_cid, + repo_id = COALESCE(pinned_cids.repo_id, EXCLUDED.repo_id)", ) .bind(sha256_hex) - .bind(pinata_cid) // fallback local cid if row is new + .bind(raw_cid) // resolver-key cid: locally-computed raw CID, never the provider CID .bind(Utc::now().to_rfc3339()) .bind(pinata_cid) + .bind(repo_id) .execute(&self.pool) .await?; Ok(()) @@ -5715,6 +6242,51 @@ mod ref_certificate_tests { ); } + /// INV-7: upgrade-path test — an existing node already past v1 must still get + /// the `pinned_cids.cid` index. It ships as its OWN v11 migration (not appended + /// to the applied v1 bundle), so dropping the index + its `schema_migrations` + /// row and re-running migrations must recreate it, exercising the real code + /// path rather than hand-copying the SQL. + #[sqlx::test] + async fn v18_pinned_cids_cid_index_applies_on_upgrade(pool: PgPool) { + async fn index_exists(pool: &PgPool) -> bool { + sqlx::query_scalar::<_, bool>( + "SELECT EXISTS(SELECT 1 FROM pg_indexes WHERE indexname = 'idx_pinned_cids_cid')", + ) + .fetch_one(pool) + .await + .unwrap() + } + + let db = Db::for_testing(pool.clone()); + db.run_migrations().await.unwrap(); + assert!( + index_exists(&pool).await, + "fresh migration chain creates the index" + ); + + // Simulate a node at pre-v18: drop the index and its migration record. + sqlx::query("DROP INDEX IF EXISTS idx_pinned_cids_cid") + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version = 18") + .execute(&pool) + .await + .unwrap(); + assert!( + !index_exists(&pool).await, + "precondition: index and its migration record removed" + ); + + // Re-run migrations: v11 re-applies and recreates the index on the upgrade. + db.run_migrations().await.unwrap(); + assert!( + index_exists(&pool).await, + "v11 must recreate idx_pinned_cids_cid on an upgrading node" + ); + } + /// INV-7: upgrade-path test — seed a database at v9 with duplicate /// ref_certificates, then let the real v10 migration fire via /// run_migrations(). This exercises the migration code path rather than diff --git a/crates/gitlawb-node/src/encrypted_pin.rs b/crates/gitlawb-node/src/encrypted_pin.rs index 40e444e5..19b80651 100644 --- a/crates/gitlawb-node/src/encrypted_pin.rs +++ b/crates/gitlawb-node/src/encrypted_pin.rs @@ -170,7 +170,7 @@ pub async fn encrypt_and_pin( continue; } }; - let cid = match crate::ipfs_pin::pin_git_object(ipfs_api, oid, &envelope).await { + let cid = match crate::ipfs_pin::pin_git_object(ipfs_api, oid, &envelope, None).await { Ok(c) if !c.is_empty() => c, Ok(_) => { tracing::warn!(oid = %oid, "pin_git_object returned no cid; skipping"); diff --git a/crates/gitlawb-node/src/error.rs b/crates/gitlawb-node/src/error.rs index bdd5aec9..c13b92b4 100644 --- a/crates/gitlawb-node/src/error.rs +++ b/crates/gitlawb-node/src/error.rs @@ -41,12 +41,18 @@ pub enum AppError { #[error("incomplete: {0}")] Incomplete(String), + #[error("search incomplete: {0}")] + SearchIncomplete(String), + #[error("git error: {0}")] Git(String), #[error("git service timed out: {0}")] Timeout(String), + #[error("server overloaded: {0}")] + Overloaded(String), + #[error("database error: {0}")] Db(#[from] sqlx::Error), @@ -138,6 +144,15 @@ impl IntoResponse for AppError { AppError::Incomplete(msg) => { (StatusCode::UNPROCESSABLE_ENTITY, "incomplete", msg.clone()) } + // A bounded search that could not complete (the CID resolver hit its + // legacy-probe or walk ceiling), distinct from the 404 that asserts a + // definitive not-found: absence was NOT proven, so the caller should + // retry rather than treat it as gone (#173, F2). 503, retryable. + AppError::SearchIncomplete(msg) => ( + StatusCode::SERVICE_UNAVAILABLE, + "search_incomplete", + msg.clone(), + ), AppError::Git(msg) => (StatusCode::INTERNAL_SERVER_ERROR, "git_error", msg.clone()), // 504, distinct from the 500 git_error and from the read-gate's 404 / // the auth 401, so the client can tell a deadline from a failure. @@ -147,6 +162,12 @@ impl IntoResponse for AppError { DB_UNAVAILABLE_CODE, DB_UNAVAILABLE_MESSAGE.into(), ), + // 503 with a Retry-After (attached after this match — the shared tail + // can't carry per-variant headers). This is the single place Overloaded + // becomes a response, so it can never ship a 503 without the retry hint. + AppError::Overloaded(msg) => { + (StatusCode::SERVICE_UNAVAILABLE, "overloaded", msg.clone()) + } AppError::Db(e) => (StatusCode::INTERNAL_SERVER_ERROR, "db_error", e.to_string()), AppError::Internal(e) => ( StatusCode::INTERNAL_SERVER_ERROR, @@ -160,7 +181,21 @@ impl IntoResponse for AppError { "message": message, })); - (status, body).into_response() + let mut resp = (status, body).into_response(); + // Both retryable 503s advertise when to retry: Overloaded (capacity shed) and + // SearchIncomplete (a bounded CID search cut short by a cap — retry may complete + // it). They ride the shared tail above for body/status, so the header is attached + // here rather than in bespoke early returns, keeping each variant handled once. + if matches!( + self, + AppError::Overloaded(_) | AppError::SearchIncomplete(_) + ) { + resp.headers_mut().insert( + axum::http::header::RETRY_AFTER, + axum::http::HeaderValue::from_static("1"), + ); + } + resp } } @@ -182,4 +217,14 @@ mod tests { StatusCode::INTERNAL_SERVER_ERROR ); } + + #[test] + fn overloaded_maps_to_503_with_retry_after() { + let resp = AppError::Overloaded("x".into()).into_response(); + assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + resp.headers().get("retry-after").unwrap().to_str().unwrap(), + "1" + ); + } } diff --git a/crates/gitlawb-node/src/git/push_delta.rs b/crates/gitlawb-node/src/git/push_delta.rs index 7ab00816..0b569693 100644 --- a/crates/gitlawb-node/src/git/push_delta.rs +++ b/crates/gitlawb-node/src/git/push_delta.rs @@ -29,9 +29,9 @@ use std::collections::HashSet; use std::path::{Path, PathBuf}; -use std::process::Command; +use std::time::{Duration, Instant}; -use anyhow::{bail, Context, Result}; +use anyhow::Result; /// Env var that forces the push path to always full-scan, bypassing the delta /// optimization (KTD7 kill-switch). Reuses the already-tested fallback branch @@ -64,11 +64,21 @@ pub enum PinCandidates { /// Fail-closed: any condition where the introduced set cannot be safely /// determined returns [`PinCandidates::FullScanRequired`] rather than a partial /// set, so the caller full-scans instead of silently under-pinning. -pub fn resolve_push_delta(repo_path: &Path, new_tips: &[&str], old_tips: &[&str]) -> PinCandidates { - // KTD7 kill-switch: force the (already-tested) full-scan fallback. The env - // read is split out from the pure logic so the resolver stays unit-testable - // without touching process-global state. - resolve_push_delta_inner(repo_path, new_tips, old_tips, force_full_scan()) +pub fn resolve_push_delta( + repo_path: &Path, + new_tips: &[&str], + old_tips: &[&str], + git_bin: &str, + deadline: Instant, +) -> PinCandidates { + resolve_push_delta_inner( + repo_path, + new_tips, + old_tips, + force_full_scan(), + git_bin, + deadline, + ) } /// Whether the force-full-scan kill-switch env var is set. @@ -83,6 +93,8 @@ fn resolve_push_delta_inner( new_tips: &[&str], old_tips: &[&str], force_full_scan: bool, + git_bin: &str, + deadline: Instant, ) -> PinCandidates { if force_full_scan { tracing::debug!("{FORCE_FULL_SCAN_ENV} set — forcing full scan"); @@ -102,7 +114,7 @@ fn resolve_push_delta_inner( // commit). Bare `cat-file -t` returns `tag` for an annotated tag, and // `for-each-ref %(*objecttype)` peels only one level — neither is correct. for tip in new_tips { - match peeled_object_type(repo_path, tip) { + match peeled_object_type(repo_path, tip, git_bin, deadline) { Some(t) if t == "commit" => {} other => { tracing::debug!( @@ -115,7 +127,7 @@ fn resolve_push_delta_inner( } } - match rev_list_delta(repo_path, new_tips, old_tips) { + match rev_list_delta(repo_path, new_tips, old_tips, git_bin, deadline) { Ok(oids) => PinCandidates::Delta(oids), Err(e) => { tracing::debug!(err = %e, "push-delta rev-list failed — forcing full scan"); @@ -126,41 +138,43 @@ fn resolve_push_delta_inner( /// Return the fully-peeled object type of `sha` (e.g. `commit`, `tree`, /// `blob`), or `None` if the object is missing/unpeelable or git errored. -fn peeled_object_type(repo_path: &Path, sha: &str) -> Option { - let output = Command::new("git") - .args(["cat-file", "-t", &format!("{sha}^{{}}")]) - .current_dir(repo_path) - .output() - .ok()?; - if !output.status.success() { - return None; - } - Some(String::from_utf8_lossy(&output.stdout).trim().to_string()) +fn peeled_object_type( + repo_path: &Path, + sha: &str, + git_bin: &str, + deadline: Instant, +) -> Option { + let peel = format!("{sha}^{{}}"); + let out = crate::git::visibility_pack::run_bounded_git( + git_bin, + &["cat-file", "-t", &peel], + repo_path, + b"", + deadline, + ) + .ok()?; + Some(String::from_utf8_lossy(&out).trim().to_string()) } /// Run `git rev-list --objects --no-object-names --not ` and return /// the bare OID set. Decides on `status.success()` *before* parsing stdout, so /// a walk that prints a valid prefix then errors mid-walk is discarded. -fn rev_list_delta(repo_path: &Path, new_tips: &[&str], old_tips: &[&str]) -> Result> { +fn rev_list_delta( + repo_path: &Path, + new_tips: &[&str], + old_tips: &[&str], + git_bin: &str, + deadline: Instant, +) -> Result> { let mut args: Vec<&str> = vec!["rev-list", "--objects", "--no-object-names"]; args.extend_from_slice(new_tips); if !old_tips.is_empty() { args.push("--not"); args.extend_from_slice(old_tips); } - - let output = Command::new("git") - .args(&args) - .current_dir(repo_path) - .output() - .context("failed to run git rev-list for push delta")?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - bail!("git rev-list failed: {stderr}"); - } - - let stdout = String::from_utf8_lossy(&output.stdout); + let out = + crate::git::visibility_pack::run_bounded_git(git_bin, &args, repo_path, b"", deadline)?; + let stdout = String::from_utf8_lossy(&out); Ok(stdout .lines() .map(|l| l.trim().to_string()) @@ -175,23 +189,19 @@ fn rev_list_delta(repo_path: &Path, new_tips: &[&str], old_tips: &[&str]) -> Res /// reconciliation sweep relies on. It returns *all* objects (including /// unreachable/dangling ones), which is what the sweep needs to catch /// stragglers — do not swap it for a reachability walk. -pub fn list_all_objects(repo_path: &Path) -> Result> { - let output = Command::new("git") - .args([ +pub fn list_all_objects(repo_path: &Path, git_bin: &str, deadline: Instant) -> Result> { + let out = crate::git::visibility_pack::run_bounded_git( + git_bin, + &[ "cat-file", "--batch-all-objects", "--batch-check=%(objectname)", - ]) - .current_dir(repo_path) - .output() - .context("failed to run git cat-file")?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - bail!("git cat-file failed: {stderr}"); - } - - let stdout = String::from_utf8_lossy(&output.stdout); + ], + repo_path, + b"", + deadline, + )?; + let stdout = String::from_utf8_lossy(&out); Ok(stdout .lines() .map(|l| l.trim().to_string()) @@ -203,23 +213,23 @@ pub fn list_all_objects(repo_path: &Path) -> Result> { /// `--batch-check='%(objectname) %(objecttype)'`. The pin path's fail-closed /// filter needs to tell blobs (content, withholdable) from commits/trees /// (structural, never withheld) without typing the candidate list itself. -pub fn list_all_objects_with_type(repo_path: &Path) -> Result> { - let output = Command::new("git") - .args([ +pub fn list_all_objects_with_type( + repo_path: &Path, + git_bin: &str, + deadline: Instant, +) -> Result> { + let out = crate::git::visibility_pack::run_bounded_git( + git_bin, + &[ "cat-file", "--batch-all-objects", "--batch-check=%(objectname) %(objecttype)", - ]) - .current_dir(repo_path) - .output() - .context("failed to run git cat-file")?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - bail!("git cat-file failed: {stderr}"); - } - - let stdout = String::from_utf8_lossy(&output.stdout); + ], + repo_path, + b"", + deadline, + )?; + let stdout = String::from_utf8_lossy(&out); Ok(stdout .lines() .filter_map(|l| { @@ -235,8 +245,12 @@ pub fn list_all_objects_with_type(repo_path: &Path) -> Result Result> { - Ok(list_all_objects_with_type(repo_path)? +pub fn all_blob_oids( + repo_path: &Path, + git_bin: &str, + deadline: Instant, +) -> Result> { + Ok(list_all_objects_with_type(repo_path, git_bin, deadline)? .into_iter() .filter(|(_, ty)| ty == "blob") .map(|(oid, _)| oid) @@ -268,22 +282,68 @@ pub struct PinCandidateSet { /// it can never leak because the withheld/fail-closed filter still runs on /// whatever set is returned. `full_scan` rides on the returned set so the caller /// knows when the dangling-inclusive filter is required. +/// +/// `scan_sem` is the post-receive scan admission pool (`git_encrypt_semaphore`, +/// #174 F4): both git-spawning stages — the per-tip `cat-file` probe + delta +/// rev-list, and the full-scan fallback — run under ONE permit held for the +/// whole blocking scan. The deletion-only fast path (no new tips) spawns no git +/// and never parks. +/// +/// `force_full_scan` forces the full-scan fallback regardless of the tips — the +/// coalesced-drain `PendingWork::FullScan` marker (#174 F5). It composes with +/// the env kill-switch (either forces), and it disqualifies the empty-tips fast +/// path: a forced-scan call with no tips must still enumerate the repo, never +/// silently resolve to an empty delta (which would pin nothing and lose the +/// drained pushes' work — the F5 bug shape). pub async fn resolve_candidates_for_push( + scan_sem: std::sync::Arc, repo_path: PathBuf, new_tips: Vec, old_tips: Vec, + git_bin: String, + timeout: Duration, + force_full_scan: bool, ) -> PinCandidateSet { + let force = force_full_scan || self::force_full_scan(); + // No-git fast path: a deletion-only push resolves to an empty delta without + // spawning any git child, so it must not park on the scan pool. A forced + // full scan (caller flag or kill-switch) disqualifies it — the scan spawns git. + if new_tips.is_empty() && !force { + tracing::info!(delta = 0usize, repo = %repo_path.display(), "pin candidate set from push delta"); + return PinCandidateSet { + candidates: Vec::new(), + full_scan: false, + }; + } + // Scan admission (#174 F4): DEFER, never shed — a dropped scan would + // silently under-pin this push. The permit moves into the blocking closure + // so a started scan always completes holding it. Residuals at + // `acquire_scan_permit`. + let permit = + crate::state::acquire_scan_permit(scan_sem, &repo_path, "pin-candidate scan").await; tokio::task::spawn_blocking(move || { + let _permit = permit; + // ONE shared deadline for the whole scan, per jatmn ("the same deadline"). + let deadline = Instant::now() + timeout; let new_refs: Vec<&str> = new_tips.iter().map(String::as_str).collect(); let old_refs: Vec<&str> = old_tips.iter().map(String::as_str).collect(); - match resolve_push_delta(&repo_path, &new_refs, &old_refs) { + // `force` already folds in the env kill-switch, so the forced arm skips the + // delta machinery outright; the unforced arm goes through the normal + // resolver (whose own env read is false here by construction). + let resolved = if force { + tracing::debug!("full scan forced (coalesced-drain marker or kill-switch)"); + PinCandidates::FullScanRequired + } else { + resolve_push_delta(&repo_path, &new_refs, &old_refs, &git_bin, deadline) + }; + match resolved { PinCandidates::Delta(objs) => { tracing::info!(delta = objs.len(), repo = %repo_path.display(), "pin candidate set from push delta"); PinCandidateSet { candidates: objs, full_scan: false } } PinCandidates::FullScanRequired => { tracing::warn!(repo = %repo_path.display(), "pin delta unavailable (non-commit tip, git error, or force-full-scan) — full-scan fallback"); - match list_all_objects(&repo_path) { + match list_all_objects(&repo_path, &git_bin, deadline) { Ok(objs) => PinCandidateSet { candidates: objs, full_scan: true }, Err(e) => { tracing::warn!(repo = %repo_path.display(), err = %e, "full-scan fallback failed; pinning nothing this push (reconciliation sweep backstops)"); @@ -304,8 +364,13 @@ pub async fn resolve_candidates_for_push( mod tests { use super::*; use std::collections::HashSet; + use std::process::Command; use tempfile::TempDir; + fn td() -> std::time::Instant { + std::time::Instant::now() + std::time::Duration::from_secs(600) + } + /// Minimal git helper for building test repos. struct Repo { _td: TempDir, @@ -363,9 +428,10 @@ mod tests { let repo = Repo::new(); let c1 = repo.commit_file("a.txt", "one\n"); let c2 = repo.commit_file("b.txt", "two\n"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&c2], &[&c1])) - .into_iter() - .collect(); + let got: HashSet = + delta(resolve_push_delta(&repo.path, &[&c2], &[&c1], "git", td())) + .into_iter() + .collect(); // The new blob b.txt and commit c2 are in the delta; the old blob a.txt // and commit c1 are not. let new_blob = repo.rev("HEAD:b.txt"); @@ -383,7 +449,7 @@ mod tests { // genuinely new objects, never fewer. let repo = Repo::new(); let c1 = repo.commit_file("a.txt", "one\n"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&c1], &[])) + let got: HashSet = delta(resolve_push_delta(&repo.path, &[&c1], &[], "git", td())) .into_iter() .collect(); assert!(got.contains(&c1)); @@ -399,9 +465,15 @@ mod tests { // Rewrite history: reset to base, commit a different file. repo.git(&["reset", "-q", "--hard", &base]); let new_tip = repo.commit_file("c.txt", "three\n"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&new_tip], &[&old_tip])) - .into_iter() - .collect(); + let got: HashSet = delta(resolve_push_delta( + &repo.path, + &[&new_tip], + &[&old_tip], + "git", + td(), + )) + .into_iter() + .collect(); assert!(got.contains(&new_tip), "new tip in delta"); assert!(got.contains(&repo.rev(&format!("{new_tip}:c.txt")))); // No error; force-push computes new-minus-old cleanly. @@ -413,7 +485,7 @@ mod tests { repo.commit_file("a.txt", "one\n"); // All updates were deletions => new_tips empty after the caller strips zeros. assert_eq!( - resolve_push_delta(&repo.path, &[], &[ZERO]), + resolve_push_delta(&repo.path, &[], &[ZERO], "git", td()), PinCandidates::Delta(Vec::new()) ); } @@ -424,7 +496,7 @@ mod tests { repo.commit_file("a.txt", "one\n"); let blob = repo.rev("HEAD:a.txt"); assert_eq!( - resolve_push_delta(&repo.path, &[&blob], &[]), + resolve_push_delta(&repo.path, &[&blob], &[], "git", td()), PinCandidates::FullScanRequired, "a blob tip must force full scan (rev-list would exit 0 and walk it)" ); @@ -436,7 +508,7 @@ mod tests { repo.commit_file("a.txt", "one\n"); let tree = repo.rev("HEAD^{tree}"); assert_eq!( - resolve_push_delta(&repo.path, &[&tree], &[]), + resolve_push_delta(&repo.path, &[&tree], &[], "git", td()), PinCandidates::FullScanRequired ); } @@ -449,7 +521,7 @@ mod tests { repo.git(&["tag", "-a", "treetag", "-m", "x", &tree]); let tag = repo.rev("treetag"); assert_eq!( - resolve_push_delta(&repo.path, &[&tag], &[]), + resolve_push_delta(&repo.path, &[&tag], &[], "git", td()), PinCandidates::FullScanRequired, "annotated tag peeling to a tree must force full scan" ); @@ -465,7 +537,7 @@ mod tests { repo.git(&["tag", "-a", "t2", "-m", "x", &t1]); let t2 = repo.rev("t2"); assert_eq!( - resolve_push_delta(&repo.path, &[&t2], &[]), + resolve_push_delta(&repo.path, &[&t2], &[], "git", td()), PinCandidates::FullScanRequired ); } @@ -480,7 +552,7 @@ mod tests { let c1 = repo.commit_file("a.txt", "one\n"); repo.git(&["tag", "-a", "rel", "-m", "release", &c1]); let tag = repo.rev("rel"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&tag], &[])) + let got: HashSet = delta(resolve_push_delta(&repo.path, &[&tag], &[], "git", td())) .into_iter() .collect(); assert!( @@ -501,7 +573,7 @@ mod tests { let t1 = repo.rev("t1"); repo.git(&["tag", "-a", "t2", "-m", "x", &t1]); let t2 = repo.rev("t2"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&t2], &[])) + let got: HashSet = delta(resolve_push_delta(&repo.path, &[&t2], &[], "git", td())) .into_iter() .collect(); assert!( @@ -516,9 +588,16 @@ mod tests { let repo = Repo::new(); let c1 = repo.commit_file("a.txt", "one\n"); let c2 = repo.commit_file("b.txt", "two\n"); - let set = - resolve_candidates_for_push(repo.path.clone(), vec![c2.clone()], vec![c1.clone()]) - .await; + let set = resolve_candidates_for_push( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), + repo.path.clone(), + vec![c2.clone()], + vec![c1.clone()], + "git".to_string(), + std::time::Duration::from_secs(600), + false, + ) + .await; assert!(!set.full_scan, "happy-path delta is not a full scan"); let got: HashSet = set.candidates.into_iter().collect(); let new_blob = repo.rev("HEAD:b.txt"); @@ -536,20 +615,206 @@ mod tests { let repo = Repo::new(); repo.commit_file("a.txt", "one\n"); let blob = repo.rev("HEAD:a.txt"); - let all: HashSet = list_all_objects(&repo.path).unwrap().into_iter().collect(); - let set = resolve_candidates_for_push(repo.path.clone(), vec![blob], vec![]).await; + let all: HashSet = list_all_objects(&repo.path, "git", td()) + .unwrap() + .into_iter() + .collect(); + let set = resolve_candidates_for_push( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), + repo.path.clone(), + vec![blob], + vec![], + "git".to_string(), + std::time::Duration::from_secs(600), + false, + ) + .await; assert!(set.full_scan, "non-commit tip is signalled as a full scan"); let got: HashSet = set.candidates.into_iter().collect(); assert_eq!(got, all, "non-commit tip falls back to full repo scan"); } + /// F4 defer proof 2: `resolve_candidates_for_push`'s git stages (the per-tip + /// cat-file type probe + delta rev-list, and the full-scan fallback) run under a + /// scan-admission permit: with a zero-permit pool the call parks and spawns no + /// git; once a permit is available the SAME call runs (defer, not shed). On + /// ungated code the git runs regardless of the pool (RED). + #[cfg(unix)] + #[tokio::test] + async fn resolve_candidates_defers_when_scan_pool_exhausted() { + use std::os::unix::fs::PermissionsExt; + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + + let dir = tempfile::TempDir::new().unwrap(); + let marker = dir.path().join("git.ran"); + // Fake git records ANY invocation; cat-file reports a commit tip so the + // delta stage proceeds, rev-list yields an empty delta. + let fake = dir.path().join("fakegit"); + std::fs::write( + &fake, + format!( + "#!/bin/sh\necho ran >> \"{}\"\ncase \"$1\" in\n cat-file) echo commit ;;\n *) : ;;\nesac\nexit 0\n", + marker.display() + ), + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + let git_bin = fake.to_str().unwrap().to_string(); + let tip = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef".to_string(); + + let sem: Arc = Arc::new(Semaphore::new(0)); + let blocked = tokio::time::timeout( + Duration::from_millis(500), + resolve_candidates_for_push( + sem.clone(), + dir.path().to_path_buf(), + vec![tip.clone()], + vec![], + git_bin.clone(), + Duration::from_secs(5), + false, + ), + ) + .await; + assert!( + blocked.is_err(), + "the pin-candidate scan must defer (park on admission) when the pool is exhausted" + ); + assert!( + !marker.exists(), + "the scan's git must not spawn while its admission permit is unavailable (F4)" + ); + + // Release admission: the SAME scan now runs (defer, not shed). + sem.add_permits(1); + let set = resolve_candidates_for_push( + sem, + dir.path().to_path_buf(), + vec![tip], + vec![], + git_bin, + Duration::from_secs(5), + false, + ) + .await; + assert!( + marker.exists(), + "once admission is available the deferred scan runs its git" + ); + assert!(!set.full_scan, "commit tip + empty rev-list is a delta"); + assert!(set.candidates.is_empty()); + } + + /// F4 fast-path negative arm: a deletion-only push (no new tips, kill-switch + /// off) computes its empty delta without spawning ANY git child, so it must + /// complete without acquiring from the scan pool — even at zero permits. The + /// per-tip cat-file probe means every push with a non-empty new tip DOES spawn + /// git, so this is the only genuinely git-free stage. + #[cfg(unix)] + #[tokio::test] + async fn resolve_candidates_no_git_fast_path_skips_admission() { + use std::os::unix::fs::PermissionsExt; + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + + let dir = tempfile::TempDir::new().unwrap(); + let marker = dir.path().join("git.ran"); + let fake = dir.path().join("fakegit"); + std::fs::write( + &fake, + format!("#!/bin/sh\necho ran >> \"{}\"\nexit 0\n", marker.display()), + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + + let set = tokio::time::timeout( + Duration::from_millis(500), + resolve_candidates_for_push( + Arc::new(Semaphore::new(0)), + dir.path().to_path_buf(), + vec![], + vec!["deadbeefdeadbeefdeadbeefdeadbeefdeadbeef".to_string()], + fake.to_str().unwrap().to_string(), + Duration::from_secs(5), + false, + ), + ) + .await + .expect("the no-new-tips fast path must not park on the scan pool"); + assert_eq!( + set, + PinCandidateSet { + candidates: Vec::new(), + full_scan: false + } + ); + assert!(!marker.exists(), "the fast path must spawn no git at all"); + } + + /// #174 F5: the coalesced-drain FullScan marker is signalled via the explicit + /// `force_full_scan` flag, and a flagged call with NO tips must still enumerate + /// the whole repo (full_scan=true, non-empty candidates). The RED arm of the + /// encoding question: if the marker were encoded as a plain empty-tips call + /// (flag off — the fast path), the drain would resolve to an empty delta and + /// pin nothing, silently losing the coalesced pushes' work. + #[tokio::test] + async fn forced_full_scan_with_no_tips_enumerates_the_repo() { + let repo = Repo::new(); + repo.commit_file("a.txt", "one\n"); + let all: HashSet = list_all_objects(&repo.path, "git", td()) + .unwrap() + .into_iter() + .collect(); + assert!(!all.is_empty(), "fixture repo has objects"); + + let set = resolve_candidates_for_push( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), + repo.path.clone(), + vec![], + vec![], + "git".to_string(), + std::time::Duration::from_secs(600), + true, + ) + .await; + assert!(set.full_scan, "the forced call is signalled as a full scan"); + let got: HashSet = set.candidates.into_iter().collect(); + assert_eq!( + got, all, + "a forced full scan with no tips enumerates the repo — never an empty delta" + ); + + // The discriminator: the SAME empty-tips call without the flag is the + // deletion-only fast path (empty delta, pin nothing). The two must differ, + // or the marker encoding has collapsed into the silent-loss shape. + let unforced = resolve_candidates_for_push( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), + repo.path.clone(), + vec![], + vec![], + "git".to_string(), + std::time::Duration::from_secs(600), + false, + ) + .await; + assert!(!unforced.full_scan); + assert!(unforced.candidates.is_empty()); + } + #[test] fn missing_oid_tip_forces_full_scan() { let repo = Repo::new(); repo.commit_file("a.txt", "one\n"); let bogus = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"; assert_eq!( - resolve_push_delta(&repo.path, &[bogus], &[]), + resolve_push_delta(&repo.path, &[bogus], &[], "git", td()), PinCandidates::FullScanRequired, "a missing/corrupt tip OID must force full scan" ); @@ -564,7 +829,7 @@ mod tests { let c1 = repo.commit_file("a.txt", "one\n"); let c2 = repo.commit_file("b.txt", "two\n"); let old_tree = repo.rev(&format!("{c1}^{{tree}}")); - let result = resolve_push_delta(&repo.path, &[&c2], &[&old_tree]); + let result = resolve_push_delta(&repo.path, &[&c2], &[&old_tree], "git", td()); match result { PinCandidates::FullScanRequired => {} // safe: caller full-scans PinCandidates::Delta(objs) => { @@ -583,10 +848,15 @@ mod tests { // branch2 from base advances independently repo.git(&["checkout", "-q", "-b", "branch2", &base]); let b2 = repo.commit_file("c.txt", "three\n"); - let got: HashSet = - delta(resolve_push_delta(&repo.path, &[&b1, &b2], &[&base, &base])) - .into_iter() - .collect(); + let got: HashSet = delta(resolve_push_delta( + &repo.path, + &[&b1, &b2], + &[&base, &base], + "git", + td(), + )) + .into_iter() + .collect(); assert!(got.contains(&b1), "branch1 new commit"); assert!(got.contains(&b2), "branch2 new commit"); } @@ -595,7 +865,7 @@ mod tests { fn empty_repo_no_tips_yields_empty_delta() { let repo = Repo::new(); assert_eq!( - resolve_push_delta(&repo.path, &[], &[]), + resolve_push_delta(&repo.path, &[], &[], "git", td()), PinCandidates::Delta(Vec::new()) ); } @@ -609,12 +879,12 @@ mod tests { let repo = Repo::new(); let c1 = repo.commit_file("a.txt", "one\n"); assert_eq!( - resolve_push_delta_inner(&repo.path, &[&c1], &[], true), + resolve_push_delta_inner(&repo.path, &[&c1], &[], true, "git", td()), PinCandidates::FullScanRequired ); // And with the flag off, the same push yields a Delta. assert!(matches!( - resolve_push_delta_inner(&repo.path, &[&c1], &[], false), + resolve_push_delta_inner(&repo.path, &[&c1], &[], false, "git", td()), PinCandidates::Delta(_) )); } @@ -624,7 +894,7 @@ mod tests { let repo = Repo::new(); repo.commit_file("a.txt", "one\n"); repo.commit_file("b.txt", "two\n"); - let all = list_all_objects(&repo.path).unwrap(); + let all = list_all_objects(&repo.path, "git", td()).unwrap(); // 2 commits + 2 trees + 2 blobs = 6 objects. assert_eq!(all.len(), 6, "got: {all:?}"); } @@ -642,7 +912,7 @@ mod tests { std::fs::write(repo.path.join("orphan.bin"), b"dangling\n").unwrap(); let dangling = repo.git(&["hash-object", "-w", "orphan.bin"]); - let blobs = all_blob_oids(&repo.path).unwrap(); + let blobs = all_blob_oids(&repo.path, "git", td()).unwrap(); assert!(blobs.contains(&reachable_blob), "reachable blob present"); assert!( blobs.contains(&dangling), @@ -652,7 +922,7 @@ mod tests { assert!(!blobs.contains(&tree), "tree is not a blob"); // The typed lister tags each object; spot-check the dangling blob's type. - let typed = list_all_objects_with_type(&repo.path).unwrap(); + let typed = list_all_objects_with_type(&repo.path, "git", td()).unwrap(); assert!( typed .iter() @@ -660,4 +930,96 @@ mod tests { "dangling object is typed as a blob" ); } + + #[cfg(unix)] + #[test] + fn list_all_objects_times_out_on_a_hung_git_instead_of_blocking() { + use std::os::unix::fs::PermissionsExt; + use std::time::{Duration, Instant}; + let dir = tempfile::TempDir::new().unwrap(); + // Fake git: hang on cat-file (bounded to 30s so a broken test can't wedge). + let fake = dir.path().join("fakegit"); + std::fs::write( + &fake, + "#!/bin/sh\ncase \"$1\" in\n cat-file) i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;;\n *) : ;;\nesac\n", + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + let git_bin = fake.to_str().unwrap().to_string(); + + let (tx, rx) = std::sync::mpsc::channel(); + let path = dir.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(list_all_objects( + &path, + &git_bin, + Instant::now() + Duration::from_millis(150), + )); + }); + let res = rx.recv_timeout(Duration::from_secs(10)).expect( + "list_all_objects must return within the watchdog budget, not block on a hung git", + ); + assert!( + res.is_err(), + "a hung git must make list_all_objects error out, not hang" + ); + } + + /// #174 finding 2: the DELTA-path children — `peeled_object_type` (cat-file), + /// `rev_list_delta` (rev-list) — and `list_all_objects_with_type` (cat-file) are + /// the remaining candidate-discovery execs (the common push path). Each must + /// return within the watchdog budget on a hung git rather than block and pin the + /// write permit. Revert any one to a bare `Command::output()` and its arm blocks + /// past the recv budget (RED). + #[cfg(unix)] + #[test] + fn delta_path_exec_fns_time_out_on_a_hung_git() { + use std::os::unix::fs::PermissionsExt; + use std::time::{Duration, Instant}; + let dir = tempfile::TempDir::new().unwrap(); + // Fake git hangs on BOTH cat-file and rev-list (bounded 30s so a broken + // test can't leak a permanent orphan or wedge the suite). + let fake = dir.path().join("fakegit"); + std::fs::write( + &fake, + "#!/bin/sh\ncase \"$1\" in\n cat-file|rev-list) i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;;\n *) : ;;\nesac\n", + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + let git_bin = fake.to_str().unwrap().to_string(); + let path = dir.path().to_path_buf(); + + // Run `f` on a thread and require it to RETURN (not block) within 10s. + fn returns_within(f: impl FnOnce() -> T + Send + 'static) -> T { + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = tx.send(f()); + }); + rx.recv_timeout(Duration::from_secs(10)).expect( + "a delta-path exec fn must return within the watchdog budget, not block on a hung git", + ) + } + let dl = || Instant::now() + Duration::from_millis(150); + let sha = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"; + + let (p, g, d) = (path.clone(), git_bin.clone(), dl()); + assert!( + returns_within(move || peeled_object_type(&p, sha, &g, d)).is_none(), + "peeled_object_type must time out to None on a hung cat-file" + ); + let (p, g, d) = (path.clone(), git_bin.clone(), dl()); + assert!( + returns_within(move || rev_list_delta(&p, &[sha], &[], &g, d)).is_err(), + "rev_list_delta must error out on a hung rev-list" + ); + let (p, g, d) = (path.clone(), git_bin.clone(), dl()); + assert!( + returns_within(move || list_all_objects_with_type(&p, &g, d)).is_err(), + "list_all_objects_with_type must error out on a hung cat-file" + ); + } } diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 65058016..70a32874 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -11,9 +11,12 @@ use std::collections::HashSet; use std::path::{Path, PathBuf}; use std::sync::Arc; +use std::time::Duration; use anyhow::{Context, Result}; -use sqlx::PgPool; +use sqlx::pool::PoolConnection; +use sqlx::postgres::PgPoolOptions; +use sqlx::{PgPool, Postgres}; use tokio::sync::Mutex; use tracing::{debug, info, warn}; @@ -25,30 +28,105 @@ use super::tigris::TigrisClient; pub struct RepoStore { repos_dir: PathBuf, tigris: Option, - /// Shared Postgres pool for advisory locks. - pool: PgPool, + /// Dedicated Postgres pool for repo write advisory locks, built by + /// `build_lock_pool` (see there for why it is separate and why it carries an + /// `after_release` hook). Never use this for ordinary queries. + lock_pool: PgPool, /// Tracks repos already confirmed to exist in Tigris — avoids redundant /// HEAD checks and background uploads for repos we've already migrated. migrated: Arc>>, + /// Test-only stall injected at the head of `acquire_write`'s Tigris phase, + /// i.e. AFTER the advisory lock is taken and BEFORE the guard exists. That + /// window is exactly where the outer `tokio::time::timeout` in + /// `api/repos.rs` can drop the future (#173). `TigrisClient` takes its + /// endpoint from process-wide AWS env vars and has no injectable seam, so + /// this flag is the smallest way to hold a real `acquire_write` open in that + /// window and cancel it there. + #[cfg(test)] + tigris_stall: Option, + /// Test-only counter of how many times a write guard from this store REACHED the + /// Tigris upload site in `release` (the point past the `success` check, where a + /// configured client would be uploaded to). It counts the decision, not a network + /// call: `TigrisClient` takes its endpoint from process-wide AWS env vars and has no + /// injectable seam, so every test runs with `tigris: None` and a counter inside the + /// `Some` arm could never move. Reaching the site is the property under test anyway: + /// an interrupted push must not publish a half-applied repo, and the disconnect path + /// must therefore never get here (#173 F2). + /// + /// Per store rather than a process global, so cases running in parallel do not see + /// each other's uploads, and an `Arc` rather than a `thread_local` because the guard + /// is released from a detached task on another worker thread. Same test-only counter + /// idiom as `ipfs_pin::note_legacy_repair_read`. + #[cfg(test)] + upload_site_reached: Arc, + /// Test-only seam: armed here, copied into every `RepoWriteGuard` this store + /// hands out, so a test that only holds the `AppState` (not the guard) can + /// still park `release` at its pre-unlock point. See + /// `RepoWriteGuard::test_pre_unlock_gate`. Never set outside tests. + #[cfg(test)] + pre_unlock_gate: Option>, } impl RepoStore { + /// Derives its own lock pool from `pool`, so callers that only have the main + /// pool (tests, `for_testing` sites in other modules) still get the + /// `after_release` semantics `acquire_write` depends on. #[cfg(test)] pub fn for_testing(repos_dir: PathBuf, pool: PgPool) -> Self { - Self { + Self::new( repos_dir, - tigris: None, - pool, - migrated: Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new())), - } + None, + build_lock_pool(&pool, 8, Duration::from_secs(5)), + ) + } + + /// Test-only: every guard from this store parks in `release` right before the + /// `pg_advisory_unlock` await, until `gate` is notified. Dropping the future + /// while it is parked reproduces a client disconnect inside `release`. + #[cfg(test)] + pub fn with_pre_unlock_gate(mut self, gate: Arc) -> Self { + self.pre_unlock_gate = Some(gate); + self + } + + /// Test-only: the dedicated advisory-lock pool this store runs its write locks + /// on. `for_testing` DERIVES it from the pool it is handed (see `build_lock_pool`), + /// so a test that wants to observe what happened to a guard's connection has to + /// look here, not at the pool it passed in. + #[cfg(test)] + pub(crate) fn lock_pool(&self) -> &PgPool { + &self.lock_pool } - pub fn new(repos_dir: PathBuf, tigris: Option, pool: PgPool) -> Self { + /// Test-only: see `tigris_stall`. + #[cfg(test)] + pub fn with_tigris_stall(mut self, stall: Duration) -> Self { + self.tigris_stall = Some(stall); + self + } + + /// Test-only: how many write guards from this store have reached the Tigris upload + /// site. See [`RepoStore::upload_site_reached`]. + #[cfg(test)] + pub fn tigris_upload_site_reached(&self) -> usize { + self.upload_site_reached + .load(std::sync::atomic::Ordering::SeqCst) + } + + /// `lock_pool` must come from `build_lock_pool`; a plain pool leaks advisory + /// locks on cancellation. + pub fn new(repos_dir: PathBuf, tigris: Option, lock_pool: PgPool) -> Self { Self { repos_dir, tigris, - pool, + lock_pool, migrated: Arc::new(Mutex::new(HashSet::new())), + #[cfg(test)] + tigris_stall: None, + #[cfg(test)] + upload_site_reached: Arc::new(std::sync::atomic::AtomicUsize::new(0)), + #[cfg(test)] + pre_unlock_gate: None, } } @@ -157,29 +235,70 @@ impl RepoStore { let (owner_slug, local_path) = self.local_path(owner_did, repo_name)?; let lock_key = advisory_lock_key(&owner_slug, repo_name); - // Acquire Postgres advisory lock with retry using pg_try_advisory_lock - // to avoid blocking indefinitely on stale locks from crashed connections. - let mut acquired = false; + // Acquire the Postgres advisory lock with retry, using pg_try_advisory_lock so a + // stale lock from a crashed connection can't block us indefinitely. + // + // The connection is checked out INSIDE the loop and RETURNED before each sleep. + // Only the connection that actually took the lock is retained. Two constraints + // pull in opposite directions here, and this is what satisfies both: + // + // * Session ownership. A session-level advisory lock belongs to the CONNECTION + // that took it, so the lock and its `pg_advisory_unlock` must run on the same + // one. Running them through the pool (`fetch_one(&self.pool)`) lets them land + // on different connections: the unlock silently returns false and the lock + // leaks, while a competing acquire that happens to draw the holding + // connection re-enters the lock and two pushes to one repo run concurrently. + // Hence: keep the connection that WON. + // * Occupancy. Holding a connection across the ~60 one-second sleeps would let + // one spinning acquire park a lock-pool connection for a minute. That is not + // just a push-path concern: `api/issues.rs` and `api/pulls.rs` reach + // acquire_write holding no concurrency permit at all, so a caller could park + // the whole pool and starve authenticated pushes on every repo (#173 F1). + // Hence: return the connection when we LOSE, before sleeping. + // + // Returning a losing connection is safe with respect to the cancellation design: + // `after_release` runs `pg_advisory_unlock_all()`, a no-op on a connection that + // took nothing, so it cannot disturb a lock held by any other connection + // (proven by `returning_an_unlocked_connection_does_not_clear_another_connections_lock`). + // + // Cancellation safety is unchanged: the future can only be dropped while a + // connection is checked out, and dropping it runs the same `after_release` hook, + // which clears whatever lock it had just taken (#173 U1). + let mut lock_conn = None; for attempt in 0..60 { + let mut conn = self.lock_pool.acquire().await.map_err(|e| { + anyhow::Error::new(LockPoolBusy) + .context(format!("checking out a lock-pool connection: {e}")) + })?; let row: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") .bind(lock_key) - .fetch_one(&self.pool) + .fetch_one(&mut *conn) .await .context("trying advisory lock")?; if row.0 { - acquired = true; + lock_conn = Some(conn); break; } + // Lost the race: give the connection back so a spinning acquire occupies + // nothing while it waits. + drop(conn); if attempt < 59 { tokio::time::sleep(std::time::Duration::from_secs(1)).await; } } - if !acquired { + let Some(lock_conn) = lock_conn else { anyhow::bail!("could not acquire advisory lock after 60s — possible stale lock for {owner_slug}/{repo_name}"); + }; + + #[cfg(test)] + if let Some(stall) = self.tigris_stall { + tokio::time::sleep(stall).await; } - // Always download the latest from Tigris before writing. - // Local disk may be stale if another machine pushed since our last access. + // Always download the latest from Tigris before writing. Local disk may be + // stale if another machine pushed since our last access. The lock connection + // is already held, so a cancellation here returns it through `after_release`, + // which clears the lock. if let Some(ref tigris) = self.tigris { if tigris.exists(&owner_slug, repo_name).await.unwrap_or(false) { debug!(repo = %repo_name, "write acquire: downloading latest from tigris"); @@ -202,8 +321,13 @@ impl RepoStore { repo_name: repo_name.to_string(), local_path, lock_key, - pool: self.pool.clone(), + lock_conn: Some(lock_conn), + released: false, tigris: self.tigris.clone(), + #[cfg(test)] + upload_site_reached: Arc::clone(&self.upload_site_reached), + #[cfg(test)] + test_pre_unlock_gate: self.pre_unlock_gate.clone(), }) } @@ -257,40 +381,61 @@ impl RepoStore { /// segment is rejected. This is the CodeQL-recognised barrier /// pattern for `rust/path-injection`. fn local_path(&self, owner_did: &str, repo_name: &str) -> Result<(String, PathBuf)> { - validate_path_components(owner_did, repo_name)?; - let owner_slug = owner_did.replace([':', '/'], "_"); - let local_path = self - .repos_dir - .join(&owner_slug) - .join(format!("{repo_name}.git")); - - if !local_path.starts_with(&self.repos_dir) { - anyhow::bail!( - "computed repo path escaped repos_dir: {}", - local_path.display() - ); - } + let local_path = validated_repo_disk_path(&self.repos_dir, owner_did, repo_name)?; + Ok((owner_slug, local_path)) + } +} - // Explicit component walk — sanitisation barrier that static analysers - // (CodeQL `rust/path-injection`) recognise. The path must be composed - // entirely of Normal segments after the root prefix; any ParentDir or - // CurDir component is a traversal attempt. - for component in local_path.components() { - use std::path::Component; - match component { - Component::Prefix(_) | Component::RootDir | Component::Normal(_) => {} - Component::ParentDir => { - anyhow::bail!("path contains parent-directory component"); - } - Component::CurDir => { - anyhow::bail!("path contains current-directory component"); - } +/// The three-layer validated form of `store::repo_disk_path`, with NO Tigris fetch and +/// no `RepoStore` (#173 round 11, F3). Extracted from `RepoStore::local_path` so a +/// second caller that must not pull a cold repo, the U4 legacy provider-CID sweep, gets +/// the same barrier instead of the raw join. `local_path` is now a thin wrapper over +/// this, so the two cannot drift. +/// +/// Three-layer defence against path traversal: +/// 1. Strict allowlist on `owner_did` and `repo_name` (no `..`, slashes, +/// null bytes, leading dots; length-bounded). +/// 2. The joined path must remain rooted at `repos_dir`. +/// 3. Every component of the joined path must be `Component::Normal` +/// (or the prefix/root from `repos_dir`); any `ParentDir`/`CurDir` +/// segment is rejected. This is the CodeQL-recognised barrier +/// pattern for `rust/path-injection`. +pub(crate) fn validated_repo_disk_path( + repos_dir: &Path, + owner_did: &str, + repo_name: &str, +) -> Result { + validate_path_components(owner_did, repo_name)?; + + let owner_slug = owner_did.replace([':', '/'], "_"); + let local_path = repos_dir.join(&owner_slug).join(format!("{repo_name}.git")); + + if !local_path.starts_with(repos_dir) { + anyhow::bail!( + "computed repo path escaped repos_dir: {}", + local_path.display() + ); + } + + // Explicit component walk — sanitisation barrier that static analysers + // (CodeQL `rust/path-injection`) recognise. The path must be composed + // entirely of Normal segments after the root prefix; any ParentDir or + // CurDir component is a traversal attempt. + for component in local_path.components() { + use std::path::Component; + match component { + Component::Prefix(_) | Component::RootDir | Component::Normal(_) => {} + Component::ParentDir => { + anyhow::bail!("path contains parent-directory component"); + } + Component::CurDir => { + anyhow::bail!("path contains current-directory component"); } } - - Ok((owner_slug, local_path)) } + + Ok(local_path) } /// Strict allowlist validator for `owner_did` and `repo_name`. @@ -472,6 +617,19 @@ fn validate_repo_name(repo_name: &str) -> Result<()> { Ok(()) } +/// Error marker for "no lock-pool connection was available in time". +/// +/// Carried through the `anyhow` chain (like [`smart_http::GitServiceTimeout`]) so the +/// HTTP handler can `downcast_ref` it and shed a 503 + Retry-After instead of the +/// generic 500 a git error maps to: an exhausted lock pool is a CAPACITY signal, and +/// telling the client to retry shortly is the same shed semantics the surrounding +/// admission code already uses (#173 F1). +/// +/// [`smart_http::GitServiceTimeout`]: crate::git::smart_http::GitServiceTimeout +#[derive(Debug, thiserror::Error)] +#[error("no lock-pool connection available")] +pub struct LockPoolBusy; + /// Guard returned by `acquire_write()`. Holds the Postgres advisory lock and /// uploads to Tigris + releases the lock on `release()`. pub struct RepoWriteGuard { @@ -479,8 +637,74 @@ pub struct RepoWriteGuard { repo_name: String, pub local_path: PathBuf, lock_key: i64, - pool: PgPool, + /// The lock-pool connection that TOOK the advisory lock. It must be the one + /// that releases it (session locks are owned by their connection), and + /// holding it here is also what makes a guard dropped without `release` + /// safe: the drop returns the connection through the pool's `after_release` + /// hook, which runs `pg_advisory_unlock_all()`. + /// + /// `Option` because that hook is not a complete answer. When the unlock ERRORS + /// on a live session (a statement timeout, an admin cancel, an aborted + /// transaction), `after_release` issues its `pg_advisory_unlock_all()` on the + /// SAME broken session and it fails too, so the connection goes back to the pool + /// still holding the lock and nothing ever clears it (measured: never freed in + /// 15s, #174 F3b). Those paths `take()` the connection and close it instead; + /// ending the session is what actually frees the lock. `None` only after such a + /// disposal, or after `Drop` has moved it into the detached unlock. + lock_conn: Option>, + /// Set once `release` has run its unlock, making the `Drop` backstop inert. A + /// guard is only ever constructed with the lock already held, so there is no + /// "never locked" state to track alongside it. + released: bool, tigris: Option, + /// Shared with the store that handed this guard out; see + /// [`RepoStore::upload_site_reached`]. + #[cfg(test)] + upload_site_reached: Arc, + /// Test-only seam: when set, `release` parks on this gate at the exact point it + /// is about to await `pg_advisory_unlock` (connection still owned, not yet + /// returned to the lock pool). Dropping the `release` future while it is parked + /// reproduces a mid-unlock cancellation, so a test can assert the lock is still + /// freed: the drop returns the connection through the pool's `after_release` + /// hook, which runs `pg_advisory_unlock_all()`. Never set outside tests. + #[cfg(test)] + test_pre_unlock_gate: Option>, +} + +/// Deadline for tearing down the connection that saw a failing `pg_advisory_unlock`. +/// Long enough that a healthy socket always finishes well inside it, short enough that +/// a blackholed one does not pin admission resources for a TCP timeout. +const UNLOCK_ERROR_CLOSE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); + +/// Await `close` under a deadline (#174 F3c). +/// +/// `release` awaits this INLINE while the global write permit, the per-source permit +/// and the write lease are all still held, and sqlx puts no deadline on `close()`: +/// it writes Terminate and then tears the socket down. The branch that reaches here is +/// by definition a connection whose last statement errored, and a blackholed TCP path +/// to Postgres (a cloud failover that drops packets without an RST) is a plausible +/// cause, so an unbounded await here parks every later push to the repo behind three +/// pinned admission resources until the steal bound. +/// +/// On elapsed the future is simply dropped, which drops the `PoolConnection` it owns. +/// Dropping it closes the socket, and closing the socket is what actually ends the +/// session and makes Postgres release the lock, so the deadline costs nothing the +/// graceful path was buying. +async fn close_conn_bounded( + repo_name: &str, + close: impl std::future::Future>, +) { + match tokio::time::timeout(UNLOCK_ERROR_CLOSE_TIMEOUT, close).await { + Ok(Ok(())) => {} + Ok(Err(e)) => { + warn!(repo = %repo_name, err = %e, + "closing the write-lock connection failed, the session teardown still frees the lock server-side"); + } + Err(_) => { + warn!(repo = %repo_name, timeout_secs = UNLOCK_ERROR_CLOSE_TIMEOUT.as_secs(), + "closing the write-lock connection timed out, dropping it instead; the socket goes down either way, which is what frees the lock server-side"); + } + } } impl RepoWriteGuard { @@ -494,9 +718,16 @@ impl RepoWriteGuard { /// half-applied or otherwise inconsistent repo would propagate corruption to /// Tigris (and to every node that later downloads it). The lock is always /// released regardless, to avoid stale locks blocking future writes. - pub async fn release(self, success: bool) { + pub async fn release(mut self, success: bool) { // Upload to Tigris only on success. if success { + // The upload site, recorded for tests before the client is consulted: with + // no injectable seam on `TigrisClient` a counter inside the arm below could + // never move, and it is reaching this point at all that an interrupted push + // must not do (#173 F2). + #[cfg(test)] + self.upload_site_reached + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); if let Some(ref tigris) = self.tigris { if let Err(e) = tigris .upload(&self.owner_slug, &self.repo_name, &self.local_path) @@ -509,14 +740,140 @@ impl RepoWriteGuard { warn!(repo = %self.repo_name, "write failed — skipping tigris upload to avoid propagating an inconsistent repo"); } - // Release advisory lock - let _ = sqlx::query("SELECT pg_advisory_unlock($1)") - .bind(self.lock_key) - .execute(&self.pool) - .await; + // Test-only: park right before the unlock await so a test can drop this + // future mid-unlock, with the connection still owned. + #[cfg(test)] + if let Some(gate) = self.test_pre_unlock_gate.clone() { + gate.notified().await; + } + // Release the advisory lock on the connection that took it. Anything else + // (a fresh `&pool` checkout) is a no-op that returns false: Postgres + // scopes a session lock to its owning connection. + // + // Unlock through the connection while it is STILL owned by `self`; do not + // `take()` it first. A cancellation during this await then drops `self` with + // the connection still in place, so it returns to the lock pool and + // `after_release` clears the lock (#174 F4). + let unlock = match self.lock_conn.as_deref_mut() { + Some(conn) => Some( + sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(self.lock_key) + .execute(&mut *conn) + .await, + ), + None => None, + }; + // An unlock that ERRORS is a different failure from a cancellation: the await + // resolved, so the session is alive and still holds the lock. Returning that + // connection to the pool does NOT recover it, because `after_release` runs its + // `pg_advisory_unlock_all()` on the same broken session and fails identically + // (#174 F3b). Close it: ending the session is what frees the lock. + if let Some(Err(e)) = unlock { + warn!(repo = %self.repo_name, err = %e, + "advisory unlock failed, closing the connection so the session ends and postgres drops the lock"); + if let Some(conn) = self.lock_conn.take() { + close_conn_bounded(&self.repo_name, conn.close()).await; + } + } + // On the clean path, dropping `self` returns the connection to the lock pool, + // where `after_release` sweeps anything the unlock above missed. + self.released = true; + } +} + +impl Drop for RepoWriteGuard { + /// Backstop for a guard dropped WITHOUT `release` (a cancelled `acquire_write`, a + /// handler future dropped before the release call). The pool's `after_release` + /// hook covers the ordinary case on its own, but not one: if the detached unlock + /// ERRORS on a live session, the hook's `pg_advisory_unlock_all()` fails the same + /// way and the connection returns to the pool still holding the lock (#174 F3b). + /// So the unlock runs here and disposes of the connection when it errors. + /// + /// `Drop` cannot await, so the unlock is spawned; it runs on the same session, + /// which is what makes it effective. With no runtime to spawn onto there is + /// nothing that can unlock, so the connection is detached and dropped instead: + /// closing the socket ends the session, and that frees the lock server-side. + fn drop(&mut self) { + if self.released { + return; + } + let Some(mut conn) = self.lock_conn.take() else { + return; + }; + let lock_key = self.lock_key; + let repo_name = self.repo_name.clone(); + match tokio::runtime::Handle::try_current() { + Ok(handle) => { + handle.spawn(async move { + let unlock = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(lock_key) + .execute(&mut *conn) + .await; + // Same failure as `release`'s, one level down: the await RESOLVED + // with an error, so the session is alive and still holds the lock. + // Ending this block would drop `conn` and RETURN it to the pool, + // where `after_release` fails identically. Close it instead. + if let Err(e) = unlock { + warn!(repo = %repo_name, err = %e, "detached advisory-unlock on write-guard drop failed, closing the connection so the session ends and postgres drops the lock"); + close_conn_bounded(&repo_name, conn.close()).await; + } + }); + } + Err(_) => { + // `PoolConnection`'s own drop spawns its return-to-pool task, which + // panics with no runtime. `detach` gives up the pool slot and yields a + // plain `PgConnection`; dropping that closes the socket, which ends the + // session and is what frees the lock. + drop(conn.detach()); + warn!( + repo = %repo_name, + "RepoWriteGuard dropped off a Tokio runtime; no detached unlock is \ + possible, so the pinned connection is disposed of instead: ending \ + the session is what releases the advisory lock" + ); + } + } } } +/// Build the dedicated advisory-lock pool a `RepoStore` runs its write locks on. +/// Connect options are cloned off an existing pool so callers need not re-parse +/// the database URL; the pool is lazy, so no connection is opened here. +/// +/// Two properties, both load-bearing: +/// +/// * The `after_release` hook runs `pg_advisory_unlock_all()` before a +/// connection goes back into the pool. sqlx's `PoolConnection::drop` spawns +/// `return_to_pool()`, which invokes this hook, so a connection dropped by +/// CANCELLATION still clears its locks. That is what keeps an `acquire_write` +/// killed mid-Tigris by the caller's `tokio::time::timeout` from leaking a +/// lock and wedging every later push to that repo (#173). Note the hook runs +/// from that spawned task, so the unlock is asynchronous with respect to the +/// drop: the lock clears shortly after the connection goes away, not +/// synchronously with it. +/// * It is a SEPARATE pool from the main query pool, not a slice of it. A push +/// holds its lock connection for the whole receive-pack, and +/// `db_max_connections` (default 20) is well below +/// `max_concurrent_git_pushes` (default 32), so drawing these from the main +/// pool would starve every other query during a push burst. +/// +/// `acquire_timeout` bounds the wait when every lock-pool connection is busy, so +/// exhaustion surfaces as a clean error rather than an unbounded hang. +pub fn build_lock_pool(source: &PgPool, max_connections: u32, acquire_timeout: Duration) -> PgPool { + PgPoolOptions::new() + .max_connections(max_connections) + .acquire_timeout(acquire_timeout) + .after_release(|conn, _meta| { + Box::pin(async move { + sqlx::query("SELECT pg_advisory_unlock_all()") + .execute(&mut *conn) + .await?; + Ok(true) + }) + }) + .connect_lazy_with((*source.connect_options()).clone()) +} + /// Compute a stable i64 hash for a Postgres advisory lock key. fn advisory_lock_key(owner_slug: &str, repo_name: &str) -> i64 { use std::hash::{Hash, Hasher}; @@ -529,6 +886,503 @@ fn advisory_lock_key(owner_slug: &str, repo_name: &str) -> i64 { #[cfg(test)] mod tests { use super::*; + use std::time::Duration; + + // ── advisory-lock test helpers (#173 U1) ─────────────────────────────── + + /// Postgres advisory locks live in a CLUSTER-wide space, not a per-database + /// one, so two `#[sqlx::test]` cases running against their own temporary + /// databases still share the key space. Every lock test therefore mints its + /// own key instead of reusing a fixed constant. + fn unique_lock_key() -> i64 { + use std::sync::atomic::{AtomicI64, Ordering}; + static NEXT: AtomicI64 = AtomicI64::new(0); + let n = NEXT.fetch_add(1, Ordering::Relaxed); + ((std::process::id() as i64) << 24) | (n & 0xff_ffff) + } + + /// A plain pool (no `after_release` hook, no idle timeout) at the same + /// database as the `#[sqlx::test]` pool. Two separate reasons these tests + /// cannot just use the pool the harness hands them: + /// + /// 1. Observing lock state has to happen from a session that is definitely + /// not the one under test. Session advisory locks are re-entrant, so + /// `pg_try_advisory_lock` on the very connection that already holds the key + /// returns true, and a same-pool probe silently reports a leaked lock free. + /// 2. The harness pool sets `idle_timeout(1s)`, so a connection returned to it + /// is closed about a second later and Postgres drops every lock that + /// session held. That would mask exactly the leak these tests exist to + /// catch, so the store under test runs on one of these too. + fn sibling_pool(pool: &PgPool, max_connections: u32) -> PgPool { + sqlx::postgres::PgPoolOptions::new() + .max_connections(max_connections) + .connect_lazy_with((*pool.connect_options()).clone()) + } + + /// Probe the lock from a connection that is NOT the one under test. Session + /// advisory locks are re-entrant within their own session, so a check from the + /// holding connection would pass vacuously and prove nothing. + async fn lock_is_free_elsewhere(pool: &PgPool, key: i64) -> bool { + let mut probe = pool.acquire().await.expect("probe connection"); + let taken: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *probe) + .await + .expect("probe try-lock"); + if taken.0 { + sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *probe) + .await + .expect("probe unlock"); + } + taken.0 + } + + /// `after_release` runs from the task sqlx spawns in `PoolConnection::drop`, + /// so the unlock is ASYNCHRONOUS with respect to the drop. Callers must poll + /// rather than assume the lock is gone the instant the connection goes away. + async fn wait_until_free(pool: &PgPool, key: i64, within: Duration) -> bool { + let deadline = std::time::Instant::now() + within; + loop { + if lock_is_free_elsewhere(pool, key).await { + return true; + } + if std::time::Instant::now() >= deadline { + return false; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + } + + // ── DESIGN GATE ──────────────────────────────────────────────────────── + // The whole cancellation-safety design rests on one sqlx behaviour: + // `PoolConnection::drop` spawns `return_to_pool()`, which invokes the pool's + // `after_release` hook before the connection is reused. If that holds, a + // connection dropped by cancellation still runs `pg_advisory_unlock_all()` + // and the lock cannot leak. This test proves it by execution, through the + // production `build_lock_pool` so that stripping the hook there turns it red. + + #[sqlx::test] + async fn dropped_pool_connection_runs_after_release_and_clears_locks(pool: PgPool) { + let key = unique_lock_key(); + let lock_pool = build_lock_pool(&pool, 4, Duration::from_secs(5)); + + { + let mut conn = lock_pool.acquire().await.expect("lock-pool connection"); + let taken: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *conn) + .await + .expect("try-lock"); + assert!(taken.0, "first try-lock must succeed"); + assert!( + !lock_is_free_elsewhere(&pool, key).await, + "lock must be observably HELD from another session while the connection lives" + ); + // Drop WITHOUT calling pg_advisory_unlock: this models cancellation. + } + + assert!( + wait_until_free(&pool, key, Duration::from_secs(5)).await, + "after_release must clear the advisory lock of a dropped connection" + ); + } + + // ── acquire_write cancellation safety (#173 U1) ──────────────────────── + + /// The reviewer's named regression. `api/repos.rs` wraps `acquire_write` in a + /// `tokio::time::timeout`; when that fires during the Tigris phase the future + /// is dropped after the advisory lock was taken and before `RepoWriteGuard` + /// (the only thing that unlocks) exists. The lock then leaks and every later + /// push to the same repo spins the 60-attempt / 60s ceiling and fails. + #[sqlx::test] + async fn cancelled_acquire_write_mid_tigris_does_not_leak_the_lock(pool: PgPool) { + let repos_dir = PathBuf::from("/tmp/gitlawb-test-repos"); + let owner = "did:key:z6MkCancelMidTigris"; + let repo = "cancel-mid-tigris"; + + let store_pool = sibling_pool(&pool, 8); + let stalling = RepoStore::for_testing(repos_dir.clone(), store_pool.clone()) + .with_tigris_stall(Duration::from_secs(30)); + let cancelled = tokio::time::timeout( + Duration::from_millis(500), + stalling.acquire_write(owner, repo), + ) + .await; + assert!( + cancelled.is_err(), + "the acquire must still be inside the Tigris phase when the timeout fires" + ); + + // Observed from an independent session, so the check cannot be satisfied + // by re-entrancy on whichever pooled connection happens to be handed back. + let probe = sibling_pool(&pool, 2); + let key = advisory_lock_key(&owner.replace([':', '/'], "_"), repo); + assert!( + wait_until_free(&probe, key, Duration::from_secs(5)).await, + "a cancelled acquire_write must leave no advisory lock held" + ); + + // A subsequent acquire for the SAME repo must succeed promptly. Before the + // fix it blocks on the leaked lock until the 60-attempt ceiling. + let store = RepoStore::for_testing(repos_dir, store_pool); + let guard = tokio::time::timeout(Duration::from_secs(5), store.acquire_write(owner, repo)) + .await + .expect("second acquire_write must not block on a leaked lock") + .expect("second acquire_write must succeed"); + guard.release(false).await; + } + + /// Cancellation BEFORE the lock is taken must leave nothing behind: no lock, + /// and no lock-pool connection stranded. The lock pool here holds exactly one + /// connection, so a stranded one would make the follow-up acquire time out + /// waiting for a checkout. + #[sqlx::test] + async fn cancelled_acquire_write_before_the_lock_leaves_nothing_held(pool: PgPool) { + let probe = sibling_pool(&pool, 2); + let owner = "did:key:z6MkCancelEarly"; + let repo = "cancel-early"; + let key = advisory_lock_key(&owner.replace([':', '/'], "_"), repo); + + let store = RepoStore::new( + PathBuf::from("/tmp/gitlawb-test-repos"), + None, + build_lock_pool(&pool, 1, Duration::from_secs(3)), + ); + + // A zero deadline polls the future once, which gets it no further than the + // first await (the pool checkout / the first try-lock round trip), so it is + // cancelled before any lock can be taken. + let cancelled = + tokio::time::timeout(Duration::ZERO, store.acquire_write(owner, repo)).await; + assert!(cancelled.is_err(), "the acquire must be cancelled"); + + assert!( + lock_is_free_elsewhere(&probe, key).await, + "no lock may be held when the acquire never got that far" + ); + + // The single lock-pool connection must be back: if cancellation stranded + // it, this checkout blocks until the 3s acquire timeout and fails. + let guard = tokio::time::timeout(Duration::from_secs(2), store.acquire_write(owner, repo)) + .await + .expect("the lock-pool connection must have been returned") + .expect("acquire after cancellation"); + guard.release(false).await; + } + + /// Lock-pool exhaustion is a bounded wait and a clean error, never a panic and + /// never an unbounded hang. + #[sqlx::test] + async fn lock_pool_exhaustion_is_a_bounded_error(pool: PgPool) { + let owner = "did:key:z6MkExhaustion"; + let store = RepoStore::new( + PathBuf::from("/tmp/gitlawb-test-repos"), + None, + build_lock_pool(&pool, 1, Duration::from_secs(2)), + ); + + let held = store + .acquire_write(owner, "exhaust-a") + .await + .expect("first acquire"); + + // Different repo, so this is not the advisory lock queueing: the only + // connection in the lock pool is checked out by `held`. + let started = std::time::Instant::now(); + let err = tokio::time::timeout( + Duration::from_secs(10), + store.acquire_write(owner, "exhaust-b"), + ) + .await + .expect("the wait must be bounded by the pool acquire timeout"); + let err = match err { + Ok(_) => panic!("an exhausted lock pool must surface an error, not a guard"), + Err(e) => e, + }; + assert!( + started.elapsed() < Duration::from_secs(6), + "the error must arrive on the acquire timeout, not after a long hang" + ); + assert!( + err.to_string().contains("lock-pool connection"), + "the error must name the lock-pool checkout, got: {err}" + ); + + held.release(false).await; + } + + /// #173 F1 (RED-before/GREEN-after). A contended `acquire_write` spins for up to + /// 60 one-second attempts. It must not OCCUPY a lock-pool connection for that whole + /// spin: `acquire_write` has non-push callers (`api/issues.rs`, `api/pulls.rs`) that + /// hold no concurrency permit, so any self-minted did:key could otherwise park a + /// connection per call and starve authenticated pushes on EVERY repo. + /// + /// Lock pool of exactly 2, two spinners. Pre-fix (checkout hoisted above the retry + /// loop) they pin both connections for the full spin and an UNCONTENDED acquire on a + /// third repo dies on the pool acquire timeout. Post-fix each spinner returns its + /// connection before sleeping, so it occupies ~0 and the uncontended acquire sails + /// through. + #[sqlx::test] + async fn a_spinning_acquire_write_does_not_occupy_a_lock_pool_connection(pool: PgPool) { + let owner = "did:key:z6MkSpinOccupancy"; + let owner_slug = owner.replace([':', '/'], "_"); + let store = RepoStore::new( + PathBuf::from("/tmp/gitlawb-test-repos"), + None, + build_lock_pool(&pool, 2, Duration::from_secs(2)), + ); + + // An independent session holds both contended keys, so the spinners' try-locks + // return false on every iteration and they stay in the retry loop. + let holder = sibling_pool(&pool, 2); + let mut held_conn = holder.acquire().await.expect("holder connection"); + for repo in ["spin-a", "spin-b"] { + let taken: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(advisory_lock_key(&owner_slug, repo)) + .fetch_one(&mut *held_conn) + .await + .expect("holder try-lock"); + assert!(taken.0, "the holder must own {repo}'s key"); + } + + let mut spinners = Vec::new(); + for repo in ["spin-a", "spin-b"] { + let store = store.clone(); + spinners.push(tokio::spawn(async move { + store.acquire_write(owner, repo).await + })); + } + // Let both reach the spin (each has done at least one failed try-lock by now). + tokio::time::sleep(Duration::from_millis(500)).await; + + let started = std::time::Instant::now(); + let uncontended = tokio::time::timeout( + Duration::from_secs(10), + store.acquire_write(owner, "spin-free"), + ) + .await + .expect("the uncontended acquire must return, not hang"); + let elapsed = started.elapsed(); + let free_guard = uncontended.unwrap_or_else(|e| { + panic!( + "an UNCONTENDED acquire_write on a DIFFERENT repo must not be starved by \ + spinners holding the lock pool; got: {e}" + ) + }); + assert!( + elapsed < Duration::from_secs(2), + "the uncontended acquire must not queue behind the spinners for the pool \ + acquire timeout; took {elapsed:?}" + ); + free_guard.release(false).await; + + // The drop-and-retake cycle must still END in a real, exclusive lock: free + // spin-a's key and the spinner that was cycling connections must take it. + sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(advisory_lock_key(&owner_slug, "spin-a")) + .execute(&mut *held_conn) + .await + .expect("release spin-a"); + let winner = tokio::time::timeout(Duration::from_secs(15), spinners.remove(0)) + .await + .expect("the spinner must finish once its key frees") + .expect("spinner task") + .expect("the spinner must acquire once the key frees"); + let probe = sibling_pool(&pool, 2); + assert!( + !lock_is_free_elsewhere(&probe, advisory_lock_key(&owner_slug, "spin-a")).await, + "the lock a spinner finally took must be observably held from another session" + ); + winner.release(false).await; + + for s in spinners { + s.abort(); + } + sqlx::query("SELECT pg_advisory_unlock_all()") + .execute(&mut *held_conn) + .await + .expect("release the remaining holder lock"); + } + + /// #173 F1, the property the fix rests on: returning a lock-pool connection that + /// holds NOTHING runs `after_release`'s `pg_advisory_unlock_all()`, which is a no-op + /// and must not disturb a lock held on a DIFFERENT connection of the same pool. + /// Session advisory locks are per connection, so this is by construction, but the + /// spin fix depends on it, so it is proven by execution rather than assumed. + #[sqlx::test] + async fn returning_an_unlocked_connection_does_not_clear_another_connections_lock( + pool: PgPool, + ) { + let owner = "did:key:z6MkNoOpUnlockAll"; + let repo = "noop-unlock"; + let key = advisory_lock_key(&owner.replace([':', '/'], "_"), repo); + let probe = sibling_pool(&pool, 2); + let lock_pool = build_lock_pool(&pool, 4, Duration::from_secs(5)); + let store = RepoStore::new( + PathBuf::from("/tmp/gitlawb-test-repos"), + None, + lock_pool.clone(), + ); + + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + + // Churn the pool: check out and drop connections that hold no lock, exactly what + // a spinning acquire now does between attempts. Each return fires + // pg_advisory_unlock_all() on that connection. + for _ in 0..10 { + let mut conn = lock_pool.acquire().await.expect("churn checkout"); + let _: (i32,) = sqlx::query_as("SELECT 1") + .fetch_one(&mut *conn) + .await + .expect("churn query"); + drop(conn); + tokio::time::sleep(Duration::from_millis(10)).await; + } + + assert!( + !lock_is_free_elsewhere(&probe, key).await, + "a held write lock must survive other lock-pool connections being returned" + ); + guard.release(true).await; + assert!( + lock_is_free_elsewhere(&probe, key).await, + "release must still free the lock after the churn" + ); + } + + /// #173 F1: lock-pool exhaustion is a DISTINCT error the handler can shed as a 503, + /// not a generic git 500. Both directions: an exhausted pool downcasts to + /// [`LockPoolBusy`], and an unrelated failure (a rejected repo name) does not. + #[sqlx::test] + async fn lock_pool_exhaustion_is_a_distinct_downcastable_error(pool: PgPool) { + let owner = "did:key:z6MkBusyDowncast"; + let store = RepoStore::new( + PathBuf::from("/tmp/gitlawb-test-repos"), + None, + build_lock_pool(&pool, 1, Duration::from_secs(1)), + ); + let held = store + .acquire_write(owner, "busy-a") + .await + .expect("first acquire"); + + let err = match store.acquire_write(owner, "busy-b").await { + Ok(_) => panic!("an exhausted lock pool must error, not hand back a guard"), + Err(e) => e, + }; + assert!( + err.downcast_ref::().is_some(), + "lock-pool exhaustion must be downcastable so the handler sheds 503, got: {err}" + ); + + // MUST-NOT: an ordinary rejection is not a capacity signal. + let other = match store.acquire_write(owner, "../escape").await { + Ok(_) => panic!("a traversal repo name must be rejected"), + Err(e) => e, + }; + assert!( + other.downcast_ref::().is_none(), + "a validation failure must not masquerade as lock-pool capacity, got: {other}" + ); + + held.release(false).await; + } + + /// Round trip: the lock is observably HELD between acquire and release, and + /// observably FREE after. Both checks run from an independent session; from + /// the holding session they would pass vacuously (session locks are + /// re-entrant) and would not notice an unlock that landed on the wrong + /// connection. + #[sqlx::test] + async fn acquire_write_holds_the_lock_until_release(pool: PgPool) { + let probe = sibling_pool(&pool, 2); + let owner = "did:key:z6MkRoundTrip"; + let repo = "round-trip"; + let key = advisory_lock_key(&owner.replace([':', '/'], "_"), repo); + + let store = RepoStore::for_testing(PathBuf::from("/tmp/gitlawb-test-repos"), pool.clone()); + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + assert!( + !lock_is_free_elsewhere(&probe, key).await, + "the lock must be held while the guard is alive" + ); + + // No polling here, deliberately. `release` must free the lock SYNCHRONOUSLY, + // which it can only do by unlocking on the connection that took it; a + // `pg_advisory_unlock` sent through the pool would land on some other + // session and return false. The `after_release` hook is a net for the + // cancellation path and fires from a spawned task well after this point, so + // it must not be what makes this assertion pass. + guard.release(true).await; + assert!( + lock_is_free_elsewhere(&probe, key).await, + "release must free the lock as seen from another session" + ); + } + + /// `release(false)` skips the Tigris upload but must still free the lock; a + /// failed write that kept the lock would wedge the repo. + #[sqlx::test] + async fn release_after_failed_write_still_frees_the_lock(pool: PgPool) { + let probe = sibling_pool(&pool, 2); + let owner = "did:key:z6MkFailedWrite"; + let repo = "failed-write"; + let key = advisory_lock_key(&owner.replace([':', '/'], "_"), repo); + + let store = RepoStore::for_testing(PathBuf::from("/tmp/gitlawb-test-repos"), pool.clone()); + let guard = store.acquire_write(owner, repo).await.expect("acquire"); + guard.release(false).await; + + assert!( + wait_until_free(&probe, key, Duration::from_secs(5)).await, + "release(success = false) must still free the lock" + ); + } + + /// The lock is per repo: a second acquire for the SAME repo waits for the + /// first to release, while a different repo proceeds straight through. + #[sqlx::test] + async fn same_repo_acquires_serialize_and_different_repos_do_not(pool: PgPool) { + let repos_dir = PathBuf::from("/tmp/gitlawb-test-repos"); + let owner = "did:key:z6MkSerialize"; + let store = RepoStore::for_testing(repos_dir, pool.clone()); + + let first = store + .acquire_write(owner, "serialize-a") + .await + .expect("first acquire"); + + // Different repo: unaffected by the held lock. + let other = tokio::time::timeout( + Duration::from_secs(2), + store.acquire_write(owner, "serialize-b"), + ) + .await + .expect("a different repo must not wait on this lock") + .expect("acquire other repo"); + other.release(false).await; + + // Same repo: must not acquire while `first` is alive. + let contender = tokio::spawn({ + let store = store.clone(); + async move { store.acquire_write(owner, "serialize-a").await } + }); + tokio::time::sleep(Duration::from_millis(1500)).await; + assert!( + !contender.is_finished(), + "a second acquire for the same repo must block while the first guard lives" + ); + + first.release(false).await; + let second = tokio::time::timeout(Duration::from_secs(10), contender) + .await + .expect("contender must finish once the lock is free") + .expect("contender task") + .expect("contender acquire"); + second.release(false).await; + } // ── sync slug validation (#272) ──────────────────────────────────────── @@ -932,4 +1786,586 @@ mod tests { ); } } + + // ── advisory-lock cancellation-safety (#174 F1, RED-before/GREEN-after) ── + + /// F1 (P1): dropping a `RepoWriteGuard` WITHOUT calling `release()` — the + /// state a `tokio::time::timeout` cancellation leaves `acquire_write` in when + /// it fires during the Tigris await — must still release the session advisory + /// lock. A checker connection is held OUT of the pool first, so `acquire_write` + /// is forced onto a distinct session; the checker (a different session) then + /// probes the lock, so advisory-lock re-entrancy cannot mask a leak. + /// + /// Load-bearing: RED today (no `Drop` releases the lock → held by + /// `acquire_write`'s session → checker's `pg_try_advisory_lock` returns + /// false). GREEN after the connection-affine `Drop` backstop. + #[sqlx::test] + async fn write_guard_drop_without_release_frees_the_lock(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone()); + let owner = "did:key:z6MkDropBackstopProofAAAAAAAAAAAAAAAAAAAAAA"; + let name = "leaktest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + // Distinct session for the probe: hold it out of the pool BEFORE acquiring, + // so acquire_write cannot use it and a re-entrant probe cannot falsely read free. + let mut checker = pool.acquire().await.expect("checker connection"); + + let guard = store.acquire_write(owner, name).await.expect("acquire"); + // The cancellation shape: drop without release(). + drop(guard); + // Let the detached unlock task run. + tokio::time::sleep(std::time::Duration::from_millis(400)).await; + + let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *checker) + .await + .unwrap(); + assert!( + free, + "advisory lock must be released when the guard is dropped without release()" + ); + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } + + /// F1 (latent non-affine release): `release()` must unlock on the SAME + /// session that locked, so the lock is freed regardless of which pooled + /// connection would service a fresh query. Observed from a distinct session. + #[sqlx::test] + async fn write_guard_release_frees_the_lock_from_a_distinct_session(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone()); + let owner = "did:key:z6MkAffineReleaseProofBBBBBBBBBBBBBBBBBBBB"; + let name = "affinetest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + let mut checker = pool.acquire().await.expect("checker connection"); + let guard = store.acquire_write(owner, name).await.expect("acquire"); + guard.release(false).await; + + let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *checker) + .await + .unwrap(); + assert!( + free, + "release() must free the advisory lock via connection-affine unlock" + ); + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } + + // ── cancellation-safe unlock (#174 F4, RED-before/GREEN-after) ────────── + + /// F4 (P1): a cancellation DURING the unlock await must still free the session + /// advisory lock. The guard owns the lock-pool connection that took the lock, so + /// dropping the parked `release` future returns that connection to the pool, + /// where the `after_release` hook runs `pg_advisory_unlock_all()` and clears + /// whatever the interrupted unlock did not. A test-only gate parks `release` at + /// the exact pre-unlock point; dropping the future there reproduces the + /// cancellation. + /// + /// Load-bearing: build the store's lock pool WITHOUT the `after_release` hook + /// and this goes RED, since the connection then returns to the pool still + /// holding the session lock and the checker's `pg_try_advisory_lock` returns + /// false. + #[sqlx::test] + async fn write_guard_release_cancelled_mid_unlock_frees_the_lock(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone()); + let owner = "did:key:z6MkCancelMidUnlockProofCCCCCCCCCCCCCCCCCC"; + let name = "canceltest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + // Distinct session for the probe, held out of the pool before acquiring. + let mut checker = pool.acquire().await.expect("checker connection"); + + let mut guard = store.acquire_write(owner, name).await.expect("acquire"); + // Arm the pre-unlock gate; it is never notified, so `release` parks on it + // with the connection still owned and `released` still false. + let gate = Arc::new(tokio::sync::Notify::new()); + guard.test_pre_unlock_gate = Some(gate); + + // Box the future so we can drop it ourselves (tokio::pin! keeps it alive to + // end of scope, which would defer the guard's Drop past the assertions). + let mut fut = Box::pin(guard.release(false)); + let parked = + tokio::time::timeout(std::time::Duration::from_millis(300), fut.as_mut()).await; + assert!( + parked.is_err(), + "release should park on the pre-unlock gate, not complete" + ); + // Cancel mid-unlock: dropping the boxed future runs RepoWriteGuard::drop. + drop(fut); + + // Let the Drop backstop's detached unlock task run. + tokio::time::sleep(std::time::Duration::from_millis(400)).await; + + let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *checker) + .await + .unwrap(); + assert!( + free, + "advisory lock must be freed when release is cancelled mid-unlock — the \ + connection must stay owned by the guard so Drop's backstop can run" + ); + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } + + /// F4: the ordinary success path still frees the lock, and a second write + /// acquire on the same repo returns promptly (no stale-lock retry loop). + #[sqlx::test] + async fn write_guard_release_true_frees_lock_and_second_acquire_succeeds(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone()); + let owner = "did:key:z6MkReleaseTrueProofDDDDDDDDDDDDDDDDDDDDDD"; + let name = "reltruetest"; + + let guard = store + .acquire_write(owner, name) + .await + .expect("first acquire"); + guard.release(true).await; + + let again = tokio::time::timeout( + std::time::Duration::from_secs(2), + store.acquire_write(owner, name), + ) + .await + .expect("second acquire_write must not hit the ~60s stale-lock retry loop") + .expect("second acquire"); + again.release(true).await; + } + + // ── unlock error disposes the connection (#174 F3b, RED-before/GREEN-after) ─ + + /// Put the guard's pinned connection into a failed-transaction state, so the + /// next statement on it errors while the SESSION stays alive and keeps holding + /// the session-level advisory lock (those survive a transaction abort; only + /// `pg_advisory_xact_lock` would not). This is the smallest injection that + /// reproduces F3b's shape: `pg_advisory_unlock` returning `Err` on a live, + /// still-locking session. No production seam is needed because the tests live + /// in this module and can reach `conn` directly. + async fn poison_guard_connection(guard: &mut RepoWriteGuard) { + let conn = guard + .lock_conn + .as_deref_mut() + .expect("guard holds its connection before release"); + sqlx::query("BEGIN") + .execute(&mut *conn) + .await + .expect("open a transaction on the guard connection"); + let poisoned = sqlx::query("SELECT 1 / 0").execute(&mut *conn).await; + assert!( + poisoned.is_err(), + "the poison statement must fail so the transaction is aborted" + ); + } + + /// How long the polls below may wait for an asynchronous server-side effect. + /// Postgres frees a session advisory lock when the backend exits, which happens + /// asynchronously to our socket close, so these are polled to a generous deadline + /// rather than slept for a fixed interval: a constant sleep is a flake under load, + /// and one that is long enough to be safe is dead time on every run. + const RELEASE_POLL_DEADLINE: std::time::Duration = std::time::Duration::from_secs(20); + + /// Poll `cond` until it holds, failing at the deadline so a regression fails the + /// test rather than hanging the suite. + async fn wait_until(mut cond: impl FnMut() -> bool, what: &str) { + let deadline = std::time::Instant::now() + RELEASE_POLL_DEADLINE; + while !cond() { + assert!( + std::time::Instant::now() < deadline, + "timed out waiting for {what}" + ); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + } + + /// Poll the advisory lock from `checker` until it is free. `pg_try_advisory_lock` + /// ACQUIRES on success, so a true result both answers the question and leaves the + /// checker session holding the lock; the caller unlocks it. + async fn wait_until_lock_free(checker: &mut sqlx::PgConnection, key: i64, what: &str) { + let deadline = std::time::Instant::now() + RELEASE_POLL_DEADLINE; + loop { + let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *checker) + .await + .unwrap(); + if free { + return; + } + assert!( + std::time::Instant::now() < deadline, + "timed out waiting for {what}" + ); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + } + + /// F3c (P2): the connection teardown on the failing-unlock path must be BOUNDED. + /// `release` awaits it inline while the global write permit, the per-source permit + /// and the write lease are all still held, and sqlx's `close()` carries no deadline + /// of its own, so a blackholed socket would park every later push to that repo + /// behind three pinned admission resources. + /// + /// What this covers: the deadline itself. A close that never resolves still lets + /// `close_conn_bounded` return, which is the property `release` depends on. What it + /// does NOT cover, and is reasoned rather than run: that sqlx's own `close()` is + /// what stalls in production. Making a real `PgConnection::close` hang needs a + /// blackholed TCP path to Postgres, and the flip has to land after the unlock + /// statement round-trips but before the Terminate write, which is not a seam this + /// module exposes. A never-resolving future is the faithful stand-in for that + /// close, and the F3b tests above already cover that `release` really routes its + /// close through here. + /// + /// Time is paused, so nothing here depends on wall clock: the runtime auto-advances + /// to the next timer, and the assertion is on which timer fired, not on elapsed + /// time. The outer bound is what turns a removed deadline into a failure rather + /// than a hung suite. + /// + /// Load-bearing: drop the `tokio::time::timeout` in `close_conn_bounded` and the + /// inner future never resolves, so the outer bound fires and this fails. + #[tokio::test(start_paused = true)] + async fn unlock_error_connection_close_is_bounded() { + let hanging = std::future::pending::>(); + let outcome = tokio::time::timeout( + UNLOCK_ERROR_CLOSE_TIMEOUT * 4, + close_conn_bounded("boundedclosetest", hanging), + ) + .await; + assert!( + outcome.is_ok(), + "a connection close that never completes must not hold the write lease and \ + both admission permits open-endedly: close_conn_bounded must give up and \ + drop the connection" + ); + } + + /// U8, the off-runtime arm: with no Tokio runtime there is nothing to spawn the + /// unlock onto, and the connection has already been taken out of the guard, so + /// dropping it with no unlock attempted returns it to the pool with the session + /// lock still held. Dropping a `PoolConnection` off a runtime is worse than that: + /// sqlx's return-to-pool path spawns, and its no-runtime fallback panics, so that + /// arm also panics in a destructor. + /// + /// Reached by dropping the guard on a plain `std::thread`, where + /// `Handle::try_current()` fails. + /// + /// Load-bearing: replace the `detach` arm with a plain `drop(conn)` and the join + /// sees sqlx's "requires a Tokio context" panic; `detach` gives up the pool slot, + /// so nothing is spawned and dropping the detached connection closes the socket, + /// which ends the session and frees the lock. + #[sqlx::test] + async fn write_guard_dropped_off_runtime_disposes_the_connection(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let store_pool = pool_without_idle_reaper(&pool).await; + let store = RepoStore::for_testing(dir.path().to_path_buf(), store_pool.clone()); + let owner = "did:key:z6MkDropOffRuntimeProofKKKKKKKKKKKKKKKKKK"; + let name = "dropoffruntimetest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + let mut checker = pool.acquire().await.expect("checker connection"); + let guard = store.acquire_write(owner, name).await.expect("acquire"); + // The guard's connection lives in the store's DERIVED lock pool, not the pool + // handed to `for_testing`; see `RepoStore::lock_pool`. + let lock_pool = store.lock_pool().clone(); + let size_before = lock_pool.size(); + assert!(size_before > 0, "the lock pool owns the guard's connection"); + + let dropped = std::thread::spawn(move || drop(guard)).join(); + assert!( + dropped.is_ok(), + "dropping a write guard off a Tokio runtime must not panic" + ); + + wait_until( + || lock_pool.size() == size_before - 1, + "the connection of a guard dropped off a runtime to be disposed of rather \ + than returned to the pool with no unlock attempted", + ) + .await; + wait_until_lock_free( + &mut checker, + key, + "a guard dropped off a runtime to end its session so postgres drops the lock", + ) + .await; + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } + + /// A second pool over the same test database with the idle reaper DISABLED. + /// + /// `#[sqlx::test]`'s own pool sets `idle_timeout(1s)`, so a connection returned to + /// it is closed by the reaper about a second later, which ends the session and + /// frees the advisory lock all on its own. The old fixed 400ms sleep landed inside + /// that window by luck; polling to a deadline long enough to be flake-proof would + /// land outside it and go green whether or not `release` disposed of the + /// connection, so the tests below would stop testing anything (measured: with the + /// reaper in play, the disposal shows up at ~2s even with the fix reverted). With + /// no reaper, `release` is the only thing that can end that session, so the poll + /// measures exactly the property these two tests exist for. + async fn pool_without_idle_reaper(pool: &sqlx::PgPool) -> sqlx::PgPool { + sqlx::postgres::PgPoolOptions::new() + .max_connections(5) + .idle_timeout(None) + .max_lifetime(None) + .connect_with(pool.connect_options().as_ref().clone()) + .await + .expect("a second pool over the test database") + } + + /// F3b (P1): when `pg_advisory_unlock` ERRORS while the session is still alive + /// (statement timeout, admin cancel, aborted transaction), the lock must not + /// survive `release`. The old code discarded the error with `let _ =` and set + /// `released = true` anyway, so `Drop` early-returned and the `PoolConnection` + /// went back to the pool still holding the session lock. + /// + /// Observed from a SEPARATE connection held out of the pool before acquiring: + /// session advisory locks are re-entrant and counted, so probing from the + /// holding session (or via a fresh `acquire_write` that may be handed the same + /// connection) would report free whether or not the fix is present. + /// + /// Load-bearing: RED before the fix (lock still held → `pg_try_advisory_lock` + /// returns false), GREEN after (the errored connection is closed, so the + /// session ends and Postgres drops the lock). + #[sqlx::test] + async fn write_guard_release_with_failing_unlock_frees_the_lock(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let store_pool = pool_without_idle_reaper(&pool).await; + let store = RepoStore::for_testing(dir.path().to_path_buf(), store_pool.clone()); + let owner = "did:key:z6MkUnlockErrorProofFFFFFFFFFFFFFFFFFFFFFF"; + let name = "unlockerrtest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + // Distinct session for the probe, held out of the pool before acquiring. + let mut checker = pool.acquire().await.expect("checker connection"); + + let mut guard = store.acquire_write(owner, name).await.expect("acquire"); + poison_guard_connection(&mut guard).await; + + // Sanity: the poisoned session is still alive and still holds the lock, so + // the assertion below measures the release path and not a dead session. + let (free_before,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *checker) + .await + .unwrap(); + assert!( + !free_before, + "the poisoned session must still hold the lock before release" + ); + + guard.release(false).await; + + // Postgres drops the lock when the disposed session's backend exits, which is + // asynchronous to our socket close: poll for it rather than sleeping a + // constant. The deadline is what keeps this load-bearing: a release that + // leaves the lock held never satisfies the probe and fails here. + wait_until_lock_free( + &mut checker, + key, + "an errored pg_advisory_unlock must not leave the lock held: release must \ + dispose of the connection so the session ends", + ) + .await; + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } + + /// F3b: the connection that saw the unlock error must leave the pool entirely, + /// rather than being handed to the next caller while still holding the lock. + /// `PgPool::size()` counts the connections the pool owns, so closing the + /// errored one is observable as a drop in that count. + #[sqlx::test] + async fn write_guard_release_with_failing_unlock_does_not_return_the_connection( + pool: sqlx::PgPool, + ) { + let dir = tempfile::TempDir::new().unwrap(); + let store_pool = pool_without_idle_reaper(&pool).await; + let store = RepoStore::for_testing(dir.path().to_path_buf(), store_pool.clone()); + let owner = "did:key:z6MkUnlockErrorPoolProofGGGGGGGGGGGGGGGGGG"; + let name = "unlockerrpooltest"; + + let mut guard = store.acquire_write(owner, name).await.expect("acquire"); + poison_guard_connection(&mut guard).await; + let lock_pool = store.lock_pool().clone(); + let size_before = lock_pool.size(); + assert!(size_before > 0, "the lock pool owns the guard's connection"); + + guard.release(false).await; + + // The pool's size drops when the closed connection's slot is given up, which + // is not synchronous with `release` returning: poll rather than sleep. + wait_until( + || lock_pool.size() == size_before - 1, + "the connection that saw the unlock error to be closed rather than returned \ + to the pool still holding the session lock", + ) + .await; + } + + /// F3b regression guard on the success path: a normal unlock keeps the + /// connection in the pool and marks the guard released, so the disposal branch + /// is confined to the error case. + #[sqlx::test] + async fn write_guard_release_success_keeps_the_connection_and_frees_the_lock( + pool: sqlx::PgPool, + ) { + let dir = tempfile::TempDir::new().unwrap(); + let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone()); + let owner = "did:key:z6MkUnlockOkProofHHHHHHHHHHHHHHHHHHHHHHHH"; + let name = "unlockoktest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + let mut checker = pool.acquire().await.expect("checker connection"); + let guard = store.acquire_write(owner, name).await.expect("acquire"); + let size_before = pool.size(); + + guard.release(false).await; + tokio::time::sleep(std::time::Duration::from_millis(400)).await; + + assert_eq!( + pool.size(), + size_before, + "a successful unlock must leave the connection in the pool" + ); + let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *checker) + .await + .unwrap(); + assert!(free, "the success path must still free the lock"); + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } + + // ── the Drop backstop disposes its connection too (#174 U8) ───────────── + + /// U8 (P2): the `Drop` backstop carries the same hazard F3b closed in `release`. + /// When the detached `pg_advisory_unlock` ERRORS on a live session, the async + /// block ends and drops the moved `PoolConnection`, which RETURNS it to the pool + /// while that session may still hold the lock, handing the next caller a + /// connection holding a lock nobody tracks. The errored connection must be closed + /// instead: that keeps it out of the pool and ends the session, which is what + /// frees the lock server-side. + /// + /// Run against a pool with the idle reaper disabled, for the reason spelled out on + /// `pool_without_idle_reaper`: with the reaper in play the session dies on its own + /// about a second later and the assertion stops measuring the disposal. + /// + /// Load-bearing: RED before the fix (the connection goes back to the pool, so the + /// size never drops and this times out), GREEN after. + #[sqlx::test] + async fn write_guard_drop_with_failing_unlock_does_not_return_the_connection( + pool: sqlx::PgPool, + ) { + let dir = tempfile::TempDir::new().unwrap(); + let store_pool = pool_without_idle_reaper(&pool).await; + let store = RepoStore::for_testing(dir.path().to_path_buf(), store_pool.clone()); + let owner = "did:key:z6MkDropUnlockErrProofIIIIIIIIIIIIIIIIIIII"; + let name = "dropunlockerrtest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + // Distinct session for the probe, held out of the store's pool entirely. + let mut checker = pool.acquire().await.expect("checker connection"); + + let mut guard = store.acquire_write(owner, name).await.expect("acquire"); + poison_guard_connection(&mut guard).await; + let lock_pool = store.lock_pool().clone(); + let size_before = lock_pool.size(); + assert!(size_before > 0, "the lock pool owns the guard's connection"); + + // The backstop shape: dropped without release(), with an unlock that errors. + drop(guard); + + wait_until( + || lock_pool.size() == size_before - 1, + "the connection whose detached unlock errored to be closed rather than \ + returned to the pool still holding the session lock", + ) + .await; + wait_until_lock_free( + &mut checker, + key, + "an errored detached unlock must not leave the lock held: Drop must dispose \ + of the connection so the session ends", + ) + .await; + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } + + /// U8 regression guard on the success path: a detached unlock that SUCCEEDS must + /// still return the connection to the pool. Without this, "close the connection on + /// Drop" could be widened to "always close" and the test above would not notice. + #[sqlx::test] + async fn write_guard_drop_with_successful_unlock_keeps_the_connection(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let store_pool = pool_without_idle_reaper(&pool).await; + let store = RepoStore::for_testing(dir.path().to_path_buf(), store_pool.clone()); + let owner = "did:key:z6MkDropUnlockOkProofJJJJJJJJJJJJJJJJJJJJ"; + let name = "dropunlockoktest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + let mut checker = pool.acquire().await.expect("checker connection"); + let guard = store.acquire_write(owner, name).await.expect("acquire"); + let lock_pool = store.lock_pool().clone(); + let size_before = lock_pool.size(); + assert!(size_before > 0, "the lock pool owns the guard's connection"); + + drop(guard); + + // The connection goes back only once the detached unlock task has finished. + wait_until( + || lock_pool.num_idle() > 0, + "the detached unlock to finish and hand the connection back", + ) + .await; + assert_eq!( + lock_pool.size(), + size_before, + "a successful detached unlock must leave the connection in the pool" + ); + wait_until_lock_free( + &mut checker, + key, + "the Drop backstop's successful unlock to free the lock", + ) + .await; + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } } diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index eeb35b99..213b76f2 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -6,29 +6,116 @@ use bytes::Bytes; use std::collections::HashSet; use std::path::Path; use std::process::Stdio; -use std::time::Duration; +use std::time::{Duration, Instant}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::process::Command; +/// Owns the served-git admission permits (global + per-source) for the lifetime of +/// the work they admitted, so admission is released only when that work is truly +/// done — not the instant the handler future drops on a client disconnect. +/// +/// A drop-only wrapper: nothing here inspects what it holds. The handler MOVEs its +/// permits in and keeps no copy (a retained copy would drop early and release admission +/// the moment the future is dropped, defeating the guard). It is threaded into +/// `drive_git_child`, whose [`KillGroupOnDrop`] moves it into the detached reaper on +/// disconnect, so both permits drop only after the process group is confirmed reaped +/// (`kill(-pgid,0)==ESRCH`) rather than while the group is still alive holding PIDs past +/// the concurrency cap (#174 P1-a, plain-spawn residual). +/// +/// The `be0cdd6` path-scoped upload-pack walk already applies this discipline by +/// moving its permits into the `spawn_blocking`; this generalizes it to the plain +/// (non-path-scoped) `info_refs` / `run_git_service` spawn paths and, via the +/// hand-back contract on `drive_git_child`, across both stages of the filtered +/// (`upload_pack_excluding`) pack build (F1). +pub struct AdmissionGuard { + // Boxed so the concrete permit types stay private to the caller (repos.rs) — the + // guard only needs to hold them and drop them, never inspect them. `Send + + // 'static` so the guard can move into the detached reaper task. + _global: Option>, + _caller: Option>, + // Any further work-scoped hold that must outlive the process group; see `with_hold`. + _hold: Option>, + // Per-repo write lease (#174 U2/F3), `Some` ONLY on the receive-pack write path + // (via [`with_lease`](Self::with_lease)); `None` on every read path and every + // non-receive-pack write path. It rides this guard into `KillGroupOnDrop`'s detached + // reaper, so on a client disconnect the lease frees only after the disconnected + // push's git group is reaped — serializing a second same-node push against the + // still-writing group. A `RepoWriteLease` clone is `Send + 'static` and holds no pg + // connection, so it travels with the reaper cleanly. A stray `Some` on a READ path + // would wrongly serialize upload-pack against pushes, hence the None-everywhere-else + // discipline. + _lease: Option, +} + +impl AdmissionGuard { + /// Take ownership of the global permit and an optional per-caller permit. Both are + /// erased to `Box` — the guard's only job is to hold them until it drops. + /// No lease is attached here; the receive-pack write path adds one via + /// [`with_lease`](Self::with_lease), so every other call site is lease-free. + pub fn new(global: impl Send + 'static, caller: Option) -> Self { + Self { + _global: Some(Box::new(global)), + _caller: caller.map(|c| Box::new(c) as Box), + _hold: None, + _lease: None, + } + } + + /// Attach a further hold that must not be released until the process group is + /// reaped, and ride it through the same seam as the permits. + /// + /// The push handler uses this for the repo WRITE LOCK (#173 F2). Its + /// `guard.release(..)` line is only reached if `receive_pack` returns, so on a client + /// disconnect the lock used to be freed by the dropped future while the detached + /// reaper was still giving the group its SIGTERM grace, admitting a second + /// `receive-pack` on the same repo. Carrying the lock here holds it until the group + /// is ESRCH-confirmed gone, which is the same invariant the timeout path already + /// keeps ("a caller releasing a write lock can't race them", `reap_group_on_timeout`). + pub fn with_hold(mut self, hold: impl Send + 'static) -> Self { + self._hold = Some(Box::new(hold)); + self + } + + /// Attach the per-repo write lease (#174 U2/F3). Called ONLY on the receive-pack + /// write path, so the lease rides the disconnect reaper; read paths never call this. + pub fn with_lease(mut self, lease: crate::state::RepoWriteLease) -> Self { + self._lease = Some(lease); + self + } +} + /// Handle `GET /:owner/:repo/info/refs?service=git-upload-pack` /// or `?service=git-receive-pack` /// /// This is the ref advertisement — the first step of a clone or push. -pub async fn info_refs(repo_path: &Path, service: &str) -> Result { +/// +/// `git_bin` is injectable purely so the process-group teardown can be driven by a +/// fake `git` in tests (production passes `"git"`). `timeout` bounds the whole child +/// interaction: previously the advertisement ran a bare `Command::output()` with no +/// deadline and no teardown, so a hung git pinned its concurrency slot indefinitely +/// and a client disconnect orphaned the child (#174). It now shares +/// [`drive_git_child`]'s timeout + `process_group(0)` + [`KillGroupOnDrop`] teardown. +pub async fn info_refs( + git_bin: &str, + service: &str, + repo_path: &Path, + timeout: Duration, + admission: Option, +) -> Result { validate_service(service)?; - let output = Command::new("git") + let mut command = Command::new(git_bin); + command .arg(service_to_command(service)) .arg("--stateless-rpc") .arg("--advertise-refs") - .arg(repo_path) - .output() - .await?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - bail!("git {service} --advertise-refs failed: {stderr}"); - } + .arg(repo_path); + // No request body: advertise-refs does not read stdin. + let (stdout, admission) = + drive_git_child(command, Bytes::new(), timeout, "advertise-refs", admission).await?; + // Single-stage op: the advertisement's group is reaped by now; release admission + // rather than holding it across the pure response framing below. + drop(admission); let content_type = format!("application/x-{service}-advertisement"); @@ -38,7 +125,7 @@ pub async fn info_refs(repo_path: &Path, service: &str) -> Result { let mut body = Vec::new(); body.extend_from_slice(&pkt_service); body.extend_from_slice(flush); - body.extend_from_slice(&output.stdout); + body.extend_from_slice(&stdout); Ok(Response::builder() .status(StatusCode::OK) @@ -53,12 +140,21 @@ pub async fn info_refs(repo_path: &Path, service: &str) -> Result { /// Serves pack data for a clone or fetch. This is stateless — the entire /// negotiation happens in a single request/response. pub async fn upload_pack( + git_bin: &str, repo_path: &Path, request_body: Bytes, timeout: Duration, + admission: Option, ) -> Result { - let output = - run_git_service("git", "git-upload-pack", repo_path, request_body, timeout).await?; + let output = run_git_service( + git_bin, + "git-upload-pack", + repo_path, + request_body, + timeout, + admission, + ) + .await?; Ok(Response::builder() .status(StatusCode::OK) @@ -72,12 +168,21 @@ pub async fn upload_pack( /// Accepts a push. The caller MUST verify HTTP Signature auth before /// calling this function. pub async fn receive_pack( + git_bin: &str, repo_path: &Path, request_body: Bytes, timeout: Duration, + admission: Option, ) -> Result { - let output = - run_git_service("git", "git-receive-pack", repo_path, request_body, timeout).await?; + let output = run_git_service( + git_bin, + "git-receive-pack", + repo_path, + request_body, + timeout, + admission, + ) + .await?; Ok(Response::builder() .status(StatusCode::OK) @@ -100,24 +205,86 @@ pub async fn receive_pack( /// `wait_with_output()` returns, so a request that completed cleanly never signals. #[cfg(unix)] struct KillGroupOnDrop { + // Holds the child + its pgid while armed. The interaction drives the child through + // `child_mut()`; the success/timeout paths call `disarm` once they have reaped it. + // On drop — a client disconnect that drops the whole request future — the guard + // launches a detached reaper that OWNS the child and runs the full + // SIGTERM -> grace -> SIGKILL -> reap sequence (`reap_group_on_timeout`), as strong + // as the timeout path, so a SIGTERM-ignoring group member is SIGKILLed and reaped + // rather than left running until EPIPE to accumulate past the concurrency cap + // (#174 P1-c). Owning the tokio `Child` in the reaper (rather than a raw + // `waitpid(-pgid)`) keeps a single reaper of the leader, so it never races tokio's + // own SIGCHLD-driven orphan reaper. + child: Option, pgid: Option, + // The global + per-source admission permits for this op, if any. repos.rs moves + // its permits in here (directly on the plain spawn paths, via the stage threading + // on the filtered rev-list/pack-objects ones) so admission is released only when + // the group is confirmed reaped, on every exit: complete, timeout, or + // client-disconnect (#174 P1-a, F1). `None` when the op carries no admission + // (e.g. the smart_http test harness). + admission: Option, } #[cfg(unix)] impl KillGroupOnDrop { - fn disarm(&mut self) { + /// The child, while armed. The interaction drives stdin/stdout/`wait` through this. + fn child_mut(&mut self) -> &mut tokio::process::Child { + self.child.as_mut().expect("child present while armed") + } + + /// Disarm on the success/timeout path: the body has already reaped the child (its + /// `wait()` returned, or `reap_group_on_timeout` ran), so drop the handle and clear + /// the pgid, leaving the guard's Drop a no-op. Returns the admission guard: the + /// group is already reaped on both callers of this, so the success path hands it + /// back to drive_git_child's caller (a multi-stage build carries it to the next + /// stage) and the timeout path drops it there and then. + fn disarm(&mut self) -> Option { self.pgid = None; + self.child = None; + self.admission.take() } } #[cfg(unix)] impl Drop for KillGroupOnDrop { fn drop(&mut self) { - if let Some(pgid) = self.pgid { - // SAFETY: kill(2) takes only integer arguments and borrows no Rust - // memory. Signalling a stale group just returns ESRCH, which we ignore. - unsafe { - libc::kill(-pgid, libc::SIGTERM); + // Move any admission guard out first so it travels with the reaper (or drops + // here on the no-child / no-runtime fallback), never before the group is gone. + let admission = self.admission.take(); + let (Some(mut child), Some(pgid)) = (self.child.take(), self.pgid) else { + // Nothing to reap (already disarmed); dropping `admission` here is correct — + // the group is already gone. + return; + }; + // A sync Drop cannot await, so launch a detached reaper that owns the child and + // runs the same teardown as the timeout path (TERM -> grace -> SIGKILL -> reap + // the whole group). Owning the tokio `Child` means this task is the sole reaper + // of the leader, so it cannot race tokio's orphan reaper. Prefer the current + // runtime handle; if dropped outside a runtime, fall back to a best-effort + // synchronous SIGTERM so the group is at least signalled. + match tokio::runtime::Handle::try_current() { + Ok(handle) => { + handle.spawn(async move { + reap_group_on_timeout(&mut child).await; + // Release admission only now: the group is ESRCH-confirmed gone (or + // hit the ~4s D-state hard cap, past which nothing in userspace can + // free the PIDs anyway). Holding the permits until here is what + // stops disconnect-spam from admitting replacements while the prior + // group is still alive (#174 P1-a). `admission` is moved in and + // dropped when this closure ends. + drop(admission); + }); + } + Err(_) => { + // SAFETY: kill(2) takes only integers and borrows no Rust memory; + // ESRCH on an already-gone group is ignored. + unsafe { + libc::kill(-pgid, libc::SIGTERM); + } + // No runtime to await the reap; drop admission best-effort after the + // synchronous signal (the fallback path is not on the hot request path). + drop(admission); } } } @@ -202,12 +369,44 @@ async fn run_git_service( repo_path: &Path, input: Bytes, timeout: Duration, + admission: Option, ) -> Result> { let mut command = Command::new(git_bin); command .arg(service_to_command(service)) .arg("--stateless-rpc") - .arg(repo_path) + .arg(repo_path); + let (out, admission) = drive_git_child(command, input, timeout, service, admission).await?; + // Single-stage op: the child group is already reaped when drive_git_child + // returns, so releasing admission here keeps the permits held exactly for the + // process lifetime, no longer. + drop(admission); + Ok(out) +} + +/// Drive a spawned git child under `timeout` with process-group teardown, returning +/// its stdout together with the admission guard. Shared core for +/// [`run_git_service`], [`info_refs`], and the filtered-pack stages: the caller +/// passes a `Command` with its args set; this adds piped stdio and `process_group(0)`. +/// On the deadline the whole group is torn down and reaped before returning +/// [`GitServiceTimeout`]; on a dropped future (client disconnect) the +/// [`KillGroupOnDrop`] guard fires. `input` is written to the child's stdin (empty +/// for the advertise-refs path, which has no request body); `what` labels errors. +/// +/// Admission contract: on success the guard is handed BACK so a multi-stage caller +/// (the filtered-pack build) can carry one admission across consecutive children +/// instead of releasing it between stages. On every Err return (timeout, spawn +/// failure, interaction error, non-zero exit) the guard is dropped internally, and +/// only after the child is reaped: an Err cannot carry the guard, and by then the +/// group is confirmed gone, so releasing admission there is correct. +async fn drive_git_child( + mut command: Command, + input: Bytes, + timeout: Duration, + what: &str, + admission: Option, +) -> Result<(Vec, Option)> { + command .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -219,20 +418,33 @@ async fn run_git_service( let mut child = command.spawn()?; - // Arm the group-kill guard for the lifetime of the request. With - // process_group(0) the child is its own group leader, so pgid == its pid. - // This fires on a client disconnect (the whole future is dropped mid-request). + // Own the pipes so the child stays reap-able: wait_with_output would consume it, + // but on a timeout we must actively reap the group first (see + // reap_group_on_timeout), and on a disconnect the guard's detached reaper does. + // Take the pipes before the child moves into the guard below. + let mut stdin = child.stdin.take(); + let mut stdout = child.stdout.take().context("git stdout was not piped")?; + let mut stderr = child.stderr.take().context("git stderr was not piped")?; + + // Arm the group-kill guard for the lifetime of the request. With process_group(0) + // the child is its own group leader, so pgid == its pid. On a client disconnect + // (the whole future is dropped mid-request) the guard's Drop launches a detached + // reaper that OWNS the child and runs the full TERM/grace/SIGKILL/reap — as strong + // as the timeout path below (#174 P1-c). The guard owns the child; the interaction + // drives it through `child_mut()`. + #[cfg(unix)] + let pgid = child.id().map(|id| id as i32); #[cfg(unix)] let mut group_guard = KillGroupOnDrop { - pgid: child.id().map(|id| id as i32), + child: Some(child), + pgid, + admission, }; + // On non-unix there is no process-group teardown; the `admission` parameter + // itself stays live for the child's whole interaction (nothing moves it until + // the match below), so the success path hands it back like the unix path and + // the timeout path drops it after the child is waited on. - // Own the pipes so `child` stays reap-able after a timeout: wait_with_output - // would consume it, but on timeout we must actively reap the group before - // returning (see reap_group_on_timeout). - let mut stdin = child.stdin.take(); - let mut stdout = child.stdout.take().context("git stdout was not piped")?; - let mut stderr = child.stderr.take().context("git stderr was not piped")?; let mut out = Vec::new(); let mut err = Vec::new(); @@ -249,11 +461,15 @@ async fn run_git_service( None => Ok(()), } }; + #[cfg(unix)] + let child_ref = group_guard.child_mut(); + #[cfg(not(unix))] + let child_ref = &mut child; let (write_result, r_out, r_err, status) = tokio::join!( write, stdout.read_to_end(&mut out), stderr.read_to_end(&mut err), - child.wait(), + child_ref.wait(), ); r_out?; r_err?; @@ -261,28 +477,34 @@ async fn run_git_service( }; let timed = tokio::time::timeout(timeout, interact).await; - let (write_result, status) = match timed { + let (write_result, status, admission) = match timed { Ok(result) => { // The join runs all arms to completion, so the child is reaped: disarm // before surfacing any interaction error (a read/wait error), else the - // guard's drop would fire SIGTERM on the reaped, possibly-reused pgid. + // guard's drop would reap an already-reaped child / signal a reused pgid. + // Disarming hands the admission guard back; it rides to the Ok return + // below, and every error return between here and there drops it, always + // AFTER the reap, the earliest provably-free point. #[cfg(unix)] - group_guard.disarm(); - result? + let admission = group_guard.disarm(); + let (write_result, status) = result?; + (write_result, status, admission) } Err(_elapsed) => { // Timeout: tear the whole group down and reap it before returning so a // caller releasing a write lock can't race a still-live git. Then disarm - // the (now redundant) guard so its drop can't hit a reused pgid. + // the (now redundant) guard so its drop can't hit a reused pgid. The + // returned admission guard drops here, AFTER the group is confirmed reaped. #[cfg(unix)] { - reap_group_on_timeout(&mut child).await; - group_guard.disarm(); + reap_group_on_timeout(group_guard.child_mut()).await; + drop(group_guard.disarm()); } #[cfg(not(unix))] { let _ = child.start_kill(); let _ = tokio::time::timeout(Duration::from_secs(2), child.wait()).await; + drop(admission); } return Err(GitServiceTimeout.into()); } @@ -291,15 +513,16 @@ async fn run_git_service( // Surface git's own failure (its stderr, which the handler may classify as a // 400) before any stdin-write error: when git rejects a malformed body it // exits non-zero and closes stdin, so the write's EPIPE would otherwise mask - // the real cause. + // the real cause. `admission` drops on these error returns; the child is + // already reaped by the completed join above. if !status.success() { let stderr = String::from_utf8_lossy(&err); - bail!("{service} failed: {stderr}"); + bail!("{what} failed: {stderr}"); } write_result.context("failed to write to git stdin")?; - Ok(out) + Ok((out, admission)) } fn service_to_command(service: &str) -> &str { @@ -324,62 +547,92 @@ fn pkt_line(data: &str) -> Vec { format!("{len:04x}{data}").into_bytes() } -/// Build a packfile containing every object reachable from all refs EXCEPT the -/// given blob OIDs. Commits and trees are always included, so SHAs stay intact; -/// only the named blobs are dropped. -pub fn build_filtered_pack(repo_path: &Path, withheld: &HashSet) -> Result> { - // All reachable objects as "oid [path]" lines. - let rev = std::process::Command::new("git") +/// Run `rev-list --objects --all` under `timeout` and return the reachable object +/// ids minus the withheld blobs. Runs under [`drive_git_child`] (async, with the +/// `tokio::time::timeout` + `process_group(0)` + [`KillGroupOnDrop`] teardown), so a +/// hung/slow enumeration is duration-bounded and reaped on client disconnect just +/// like the pack-objects stage. A bare blocking `Command::output()` inside a +/// `spawn_blocking` is uncancellable, so a hung rev-list would pin the endpoint's +/// concurrency permit for the whole hang (#174). `git_bin` is injectable for the +/// same fake-git testing reason as `run_git_service`. +async fn rev_list_keep( + git_bin: &str, + repo_path: &Path, + withheld: &HashSet, + timeout: Duration, + admission: Option, +) -> Result<(Vec, Option)> { + let mut command = Command::new(git_bin); + command .args(["rev-list", "--objects", "--all"]) - .current_dir(repo_path) - .output()?; - if !rev.status.success() { - bail!( - "git rev-list failed: {}", - String::from_utf8_lossy(&rev.stderr) - ); - } + .current_dir(repo_path); + // The filtered-serve caller's admission rides through drive_git_child so a + // disconnect mid-enumeration keeps the permits held until the rev-list group is + // reaped; on success the guard comes back for the pack-objects stage (F1). + let (stdout, admission) = + drive_git_child(command, Bytes::new(), timeout, "rev-list", admission).await?; let mut keep = Vec::new(); - for line in String::from_utf8_lossy(&rev.stdout).lines() { + for line in String::from_utf8_lossy(&stdout).lines() { let oid = line.split_whitespace().next().unwrap_or(""); if oid.is_empty() || withheld.contains(oid) { continue; } keep.push(oid.to_string()); } - let mut child = std::process::Command::new("git") + Ok((keep, admission)) +} + +/// Build a packfile containing every object reachable from all refs EXCEPT the +/// given blob OIDs. Commits and trees are always included, so SHAs stay intact; +/// only the named blobs are dropped. +/// +/// Both git stages — the `rev-list` enumeration and the streaming `pack-objects` +/// build — run under [`drive_git_child`] sharing one deadline, so a hung/slow git at +/// either stage is duration-bounded and its process group is reaped on client +/// disconnect. An outer `tokio::time::timeout` around a `spawn_blocking` cannot +/// cancel the blocking thread, so neither stage may live off the async side +/// (#174, KTD5). +/// +/// `admission` carries the filtered-serve permits across BOTH stages: stage 1 hands +/// it back on success, stage 2 takes it, and the caller receives it after stage 2 so +/// release happens only once the last child group is reaped. On a disconnect the +/// guard rides the active stage's detached reaper instead of dropping with the +/// future (F1). A stage that fails or times out (including a stage 2 whose +/// remaining budget has saturated to zero) drops the guard inside drive_git_child, +/// after that stage's reap. +pub async fn build_filtered_pack( + git_bin: &str, + repo_path: &Path, + withheld: &HashSet, + timeout: Duration, + admission: Option, +) -> Result<(Vec, Option)> { + // One deadline spans both git stages so a slow rev-list eats into the pack + // budget rather than granting each stage a fresh `timeout` (2x the permit hold). + let deadline = Instant::now() + timeout; + let (keep, admission) = rev_list_keep( + git_bin, + repo_path, + withheld, + deadline.saturating_duration_since(Instant::now()), + admission, + ) + .await?; + let mut data = keep.join("\n").into_bytes(); + data.push(b'\n'); + let mut command = Command::new(git_bin); + command .args(["pack-objects", "--stdout"]) - .current_dir(repo_path) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; - // Feed the object ids on stdin, but always reap the child afterward even if - // the write fails or stdin is missing, so an error can't drop the Child - // unwaited and leak a zombie (#53). - let write_result: std::io::Result<()> = { - use std::io::Write as _; - match child.stdin.take() { - Some(mut stdin) => { - let mut data = keep.join("\n").into_bytes(); - data.push(b'\n'); - stdin.write_all(&data) - } - None => Err(std::io::Error::new( - std::io::ErrorKind::BrokenPipe, - "git pack-objects stdin unavailable", - )), - } - }; - let out = child.wait_with_output()?; - write_result.context("failed to write object ids to git pack-objects stdin")?; - if !out.status.success() { - bail!( - "git pack-objects failed: {}", - String::from_utf8_lossy(&out.stderr) - ); - } - Ok(out.stdout) + .current_dir(repo_path); + let (out, admission) = drive_git_child( + command, + Bytes::from(data), + deadline.saturating_duration_since(Instant::now()), + "pack-objects", + admission, + ) + .await?; + Ok((out, admission)) } /// Serve a clone/fetch with the withheld blobs removed from the response pack. @@ -406,20 +659,24 @@ pub fn build_filtered_pack(repo_path: &Path, withheld: &HashSet) -> Resu /// instead of a thin delta. Honoring negotiation for smaller fetch packs is an /// optimization follow-up, not a correctness requirement. pub async fn upload_pack_excluding( + git_bin: &str, repo_path: &Path, request_body: Bytes, withheld: &HashSet, + timeout: Duration, + admission: Option, ) -> Result { - // build_filtered_pack shells out to git (rev-list, pack-objects) with - // blocking std::process I/O; run it off the async worker so a large repo's - // pack build does not stall the tokio runtime. - let pack = { - let repo_path = repo_path.to_path_buf(); - let withheld = withheld.clone(); - tokio::task::spawn_blocking(move || build_filtered_pack(&repo_path, &withheld)) - .await - .context("filtered-pack build task panicked")?? - }; + // Both filtered-pack stages run async under drive_git_child (duration-bounded, + // process group reaped on disconnect, #174), and `admission` threads through + // them so the caller's permits release only after the active stage's group is + // reaped, never the instant a disconnect drops this future (F1). `git_bin` is + // injectable for the same fake-git testing reason as `run_git_service` + // (production passes the configured git binary). + let (pack, admission) = + build_filtered_pack(git_bin, repo_path, withheld, timeout, admission).await?; + // Both git stages are done and their groups reaped; release admission before the + // pure in-memory response framing below. + drop(admission); // The client lists its capabilities on the first `want` line. Honor // side-band-64k when offered (every modern smart-HTTP client offers it); @@ -538,7 +795,10 @@ mod tests { let mut withheld = std::collections::HashSet::new(); withheld.insert(secret.clone()); - let pack = build_filtered_pack(&bare, &withheld).unwrap(); + let (pack, _admission) = + build_filtered_pack("git", &bare, &withheld, Duration::from_secs(30), None) + .await + .unwrap(); let ids = pack_object_ids(&pack); assert!(ids.contains(&public), "public blob must be in the pack"); assert!( @@ -600,7 +860,10 @@ mod tests { b"0098want 0000000000000000000000000000000000000000 \ side-band-64k ofs-delta agent=git/2\n00000009done\n", ); - let resp = upload_pack_excluding(&bare, req, &withheld).await.unwrap(); + let resp = + upload_pack_excluding("git", &bare, req, &withheld, Duration::from_secs(30), None) + .await + .unwrap(); let body = to_bytes(resp.into_body(), usize::MAX).await.unwrap(); let ids = pack_object_ids(&extract_pack(&body)); assert!( @@ -650,16 +913,25 @@ mod tests { axum::extract::Query(q): axum::extract::Query>, ) -> Response { let service = q.get("service").cloned().unwrap_or_default(); - info_refs(&st.repo, &service).await.unwrap() + info_refs("git", &service, &st.repo, Duration::from_secs(30), None) + .await + .unwrap() } async fn pack_handler( axum::extract::State(st): axum::extract::State>, body: Bytes, ) -> Response { - upload_pack_excluding(&st.repo, body, &st.withheld) - .await - .unwrap() + upload_pack_excluding( + "git", + &st.repo, + body, + &st.withheld, + Duration::from_secs(30), + None, + ) + .await + .unwrap() } /// Spawn the server for `bare`, withholding `withheld`. Returns the clone URL @@ -920,23 +1192,37 @@ mod tests { #[cfg(unix)] #[tokio::test] async fn kill_group_guard_terminates_child_on_drop() { - let mut child = tokio::process::Command::new("sleep") + let child = tokio::process::Command::new("sleep") .arg("300") .process_group(0) .spawn() .unwrap(); - let pgid = child.id().map(|id| id as i32); + let pid = child.id().unwrap() as i32; { - let _guard = KillGroupOnDrop { pgid }; - } // guard drops here -> SIGTERM to the group + // The guard owns the child; on drop its detached reaper TERM/grace/KILL/ + // reaps the group (a plain sleep dies on the first SIGTERM). + let _guard = KillGroupOnDrop { + child: Some(child), + pgid: Some(pid), + admission: None, + }; + } - use std::os::unix::process::ExitStatusExt; - let status = child.wait().await.unwrap(); - assert_eq!( - status.signal(), - Some(libc::SIGTERM), - "child must be terminated by SIGTERM via its process group" + let mut gone = false; + for _ in 0..300 { + if !alive(pid) { + gone = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + unsafe { + libc::kill(pid, libc::SIGKILL); + } + assert!( + gone, + "child must be terminated and reaped via its process group on guard drop" ); } @@ -954,9 +1240,10 @@ mod tests { .process_group(0) .spawn() .unwrap(); - let pgid = child.id().map(|id| id as i32); + let pid = child.id().unwrap() as i32; - // Read the backgrounded grandchild's pid from the first stdout line. + // Read the backgrounded grandchild's pid from the first stdout line, before + // the child moves into the guard. let mut stdout = child.stdout.take().unwrap(); let mut buf = Vec::new(); loop { @@ -971,20 +1258,26 @@ mod tests { assert!(alive(grandchild), "grandchild should be running"); { - let _guard = KillGroupOnDrop { pgid }; - } // group SIGTERM reaches sh AND the sleep grandchild - - let _ = child.wait().await; // reap sh + // The guard owns the child; on drop the detached reaper group-kills sh AND + // the sleep grandchild. + let _guard = KillGroupOnDrop { + child: Some(child), + pgid: Some(pid), + admission: None, + }; + } - // The grandchild reparents to init and is reaped; poll until it's gone. let mut gone = false; - for _ in 0..200 { + for _ in 0..300 { if !alive(grandchild) { gone = true; break; } tokio::time::sleep(std::time::Duration::from_millis(10)).await; } + unsafe { + libc::kill(grandchild, libc::SIGKILL); + } assert!( gone, "grandchild must be terminated by the group signal (#53)" @@ -996,27 +1289,32 @@ mod tests { async fn kill_group_guard_disarmed_does_not_kill() { // A request that completed cleanly disarms the guard; dropping it must not // signal anything. - let mut child = tokio::process::Command::new("sleep") + let child = tokio::process::Command::new("sleep") .arg("300") .process_group(0) .spawn() .unwrap(); + let pid = child.id().unwrap() as i32; { let mut guard = KillGroupOnDrop { - pgid: child.id().map(|id| id as i32), + child: Some(child), + pgid: Some(pid), + admission: None, }; guard.disarm(); - } // disarmed -> no kill + } // disarmed -> no reaper, no kill - assert!( - child.try_wait().unwrap().is_none(), - "disarmed guard must not kill the child" - ); + // Give any erroneously-spawned reaper a chance to run, then assert alive. + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + assert!(alive(pid), "disarmed guard must not kill the child"); - // Clean up the still-running child. - let _ = child.kill().await; - let _ = child.wait().await; + // Clean up the still-running child (disarm dropped the handle, so reap by pid). + unsafe { + libc::kill(pid, libc::SIGKILL); + let mut status = 0; + libc::waitpid(pid, &mut status, 0); + } } // ── #62 PR1: end-to-end teardown wiring through run_git_service ───────── @@ -1120,6 +1418,66 @@ mod tests { ); } + /// True when `err` is the transient ETXTBSY exec race a freshly-written fake + /// `git` hits under fork-storm load — a concurrent worker forked while this + /// file's write fd was still open, so `execve` sees it as busy (the same race + /// `fake_git_run_with_pids` retries). Deliberately narrow: only this raw-OS + /// error is retried, so a wrong error *type* (e.g. a missing async bound + /// surfacing as anything other than `GitServiceTimeout`) still fails loudly at + /// the caller's assertion instead of being swallowed. `spawn()?` propagates the + /// `io::Error` with no context, so it survives as the anyhow root. + #[cfg(unix)] + fn is_transient_exec_race(err: &anyhow::Error) -> bool { + // ETXTBSY == 26 on Linux and the BSDs/macOS; std has no stable ErrorKind. + err.downcast_ref::() + .and_then(std::io::Error::raw_os_error) + == Some(26) + } + + /// Drive `build_filtered_pack` under a per-attempt watchdog, retrying ONLY the + /// transient ETXTBSY exec race and returning the terminal error for the caller + /// to classify. Every attempt keeps its own outer `tokio::time::timeout`, so a + /// MISSING async bound — the regression the `build_filtered_pack_times_out_*` + /// tests guard — still trips the watchdog loudly on every attempt: the retry + /// can only absorb a fast exec failure, never a hang (a hang never returns, so + /// it can never reach the retry decision). + #[cfg(unix)] + async fn build_filtered_pack_or_exec_race_retry( + git_bin: &str, + repo_path: &std::path::Path, + withheld: &HashSet, + stage_timeout: Duration, + ) -> anyhow::Error { + for i in 0..FAKE_GIT_RETRY_ATTEMPTS { + let result = tokio::time::timeout( + Duration::from_secs(10), + build_filtered_pack(git_bin, repo_path, withheld, stage_timeout, None), + ) + .await + .expect( + "build_filtered_pack must return within the watchdog — the git stage \ + must be timeout-bounded, not an uncancellable spawn_blocking", + ); + // `match` rather than `expect_err`: the Ok arm carries an + // AdmissionGuard, which has no Debug impl for expect_err to print. + let err = match result { + Ok(_) => panic!("a hung git stage must return an error, not hang"), + Err(e) => e, + }; + if is_transient_exec_race(&err) { + // Fresh-fake-git ETXTBSY: back off (growing) so a bursty fork-pressure + // spike subsides before retrying, per fake_git_run_with_pids. + tokio::time::sleep(Duration::from_millis(FAKE_GIT_BACKOFF_STEP_MS * (i + 1))).await; + continue; + } + return err; + } + panic!( + "fake git kept hitting ETXTBSY after {FAKE_GIT_RETRY_ATTEMPTS} attempts \ + (persistent exec failure, not a transient parallel-runner miss)" + ); + } + // Dropping the request future mid-flight (client disconnect) must SIGTERM the // whole group so git AND its pack-objects grandchild die together. Goes RED // if `process_group(0)` or the guard-arming is removed: without its own @@ -1154,6 +1512,7 @@ mod tests { tmp.path(), Bytes::new(), Duration::from_secs(60), + None, )); // Advance the future a slice at a time until the fake records its @@ -1225,6 +1584,601 @@ mod tests { ); } + /// #174 U2 (P1-c, RED-before/GREEN-after): on a client disconnect the teardown must + /// be as strong as the timeout path — a group member that IGNORES SIGTERM is still + /// SIGKILLed and reaped, not left running to accumulate past the concurrency cap. + /// The old `KillGroupOnDrop::drop` sent a lone SIGTERM with no escalation or reap, + /// so a SIGTERM-ignoring descendant survived the disconnect (RED). The fix launches + /// a detached reaper owning the child that runs the full TERM/grace/SIGKILL/reap. + /// This is distinct from the well-behaved-grandchild test above, whose `sleep` dies + /// on the first SIGTERM. + #[cfg(unix)] + #[tokio::test] + async fn run_git_service_sigkills_a_sigterm_ignoring_child_on_disconnect() { + let tmp = tempfile::TempDir::new().unwrap(); + let descfile = tmp.path().join("desc.pid"); + // Leader (dies on the group SIGTERM) spawns a descendant that traps SIGTERM, + // records its own pid, and loops ~30s (bounded so a RED run leaks no permanent + // orphan; the assertion fires well before then). + let body = format!( + "#!/bin/sh\n\ + sh -c 'trap \"\" TERM; echo $$ > \"{}\"; i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done' &\n\ + wait\n", + descfile.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + + let mut fut = Box::pin(run_git_service( + git_bin.to_str().unwrap(), + "git-upload-pack", + tmp.path(), + Bytes::new(), + Duration::from_secs(60), + None, + )); + // Drive the future a slice at a time until the descendant records its pid. + let mut desc: Option = None; + for _ in 0..500 { + let _ = tokio::time::timeout(Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&descfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + desc = Some(p); + break; + } + } + let desc = desc.expect("the fake leader must have spawned and recorded a descendant"); + let _cleanup = ReapOnPanic(vec![desc]); + assert!(alive(desc), "descendant should be running before the drop"); + + // Client disconnect: drop the request future. The guard's detached reaper must + // escalate SIGTERM -> SIGKILL and reap the SIGTERM-ignoring descendant. + drop(fut); + + let mut gone = false; + for _ in 0..500 { + if !alive(desc) { + gone = true; + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + // Kill regardless so a RED run leaks no orphan. + unsafe { + libc::kill(desc, libc::SIGKILL); + } + assert!( + gone, + "a SIGTERM-ignoring descendant (pid {desc}) must be SIGKILLed and reaped on \ + disconnect, not left running (old KillGroupOnDrop sent SIGTERM only)" + ); + } + + // #174 (SC3): the info/refs advertisement is now duration-bounded — previously + // it ran a bare `Command::output()` with no deadline, so a hung git pinned its + // concurrency slot forever. A hung fake git must abort with GitServiceTimeout + // (which the handler maps to 504). The outer watchdog turns a missing timeout + // into a loud failure instead of hanging the suite. info_refs shares + // drive_git_child, so its disconnect/group-teardown is the same code proven by + // run_git_service_tears_down_group_when_future_dropped above. + #[cfg(unix)] + #[tokio::test] + async fn info_refs_times_out_a_hung_advertisement() { + let tmp = tempfile::TempDir::new().unwrap(); + // A fake git that hangs forever instead of advertising refs. + let git_bin = write_fake_git(tmp.path(), "#!/bin/sh\nsleep 300\n"); + + let result = tokio::time::timeout( + Duration::from_secs(10), + info_refs( + git_bin.to_str().unwrap(), + "git-upload-pack", + tmp.path(), + Duration::from_millis(200), + None, + ), + ) + .await + .expect("info_refs must return within the watchdog — its own timeout must fire"); + let err = result.expect_err("a hung advertisement must return an error, not hang"); + assert!( + err.downcast_ref::().is_some(), + "a hung advertisement must abort with GitServiceTimeout (-> 504), got: {err}" + ); + } + + // #174 (SC3, KTD5): the filtered-pack build's streaming pack-objects stage is + // duration-bounded on the ASYNC side. The old build ran the whole thing in a + // spawn_blocking, so an outer tokio timeout could not cancel the blocking thread + // and a disconnect orphaned the git child. A fake git that returns objects fast + // on rev-list but hangs on pack-objects must now abort with GitServiceTimeout; + // the watchdog turns a missing bound into a loud failure. The stage runs under + // drive_git_child, so its disconnect/group-teardown is the same code proven by + // run_git_service_tears_down_group_when_future_dropped. + #[cfg(unix)] + #[tokio::test] + async fn build_filtered_pack_times_out_a_hung_pack_objects() { + let tmp = tempfile::TempDir::new().unwrap(); + // rev-list returns one oid fast; pack-objects hangs forever. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) echo deadbeefdeadbeefdeadbeefdeadbeefdeadbeef ;;\n pack-objects) sleep 300 ;;\n *) exit 1 ;;\nesac\n"; + let git_bin = write_fake_git(tmp.path(), body); + let withheld = HashSet::new(); + + // Retry only the transient ETXTBSY exec race (see the helper); the + // per-attempt watchdog keeps the missing-bound regression loud. + let err = build_filtered_pack_or_exec_race_retry( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_millis(200), + ) + .await; + assert!( + err.downcast_ref::().is_some(), + "a hung pack-objects must abort with GitServiceTimeout, got: {err}" + ); + } + + // #174 (SC3, KTD5): the filtered-pack build's rev-list ENUMERATION stage is now + // duration-bounded on the ASYNC side too. It previously ran inside a + // spawn_blocking via a bare `Command::output()`, which an outer tokio timeout + // cannot cancel, so a hung/slow rev-list pinned the endpoint's concurrency permit + // for the whole hang. A fake git that hangs on rev-list must now abort with + // GitServiceTimeout; the watchdog turns a missing bound into a loud failure. + // rev-list runs under drive_git_child, so its disconnect/group-teardown is the + // same code proven by run_git_service_tears_down_group_when_future_dropped. + #[cfg(unix)] + #[tokio::test] + async fn build_filtered_pack_times_out_a_hung_rev_list() { + let tmp = tempfile::TempDir::new().unwrap(); + // rev-list hangs forever; pack-objects would return fast if it were reached. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) sleep 300 ;;\n pack-objects) printf '' ;;\n *) exit 1 ;;\nesac\n"; + let git_bin = write_fake_git(tmp.path(), body); + let withheld = HashSet::new(); + + // Retry only the transient ETXTBSY exec race (see the helper); the + // per-attempt watchdog keeps the missing-bound regression loud. + let err = build_filtered_pack_or_exec_race_retry( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_millis(200), + ) + .await; + assert!( + err.downcast_ref::().is_some(), + "a hung rev-list must abort with GitServiceTimeout, got: {err}" + ); + } + + // #174 U1 (R2, KTD3, RED-before/GREEN-after): the path-scoped filtered-pack serve + // must hold read + per-caller admission until its pack-objects process group is + // reaped on a client disconnect, exactly as the plain upload_pack path does. Before + // the fix build_filtered_pack took no AdmissionGuard and the handler's `_hold` + // permits dropped the instant the request future was dropped, so disconnect-spam on + // a path-scoped repo could hold PIDs past the concurrency cap while the permits were + // already free (#174 P1-a, on the filtered path the plain path had already closed). + // + // A real AdmissionGuard built from two owned semaphore permits rides + // rev-list -> pack-objects. We drive the future until pack-objects has forked its + // grandchild (the streaming pack-writer stand-in), assert the permits are still held + // mid-serve, then DROP the future (client disconnect) and assert the permits are + // released only AFTER the group is ESRCH-confirmed gone — never while it is alive. + // Goes RED if the guard is not threaded into the pack-objects stage (it would then + // drop after rev-list, freeing the permits mid-serve or on the bare future drop). + #[cfg(unix)] + #[tokio::test] + async fn filtered_pack_holds_admission_until_group_reaped_on_disconnect() { + use std::sync::Arc; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let pidfile = tmp.path().join("pids"); + // rev-list returns one oid fast; pack-objects forks a grandchild (the streaming + // writer stand-in), records leader+grandchild pids, then hangs so the future + // parks mid-serve with the guard owned by the pack-objects KillGroupOnDrop. The + // grandchild inherits (holds open) the stdout pipe, so drive_git_child's + // read_to_end blocks and the future stays pending until we drop it. + let body = format!( + "#!/bin/sh\ncase \"$1\" in\n rev-list) echo deadbeefdeadbeefdeadbeefdeadbeefdeadbeef ;;\n pack-objects) sleep 300 &\nprintf '%s\\n%s\\n' \"$$\" \"$!\" > \"{}\"\nwait ;;\n *) exit 1 ;;\nesac\n", + pidfile.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + let withheld = HashSet::new(); + + // Retry the fake-git spawn race like the sibling disconnect test; each attempt + // gets a FRESH semaphore so a dropped losing attempt can't skew the winning + // attempt's permit accounting. Keep the winning attempt's future PENDING so the + // drop below exercises the client-disconnect teardown. + let (fut, sem, leader, grandchild) = { + let mut attempt = 0u64; + loop { + attempt += 1; + let _ = std::fs::remove_file(&pidfile); + // Semaphore(4): two owned permits model the handler's global-read + + // per-caller admission, leaving 2 available while the op is in flight. + let sem = Arc::new(Semaphore::new(4)); + let g = sem.clone().try_acquire_owned().unwrap(); + let c = sem.clone().try_acquire_owned().unwrap(); + let admission = AdmissionGuard::new(g, Some(c)); + let mut fut = Box::pin(build_filtered_pack( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_secs(60), + Some(admission), + )); + // Advance the future a slice at a time until the fake records its pids + // (i.e. pack-objects is running). `Ok(_)` means the future returned + // before the pidfile appeared (spawn error / early exit); stop polling + // then, since re-polling a completed future panics. + let mut pids = None; + for _ in 0..500 { + let finished = tokio::time::timeout(Duration::from_millis(10), &mut fut) + .await + .is_ok(); + if let Some(p) = read_two_pids(&pidfile) { + pids = Some(p); + break; + } + if finished { + break; + } + } + match pids { + Some((l, gch)) => break (fut, sem, l, gch), + None => { + // Transient spawn miss: drop the still-armed future so its guard + // reaps anything that spawned (and returns the permits), then + // back off before retrying. + drop(fut); + assert!( + attempt < FAKE_GIT_RETRY_ATTEMPTS, + "fake git failed to reach the pack-objects stage after \ + {FAKE_GIT_RETRY_ATTEMPTS} attempts (persistent failure, \ + not a transient parallel-runner miss)" + ); + tokio::time::sleep(Duration::from_millis( + FAKE_GIT_BACKOFF_STEP_MS * attempt, + )) + .await; + } + } + } + }; + let _cleanup = ReapOnPanic(vec![leader, grandchild]); + assert!(alive(grandchild), "grandchild must be running mid-serve"); + + // Mid-serve: the pack-objects stage owns the guard, so the two permits are still + // held. A build that dropped the guard after rev-list (unthreaded pack-objects + // stage) would have freed them here. + assert_eq!( + sem.available_permits(), + 2, + "admission permits must be held while the filtered serve is in flight" + ); + + // Client disconnect: drop the request future. The pack-objects KillGroupOnDrop + // must tear the group down AND hold the permits until the group is + // ESRCH-confirmed gone, releasing them only then. + drop(fut); + + let mut released_while_alive = false; + let mut released_after_reap = false; + for _ in 0..500 { + let released = sem.available_permits() == 4; + let group_alive = alive(grandchild); + if released && group_alive { + released_while_alive = true; + } + if released && !group_alive { + released_after_reap = true; + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert!( + !released_while_alive, + "admission permits were released while the git group was still alive — the \ + path-scoped concurrency-cap bypass (#174 P1-a) is open" + ); + assert!( + released_after_reap, + "admission permits must be released once the group is reaped on disconnect" + ); + } + + // ── F1: filtered-serve admission threaded through both pack stages ────── + // + // The handler-level disconnect regression lives in api/repos.rs + // (upload_pack_filtered_permit_held_through_group_reap_after_disconnect); these + // exercise the guard contract at the smart_http seam: held through a + // mid-rev-list disconnect, handed back on success, and recovered (post-reap) + // on every error return. + + /// F1: a client disconnect mid-REV-LIST (stage 1 of the filtered build) must + /// keep the threaded admission held until the rev-list group is reaped, and + /// stage 2 (pack-objects) must never spawn, since the dropped future cannot + /// advance to it. + #[cfg(unix)] + #[tokio::test] + async fn build_filtered_pack_holds_admission_through_rev_list_reap_on_disconnect() { + let tmp = tempfile::TempDir::new().unwrap(); + let descfile = tmp.path().join("desc.pid"); + let packfile = tmp.path().join("pack.ran"); + // rev-list: SIGTERM-trapping descendant records its pid and loops (bounded + // so a broken fix leaks no permanent orphan); the leader waits, keeping the + // interaction pending. pack-objects: records that it ran (it must never run). + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + rev-list)\n\ + sh -c 'trap \"\" TERM; echo $$ > \"{desc}\"; i=0; while [ $i -lt 20 ]; do sleep 1; i=$((i+1)); done' &\n\ + wait ;;\n\ + pack-objects) : > \"{pack}\" ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + desc = descfile.display(), + pack = packfile.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + let guard = AdmissionGuard::new(sem.clone().try_acquire_owned().unwrap(), None::<()>); + let withheld = HashSet::new(); + + let mut fut = Box::pin(build_filtered_pack( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_secs(60), + Some(guard), + )); + // Drive until the rev-list descendant records its pid. + let mut desc: Option = None; + for _ in 0..500 { + let _ = tokio::time::timeout(Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&descfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + desc = Some(p); + break; + } + } + let desc = desc.expect("the fake rev-list must have spawned its descendant"); + let _cleanup = ReapOnPanic(vec![desc]); + assert_eq!( + sem.available_permits(), + 0, + "admission held while rev-list runs" + ); + + // Client disconnect mid-enumeration. + drop(fut); + + // Load-bearing: the permit must stay held while the SIGTERM-ignoring group + // member is still alive (check before the reaper's ~2s SIGKILL escalation). + tokio::time::sleep(Duration::from_millis(100)).await; + assert!( + alive(desc), + "the SIGTERM-ignoring descendant must still be alive during the hold window" + ); + assert_eq!( + sem.available_permits(), + 0, + "on disconnect mid-rev-list the admission must be held until the group is \ + reaped, not released the instant the future drops (F1)" + ); + + let mut freed = false; + for _ in 0..400 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + assert!( + freed, + "the reaper must release admission once the rev-list group is gone" + ); + assert!( + !packfile.exists(), + "pack-objects must never spawn after a disconnect mid-rev-list" + ); + } + + /// F1 success path: a filtered serve that completes hands the guard back + /// through both stages and releases admission when the serve returns. No leak, + /// no double-release. + #[tokio::test] + async fn upload_pack_excluding_releases_admission_after_success() { + let td = TempDir::new().unwrap(); + let (_work, bare, secret_oid, _public_oid) = fixture_with_secret(&td); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + let guard = AdmissionGuard::new(sem.clone().try_acquire_owned().unwrap(), None::<()>); + let withheld = HashSet::from([secret_oid]); + let resp = upload_pack_excluding( + "git", + &bare, + Bytes::from_static(b"0000"), + &withheld, + Duration::from_secs(30), + Some(guard), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!( + sem.available_permits(), + 1, + "admission must be released once the filtered serve returns" + ); + } + + /// F1 error path: a rev-list that exits non-zero fails the build AND recovers + /// the threaded admission: drive_git_child drops the guard only after the + /// completed join has reaped the child, and the error return must not leak it. + #[cfg(unix)] + #[tokio::test] + async fn build_filtered_pack_releases_admission_after_rev_list_failure() { + let tmp = tempfile::TempDir::new().unwrap(); + let body = + "#!/bin/sh\ncase \"$1\" in\n rev-list) echo boom >&2; exit 2 ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + let withheld = HashSet::new(); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + + for i in 0..FAKE_GIT_RETRY_ATTEMPTS { + let guard = AdmissionGuard::new(sem.clone().try_acquire_owned().unwrap(), None::<()>); + let result = build_filtered_pack( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_secs(30), + Some(guard), + ) + .await; + let err = match result { + Ok(_) => panic!("a non-zero rev-list exit must fail the build"), + Err(e) => e, + }; + // Load-bearing on EVERY attempt: even a transient spawn-failure return + // must recover the permit (the guard drops before the Err surfaces). + assert_eq!( + sem.available_permits(), + 1, + "the admission permit must be recovered on the error return, not leaked" + ); + if is_transient_exec_race(&err) { + tokio::time::sleep(Duration::from_millis(FAKE_GIT_BACKOFF_STEP_MS * (i + 1))).await; + continue; + } + assert!( + err.to_string().contains("rev-list failed"), + "the failure must surface rev-list's own error, got: {err}" + ); + return; + } + panic!( + "fake git kept hitting ETXTBSY after {FAKE_GIT_RETRY_ATTEMPTS} attempts \ + (persistent exec failure, not a transient parallel-runner miss)" + ); + } + + /// F1: a pack-objects stage that times out (stage 2 of the filtered build, + /// guard threaded through) recovers the admission after the reap: the timeout + /// Err cannot carry the guard, so drive_git_child must drop it post-reap. + #[cfg(unix)] + #[tokio::test] + async fn build_filtered_pack_recovers_admission_after_pack_objects_timeout() { + let tmp = tempfile::TempDir::new().unwrap(); + // rev-list returns one oid fast (guard handed back); pack-objects hangs. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) echo deadbeefdeadbeefdeadbeefdeadbeefdeadbeef ;;\n pack-objects) sleep 300 ;;\n *) exit 1 ;;\nesac\n"; + let git_bin = write_fake_git(tmp.path(), body); + let withheld = HashSet::new(); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + + for i in 0..FAKE_GIT_RETRY_ATTEMPTS { + let guard = AdmissionGuard::new(sem.clone().try_acquire_owned().unwrap(), None::<()>); + let result = tokio::time::timeout( + Duration::from_secs(10), + build_filtered_pack( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_millis(200), + Some(guard), + ), + ) + .await + .expect( + "build_filtered_pack must return within the watchdog; the git stage \ + must be timeout-bounded", + ); + let err = match result { + Ok(_) => panic!("a hung pack-objects must fail the build"), + Err(e) => e, + }; + assert_eq!( + sem.available_permits(), + 1, + "admission must be recovered after the stage-2 timeout reap, not leaked" + ); + if is_transient_exec_race(&err) { + tokio::time::sleep(Duration::from_millis(FAKE_GIT_BACKOFF_STEP_MS * (i + 1))).await; + continue; + } + assert!( + err.downcast_ref::().is_some(), + "a hung pack-objects must abort with GitServiceTimeout, got: {err}" + ); + return; + } + panic!( + "fake git kept hitting ETXTBSY after {FAKE_GIT_RETRY_ATTEMPTS} attempts \ + (persistent exec failure, not a transient parallel-runner miss)" + ); + } + + /// F1 zero-budget contract: build_filtered_pack hands pack-objects + /// `deadline.saturating_duration_since(now)`, which saturates to ZERO once + /// rev-list has consumed the whole budget. A zero-budget stage must still route + /// the guard through the timeout's reap-then-drop rather than leak it: the + /// child spawns, the deadline fires immediately, the group is reaped, and only + /// then does the guard drop. + #[cfg(unix)] + #[tokio::test] + async fn drive_git_child_zero_budget_drops_admission_after_reap() { + let tmp = tempfile::TempDir::new().unwrap(); + let git_bin = write_fake_git(tmp.path(), "#!/bin/sh\nsleep 300\n"); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + + for i in 0..FAKE_GIT_RETRY_ATTEMPTS { + let guard = AdmissionGuard::new(sem.clone().try_acquire_owned().unwrap(), None::<()>); + let mut command = tokio::process::Command::new(git_bin.to_str().unwrap()); + command + .args(["pack-objects", "--stdout"]) + .current_dir(tmp.path()); + let result = tokio::time::timeout( + Duration::from_secs(10), + drive_git_child( + command, + Bytes::new(), + Duration::ZERO, + "pack-objects", + Some(guard), + ), + ) + .await + .expect("a zero-budget stage must abort via its own deadline, not hang"); + let err = match result { + Ok(_) => panic!("a zero-budget stage must return an error"), + Err(e) => e, + }; + assert_eq!( + sem.available_permits(), + 1, + "the guard must drop after the zero-budget timeout reap, not leak" + ); + if is_transient_exec_race(&err) { + tokio::time::sleep(Duration::from_millis(FAKE_GIT_BACKOFF_STEP_MS * (i + 1))).await; + continue; + } + assert!( + err.downcast_ref::().is_some(), + "a zero-budget stage must abort with GitServiceTimeout, got: {err}" + ); + return; + } + panic!( + "fake git kept hitting ETXTBSY after {FAKE_GIT_RETRY_ATTEMPTS} attempts \ + (persistent exec failure, not a transient parallel-runner miss)" + ); + } + // A request that runs to completion must DISARM the guard after reaping, so // no stray group SIGTERM fires. The fake exits non-zero (surfacing as Err) // but leaves a grandchild alive; the grandchild must survive. Goes RED if the @@ -1257,6 +2211,7 @@ mod tests { tmp.path(), Bytes::new(), Duration::from_secs(60), + None, ) }) .await; @@ -1301,6 +2256,7 @@ mod tests { tmp.path(), Bytes::new(), Duration::from_secs(60), + None, ) }) .await; @@ -1351,6 +2307,7 @@ mod tests { tmp.path(), Bytes::new(), git_timeout, + None, ), ) }) @@ -1425,6 +2382,7 @@ mod tests { tmp.path(), Bytes::new(), git_timeout, + None, ), ) }) @@ -1462,6 +2420,7 @@ mod tests { tmp.path(), Bytes::new(), Duration::from_millis(200), + None, ), ) .await; @@ -1502,6 +2461,7 @@ mod tests { tmp.path(), big, Duration::from_secs(60), + None, ) .await; diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index 229ee695..33763db2 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -271,7 +271,8 @@ pub struct TreeEntry { /// `/ipfs/` is computed from these same content bytes via /// `gitlawb_core::cid::Cid::from_git_object_bytes`. /// -/// Get just the object type. Returns `None` if the object doesn't exist. +/// Get just the object type. Returns `None` if the object doesn't exist; a +/// probe that could not examine the object store is `Err`, never `None`. pub fn object_type(repo_path: &Path, sha256_hex: &str) -> Result> { let type_output = Command::new("git") .args(["cat-file", "-t", sha256_hex]) @@ -280,6 +281,19 @@ pub fn object_type(repo_path: &Path, sha256_hex: &str) -> Result> .context("failed to run git cat-file -t")?; if !type_output.status.success() { + // A nonzero exit is an ABSENCE verdict only when git could examine the + // object store: missing-object and invalid-oid probes die with a single + // clean `fatal:` line. A broken repo dir (`fatal: not a git repository`) + // or a corrupt object (`error: inflate` / `error: unable to unpack` + // lines before the fatal) proves nothing about absence, so it must + // surface as Err — the /ipfs scan taints on Err rather than treating + // the repo as probed-clean. + let stderr = String::from_utf8_lossy(&type_output.stderr); + if stderr.contains("not a git repository") + || stderr.lines().any(|l| l.starts_with("error:")) + { + bail!("git cat-file -t failed: {}", stderr.trim()); + } return Ok(None); } @@ -306,6 +320,279 @@ pub fn read_object_content(repo_path: &Path, sha256_hex: &str, obj_type: &str) - Ok(content_output.stdout) } +/// Why an `/ipfs` existence probe could not return an absence verdict (#174 F5/U4). +/// The caller (`api::ipfs`) maps the variant to an HTTP status, and the split exists +/// only for that mapping: a `Transient` fault is retryable (503), a `Deterministic` +/// fault is terminal (500). +/// +/// The discriminator is object-store readability, NOT any English `git` wording, so a +/// future `git` message change cannot silently reclassify a fault (KTD-4): if the +/// store cannot be read (an unreadable or mid-repack pack, a removed `objects/` dir, +/// a permissions fault) the fault may clear on its own -> `Transient`; if the store IS +/// readable yet `git` still fails (a corrupt repo, a bad `.git/config`) a retry cannot +/// fix it -> `Deterministic`, so a conformant client is told not to retry-storm a +/// fresh `git cat-file` per attempt against a persistently broken repo. +#[derive(Debug)] +pub enum ProbeError { + /// Retryable (-> 503): the object store could not be read right now. + Transient(anyhow::Error), + /// Terminal (-> 500): a persistent, deterministic fault a retry cannot fix. + Deterministic(anyhow::Error), +} + +impl std::fmt::Display for ProbeError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ProbeError::Transient(e) => write!(f, "transient probe fault: {e}"), + ProbeError::Deterministic(e) => write!(f, "deterministic probe fault: {e}"), + } + } +} + +impl std::error::Error for ProbeError {} + +/// Structured outcome of one `git cat-file --batch-check` existence probe. +enum BatchProbe { + /// Present: git printed ` ` on exit 0. + Present(String), + /// A structured, CLEAN absence: git printed ` missing` on exit 0 with no + /// `error:` diagnostics on stderr. This is the ONLY signal that can become an + /// `Ok(None)` (404) absence verdict — and even then only after the store-readable + /// disambiguation below, since an unreadable pack ALSO prints a clean `missing`. + Missing, + /// git could not honestly examine the object store: a hard exit (bad config, not a + /// git repo) OR an exit-0 `missing` accompanied by `error:` diagnostics (a corrupt + /// loose object still prints `missing` on stdout but complains on stderr). Never an + /// absence verdict. Carries the raw git detail for the server log only. + Fault(anyhow::Error), +} + +/// One bounded, reaped `git cat-file --batch-check` probe (the oid is fed on stdin, so +/// no prose is matched to decide presence — the `missing` token and the exit code are +/// the structured signal). See [`object_type_bounded`] for the surrounding teardown +/// guarantees. +fn batch_check_probe( + git_bin: &str, + repo_path: &Path, + sha256_hex: &str, + deadline: std::time::Instant, +) -> std::result::Result { + let stdin = format!("{sha256_hex}\n"); + let (status, stdout, stderr) = crate::git::visibility_pack::run_bounded_git_raw( + git_bin, + &["cat-file", "--batch-check"], + repo_path, + stdin.as_bytes(), + deadline, + ) + // A spawn/timeout failure of the reaped child is not deterministic — retry it. + .map_err(ProbeError::Transient)?; + + let stderr = String::from_utf8_lossy(&stderr); + // A corrupt object makes `--batch-check` print `missing` on stdout (exit 0) yet + // emit `error:` lines on stderr; those diagnostics disqualify a clean-absence read + // regardless of the exit code, so they are checked before anything else. + let has_error_diag = stderr.lines().any(|l| l.starts_with("error:")); + + if status.success() && !has_error_diag { + let line = String::from_utf8_lossy(&stdout); + let line = line.trim(); + // ` missing` is the structured absence token; ` ` is a + // hit. Anything else on a "success" exit is unexpected and not an absence. + if line + .rsplit(' ') + .next() + .is_some_and(|last| last == "missing") + { + return Ok(BatchProbe::Missing); + } + let mut parts = line.split_whitespace(); + if let (Some(_oid), Some(ty), Some(_size)) = (parts.next(), parts.next(), parts.next()) { + return Ok(BatchProbe::Present(ty.to_string())); + } + return Ok(BatchProbe::Fault(anyhow::anyhow!( + "unexpected git cat-file --batch-check output: {line:?}" + ))); + } + + Ok(BatchProbe::Fault(anyhow::anyhow!( + "git cat-file --batch-check failed (exit {:?}): {}", + status.code(), + stderr.trim() + ))) +} + +/// Bounded, reaped variant of [`object_type`] for the async `/ipfs` serve path +/// (#174 F3/F5, #173 round-10 R1/KTD2): runs `git cat-file --batch-check` off the +/// caller's runtime through the process-group + watchdog reaper (SIGTERM -> grace -> +/// SIGKILL), so a hung or corrupt object store cannot pin a runtime worker or the +/// caller's held /ipfs walk admission past `timeout`. The bare [`object_type`] is a +/// `spawn_blocking` `Command::output` that an async timeout cannot cancel, so a wedged +/// `cat-file` there hangs for as long as git does; this twin cannot. +/// +/// Absence is keyed on `--batch-check`'s STRUCTURED ` missing` token on exit 0, +/// never on any English `fatal:` wording (KTD-4): a genuinely-absent object is the only +/// `Ok(None)` (404) path. A probe that could not honestly examine the store is a +/// [`ProbeError`], split by object-store readability into `Transient` (retryable 503) +/// and `Deterministic` (terminal 500) so the serve path can shed the right status. The +/// deadline itself arrives as a `Transient` fault, so the handler marks the search +/// truncated rather than reporting a false not-found. +pub fn object_type_bounded( + git_bin: &str, + repo_path: &Path, + sha256_hex: &str, + timeout: std::time::Duration, +) -> std::result::Result, ProbeError> { + let probe_started = std::time::Instant::now(); + let deadline = probe_started + timeout; + match batch_check_probe(git_bin, repo_path, sha256_hex, deadline)? { + BatchProbe::Present(ty) => Ok(Some(ty)), + BatchProbe::Fault(detail) => Err(classify_store_fault(repo_path, detail)), + BatchProbe::Missing => { + // A clean `missing` is the absence-vs-unreadable-pack COLLISION (#174 F5): + // a genuinely missing object AND a packed object whose pack/idx is + // unreadable (permissions, or a mid-repack race) both print an identical + // clean `missing`. Disambiguate OUT OF BAND on store readability — an + // unreadable store is not an absence verdict (taint -> retryable 503). + if !object_store_readable(repo_path) { + return Err(ProbeError::Transient(anyhow::anyhow!( + "git cat-file inconclusive: object store not readable at {} (not an absence verdict)", + repo_path.display() + ))); + } + // Only re-probe if the budget can actually pay for it. The re-probe runs the + // SAME command, so it needs roughly what the first probe took; with less than + // that left, the child is spawned only to be reaped, and the watchdog's + // SIGTERM grace plus SIGKILL settle carries this call well past `deadline` + // (measured ~2x a 1s budget before this check existed). `/ipfs/{cid}` is + // anon-reachable and an absent CID drives this branch once per repo, so an + // unaffordable re-probe is pure overshoot on a permissionless path. An + // inconclusive disambiguation is NOT an absence verdict, so taint to a + // retryable Transient rather than spawn or return a false Ok(None). + let first_probe_took = probe_started.elapsed(); + if deadline.saturating_duration_since(std::time::Instant::now()) < first_probe_took { + return Err(ProbeError::Transient(anyhow::anyhow!( + "git cat-file inconclusive: no budget left for the confirming re-probe at {} (not an absence verdict)", + repo_path.display() + ))); + } + // Store readable and budget available: re-probe once. Still `missing` on a + // confirmed-readable store is very likely truly absent (Ok(None)); a + // mid-repack race that resolved returns the type. This narrows, but cannot + // fully close, the concurrent-repack window (the readability check samples a + // different instant than the failing probe). + match batch_check_probe(git_bin, repo_path, sha256_hex, deadline)? { + BatchProbe::Present(ty) => Ok(Some(ty)), + BatchProbe::Fault(detail) => Err(classify_store_fault(repo_path, detail)), + BatchProbe::Missing => Ok(None), + } + } + } +} + +/// Classify a probe fault by object-store readability (#174 F5/U4). An unreadable store +/// may be a transient permissions/mid-repack condition (retryable 503); a readable +/// store on which git still fails is a persistent, deterministic fault — a corrupt repo +/// or a bad `.git/config` — that a retry cannot fix (terminal 500). The `detail` is +/// carried for the server log; the client-facing body is opaque (set by the caller). +fn classify_store_fault(repo_path: &Path, detail: anyhow::Error) -> ProbeError { + if object_store_readable(repo_path) { + ProbeError::Deterministic(detail) + } else { + ProbeError::Transient(detail) + } +} + +/// Best-effort check that a repo's object store is readable, used to disambiguate a +/// genuine missing-object `git cat-file` fatal from an unreadable or racing pack +/// (both emit "could not get object info"). Returns false on any unreadable +/// `objects/` dir or any pack/idx that cannot be opened (EACCES / EIO), so the +/// caller surfaces an error rather than a false absence. Cheap — a couple of readdir +/// plus open probes. It narrows, but does not close, the concurrent-repack TOCTOU: it +/// samples a different instant than the failing cat-file. +fn object_store_readable(repo_path: &Path) -> bool { + let objects = repo_path.join("objects"); + // The objects dir itself must be listable; drain the iterator so a mid-listing + // EACCES/EIO surfaces, not just the initial open. + let Ok(entries) = std::fs::read_dir(&objects) else { + return false; + }; + for entry in entries { + if entry.is_err() { + return false; + } + } + // Every pack file and its index must be openable for read. A loose-only store + // (no pack dir) is fine — the objects readdir above already proved reachability. + if let Ok(pack_entries) = std::fs::read_dir(objects.join("pack")) { + for entry in pack_entries { + let Ok(entry) = entry else { + return false; + }; + let path = entry.path(); + if matches!( + path.extension().and_then(|s| s.to_str()), + Some("pack") | Some("idx") + ) && std::fs::File::open(&path).is_err() + { + return false; + } + } + } + true +} + +/// Bounded `git cat-file -s` size read for the `GET /ipfs/{cid}` serve path (#173 +/// round-10, R1/KTD2): reads the object size WITHOUT its content (so an oversized object +/// is rejected before it is buffered, #173 F6), under +/// [`run_bounded_git`](crate::git::visibility_pack::run_bounded_git) so a wedged size +/// read is reaped at `timeout` instead of pinning the held /ipfs walk admission. +/// `Ok(Some(n))` on success, `Ok(None)` when the object is absent (a non-timeout +/// non-zero exit), `Err(GitServiceTimeout)` on the deadline. +pub fn object_size_bounded( + git_bin: &str, + repo_path: &Path, + sha256_hex: &str, + timeout: std::time::Duration, +) -> Result> { + let deadline = std::time::Instant::now() + timeout; + match crate::git::visibility_pack::run_bounded_git( + git_bin, + &["cat-file", "-s", sha256_hex], + repo_path, + b"", + deadline, + ) { + Ok(out) => Ok(String::from_utf8_lossy(&out).trim().parse::().ok()), + Err(e) if e.is::() => Err(e), + Err(_) => Ok(None), + } +} + +/// Bounded, reaped variant of [`read_object_content`] for the async `/ipfs` serve path +/// (#174 F3, #173 round-10 R1/KTD2): `git cat-file ` under +/// [`run_bounded_git`](crate::git::visibility_pack::run_bounded_git), which reaps a +/// wedged content read at `timeout` instead of letting it pin the held /ipfs walk +/// admission. Same teardown guarantees as [`object_type_bounded`]. Returns the raw +/// object bytes on success and an error (including `GitServiceTimeout` on the deadline) +/// otherwise, mirroring [`read_object_content`]. +pub fn read_object_content_bounded( + git_bin: &str, + repo_path: &Path, + sha256_hex: &str, + obj_type: &str, + timeout: std::time::Duration, +) -> Result> { + let deadline = std::time::Instant::now() + timeout; + crate::git::visibility_pack::run_bounded_git( + git_bin, + &["cat-file", obj_type, sha256_hex], + repo_path, + b"", + deadline, + ) +} + /// Read a git object by its SHA-256 hex object ID. /// /// Returns `(object_type, content_bytes)` where `content_bytes` is the raw @@ -323,6 +610,50 @@ pub fn read_object(repo_path: &Path, sha256_hex: &str) -> Result Result)>> { + let deadline = std::time::Instant::now() + timeout; + // The probe's Transient/Deterministic split exists for the `/ipfs` serve path's HTTP + // status mapping; this write-side twin has no status to shed, so unwrap to the inner + // error. That keeps a deadline overrun downcastable to `GitServiceTimeout`, which the + // pin path checks for. + let probed = object_type_bounded( + git_bin, + repo_path, + sha256_hex, + deadline.saturating_duration_since(std::time::Instant::now()), + ) + .map_err(|e| match e { + ProbeError::Transient(inner) | ProbeError::Deterministic(inner) => inner, + })?; + let obj_type = match probed { + Some(t) => t, + None => return Ok(None), + }; + let content = read_object_content_bounded( + git_bin, + repo_path, + sha256_hex, + &obj_type, + deadline.saturating_duration_since(std::time::Instant::now()), + )?; + Ok(Some((obj_type, content))) +} + /// Get the diff between two branches: changes on source_branch not in target_branch. pub fn branch_diff(repo_path: &Path, target_branch: &str, source_branch: &str) -> Result { let output = Command::new("git") @@ -501,4 +832,474 @@ mod tests { "unchanged file must not appear: {names:?}" ); } + + /// #173 round-10 (KTD2): `object_type_bounded` reaps a wedged `cat-file` child at its + /// deadline instead of blocking on it to natural exit, so a hung probe cannot pin the + /// /ipfs walk admission the owning task holds. A fake `git` records its pid and sleeps + /// far past the 1s deadline; the `run_bounded_git` watchdog (SIGTERM -> grace -> + /// SIGKILL of the process group) must kill it well before that natural exit, and the + /// call must surface `GitServiceTimeout`. REVERT PROOF (RED): swap the twin's + /// `run_bounded_git` for the bare `Command::output()` and the wedged child stays alive + /// past the deadline — the mid-flight liveness poll below reads it still running. + #[cfg(unix)] + #[test] + fn object_type_bounded_reaps_wedged_child_at_deadline() { + use std::time::Duration; + let tmp = tempfile::TempDir::new().unwrap(); + let pidfile = tmp.path().join("catfile.pid"); + // `cat-file` records its own pid then sleeps 8s (>> the 1s deadline) so the probe + // is genuinely wedged; the watchdog is what must end it. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + cat-file) echo $$ > \"{}\"; sleep 8 ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + pidfile.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + let repo = tmp.path().to_path_buf(); + let git = git_path.to_str().unwrap().to_string(); + + let alive = |pid: i32| unsafe { libc::kill(pid, 0) == 0 }; + + // The bounded probe blocks until the watchdog tears the child down, so run it on + // a worker thread and poll for the reap from here. + let handle = std::thread::spawn(move || { + super::object_type_bounded(&git, &repo, "deadbeef", Duration::from_secs(1)) + }); + + let mut pid = None; + for _ in 0..500 { + if let Some(p) = std::fs::read_to_string(&pidfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + pid = Some(p); + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + let pid = pid.expect("the fake cat-file must have spawned and recorded its pid"); + + // Past the 1s deadline + SIGTERM grace but well before the 8s natural exit: the + // watchdog must already have reaped the wedged group. A bare, unbounded read would + // leave it running here — the load-bearing RED. + std::thread::sleep(Duration::from_secs(3)); + let reaped = !alive(pid); + // Defensive reap so a RED run leaks no orphan. + unsafe { + libc::kill(pid, libc::SIGKILL); + } + assert!( + reaped, + "object_type_bounded must reap the wedged cat-file child at the deadline, \ + not leave it running to its natural exit" + ); + + let res = handle.join().expect("probe thread joins"); + let err = res.expect_err("a deadline overrun must be an error, not a value"); + // A reaped deadline is a retryable fault, not a verdict about the object, so it + // must arrive as Transient (-> 503) carrying GitServiceTimeout. + let super::ProbeError::Transient(inner) = &err else { + panic!("a deadline overrun must be a Transient probe fault, got: {err:?}"); + }; + assert!( + inner.is::(), + "a deadline overrun must surface GitServiceTimeout, got: {inner:?}" + ); + } + + /// #174 F5 (RED-before/GREEN-after): a packed object whose pack/idx is unreadable + /// makes `git cat-file -t` emit "could not get object info" — byte-identical to a + /// genuine miss. `object_type_bounded` must report absence ONLY when the object + /// store is confirmed readable; an unreadable store is Err (-> retryable 503), + /// never Ok(None) (-> a wrong 404 for a present object). + #[cfg(unix)] + #[test] + fn object_type_bounded_unreadable_pack_is_error_not_absence() { + use std::os::unix::fs::PermissionsExt; + let td = tempfile::TempDir::new().unwrap(); + let work = td.path().join("work"); + let bare = td.path().join("bare.git"); + std::fs::create_dir_all(&work).unwrap(); + let g = |args: &[&str], dir: &Path| { + assert!( + Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + g(&["init", "-q", "--object-format=sha256", "."], &work); + g(&["config", "user.email", "t@t"], &work); + g(&["config", "user.name", "t"], &work); + std::fs::write(work.join("file.txt"), b"packed f5 content\n").unwrap(); + g(&["add", "file.txt"], &work); + g(&["commit", "-qm", "c1"], &work); + let blob = String::from_utf8( + Command::new("git") + .args(["rev-parse", "HEAD:file.txt"]) + .current_dir(&work) + .output() + .unwrap() + .stdout, + ) + .unwrap() + .trim() + .to_string(); + g( + &[ + "clone", + "-q", + "--bare", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + td.path(), + ); + g(&["gc", "-q"], &bare); + + let budget = std::time::Duration::from_secs(10); + + // Readable store: the packed blob probes present; a genuine miss is Ok(None). + assert_eq!( + super::object_type_bounded("git", &bare, &blob, budget) + .unwrap() + .as_deref(), + Some("blob"), + "a packed blob on a readable store must probe present" + ); + assert!( + super::object_type_bounded("git", &bare, &"0".repeat(64), budget) + .unwrap() + .is_none(), + "a genuinely-absent object on a readable store must be Ok(None)" + ); + + // Make the pack unreadable: cat-file -t now emits the collided fatal for the + // PRESENT blob. It must surface as Err, not a false Ok(None). + let pack_dir = bare.join("objects").join("pack"); + let set_pack_mode = |mode: u32| { + for e in std::fs::read_dir(&pack_dir).unwrap() { + let p = e.unwrap().path(); + if matches!( + p.extension().and_then(|s| s.to_str()), + Some("pack") | Some("idx") + ) { + let mut perms = std::fs::metadata(&p).unwrap().permissions(); + perms.set_mode(mode); + std::fs::set_permissions(&p, perms).unwrap(); + } + } + }; + set_pack_mode(0o000); + // Root bypasses file permissions, so the chmod won't block reads there; only + // assert the error path when the pack is genuinely unreadable to this process. + let a_pack = std::fs::read_dir(&pack_dir) + .unwrap() + .filter_map(|e| e.ok()) + .map(|e| e.path()) + .find(|p| p.extension().and_then(|s| s.to_str()) == Some("pack")); + let genuinely_unreadable = a_pack + .as_ref() + .map(|p| std::fs::File::open(p).is_err()) + .unwrap_or(false); + let res = super::object_type_bounded("git", &bare, &blob, budget); + set_pack_mode(0o644); // restore so TempDir cleanup succeeds + + if genuinely_unreadable { + assert!( + res.is_err(), + "an unreadable pack must surface as Err (-> retryable 503), not Ok(None) \ + (-> a wrong 404 for a present object); got {res:?}" + ); + } + } + + /// Shared setup: a bare sha256 repo carrying one committed blob. Returns the repo + /// path and the blob's oid. + #[cfg(unix)] + fn bare_repo_with_blob(td: &std::path::Path) -> (std::path::PathBuf, String) { + let work = td.join("work"); + let bare = td.join("bare.git"); + std::fs::create_dir_all(&work).unwrap(); + let g = |args: &[&str], dir: &Path| { + assert!( + Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + g(&["init", "-q", "--object-format=sha256", "."], &work); + g(&["config", "user.email", "t@t"], &work); + g(&["config", "user.name", "t"], &work); + std::fs::write(work.join("file.txt"), b"f5 u4 content\n").unwrap(); + g(&["add", "file.txt"], &work); + g(&["commit", "-qm", "c1"], &work); + let blob = String::from_utf8( + Command::new("git") + .args(["rev-parse", "HEAD:file.txt"]) + .current_dir(&work) + .output() + .unwrap() + .stdout, + ) + .unwrap() + .trim() + .to_string(); + g( + &[ + "clone", + "-q", + "--bare", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + td, + ); + (bare, blob) + } + + /// #174 F5/U4 (RED-before/GREEN-after, the CORE regression guard): a repo with a + /// corrupt `.git/config` makes `git cat-file` die with `fatal: bad config line N` + /// (exit 128, NO `error:` line) while `objects/` stays fully readable. The old + /// `-t` path let that fall through to `Ok(None)` — a false 404 for content that + /// may well exist. `object_type_bounded` must instead classify it as a + /// DETERMINISTIC fault (a retry cannot fix it) so the serve path renders a + /// terminal 500, never a 404 and never a retryable 503. + /// + /// LOAD-BEARING: revert the classification (route `BatchProbe::Fault` on a readable + /// store back to `Ok(None)`, or drop the `has_error_diag`/exit checks so a hard + /// `fatal:` is read as `missing`) and this goes RED — the probe reports the corrupt + /// repo as an absent object. + #[cfg(unix)] + #[test] + fn object_type_bounded_bad_config_is_deterministic_not_absence() { + let td = tempfile::TempDir::new().unwrap(); + let (bare, blob) = bare_repo_with_blob(td.path()); + let budget = std::time::Duration::from_secs(10); + + // Baseline on the healthy repo: present blob probes present, genuine miss is + // the clean Ok(None) absence verdict. + assert_eq!( + super::object_type_bounded("git", &bare, &blob, budget) + .unwrap() + .as_deref(), + Some("blob"), + "a present blob on a healthy readable store must probe present" + ); + assert!( + super::object_type_bounded("git", &bare, &"0".repeat(64), budget) + .unwrap() + .is_none(), + "a genuinely-absent object on a healthy readable store must be Ok(None) (404)" + ); + + // Corrupt the config; objects/ is untouched (and stays readable). + { + use std::io::Write; + let mut cfg = std::fs::OpenOptions::new() + .append(true) + .open(bare.join("config")) + .unwrap(); + cfg.write_all(b"\n[broken section\nnot a valid = = = line\n") + .unwrap(); + } + assert!( + super::object_store_readable(&bare), + "config corruption must leave objects/ readable (that is the whole point: \ + a readable store + a git failure == deterministic, not transient)" + ); + + // Probing the PRESENT blob under the bad config must be a DETERMINISTIC fault, + // never Ok(None) (the old false 404) and never a Transient (retryable 503). + let res = super::object_type_bounded("git", &bare, &blob, budget); + assert!( + matches!(res, Err(super::ProbeError::Deterministic(_))), + "a bad-config fatal on a readable store must be a terminal Deterministic \ + fault (-> 500), never Ok(None) (-> false 404) or Transient (-> 503); got {res:?}" + ); + // And a genuinely-absent oid under the bad config is ALSO not an absence verdict. + let res_absent = super::object_type_bounded("git", &bare, &"0".repeat(64), budget); + assert!( + matches!(res_absent, Err(super::ProbeError::Deterministic(_))), + "even a would-be-absent oid must not read as Ok(None) once the config is \ + corrupt; got {res_absent:?}" + ); + } + + /// #174 F5/U4: a corrupt LOOSE object makes `git cat-file --batch-check` print + /// ` missing` on stdout (exit 0) yet emit `error:` diagnostics on stderr. The + /// clean-`missing` absence path must NOT fire here — the `error:` line disqualifies + /// a clean-absence read — so the probe surfaces a fault, not a false Ok(None) 404. + /// The object store is readable (a corrupt object file still opens), so this is a + /// Deterministic fault. LOAD-BEARING: drop the `has_error_diag` guard and a corrupt + /// object reads as `missing` -> Ok(None) -> false 404 (RED). + #[cfg(unix)] + #[test] + fn object_type_bounded_corrupt_loose_object_is_fault_not_absence() { + use std::os::unix::fs::PermissionsExt; + let td = tempfile::TempDir::new().unwrap(); + let work = td.path().join("loose"); + std::fs::create_dir_all(&work).unwrap(); + let g = |args: &[&str]| { + assert!( + Command::new("git") + .args(args) + .current_dir(&work) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + g(&["init", "-q", "--object-format=sha256", "."]); + g(&["config", "user.email", "t@t"]); + g(&["config", "user.name", "t"]); + std::fs::write(work.join("f.txt"), b"loose object content\n").unwrap(); + g(&["add", "f.txt"]); + g(&["commit", "-qm", "c1"]); + let blob = String::from_utf8( + Command::new("git") + .args(["rev-parse", "HEAD:f.txt"]) + .current_dir(&work) + .output() + .unwrap() + .stdout, + ) + .unwrap() + .trim() + .to_string(); + + // Overwrite the loose object file with non-zlib garbage (it is 0o444 by default). + let obj = work.join(".git/objects").join(&blob[0..2]).join(&blob[2..]); + let mut perms = std::fs::metadata(&obj).unwrap().permissions(); + perms.set_mode(0o644); + std::fs::set_permissions(&obj, perms).unwrap(); + std::fs::write(&obj, b"garbage not a zlib stream").unwrap(); + + let budget = std::time::Duration::from_secs(10); + let res = super::object_type_bounded("git", &work, &blob, budget); + assert!( + res.is_err(), + "a corrupt loose object (error: on stderr, `missing` on stdout) must be Err, \ + never a false Ok(None) 404; got {res:?}" + ); + } + + /// #174 U1 follow-up (RED-before/GREEN-after): the absent-CID path must not spawn a + /// confirming re-probe it cannot afford. `object_type_bounded` disambiguates a clean + /// `missing` by re-running the probe, but the re-probe took the SAME deadline with no + /// check that any budget was left, so a first probe that nearly exhausted the budget + /// still spawned a second child that could only be reaped. The watchdog's SIGTERM + /// grace plus SIGKILL settle then carried the whole call to ~2x the budget, on a + /// route an unauthenticated caller drives for every repo by spraying absent CIDs. + /// + /// Load-bearing: remove the affordability check and this goes RED on both assertions + /// (a second spawn appears, and elapsed crosses 2x the budget). Measured before the + /// fix: spawns=2, elapsed 2021ms against a 1000ms budget. + #[cfg(unix)] + #[test] + fn absent_probe_skips_a_reprobe_it_cannot_afford() { + use std::os::unix::fs::PermissionsExt; + let td = tempfile::TempDir::new().unwrap(); + let bare = td.path().join("bare.git"); + std::fs::create_dir_all(bare.join("objects/pack")).unwrap(); + let log = td.path().join("spawns.log"); + let fake = td.path().join("fakegit"); + // Burns 0.9s of a 1s budget, then reports the structured absence token cleanly. + std::fs::write( + &fake, + format!( + "#!/bin/sh\necho call >> {}\nsleep 0.9\necho 'deadbeef missing'\nexit 0\n", + log.display() + ), + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + + let budget = std::time::Duration::from_millis(1000); + + let started = std::time::Instant::now(); + let res = super::object_type_bounded(fake.to_str().unwrap(), &bare, "deadbeef", budget); + let elapsed = started.elapsed(); + let spawns = std::fs::read_to_string(&log) + .map(|s| s.lines().count()) + .unwrap_or(0); + + assert_eq!( + spawns, 1, + "a re-probe with no remaining budget must not be spawned at all; a second \ + spawn here is a child created only to be reaped, and its teardown grace is \ + what pushes this call past the deadline" + ); + assert!( + elapsed < budget + std::time::Duration::from_millis(400), + "the call must not overshoot its deadline by the reap grace; got {elapsed:?} \ + against a {budget:?} budget (pre-fix this was ~2x the budget)" + ); + assert!( + matches!(res, Err(super::ProbeError::Transient(_))), + "an unaffordable disambiguation is NOT an absence verdict: it must taint to \ + a retryable Transient, never a false Ok(None) 404; got {res:?}" + ); + } + + /// Companion must-not-regress case for the affordability check above: with an ample + /// budget the confirming re-probe MUST still run, so the #174 F5 disambiguation is + /// intact and the check did not simply disable it. Two spawns, and a clean absence. + #[cfg(unix)] + #[test] + fn absent_probe_still_reprobes_when_the_budget_allows() { + use std::os::unix::fs::PermissionsExt; + let td = tempfile::TempDir::new().unwrap(); + let bare = td.path().join("bare.git"); + std::fs::create_dir_all(bare.join("objects/pack")).unwrap(); + let log = td.path().join("spawns.log"); + let fake = td.path().join("fakegit"); + std::fs::write( + &fake, + format!( + "#!/bin/sh\necho call >> {}\necho 'deadbeef missing'\nexit 0\n", + log.display() + ), + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + + let budget = std::time::Duration::from_secs(30); + let res = super::object_type_bounded(fake.to_str().unwrap(), &bare, "deadbeef", budget); + let spawns = std::fs::read_to_string(&log) + .map(|s| s.lines().count()) + .unwrap_or(0); + assert_eq!( + spawns, 2, + "with budget to spare the confirming re-probe must still run, or the \ + absence-vs-unreadable-pack disambiguation is gone" + ); + assert!( + matches!(res, Ok(None)), + "a clean `missing` twice on a readable store is a genuine absence; got {res:?}" + ); + } } diff --git a/crates/gitlawb-node/src/git/tigris.rs b/crates/gitlawb-node/src/git/tigris.rs index ad26ddc5..cf7abfd5 100644 --- a/crates/gitlawb-node/src/git/tigris.rs +++ b/crates/gitlawb-node/src/git/tigris.rs @@ -31,6 +31,25 @@ impl TigrisClient { }) } + /// Test-only constructor with an explicit S3 endpoint, region, and static + /// credentials — no env-var reads, so parallel tests cannot race each other's + /// `AWS_*` environment the way the env-based `new` would. Lets a test point + /// the client at a non-routable endpoint to exercise acquire-stall paths. + #[cfg(test)] + pub(crate) async fn for_testing_with_endpoint(bucket: &str, endpoint_url: &str) -> Self { + let creds = aws_sdk_s3::config::Credentials::new("test", "test", None, None, "test"); + let config = aws_config::defaults(aws_config::BehaviorVersion::latest()) + .endpoint_url(endpoint_url) + .region(aws_config::Region::new("auto")) + .credentials_provider(creds) + .load() + .await; + Self { + s3: S3Client::new(&config), + bucket: bucket.to_string(), + } + } + /// S3 key for a given repo: `repos/v1/{owner_slug}/{repo_name}.tar.zst` fn repo_key(owner_slug: &str, repo_name: &str) -> String { format!("repos/v1/{owner_slug}/{repo_name}.tar.zst") diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index cb70e39c..3f4315cd 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -4,11 +4,321 @@ //! content is held back. use crate::db::VisibilityRule; -use crate::git::store; use crate::visibility::{visibility_check, Decision}; use anyhow::{Context, Result}; use std::collections::{BTreeSet, HashMap, HashSet}; use std::path::Path; +use std::process::Stdio; +use std::sync::mpsc; +use std::time::{Duration, Instant}; + +/// Fixed budget bounding the whole withheld-blob classification walk (#174 U3). +/// The walk is fast for a real repo; this bound exists to reap a hung or +/// pathologically slow git child so it cannot pin a served-git permit (the read +/// permit on the upload-pack serve path, the write permit on the receive-pack +/// post-push replication path) past the deadline. Every caller funnels through +/// `blob_paths`, so bounding here bounds both paths at one seam. Production callers +/// pass the operator-configured `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` instead; this +/// fixed budget only backs the `git_bin`-less test wrappers. +#[cfg(test)] +const WALK_TIMEOUT: Duration = Duration::from_secs(600); + +/// How long the process-group watchdog waits after SIGTERM before escalating to +/// SIGKILL, giving a well-behaved git child time to clean up its `*.lock` files. Only +/// paid on a timeout (already the exceptional path). +#[cfg(unix)] +const WATCHDOG_TERM_GRACE: Duration = Duration::from_secs(1); + +/// Run one git child under a shared `deadline` with process-group teardown, +/// BLOCKING, and return its stdout. The child runs in its own process group; a +/// watchdog thread SIGTERMs (lets git clean up its `*.lock` files), then SIGKILLs, +/// the whole group if the deadline passes before the child is reaped, so a hung or +/// slow git can pin neither a served-git permit nor a blocking thread past the +/// deadline (jatmn's "retain admission until they are reaped"). This is the +/// blocking-side counterpart of `smart_http::drive_git_child`, needed because the +/// walk's callers run it inside `spawn_blocking`, which an async timeout cannot +/// cancel. Returns [`crate::git::smart_http::GitServiceTimeout`] on the deadline so +/// the serve handler maps it to 504. `git_bin` is injectable so a fake `git` can +/// drive the teardown in tests without mutating the process-global PATH; +/// `stdin_bytes` feeds children that read stdin (empty for the arg-only children). +/// Returns true if `pid` (a process-group leader we spawned) has terminated, WITHOUT +/// reaping it. `waitid(..., WNOWAIT)` reports the exit state but leaves the child +/// waitable, so the caller's later `child.wait()` still collects the status and the +/// pid/pgid stays live until then — which is what keeps the watchdog's `kill(-pgid)` +/// teardown from ever racing a recycled pgid. Used to distinguish "the child actually +/// exited" from "the child merely closed stdout" after the drain returns (#174 P1-a). +#[cfg(unix)] +fn child_terminated_without_reaping(pid: i32) -> bool { + // SAFETY: waitid writes only into the zeroed siginfo and borrows no Rust memory; + // WNOWAIT leaves the child unreaped, WNOHANG makes the probe non-blocking. + let mut info: libc::siginfo_t = unsafe { std::mem::zeroed() }; + let rc = unsafe { + libc::waitid( + libc::P_PID, + pid as libc::id_t, + &mut info, + libc::WEXITED | libc::WNOHANG | libc::WNOWAIT, + ) + }; + // rc == 0 with si_pid == 0 means "no state change yet" (still running); a non-zero + // si_pid means the child has entered a waitable, exited state. EINTR/other errors + // (rc != 0) are treated as "not yet terminated" and the caller re-polls. + rc == 0 && unsafe { info.si_pid() } != 0 +} + +#[cfg(unix)] +pub(crate) fn run_bounded_git_raw( + git_bin: &str, + args: &[&str], + repo_path: &Path, + stdin_bytes: &[u8], + deadline: Instant, +) -> Result<(std::process::ExitStatus, Vec, Vec)> { + use std::io::{Read, Write}; + use std::os::unix::process::CommandExt; + use std::sync::mpsc::RecvTimeoutError; + + let label = args.first().copied().unwrap_or("git"); + let mut child = std::process::Command::new(git_bin) + .args(args) + .current_dir(repo_path) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .process_group(0) + .spawn() + .with_context(|| format!("failed to spawn git {label}"))?; + // With process_group(0) the child leads its own group, so pgid == its pid. + let pgid = child.id() as i32; + + // Watchdog: on the deadline, tear the WHOLE process group down — SIGTERM, a grace + // for a well-behaved child to clean up its `*.lock` files, then an UNCONDITIONAL + // SIGKILL of the group. It never stands down on leader-reap alone: a group member + // that ignores SIGTERM while the leader exits cleanly would otherwise escape the + // SIGKILL and keep running past the deadline (finding 3, #174). The main thread + // defers reaping the leader until this thread returns (see below), so the leader's + // pid is still unreaped while every `kill(-pgid)` fires and the pgid cannot have + // been recycled — which is why this no longer needs the old `reaped` short-circuit. + // Kept off the main thread because the main thread's stdout drain is exactly what + // blocks until a hung child is torn down. + let (done_tx, done_rx) = mpsc::channel::<()>(); + let watchdog = std::thread::spawn(move || -> bool { + let wait = deadline.saturating_duration_since(Instant::now()); + match done_rx.recv_timeout(wait) { + Ok(()) | Err(RecvTimeoutError::Disconnected) => false, + Err(RecvTimeoutError::Timeout) => { + // SAFETY: kill(2) takes only integers and borrows no Rust memory; + // ESRCH on an already-gone group is ignored. + unsafe { libc::kill(-pgid, libc::SIGTERM) }; + // Fixed grace: because the main thread defers the leader's reap, a + // fully-exited group still shows a zombie leader here, so polling for + // ESRCH cannot detect early completion — just wait the grace, then + // SIGKILL. On a group of only zombies the SIGKILL is a harmless no-op; + // on a SIGTERM-ignoring member it is what actually kills it. + std::thread::sleep(WATCHDOG_TERM_GRACE); + unsafe { libc::kill(-pgid, libc::SIGKILL) }; + // Brief settle so the SIGKILL is delivered before the main thread + // reaps the leader and frees the pgid. A wedged (D-state) member + // survives even SIGKILL — the documented residual, as in smart_http. + std::thread::sleep(Duration::from_millis(20)); + if unsafe { libc::kill(-pgid, 0) } == 0 { + tracing::warn!( + pgid, + "withheld-walk git survived SIGKILL past the watchdog cap (uninterruptible I/O?)" + ); + } + true + } + } + }); + + // Feed stdin on a writer thread and drain stderr on a reader thread so the main + // thread can drain stdout concurrently; writing all of stdin (or draining one + // pipe) before the others can deadlock once a pipe buffer fills. + let mut stdin = child.stdin.take(); + let input = stdin_bytes.to_vec(); + let writer = std::thread::spawn(move || { + if let Some(mut s) = stdin.take() { + let _ = s.write_all(&input); + } + }); + let mut stderr = child.stderr.take().context("git stderr was not piped")?; + let err_reader = std::thread::spawn(move || { + let mut err = Vec::new(); + let _ = stderr.read_to_end(&mut err); + err + }); + let mut stdout = child.stdout.take().context("git stdout was not piped")?; + let mut out = Vec::new(); + // Blocking drain, unblocked by the child closing stdout on exit. The watchdog's + // SIGTERM/SIGKILL is what makes a hung child exit; a git wedged in uninterruptible + // (D-state) I/O survives even SIGKILL, so this drain and the wait below can block + // until the kernel returns, pinning the walk thread and its permit. That residual + // is unreachable in userspace (no signal reaps a D-state process) and matches the + // async `reap_group_on_timeout`, which likewise only warns and gives up there. + let read_result = stdout.read_to_end(&mut out); + // The drain has returned, but that only means all stdout write ends are closed — + // NOT that the child has exited. A group member, or the leader itself, can close + // stdout and keep running; standing the watchdog down on the drain alone (as the + // old code did) would then let `child.wait()` block forever on that live child, + // past the deadline, pinning the walk thread and its permit (finding P1-a, #174). + // So stand the watchdog down only once the child has ACTUALLY terminated, detected + // WITHOUT reaping (waitid + WNOWAIT) so the leader's pid stays unreaped and its + // pgid un-recycled until the watchdog finishes and we join it below. Past the + // deadline the watchdog owns the teardown, so we stop polling and let it run the + // full SIGTERM -> grace -> SIGKILL; joining it before `child.wait()` keeps every + // `kill(-pgid)` firing while the pid is still unreaped and guarantees a + // stdout-closing-then-hanging member has been SIGKILLed rather than left running. + loop { + if child_terminated_without_reaping(pgid) { + let _ = done_tx.send(()); + break; + } + if Instant::now() >= deadline { + break; + } + std::thread::sleep(Duration::from_millis(2)); + } + let killed = watchdog.join().unwrap_or(false); + let status = child.wait().context("git wait failed")?; + let err = err_reader.join().unwrap_or_default(); + let _ = writer.join(); + read_result.context("failed to read git stdout")?; + // The watchdog runs off a wall clock that can race a child finishing right at the + // deadline. A child that exited on its own (success) is not a timeout even if the + // watchdog fired late; only a child that did not exit successfully is a genuine + // timeout, which keeps a walk completing at its budget from a spurious 504. + if killed && !status.success() { + return Err(crate::git::smart_http::GitServiceTimeout.into()); + } + Ok((status, out, err)) +} + +/// Bounded git returning only stdout, `bail!`ing on any nonzero exit. The thin +/// wrapper the walk callers use. Probes that must distinguish exit classes — +/// `git cat-file` absence vs an object-store access failure — call +/// [`run_bounded_git_raw`] and classify the status/stderr themselves. +#[cfg(unix)] +pub(crate) fn run_bounded_git( + git_bin: &str, + args: &[&str], + repo_path: &Path, + stdin_bytes: &[u8], + deadline: Instant, +) -> Result> { + let label = args.first().copied().unwrap_or("git"); + let (status, out, err) = run_bounded_git_raw(git_bin, args, repo_path, stdin_bytes, deadline)?; + if !status.success() { + anyhow::bail!("git {label} failed: {}", String::from_utf8_lossy(&err)); + } + Ok(out) +} + +/// Non-Unix fallback for [`run_bounded_git`]. Windows and other non-Unix targets +/// have no process-group teardown (`process_group(0)` / `kill(-pgid)` are Unix-only), +/// so this bounds a single child on its own: threads feed stdin and drain stderr +/// while the main thread drains stdout, and a watchdog thread kills the child at the +/// deadline (which closes stdout and unblocks the drain). The child is shared with +/// the watchdog behind a mutex that the main thread does NOT hold while draining, so +/// the watchdog can always acquire it to kill. Best-effort — it reaps only the direct +/// child, not a descendant group — which is why the hardened, group-aware path above +/// is gated to Unix, the only target the served node actually runs on (the Windows +/// release binary is best-effort / `continue-on-error` in CI). Kept in lockstep with +/// the Unix version's signature and result semantics so every caller compiles on all +/// targets (#174). +#[cfg(not(unix))] +pub(crate) fn run_bounded_git_raw( + git_bin: &str, + args: &[&str], + repo_path: &Path, + stdin_bytes: &[u8], + deadline: Instant, +) -> Result<(std::process::ExitStatus, Vec, Vec)> { + use std::io::{Read, Write}; + use std::sync::mpsc::RecvTimeoutError; + + let label = args.first().copied().unwrap_or("git"); + let mut child = std::process::Command::new(git_bin) + .args(args) + .current_dir(repo_path) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .with_context(|| format!("failed to spawn git {label}"))?; + + let mut stdin = child.stdin.take(); + let input = stdin_bytes.to_vec(); + let writer = std::thread::spawn(move || { + if let Some(mut s) = stdin.take() { + let _ = s.write_all(&input); + } + }); + let mut stderr = child.stderr.take().context("git stderr was not piped")?; + let err_reader = std::thread::spawn(move || { + let mut err = Vec::new(); + let _ = stderr.read_to_end(&mut err); + err + }); + let mut stdout = child.stdout.take().context("git stdout was not piped")?; + + // Share the child with the watchdog. The main thread drains stdout WITHOUT + // holding this lock, so the watchdog can always acquire it to kill on timeout; + // killing closes stdout and unblocks the drain below. + let child = std::sync::Arc::new(std::sync::Mutex::new(child)); + let (done_tx, done_rx) = mpsc::channel::<()>(); + let watchdog = { + let child = child.clone(); + std::thread::spawn(move || -> bool { + let wait = deadline.saturating_duration_since(Instant::now()); + match done_rx.recv_timeout(wait) { + Ok(()) | Err(RecvTimeoutError::Disconnected) => false, + Err(RecvTimeoutError::Timeout) => { + if let Ok(mut c) = child.lock() { + let _ = c.kill(); + } + true + } + } + }) + }; + + let mut out = Vec::new(); + let read_result = stdout.read_to_end(&mut out); + // The drain has returned (child exited or was killed), so taking the lock here + // cannot deadlock against the watchdog. + let status = child + .lock() + .expect("git child mutex poisoned") + .wait() + .context("git wait failed")?; + let _ = done_tx.send(()); + let killed = watchdog.join().unwrap_or(false); + let err = err_reader.join().unwrap_or_default(); + let _ = writer.join(); + read_result.context("failed to read git stdout")?; + if killed && !status.success() { + return Err(crate::git::smart_http::GitServiceTimeout.into()); + } + Ok((status, out, err)) +} + +/// Non-Unix thin wrapper matching the Unix [`run_bounded_git`] semantics. +#[cfg(not(unix))] +pub(crate) fn run_bounded_git( + git_bin: &str, + args: &[&str], + repo_path: &Path, + stdin_bytes: &[u8], + deadline: Instant, +) -> Result> { + let label = args.first().copied().unwrap_or("git"); + let (status, out, err) = run_bounded_git_raw(git_bin, args, repo_path, stdin_bytes, deadline)?; + if !status.success() { + anyhow::bail!("git {label} failed: {}", String::from_utf8_lossy(&err)); + } + Ok(out) +} /// Fail closed unless every ref ultimately resolves to a commit (a ref pointing /// directly at a blob or tree, or an annotated tag — even a nested one — of such @@ -21,19 +331,15 @@ use std::path::Path; /// Full peeling is why this is not `for-each-ref %(*objecttype)`, which /// dereferences only one tag level and so misclassifies a tag-of-a-tag-of-a- /// commit as a non-commit. -fn assert_all_refs_are_commits(repo_path: &Path) -> Result<()> { - let refs = std::process::Command::new("git") - .args(["for-each-ref", "--format=%(refname)"]) - .current_dir(repo_path) - .output() - .context("git for-each-ref failed")?; - if !refs.status.success() { - anyhow::bail!( - "git for-each-ref failed: {}", - String::from_utf8_lossy(&refs.stderr) - ); - } - let refs_stdout = String::from_utf8_lossy(&refs.stdout); +fn assert_all_refs_are_commits(repo_path: &Path, git_bin: &str, deadline: Instant) -> Result<()> { + let refs_out = run_bounded_git( + git_bin, + &["for-each-ref", "--format=%(refname)"], + repo_path, + b"", + deadline, + )?; + let refs_stdout = String::from_utf8_lossy(&refs_out); let refnames: Vec<&str> = refs_stdout .lines() .map(str::trim) @@ -43,59 +349,25 @@ fn assert_all_refs_are_commits(repo_path: &Path) -> Result<()> { return Ok(()); } - // Peel every ref in one `git cat-file --batch-check` pass: one - // `^{}` query per line, one output line per input line, in order. - // The stdin write runs on a separate thread so this thread can drain stdout - // concurrently. cat-file echoes the full query on a ` missing` line, - // so output scales with refname length (not a fixed size per ref); writing - // all of stdin before reading any stdout would deadlock both pipes once the - // child's stdout buffer fills. Dropping `stdin` at the end of the closure - // sends EOF. + // Peel every ref in one `git cat-file --batch-check` pass: one `^{}` + // query per line, one output line per input line, in order. cat-file echoes the + // full query on a ` missing` line, so output scales with refname length; + // run_bounded_git drains stdout concurrently with the stdin write, so the pipe + // cannot deadlock, and the whole peel is bounded by the shared walk deadline. let queries = refnames .iter() .map(|r| format!("{r}^{{}}")) .collect::>() .join("\n"); - use std::io::Write; - let mut child = std::process::Command::new("git") - .args(["cat-file", "--batch-check=%(objecttype)"]) - .current_dir(repo_path) - .stdin(std::process::Stdio::piped()) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) - .spawn() - .context("failed to spawn git cat-file")?; - // Feed stdin on a writer thread so this thread can drain stdout via - // wait_with_output concurrently; a None handle (the pipe vanished) becomes a - // broken-pipe write error. wait_with_output reaps the child unconditionally - // before any error is surfaced, so no path drops it unwaited (#53), and the - // writer is joined only after the drain so the join cannot deadlock. - let writer = child - .stdin - .take() - .map(|mut stdin| std::thread::spawn(move || stdin.write_all(queries.as_bytes()))); - let peel_result = child.wait_with_output(); - let write_result = match writer { - Some(handle) => handle - .join() - .map_err(|_| anyhow::anyhow!("git cat-file stdin writer thread panicked"))?, - None => Err(std::io::Error::new( - std::io::ErrorKind::BrokenPipe, - "git cat-file stdin unavailable", - )), - }; - // Surface a write error only if the process didn't already fail with a - // clearer status. - let peel = peel_result.context("git cat-file failed")?; - if !peel.status.success() { - anyhow::bail!( - "git cat-file --batch-check failed: {}", - String::from_utf8_lossy(&peel.stderr) - ); - } - write_result.context("failed to write to git cat-file stdin")?; - - let peel_stdout = String::from_utf8_lossy(&peel.stdout); + let peel_out = run_bounded_git( + git_bin, + &["cat-file", "--batch-check=%(objecttype)"], + repo_path, + queries.as_bytes(), + deadline, + )?; + + let peel_stdout = String::from_utf8_lossy(&peel_out); let types: Vec<&str> = peel_stdout.lines().map(str::trim).collect(); // A short read means at least one ref went unclassified — fail closed. if types.len() != refnames.len() { @@ -147,47 +419,46 @@ fn assert_all_refs_are_commits(repo_path: &Path) -> Result<()> { /// Fails closed: if commit enumeration or any tree walk fails, returns an error so /// the caller aborts the serve/pin rather than producing a partial (under-withheld) /// set. -fn blob_paths(repo_path: &Path) -> Result> { - assert_all_refs_are_commits(repo_path)?; +fn blob_paths(repo_path: &Path, git_bin: &str, timeout: Duration) -> Result> { + // One deadline spans the whole walk (the ref check, the HEAD probe, rev-list, + // and every per-commit ls-tree), so a slow or hung walk is bounded as a unit + // rather than granting each git child a fresh timeout. + let deadline = Instant::now() + timeout; + assert_all_refs_are_commits(repo_path, git_bin, deadline)?; // Enumerate every reachable commit, not just ref tips. `--all` walks all refs; // append HEAD so a detached HEAD (reachable by rev-list/upload-pack but in no // ref) is still classified. When HEAD does not resolve (unborn branch on an - // empty repo) `--all` alone yields nothing, which is correct — no objects exist. - let head = store::head_commit(repo_path).context("resolve HEAD failed")?; + // empty repo) `--all` alone yields nothing, which is correct: no objects exist. + // The HEAD probe is a bounded `git rev-parse --verify HEAD` (a clean exit means + // HEAD resolves), replacing the previously unbounded `store::head_commit` child. + let head_resolves = run_bounded_git( + git_bin, + &["rev-parse", "--verify", "HEAD"], + repo_path, + b"", + deadline, + ) + .is_ok(); let mut rev_args = vec!["rev-list", "--all"]; - if head.is_some() { + if head_resolves { rev_args.push("HEAD"); } - let commits = std::process::Command::new("git") - .args(&rev_args) - .current_dir(repo_path) - .output() - .context("git rev-list --all failed")?; - if !commits.status.success() { - anyhow::bail!( - "git rev-list --all failed: {}", - String::from_utf8_lossy(&commits.stderr) - ); - } - let commits_stdout = String::from_utf8_lossy(&commits.stdout); + let commits_out = run_bounded_git(git_bin, &rev_args, repo_path, b"", deadline)?; + let commits_stdout = String::from_utf8_lossy(&commits_out); let mut out: HashSet<(String, String)> = HashSet::new(); for commit in commits_stdout.lines() { let commit = commit.trim(); if commit.is_empty() { continue; } - let listing = std::process::Command::new("git") - .args(["ls-tree", "-rz", commit]) - .current_dir(repo_path) - .output() - .context("git ls-tree -rz failed")?; - if !listing.status.success() { - anyhow::bail!( - "git ls-tree -rz {commit} failed: {}", - String::from_utf8_lossy(&listing.stderr) - ); - } + let listing_out = run_bounded_git( + git_bin, + &["ls-tree", "-rz", commit], + repo_path, + b"", + deadline, + )?; // `-z` NUL-delimits records and emits paths raw; plain `git ls-tree -r` // C-quotes any path with non-ASCII or special bytes (e.g. café.txt becomes // "secret/caf\303\251.txt"), and that quoted literal would not match a @@ -198,7 +469,7 @@ fn blob_paths(repo_path: &Path) -> Result> { // path (e.g. a non-UTF-8 directory name) with U+FFFD, and the mangled string // would no longer match its deny rule — the same under-withholding class, one // layer down. Fail closed instead so the caller aborts rather than leaks. - let Ok(listing_stdout) = std::str::from_utf8(&listing.stdout) else { + let Ok(listing_stdout) = std::str::from_utf8(&listing_out) else { anyhow::bail!( "git ls-tree -rz {commit} returned a non-UTF-8 path; \ refusing to produce a partial (under-withheld) set" @@ -229,6 +500,7 @@ fn blob_paths(repo_path: &Path) -> Result> { /// /// The whole-repo "/" gate is handled by the caller before this function runs: /// if "/" denies, the caller gets a 404 and never reaches the filtered serve. +#[cfg(test)] pub fn withheld_blob_oids( repo_path: &Path, rules: &[VisibilityRule], @@ -236,7 +508,33 @@ pub fn withheld_blob_oids( owner_did: &str, caller: Option<&str>, ) -> Result> { - let pairs = blob_paths(repo_path)?; + withheld_blob_oids_bounded( + repo_path, + "git", + WALK_TIMEOUT, + rules, + is_public, + owner_did, + caller, + ) +} + +/// [`withheld_blob_oids`] with an injectable `git_bin` and walk `timeout`. Served +/// handlers call this with the operator-configured git binary and +/// `GITLAWB_GIT_SERVICE_TIMEOUT_SECS`, so the whole walk is bounded by the same +/// budget as the other served-git ops and a fake `git` can drive its teardown in +/// tests. The `git_bin`-less wrapper above keeps the fixed [`WALK_TIMEOUT`] for the +/// classification tests that run against real git. +pub fn withheld_blob_oids_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> Result> { + let pairs = blob_paths(repo_path, git_bin, timeout)?; Ok(withheld_from_pairs( &pairs, rules, is_public, owner_did, caller, )) @@ -304,6 +602,7 @@ pub fn replicable_objects(all: Vec, withheld: &HashSet) -> Vec Result> { + allowed_blob_set_for_caller_bounded( + repo_path, git_bin, timeout, rules, is_public, owner_did, None, + ) +} + /// Reachable blob OIDs that visibility ALLOWS `caller` at some path. The /// caller-aware generalization of `replicable_blob_set` (which is the anonymous /// `caller = None` case). Used by `GET /ipfs/{cid}` to gate fail-closed against @@ -325,6 +639,7 @@ pub fn replicable_blob_set( /// elsewhere (its content is readable to this caller elsewhere). Trees and /// commits are NOT included here; the caller decides per object type whether /// the allow-set applies (it does not for trees/commits — KTD3). +#[cfg(test)] pub fn allowed_blob_set_for_caller( repo_path: &Path, rules: &[VisibilityRule], @@ -332,7 +647,29 @@ pub fn allowed_blob_set_for_caller( owner_did: &str, caller: Option<&str>, ) -> Result> { - let pairs = blob_paths(repo_path)?; + allowed_blob_set_for_caller_bounded( + repo_path, + "git", + WALK_TIMEOUT, + rules, + is_public, + owner_did, + caller, + ) +} + +/// [`allowed_blob_set_for_caller`] with an injectable `git_bin` and walk `timeout`, +/// for the `GET /ipfs/{cid}` gate. +pub fn allowed_blob_set_for_caller_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> Result> { + let pairs = blob_paths(repo_path, git_bin, timeout)?; let mut allowed = HashSet::new(); for (oid, path) in &pairs { if visibility_check(rules, is_public, owner_did, caller, path) == Decision::Allow { @@ -342,6 +679,477 @@ pub fn allowed_blob_set_for_caller( Ok(allowed) } +/// The reachable-commit enumeration for the LENIENT walks (the `/ipfs/{cid}` tree +/// gate and the commit/tag reachability set): bounded `git rev-list --all [HEAD]` +/// under the caller's shared `deadline`, deliberately WITHOUT +/// `assert_all_refs_are_commits`. That guard fail-closes a repo's whole walk when +/// any ref peels to a non-commit (an annotated tag of a tree is pushable through +/// receive-pack), which would 404 every reachable tree/commit/tag CID here for a +/// legitimate reader. `rev-list --all` skips such refs cleanly, so the commit set +/// stays complete; an object reachable only via such a ref is simply excluded — +/// correctly fail-closed. Fails closed on a rev-list error. +/// +/// Safe ONLY for a caller whose output feeds a fail-closed allow-list where absence +/// = withhold: a tolerant walk there over-withholds, never leaks. NOT safe for a +/// serve/replication filter, where a missed reachable object under-withholds — +/// those go through `blob_paths`, which runs the guard first. +fn reachable_commit_oids( + repo_path: &Path, + git_bin: &str, + deadline: Instant, +) -> Result> { + // The HEAD probe is a bounded `git rev-parse --verify HEAD` (a clean exit means + // HEAD resolves), matching `blob_paths`. When HEAD does not resolve (unborn + // branch on an empty repo) `--all` alone yields nothing, which is correct. + let head_resolves = run_bounded_git( + git_bin, + &["rev-parse", "--verify", "HEAD"], + repo_path, + b"", + deadline, + ) + .is_ok(); + let mut rev_args = vec!["rev-list", "--all"]; + if head_resolves { + rev_args.push("HEAD"); + } + let out = run_bounded_git(git_bin, &rev_args, repo_path, b"", deadline)?; + Ok(String::from_utf8_lossy(&out) + .lines() + .map(|l| l.trim().to_string()) + .filter(|l| !l.is_empty()) + .collect()) +} + +/// Every `(oid, "/repo/relative/path", kind)` triple reachable from the given +/// `commits` — the shared ls-tree seam the tree walk filters (`kind == "tree"`). +/// One bounded `git ls-tree -rzt` per commit under the caller's shared `deadline`: +/// `-rzt` is byte-identical to `-rz` for blob records and additionally emits the +/// tree object for each directory at its own path. `kind` is git's object-type +/// string ("blob", "tree", or "commit" for a gitlink). The commit's ROOT tree is +/// not emitted by `ls-tree` (it lists entries *under* a tree); `tree_paths` adds +/// it. Triples are de-duplicated across commits and paths carry a leading "/" to +/// match the glob form of visibility rules ("/secret/**"). +/// +/// Fails closed: if any tree walk fails — or a path is not valid UTF-8 — it +/// returns an error so the caller aborts rather than producing a partial +/// (under-withheld) set. +fn object_paths( + repo_path: &Path, + git_bin: &str, + commits: &[String], + deadline: Instant, +) -> Result> { + let mut out: HashSet<(String, String, String)> = HashSet::new(); + for commit in commits { + let listing_out = run_bounded_git( + git_bin, + &["ls-tree", "-rzt", commit], + repo_path, + b"", + deadline, + )?; + // `-z` NUL-delimits records and emits paths raw; plain `git ls-tree -r` + // C-quotes any path with non-ASCII or special bytes (e.g. café.txt becomes + // "secret/caf\303\251.txt"), and that quoted literal would not match a + // visibility rule like "/secret/**", under-withholding the object. The TAB + // field separator survives `-z`, so the per-record parse is unchanged. + // + // Parse strictly: a lossy decode would replace an invalid byte in a denied + // path (e.g. a non-UTF-8 directory name) with U+FFFD, and the mangled string + // would no longer match its deny rule — the same under-withholding class, one + // layer down. Fail closed instead so the caller aborts rather than leaks. + let Ok(listing_stdout) = std::str::from_utf8(&listing_out) else { + anyhow::bail!( + "git ls-tree -rzt {commit} returned a non-UTF-8 path; \ + refusing to produce a partial (under-withheld) set" + ); + }; + for record in listing_stdout.split('\0') { + // " \t" + let Some((meta, path)) = record.split_once('\t') else { + continue; + }; + let mut parts = meta.split_whitespace(); + let _mode = parts.next(); + let kind = parts.next(); + let oid = parts.next(); + if let (Some(kind), Some(oid)) = (kind, oid) { + out.insert((oid.to_string(), format!("/{path}"), kind.to_string())); + } + } + } + Ok(out) +} + +/// Root tree oid of every reachable commit, at "/". `ls-tree` never emits a commit's +/// own root tree (it lists entries *under* a tree), so it is added explicitly here. +/// Resolved in ONE bounded `git log --no-walk --format=%T --stdin` pass over the +/// shared commit set — not a per-commit `rev-parse` — so a tree-set walk costs the +/// same subprocess order as the blob walk. The commit oids go on STDIN, not argv: a +/// long history has tens of thousands of reachable commits, and passing them all as +/// arguments overflows ARG_MAX so `git log` fails to spawn — which the caller treats +/// as a walk error and fail-closed 404s an authorized reader of a reachable/root +/// tree (#173 P2). `run_bounded_git` drains stdout concurrently with the stdin +/// write, so a large history cannot deadlock the pipes. A commit whose root tree git +/// cannot resolve fails the pass (bail), failing closed. +fn root_tree_pairs( + repo_path: &Path, + git_bin: &str, + commits: &[String], + deadline: Instant, +) -> Result> { + if commits.is_empty() { + return Ok(HashSet::new()); + } + let mut buf = String::with_capacity(commits.len() * 65); + for c in commits { + buf.push_str(c); + buf.push('\n'); + } + let out = run_bounded_git( + git_bin, + &["log", "--no-walk=unsorted", "--format=%T", "--stdin"], + repo_path, + buf.as_bytes(), + deadline, + )?; + let mut set = HashSet::new(); + for line in String::from_utf8_lossy(&out).lines() { + let oid = line.trim(); + if !oid.is_empty() { + set.insert((oid.to_string(), "/".to_string())); + } + } + Ok(set) +} + +/// Every `(tree_oid, "/path")` pair reachable in `repo_path`: the `kind == "tree"` +/// slice of [`object_paths`] (subtree trees at their directory paths) PLUS every +/// reachable commit's root tree at "/" (see [`root_tree_pairs`]). Computes the +/// reachable-commit set ONCE (leniently — see [`reachable_commit_oids`]; the tree +/// allowed-set feeds ONLY the `/ipfs/{cid}` tree gate, where absence = fail-closed +/// 404) and drives both the ls-tree walk and the root-tree pass from it, so the two +/// cannot diverge and neither re-enumerates. The tree analog of [`blob_paths`], +/// bounded by the same shared `deadline`. +fn tree_paths( + repo_path: &Path, + git_bin: &str, + deadline: Instant, +) -> Result> { + let commits = reachable_commit_oids(repo_path, git_bin, deadline)?; + let mut out: HashSet<(String, String)> = object_paths(repo_path, git_bin, &commits, deadline)? + .into_iter() + .filter(|(_, _, kind)| kind == "tree") + .map(|(oid, path, _)| (oid, path)) + .collect(); + out.extend(root_tree_pairs(repo_path, git_bin, &commits, deadline)?); + Ok(out) +} + +/// The OIDs from a `(oid, "/path")` listing that visibility ALLOWS `caller` at some +/// path — the shared inner loop of the blob and tree allowed-sets. An oid reachable +/// at an allowed path is kept even when also reachable at a denied one. +fn allowed_set_from_pairs<'a>( + pairs: impl IntoIterator, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> HashSet { + pairs + .into_iter() + .filter(|(_, path)| { + visibility_check(rules, is_public, owner_did, caller, path) == Decision::Allow + }) + .map(|(oid, _)| oid.clone()) + .collect() +} + +/// Reachable tree OIDs that visibility ALLOWS `caller` at some path — the tree +/// analog of [`allowed_blob_set_for_caller`]. `GET /ipfs/{cid}` gates tree objects +/// with this so the CID surface matches `get_tree`: a tree reachable only at a +/// withheld path is absent from the set and 404'd; the root tree ("/") and any tree +/// on the path to an allowed subtree are present. Fails closed on a +/// dangling/unreachable tree (never enumerated by the reachable walk, so never in +/// the set — the #126 geometry, for trees). A tree reachable at an allowed path is +/// included even when also reachable at a withheld one (its structure is visible to +/// this caller elsewhere). +#[cfg(test)] +pub fn allowed_tree_set_for_caller( + repo_path: &Path, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> Result> { + allowed_tree_set_for_caller_bounded( + repo_path, + "git", + WALK_TIMEOUT, + rules, + is_public, + owner_did, + caller, + ) +} + +/// [`allowed_tree_set_for_caller`] with an injectable `git_bin` and walk `timeout`, +/// for the `GET /ipfs/{cid}` tree gate. One deadline spans the whole walk (the HEAD +/// probe, rev-list, every per-commit ls-tree, and the root-tree pass), matching +/// `blob_paths`, so a slow or hung walk is bounded as a unit while the handler holds +/// its /ipfs walk permit (#174 F5). +pub fn allowed_tree_set_for_caller_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> Result> { + let deadline = Instant::now() + timeout; + Ok(allowed_set_from_pairs( + &tree_paths(repo_path, git_bin, deadline)?, + rules, + is_public, + owner_did, + caller, + )) +} + +/// Object bound for the annotated-tag reachability walk (#173, jatmn tag fan-out). +/// A path-scoped pinned-CID request drives this walk while holding one per-request +/// and one per-IP walk slot, so the total tag work must be finite regardless of how +/// many tag refs the repo has. 8192 is far past any real repo's annotated-tag count +/// (the Linux kernel has a few hundred), yet finite: a repo beyond it fails closed +/// (Err), matching this function's fail-closed-on-any-git-error contract, rather than +/// truncating silently (which would under-withhold a still-reachable tag object). +const MAX_TAG_OBJECTS: usize = 8192; + +/// Walk the annotated-tag chains rooted at `seeds`, inserting every tag object they +/// pass through into `set`. A tag whose target is itself a tag (tag-of-a-tag) +/// discovers the inner tag, which is walked in a later round. +/// +/// #173 (jatmn): the tag inspection is BATCHED, not one process per tag. Each round +/// feeds every not-yet-inspected tag oid to a SINGLE `git cat-file --batch` child on +/// stdin and reads back framed ` \n\n` records, so the +/// number of child processes is bounded by the tag-chain DEPTH (rounds), not the tag +/// COUNT. Oids go on stdin, never argv, so a large tag set cannot overflow ARG_MAX. +/// The child runs through [`run_bounded_git`], which drains stdout concurrently with +/// the stdin write (subsuming #173's F4 writer-thread drain — a round large enough to +/// fill both pipes cannot deadlock) and tears the child down at `deadline`, so a hung +/// cat-file cannot pin the caller's /ipfs walk permit (#174 F5). Total tag objects +/// inspected are capped at `max_tag_objects`; exceeding it is an error (fail closed), +/// not a silent truncation. Takes the bound as a parameter so a test can drive a tiny +/// value while the caller passes the real `MAX_TAG_OBJECTS`. +fn walk_tag_chain( + repo_path: &Path, + git_bin: &str, + seeds: Vec, + set: &mut HashSet, + max_tag_objects: usize, + deadline: Instant, +) -> Result<()> { + // Tag oids known but not yet inspected. Seeds may repeat / already be present; + // the `set.insert` gate below is what actually dedups and terminates cycles. + let mut pending: Vec = seeds; + let mut inspected: usize = 0; + + while !pending.is_empty() { + // Inspect only oids new to `set`; a re-seen oid was already walked. + let round: Vec = pending + .drain(..) + .filter(|oid| set.insert(oid.clone())) + .collect(); + if round.is_empty() { + break; + } + inspected += round.len(); + if inspected > max_tag_objects { + anyhow::bail!( + "annotated-tag walk exceeded the object bound ({max_tag_objects}); refusing to serve" + ); + } + + // One bounded child for the whole round: feed all oids on stdin, read the + // framed records from the returned stdout. + let mut buf = String::with_capacity(round.len() * 65); + for oid in &round { + buf.push_str(oid); + buf.push('\n'); + } + let stdout = run_bounded_git( + git_bin, + &["cat-file", "--batch"], + repo_path, + buf.as_bytes(), + deadline, + )?; + + // Parse one record per requested oid: ` \n\n`. + // A ` missing\n` record has no size/body and is anomalous here (every + // oid came from a ref tip or a prior tag body), so fail closed. + let mut i = 0usize; + for _ in 0..round.len() { + let hdr_end = stdout[i..] + .iter() + .position(|&b| b == b'\n') + .map(|p| i + p) + .context("git cat-file --batch: truncated record header")?; + let header = std::str::from_utf8(&stdout[i..hdr_end]) + .context("git cat-file --batch: non-utf8 record header")?; + i = hdr_end + 1; + let mut fields = header.split(' '); + let _oid = fields.next().unwrap_or(""); + let ty = fields.next().unwrap_or(""); + if ty == "missing" || fields.clone().next().is_none() { + anyhow::bail!("git cat-file --batch: object {header:?} missing or malformed"); + } + let size: usize = fields + .next() + .unwrap_or("") + .parse() + .context("git cat-file --batch: bad record size")?; + let body_end = i + .checked_add(size) + .filter(|&e| e <= stdout.len()) + .context("git cat-file --batch: truncated record body")?; + // Only a tag object can point at an inner tag; walk its header. + if ty == "tag" { + let body = std::str::from_utf8(&stdout[i..body_end]) + .context("git cat-file --batch: non-utf8 tag body")?; + let mut target = None; + let mut is_tag = false; + for line in body.lines() { + if let Some(oid) = line.strip_prefix("object ") { + target = Some(oid.trim().to_string()); + } else if line == "type tag" { + is_tag = true; + } else if line.is_empty() { + break; // end of header + } + } + if is_tag { + if let Some(t) = target { + pending.push(t); + } + } + } + // Skip body plus its trailing newline to the next record. + i = body_end + 1; + } + } + Ok(()) +} + +/// The reachable-commit/tag gate set for the `/ipfs/{cid}` resolver (#173, F2): +/// every reachable commit oid UNION every reachable annotated-tag OBJECT oid. A +/// DANGLING commit/tag (referenced by no ref, directly or via a tag chain) is in +/// neither part, so the resolver denies it under a path-scoped rule instead of +/// leaking its message; a reachable one still serves. +#[cfg(test)] +pub fn reachable_commit_tag_oids(repo_path: &Path) -> Result> { + reachable_commit_tag_oids_bounded(repo_path, "git", WALK_TIMEOUT) +} + +/// [`reachable_commit_tag_oids`] with an injectable `git_bin` and walk `timeout`, +/// for the `GET /ipfs/{cid}` commit/tag gate. One deadline spans the whole walk. +/// +/// Reachable commits come from bounded `git rev-list --all` (+ HEAD for the +/// detached case). Unlike the blob allowed-set, this does NOT run +/// `assert_all_refs_are_commits`: that guard fail-closes a repo's whole walk when +/// any ref peels to a non-commit (an annotated tag of a tree is pushable through +/// receive-pack), which would 404 every reachable commit/tag CID here for a +/// legitimate reader. The guard exists to stop blob/tree UNDER-withholding; it is +/// unnecessary for reachability, since a dangling object is absent from +/// `rev-list --all` and the ref walk below regardless of odd refs — so dropping it +/// recovers availability without admitting any dangling object (no leak). +/// +/// Reachable tag OBJECTS: `rev-list --all` dereferences annotated tags to commits, +/// so the tag objects are absent from it. Collect them by walking every ref tip and +/// peeling each tag's chain, so a nested tag-of-a-tag's INNER tag object (reachable +/// and pinnable, but not itself a ref tip) is included too. Fails closed on any git +/// error. +pub fn reachable_commit_tag_oids_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, +) -> Result> { + let deadline = Instant::now() + timeout; + // Reachable commits — no ref-commit assertion (see docstring). The HEAD probe + // doubles as the seed source for the tag-valued detached HEAD below: + // `rev-parse --verify HEAD` returns the tag oid UNPEELED when HEAD names a tag + // object. Failing to resolve HEAD (unborn/absent) is not fatal — there is + // simply no HEAD to walk or seed. + let head_oid: Option = run_bounded_git( + git_bin, + &["rev-parse", "--verify", "HEAD"], + repo_path, + b"", + deadline, + ) + .ok() + .map(|out| String::from_utf8_lossy(&out).trim().to_string()) + .filter(|s| !s.is_empty()); + let mut rev_args = vec!["rev-list", "--all"]; + if head_oid.is_some() { + rev_args.push("HEAD"); + } + let rev = run_bounded_git(git_bin, &rev_args, repo_path, b"", deadline)?; + let mut set: HashSet = String::from_utf8_lossy(&rev) + .lines() + .map(|l| l.trim().to_string()) + .filter(|l| !l.is_empty()) + .collect(); + + // Ref tips that are annotated tag objects seed the tag-chain walk. + let refs = run_bounded_git( + git_bin, + &["for-each-ref", "--format=%(objectname) %(objecttype)"], + repo_path, + b"", + deadline, + )?; + let mut worklist: Vec = Vec::new(); + for line in String::from_utf8_lossy(&refs).lines() { + let mut it = line.split_whitespace(); + if let (Some(oid), Some("tag")) = (it.next(), it.next()) { + worklist.push(oid.to_string()); + } + } + // A detached/direct HEAD may name an annotated tag object with no ref at that tag + // (#173 review, finding 3): `rev-list --all HEAD` above peels it to its commit and + // `for-each-ref` has no tag row, so the tag OBJECT would be omitted and its pinned + // CID would 404 for an authorized reader. Seed a tag-valued HEAD into the tag-chain + // walk; a `commit` HEAD adds nothing. A cat-file failure here only skips the seed + // (over-withholds that one tag — fail-closed), matching the original's tolerance. + if let Some(head_oid) = head_oid { + if let Ok(ty) = run_bounded_git( + git_bin, + &["cat-file", "-t", &head_oid], + repo_path, + b"", + deadline, + ) { + if String::from_utf8_lossy(&ty).trim() == "tag" { + worklist.push(head_oid); + } + } + } + // Peel every tag object's chain into `set`, adding each tag object it passes + // through. Bounded and batched (#173, jatmn tag fan-out): see `walk_tag_chain`. + walk_tag_chain( + repo_path, + git_bin, + worklist, + &mut set, + MAX_TAG_OBJECTS, + deadline, + )?; + Ok(set) +} + /// Objects safe to replicate, failing closed on blobs (#99). A candidate /// replicates iff it is NOT a blob (`all_blob_oids` — commits and trees are /// structural, never content-withheld) OR it is in `allowed_blobs` (reachable @@ -365,14 +1173,28 @@ pub fn replicable_objects_fail_closed( /// owner plus any reader DID that `visibility_check` Allows at some path the /// blob appears at. Least-privilege: a reader of one private subtree is not a /// recipient of a blob that only lives in another. +#[cfg(test)] pub fn withheld_blob_recipients( repo_path: &Path, rules: &[VisibilityRule], is_public: bool, owner_did: &str, +) -> Result>> { + withheld_blob_recipients_bounded(repo_path, "git", WALK_TIMEOUT, rules, is_public, owner_did) +} + +/// [`withheld_blob_recipients`] with an injectable `git_bin` and walk `timeout`, for +/// the receive-pack encrypt-then-pin path. +pub fn withheld_blob_recipients_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, ) -> Result>> { // One history walk feeds both the withheld set and the recipient mapping. - let pairs = blob_paths(repo_path)?; + let pairs = blob_paths(repo_path, git_bin, timeout)?; let withheld = withheld_from_pairs(&pairs, rules, is_public, owner_did, None); if withheld.is_empty() { return Ok(HashMap::new()); @@ -402,38 +1224,321 @@ pub fn withheld_blob_recipients( #[cfg(test)] mod tests { use super::*; - use crate::db::VisibilityMode; - use chrono::Utc; - use std::process::Command; - use tempfile::TempDir; - fn rule(path_glob: &str, readers: &[&str]) -> VisibilityRule { - VisibilityRule { - id: "x".into(), - repo_id: "r1".into(), - path_glob: path_glob.into(), - mode: VisibilityMode::B, - reader_dids: readers.iter().map(|s| s.to_string()).collect(), - created_by: "did:key:zOwner".into(), - created_at: Utc::now(), + /// Write an executable fake `git` shell script into `dir` and return its path, + /// so a test can drive the walk's process-group teardown without a real git and + /// without mutating the process-global PATH (the crate's only injection seam). + #[cfg(unix)] + fn write_fake_git(dir: &Path, body: &str) -> String { + use std::os::unix::fs::PermissionsExt; + let p = dir.join("fakegit"); + std::fs::write(&p, body).unwrap(); + let mut perm = std::fs::metadata(&p).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&p, perm).unwrap(); + p.to_str().unwrap().to_string() + } + + /// #174 U3: the withheld-blob walk is bounded at the shared `blob_paths` seam, so + /// a hung git child cannot pin the caller's permit past the deadline. A fake git + /// that hangs on `rev-list` must make `blob_paths` return `GitServiceTimeout` + /// within the watchdog budget (not block for the child's lifetime), and the + /// child's process group must be reaped (its recorded leader PID gone). Every + /// caller (upload-pack serve, receive-pack replication) funnels through + /// `blob_paths`, so this seam-level proof covers both permit pools. Neutralize + /// the watchdog SIGTERM and this hangs past the recv budget (RED). + #[cfg(unix)] + #[test] + fn blob_paths_times_out_and_reaps_a_hung_walk() { + use std::time::Duration; + let tmp = TempDir::new().unwrap(); + // Fast on every stage except rev-list, which records its own (group-leader) + // PID and then hangs. `sleep 30` bounds the worst case if the watchdog is + // ever broken, so a regression cannot wedge the suite for 300s. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) echo $$ > revlist.pid ; sleep 30 ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + + // Run the walk on a thread with a short budget; the recv_timeout succeeding + // is itself proof the walk did not block on the hung child. + let (tx, rx) = mpsc::channel(); + let path = tmp.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(blob_paths(&path, &git_bin, Duration::from_millis(200))); + }); + let result = rx.recv_timeout(Duration::from_secs(10)).expect( + "blob_paths must return within the watchdog budget, not hang on a stuck git child", + ); + let err = result.expect_err("a hung rev-list must abort the walk with an error"); + assert!( + err.downcast_ref::() + .is_some(), + // `{err:#}` prints the whole anyhow chain. Plain `{err}` shows only the top + // context ("failed to spawn git for-each-ref") and drops the underlying io + // error, which left a real beta-lane CI failure undiagnosable. + "a hung walk must abort with GitServiceTimeout (mapped to 504), got: {err:#}" + ); + + // The recorded process-group leader must be gone: the watchdog reaps the + // whole group before blob_paths returns, so no orphaned git lingers. + let pid: i32 = std::fs::read_to_string(tmp.path().join("revlist.pid")) + .expect("the fake git must have recorded its rev-list PID") + .trim() + .parse() + .expect("recorded PID must parse"); + let mut gone = false; + for _ in 0..200 { + // SAFETY: kill(2) with signal 0 only probes existence; ESRCH (-1) means + // the process is gone. Borrows no Rust memory. + if unsafe { libc::kill(pid, 0) } != 0 { + gone = true; + break; + } + std::thread::sleep(Duration::from_millis(10)); } + assert!( + gone, + "the hung git child (pid {pid}) must be reaped, not orphaned, after the walk aborts" + ); } - const OWNER: &str = "did:key:zOwner"; + /// #174 (F1 status-gate, vetted by execution): a child that exits SUCCESSFULLY is + /// never reported as a timeout even when the watchdog fires, so a walk finishing + /// right at its deadline is not a spurious 504. The fake only exits when signalled + /// and exits 0 on SIGTERM, so with a deadline already elapsed the watchdog always + /// reaches its kill path (killed == true) yet the child's status is success. + /// Drop the `!status.success()` guard and this returns GitServiceTimeout (RED). + #[cfg(unix)] + #[test] + fn run_bounded_git_success_at_the_deadline_is_not_a_timeout() { + use std::time::{Duration, Instant}; + let tmp = TempDir::new().unwrap(); + let body = "#!/bin/sh\ntrap 'exit 0' TERM\nsleep 30 &\nwait\n"; + let git_bin = write_fake_git(tmp.path(), body); + let out = run_bounded_git( + &git_bin, + &["rev-list"], + tmp.path(), + b"", + Instant::now() + Duration::from_millis(100), + ); + assert!( + out.is_ok(), + "a child that exited successfully must not be reported as a timeout even if the watchdog fired: {out:?}" + ); + } - /// Build a bare repo with public/a.txt and secret/b.txt at one commit. - /// Returns (tempdir, bare_path, secret_blob_oid, public_blob_oid). - fn fixture() -> (TempDir, std::path::PathBuf, String, String) { - let td = TempDir::new().unwrap(); - let work = td.path().join("work"); - let bare = td.path().join("bare.git"); - let run = |args: &[&str], dir: &Path| { - let ok = Command::new("git") - .args(args) - .current_dir(dir) - .status() - .unwrap() - .success(); + /// #174 (F3, vetted by execution): a child that IGNORES SIGTERM is still reaped + /// via the watchdog's SIGKILL escalation, so it cannot pin the walk thread or its + /// permit. The fake traps SIGTERM and keeps sleeping; run_bounded_git must still + /// return (via SIGKILL at the grace step) with a timeout error and the group must + /// be gone. (A truly uninterruptible D-state child, which no signal can reap, is + /// the documented residual this teardown, like the async twin, cannot cover.) + #[cfg(unix)] + #[test] + fn run_bounded_git_reaps_a_sigterm_ignoring_child_via_sigkill() { + use std::time::{Duration, Instant}; + let tmp = TempDir::new().unwrap(); + let body = "#!/bin/sh\ntrap '' TERM\necho $$ > pid\nwhile true; do sleep 1; done\n"; + let git_bin = write_fake_git(tmp.path(), body); + let (tx, rx) = std::sync::mpsc::channel(); + let path = tmp.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(run_bounded_git( + &git_bin, + &["rev-list"], + &path, + b"", + Instant::now() + Duration::from_millis(100), + )); + }); + let out = rx + .recv_timeout(Duration::from_secs(10)) + .expect("run_bounded_git must return via SIGKILL even for a SIGTERM-ignoring child"); + assert!( + out.is_err(), + "a SIGTERM-ignoring child killed by SIGKILL is a timeout, not a success: {out:?}" + ); + let pid: i32 = std::fs::read_to_string(tmp.path().join("pid")) + .unwrap() + .trim() + .parse() + .unwrap(); + let mut gone = false; + for _ in 0..300 { + if unsafe { libc::kill(pid, 0) } != 0 { + gone = true; + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + assert!( + gone, + "the SIGTERM-ignoring child (pid {pid}) must be reaped via SIGKILL, not left running" + ); + } + + /// #174 finding 3 (jatmn/CodeRabbit): a group MEMBER that ignores SIGTERM must + /// still be SIGKILLed even when the group LEADER exits cleanly on SIGTERM. The + /// leader traps SIGTERM to exit 0, but first spawns a descendant (`sh -c`, so its + /// `$$` is its OWN pid — a `( )` subshell's `$$` is the parent's) that ignores + /// SIGTERM and closes its inherited stdout/stderr. When the watchdog SIGTERMs the + /// group, the leader exits, its stdout closes, the main drain unblocks, and the + /// leader is reaped — the exact window a `reaped`-gated watchdog stands down in, + /// before escalating to SIGKILL. The descendant must be dead when run_bounded_git + /// returns; a teardown that stands down on leader-reap leaves it running (RED). + #[cfg(unix)] + #[test] + fn run_bounded_git_sigkills_a_sigterm_ignoring_descendant_after_leader_exits() { + use std::time::{Duration, Instant}; + let tmp = TempDir::new().unwrap(); + // Both loops are bounded (~30s) so a broken teardown cannot leak a permanent + // orphan or wedge the suite; the assertion fires well before then. + let body = "#!/bin/sh\n\ +case \"$1\" in\n\ + rev-list)\n\ + sh -c 'trap \"\" TERM; echo $$ > desc.pid; exec 1>&- 2>&-; i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done' &\n\ + trap 'exit 0' TERM\n\ + i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;;\n\ + *) : ;;\n\ +esac\n"; + let git_bin = write_fake_git(tmp.path(), body); + let (tx, rx) = std::sync::mpsc::channel(); + let path = tmp.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(run_bounded_git( + &git_bin, + &["rev-list"], + &path, + b"", + Instant::now() + Duration::from_millis(100), + )); + }); + let _ = rx + .recv_timeout(Duration::from_secs(10)) + .expect("run_bounded_git must return within the watchdog budget"); + + // Wait for the descendant to record its OWN pid, then assert it is gone. + let desc_pid_path = tmp.path().join("desc.pid"); + let mut desc: Option = None; + for _ in 0..200 { + if let Some(p) = std::fs::read_to_string(&desc_pid_path) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + desc = Some(p); + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + let desc = desc.expect("the fake leader must have spawned and recorded a descendant"); + let mut gone = false; + for _ in 0..300 { + if unsafe { libc::kill(desc, 0) } != 0 { + gone = true; + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + // Kill it regardless so a RED run leaks no orphan. + unsafe { libc::kill(desc, libc::SIGKILL) }; + assert!( + gone, + "a SIGTERM-ignoring descendant (pid {desc}) must be SIGKILLed even after the leader exits cleanly, not orphaned" + ); + } + + /// #174 U1 (P1-a, RED-before/GREEN-after): the group LEADER closes its own + /// stdout/stderr BEFORE the deadline and then keeps running. On the pre-fix code + /// the stdout drain returns EOF early, `done_tx.send` stands the watchdog down + /// before it ever fires (`recv` gets `Ok` -> `false`, no kill), and `child.wait()` + /// then blocks on the still-alive leader — pinning the walk thread and its read/ + /// write permit past the deadline, bypassing GITLAWB_GIT_SERVICE_TIMEOUT_SECS. + /// This is distinct from the descendant case above: there the leader sleeps until + /// the deadline so the watchdog DOES time out; here the drain-EOF races ahead of + /// the deadline. The fix keeps the watchdog armed until the child is actually + /// reaped, so the deadline SIGTERM still fires and the call returns within budget. + /// A pre-fix build blocks on `child.wait()` past the recv budget (RED). + #[cfg(unix)] + #[test] + fn run_bounded_git_reaps_a_leader_that_closes_stdout_then_hangs() { + use std::time::{Duration, Instant}; + let tmp = TempDir::new().unwrap(); + // rev-list records its (leader) pid, closes stdout+stderr so the drain EOFs + // immediately, then sleeps without trapping TERM. `sleep 30` bounds the worst + // case so a RED run cannot wedge the suite; the recv budget fires first. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) echo $$ > leader.pid; exec 1>&- 2>&-; sleep 30 ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + let (tx, rx) = std::sync::mpsc::channel(); + let path = tmp.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(run_bounded_git( + &git_bin, + &["rev-list"], + &path, + b"", + Instant::now() + Duration::from_millis(100), + )); + }); + let out = rx.recv_timeout(Duration::from_secs(10)).expect( + "run_bounded_git must return within the watchdog budget when the leader closes stdout then hangs, not block on child.wait()", + ); + assert!( + out.is_err(), + "a leader killed at the deadline (no TERM trap) is a timeout, not a success: {out:?}" + ); + let pid: i32 = std::fs::read_to_string(tmp.path().join("leader.pid")) + .unwrap() + .trim() + .parse() + .unwrap(); + let mut gone = false; + for _ in 0..300 { + if unsafe { libc::kill(pid, 0) } != 0 { + gone = true; + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + // Kill it regardless so a RED run leaks no orphan. + unsafe { libc::kill(pid, libc::SIGKILL) }; + assert!( + gone, + "the hung leader (pid {pid}) must be killed and reaped at the deadline, not left running" + ); + } + + use crate::db::VisibilityMode; + use chrono::Utc; + use std::process::Command; + use tempfile::TempDir; + + fn rule(path_glob: &str, readers: &[&str]) -> VisibilityRule { + VisibilityRule { + id: "x".into(), + repo_id: "r1".into(), + path_glob: path_glob.into(), + mode: VisibilityMode::B, + reader_dids: readers.iter().map(|s| s.to_string()).collect(), + created_by: "did:key:zOwner".into(), + created_at: Utc::now(), + } + } + + const OWNER: &str = "did:key:zOwner"; + + /// Build a bare repo with public/a.txt and secret/b.txt at one commit. + /// Returns (tempdir, bare_path, secret_blob_oid, public_blob_oid). + fn fixture() -> (TempDir, std::path::PathBuf, String, String) { + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + let bare = td.path().join("bare.git"); + let run = |args: &[&str], dir: &Path| { + let ok = Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(); assert!(ok, "git {args:?} failed"); }; std::fs::create_dir_all(work.join("public")).unwrap(); @@ -468,6 +1573,614 @@ mod tests { (td, bare, secret, public) } + /// #173 (jatmn round 8, F4 — load-bearing): a repo with enough annotated tags that + /// one `cat-file --batch` round fills BOTH pipes (stdin > 64 KiB of oids while the + /// child blocks on a full stdout) must not deadlock. The old order wrote the whole + /// round to stdin before draining stdout and hung indefinitely, stranding a blocking- + /// pool thread; `run_bounded_git`'s concurrent writer/drain completes. Driven with a + /// completion timeout: GREEN finishes in well under a second, RED (old order) hangs + /// and the recv_timeout fires. ~3000 tags is well past the ~2030-oid deadlock + /// threshold (41 bytes/oid, 64 KiB pipes) and under MAX_TAG_OBJECTS (8192). + /// Bulk-created via one fast-import stream so the fixture cost is one git process, + /// not 3000 `git tag -a` spawns. + #[test] + fn walk_tag_chain_large_batch_does_not_deadlock() { + use std::io::Write; + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + let bare = td.path().join("bare.git"); + let run = |args: &[&str], dir: &Path| { + assert!( + Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(), + "git {args:?} failed" + ); + }; + std::fs::create_dir_all(&work).unwrap(); + std::fs::write(work.join("f.txt"), b"x\n").unwrap(); + run(&["init", "-q"], &work); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "."], &work); + run(&["commit", "-qm", "init"], &work); + let head = { + let out = Command::new("git") + .args(["rev-parse", "HEAD"]) + .current_dir(&work) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + + // Bulk-create ~3000 annotated tags via one fast-import stream. + const N: usize = 3000; + let mut stream = String::new(); + for i in 0..N { + let msg = format!("annotated tag {i}\n"); + stream.push_str(&format!("tag t{i}\n")); + stream.push_str(&format!("from {head}\n")); + stream.push_str("tagger t 1700000000 +0000\n"); + stream.push_str(&format!("data {}\n", msg.len())); + stream.push_str(&msg); + } + let mut fi = Command::new("git") + .args(["fast-import", "--quiet"]) + .current_dir(&work) + .stdin(std::process::Stdio::piped()) + .spawn() + .unwrap(); + fi.stdin + .take() + .unwrap() + .write_all(stream.as_bytes()) + .unwrap(); + assert!(fi.wait().unwrap().success(), "fast-import failed"); + + run( + &[ + "clone", + "-q", + "--bare", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + td.path(), + ); + + // Drive the walk on a worker thread with a completion timeout. The old + // write-all-before-drain order hangs here; the fix completes near-instantly. + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = tx.send(reachable_commit_tag_oids(&bare).map(|s| s.len())); + }); + match rx.recv_timeout(std::time::Duration::from_secs(20)) { + Ok(Ok(n)) => assert!( + n >= N, + "the walk must resolve every annotated tag object (got {n}, expected >= {N})" + ), + Ok(Err(e)) => panic!("walk errored: {e}"), + Err(_) => panic!("walk_tag_chain deadlocked on a large tag batch (F4 regression)"), + } + } + + /// #173 review (finding 3): an annotated tag reachable ONLY through a tag-valued + /// detached HEAD (raw HEAD naming a tag object, with no ref at that tag) must still + /// enter `reachable_commit_tag_oids`. `rev-list --all HEAD` peels such a HEAD to its + /// commit and `for-each-ref` has no tag row, so without a HEAD tag-seed the tag + /// OBJECT is omitted and its pinned CID would 404 for an authorized reader. RED + /// before the HEAD tag-seed (the tag oid is absent); GREEN after. + #[test] + fn reachable_commit_tag_oids_includes_tag_valued_detached_head() { + use std::io::Write; + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + let bare = td.path().join("bare.git"); + let run = |args: &[&str], dir: &Path| -> String { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + std::fs::create_dir_all(&work).unwrap(); + std::fs::write(work.join("a.txt"), b"hi\n").unwrap(); + run(&["init", "-q"], &work); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "."], &work); + run(&["commit", "-qm", "seed"], &work); + run( + &[ + "clone", + "-q", + "--bare", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + td.path(), + ); + let commit = run(&["rev-parse", "HEAD"], &bare); + + // An annotated tag OBJECT in the bare ODB, with NO ref pointing at it. + let tag_body = format!( + "object {commit}\ntype commit\ntag htag\ntagger t 0 +0000\n\nHEAD-only tag\n" + ); + let tag_oid = { + let mut child = Command::new("git") + .args(["hash-object", "-t", "tag", "-w", "--stdin"]) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap(); + child + .stdin + .as_mut() + .unwrap() + .write_all(tag_body.as_bytes()) + .unwrap(); + let out = child.wait_with_output().unwrap(); + assert!(out.status.success()); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + assert_eq!(run(&["cat-file", "-t", &tag_oid], &bare), "tag"); + // Raw-write HEAD directly to the tag object (the only way this state arises; + // update-ref / checkout both refuse a non-commit HEAD). + std::fs::write(bare.join("HEAD"), format!("{tag_oid}\n")).unwrap(); + + let set = reachable_commit_tag_oids(&bare).unwrap(); + assert!( + set.contains(&tag_oid), + "a tag reachable only via a tag-valued detached HEAD must be in the reachable set" + ); + assert!( + set.contains(&commit), + "the commit the HEAD tag peels to stays reachable (no regression)" + ); + } + + /// #173: `reachable_commit_tag_oids` on an empty repo (unborn HEAD) must return an + /// empty set, not error — exercising the `rev-parse HEAD` fail branch of the + /// detached-HEAD tag seed (there is simply no HEAD to seed). + #[test] + fn reachable_commit_tag_oids_handles_unborn_head() { + let td = TempDir::new().unwrap(); + let bare = td.path().join("empty.git"); + let ok = Command::new("git") + .args(["init", "-q", "--bare", bare.to_str().unwrap()]) + .status() + .unwrap() + .success(); + assert!(ok, "git init --bare failed"); + let set = reachable_commit_tag_oids(&bare).unwrap(); + assert!( + set.is_empty(), + "an empty repo (unborn HEAD) yields an empty reachable set with no error" + ); + } + + #[test] + fn object_paths_emits_trees_and_blob_paths_is_the_blob_slice() { + let (_td, bare, secret_oid, public_oid) = fixture(); + let deadline = Instant::now() + WALK_TIMEOUT; + // The lenient enumeration; on this clean fixture it matches the strict one. + let commits = reachable_commit_oids(&bare, "git", deadline).unwrap(); + let objs = object_paths(&bare, "git", &commits, deadline).unwrap(); + + // Blob records survive the `-rzt` change, at their paths (unchanged). + assert!(objs.contains(&(secret_oid.clone(), "/secret/b.txt".into(), "blob".into()))); + assert!(objs.contains(&(public_oid.clone(), "/public/a.txt".into(), "blob".into()))); + + // The #135 addition: subtree tree objects at their directory paths. + assert!( + objs.iter().any(|(_, p, k)| k == "tree" && p == "/secret"), + "the /secret subtree tree must be emitted at its dir path" + ); + assert!( + objs.iter().any(|(_, p, k)| k == "tree" && p == "/public"), + "the /public subtree tree must be emitted at its dir path" + ); + + // blob_paths must equal the blob slice of object_paths exactly — compared as + // SETS (both walks dedup via HashSet; the collected order is nondeterministic). + let bp: HashSet<(String, String)> = blob_paths(&bare, "git", WALK_TIMEOUT) + .unwrap() + .into_iter() + .collect(); + let bp_from_obj: HashSet<(String, String)> = objs + .iter() + .filter(|(_, _, k)| k == "blob") + .map(|(o, p, _)| (o.clone(), p.clone())) + .collect(); + assert_eq!( + bp, bp_from_obj, + "blob_paths output must be byte-identical to object_paths' blob slice" + ); + } + + #[test] + fn allowed_tree_set_gates_withheld_subtree_tree() { + let (_td, bare, _s, _p) = fixture(); + let oid = |rev: &str| { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(&bare) + .output() + .unwrap(); + assert!(out.status.success(), "rev-parse {rev}"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + let secret_tree = oid("HEAD:secret"); + let public_tree = oid("HEAD:public"); + let root_tree = oid("HEAD^{tree}"); + let reader = "did:key:z6MkReader"; + let rules = [rule("/secret/**", &[reader])]; + + // anon: the withheld /secret tree is excluded; root ("/") and /public are in. + let anon = allowed_tree_set_for_caller(&bare, &rules, true, OWNER, None).unwrap(); + assert!( + !anon.contains(&secret_tree), + "withheld /secret subtree tree excluded for anon" + ); + assert!(anon.contains(&root_tree), "root tree included (path /)"); + assert!(anon.contains(&public_tree), "/public subtree tree included"); + + // listed reader: sees the /secret tree (caller-aware, not a blanket deny). + let rd = allowed_tree_set_for_caller(&bare, &rules, true, OWNER, Some(reader)).unwrap(); + assert!( + rd.contains(&secret_tree), + "listed reader sees the /secret tree" + ); + + // owner: sees every reachable tree. + let ow = allowed_tree_set_for_caller(&bare, &rules, true, OWNER, Some(OWNER)).unwrap(); + assert!( + ow.contains(&secret_tree) && ow.contains(&public_tree) && ow.contains(&root_tree), + "owner sees all reachable trees" + ); + } + + #[test] + fn allowed_tree_set_excludes_dangling_tree() { + use std::io::Write; + let (_td, bare, secret_oid, _p) = fixture(); + // A DANGLING tree: written to the ODB but referenced by no commit. Uses a + // UNIQUE entry name so its oid is content-distinct from every reachable tree + // (a content-identical tree would dedup to a reachable oid — that is T2, not + // danglingness). The reachable-only walk never enumerates it -> fail closed. + let mut child = Command::new("git") + .args(["mktree"]) + .current_dir(&bare) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + writeln!( + child.stdin.as_mut().unwrap(), + "100644 blob {secret_oid}\tdangling-only-unreferenced.txt" + ) + .unwrap(); + let out = child.wait_with_output().unwrap(); + assert!(out.status.success(), "git mktree"); + let dangling = String::from_utf8_lossy(&out.stdout).trim().to_string(); + + let rules = [rule("/secret/**", &[])]; + for caller in [None, Some(OWNER)] { + let set = allowed_tree_set_for_caller(&bare, &rules, true, OWNER, caller).unwrap(); + assert!( + !set.contains(&dangling), + "dangling tree must never be in the reachable allowed-set (caller={caller:?})" + ); + } + } + + #[test] + fn allowed_tree_set_includes_tree_shared_across_allowed_and_denied_paths() { + // T2 (content-dedup): the SAME tree oid reachable at both an allowed and a + // withheld path is INCLUDED for anon (allowed-wins) — its structure is + // visible to the caller at the allowed path. Mirrors the blob analog + // `same_blob_at_allowed_and_denied_path_is_not_withheld`. + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + std::fs::create_dir_all(work.join("pub/sub")).unwrap(); + std::fs::create_dir_all(work.join("sec/sub")).unwrap(); + std::fs::write(work.join("pub/sub/f.txt"), b"same bytes\n").unwrap(); + std::fs::write(work.join("sec/sub/f.txt"), b"same bytes\n").unwrap(); + let run = |args: &[&str]| { + assert!( + Command::new("git") + .args(args) + .current_dir(&work) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + run(&["init", "-q"]); + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + run(&["add", "."]); + run(&["commit", "-qm", "seed"]); + let oid = |rev: &str| { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(&work) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + let pub_sub = oid("HEAD:pub/sub"); + let sec_sub = oid("HEAD:sec/sub"); + assert_eq!(pub_sub, sec_sub, "identical content dedups to one tree oid"); + + // Withhold /sec from anon; the shared oid is still reachable at /pub/sub. + let rules = [rule("/sec/**", &[])]; + let anon = allowed_tree_set_for_caller(&work, &rules, true, OWNER, None).unwrap(); + assert!( + anon.contains(&pub_sub), + "a tree reachable at an allowed path is included even when also at a withheld path" + ); + } + + #[test] + fn allowed_tree_set_includes_root_trees_of_all_reachable_commits() { + // The batched root-tree pass (root_tree_pairs) must return EVERY reachable + // commit's root tree, not just HEAD's — two commits with distinct root trees + // both land in the set. Guards the git-log-over-N-commits root derivation. + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + std::fs::create_dir_all(&work).unwrap(); + let run = |args: &[&str]| { + assert!( + Command::new("git") + .args(args) + .current_dir(&work) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + run(&["init", "-q"]); + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + let oid = |rev: &str| { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(&work) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + std::fs::write(work.join("a.txt"), b"one\n").unwrap(); + run(&["add", "."]); + run(&["commit", "-qm", "c1"]); + let root1 = oid("HEAD^{tree}"); + std::fs::write(work.join("b.txt"), b"two\n").unwrap(); + run(&["add", "."]); + run(&["commit", "-qm", "c2"]); + let root2 = oid("HEAD^{tree}"); + assert_ne!(root1, root2, "the two commits have distinct root trees"); + + // Public repo, no rules: every reachable tree is allowed for anon. + let set = allowed_tree_set_for_caller(&work, &[], true, OWNER, None).unwrap(); + assert!( + set.contains(&root1) && set.contains(&root2), + "root trees of BOTH reachable commits are in the set (batched root pass)" + ); + } + + #[test] + fn root_tree_pairs_returns_every_root_tree_at_scale() { + // Parity + liveness at scale for root_tree_pairs (#173 P2): feed every + // reachable commit oid to `git log --format=%T --stdin` and collect each + // commit's root tree. With N commits that is ~N*41 bytes of oids in and + // ~N*41 bytes of %T out — past the ~64 KiB pipe buffer in both directions — + // so this exercises the large-bidirectional-IO path the 2-commit test above + // cannot, and asserts parity: every distinct root tree comes back. + // + // NOTE: this is NOT a deadlock guard. `git log --stdin` reads its whole + // revision list to EOF before emitting any %T, so the naive "write all of + // stdin, then drain stdout" form does not deadlock at any scale for this + // invocation. `run_bounded_git`'s concurrent writer/drain is cheap defensive + // isolation, not load-bearing, and this test does not claim otherwise. The + // 30s watchdog is a general liveness bound so a future regression that + // genuinely hangs fails fast here rather than stalling the suite. + const N: usize = 2500; + let td = TempDir::new().unwrap(); + let bare = td.path().join("many.git"); + assert!(Command::new("git") + .args(["init", "-q", "--bare", bare.to_str().unwrap()]) + .status() + .unwrap() + .success()); + + // fast-import a linear chain of N commits, each adding a distinct file so + // every root tree is distinct (dedup cannot shrink the output). One + // subprocess, ~1s — far cheaper than N `git commit` spawns. + let mut stream = String::new(); + for i in 0..N { + let (b, cm) = (2 * i + 1, 2 * i + 2); + let content = format!("v{i}"); + let msg = format!("c{i}"); + stream.push_str(&format!( + "blob\nmark :{b}\ndata {}\n{content}\n", + content.len() + )); + stream.push_str(&format!( + "commit refs/heads/main\nmark :{cm}\ncommitter t 0 +0000\ndata {}\n{msg}\n", + msg.len() + )); + if i > 0 { + stream.push_str(&format!("from :{}\n", 2 * (i - 1) + 2)); + } + stream.push_str(&format!("M 100644 :{b} f{i}\n\n")); + } + let mut fi = Command::new("git") + .args(["fast-import", "--quiet"]) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .spawn() + .unwrap(); + { + use std::io::Write; + fi.stdin + .take() + .unwrap() + .write_all(stream.as_bytes()) + .unwrap(); + } + assert!(fi.wait().unwrap().success(), "fast-import failed"); + + let commits = reachable_commit_oids(&bare, "git", Instant::now() + WALK_TIMEOUT).unwrap(); + assert_eq!(commits.len(), N, "all {N} commits reachable"); + + // Call root_tree_pairs directly (private, same module) under a liveness + // watchdog, then assert it returned every distinct root tree. + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = tx.send( + root_tree_pairs(&bare, "git", &commits, Instant::now() + WALK_TIMEOUT) + .map(|s| s.len()), + ); + }); + match rx.recv_timeout(std::time::Duration::from_secs(30)) { + Ok(Ok(len)) => assert_eq!(len, N, "every distinct root tree returned"), + Ok(Err(e)) => panic!("root_tree_pairs errored: {e}"), + Err(_) => panic!("root_tree_pairs did not return within 30s"), + } + } + + /// #173 (jatmn tag fan-out): the batched `git cat-file --batch` tag walk must + /// return the SAME reachable set as the old per-tag `cat-file tag` loop — every + /// commit, the outer tag object, AND the inner tag object of a tag-of-a-tag chain + /// (the inner tag is reachable but is not itself a ref tip, so it is only found by + /// peeling the outer tag's target). Behavior-preservation proof for the rewrite. + #[test] + fn reachable_commit_tag_oids_includes_nested_tag_objects() { + let (_td, bare, _secret, _public) = fixture(); + let run = |args: &[&str]| -> String { + let out = Command::new("git") + .args(args) + .current_dir(&bare) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + // v1 -> commit, v2 -> v1 (tag-of-a-tag), plus a couple of sibling tags so the + // round batches more than one oid. Capture v1's oid, then DELETE the v1 ref so + // the inner tag object survives in the ODB but is NOT a ref tip: it is then + // reachable ONLY by peeling v2's target chain. That makes the peel load-bearing + // (breaking the inner-tag enqueue drops v1 from the set), unlike leaving v1 as + // its own ref where `for-each-ref` would seed it directly. + run(&["tag", "-a", "-m", "inner", "v1", "HEAD"]); + run(&["tag", "-a", "-m", "outer", "v2", "v1"]); + run(&["tag", "-a", "-m", "s1", "s1", "HEAD"]); + run(&["tag", "-a", "-m", "s2", "s2", "HEAD"]); + let commit = run(&["rev-parse", "HEAD"]); + let v1 = run(&["rev-parse", "v1"]); + let v2 = run(&["rev-parse", "v2"]); + let s1 = run(&["rev-parse", "s1"]); + let s2 = run(&["rev-parse", "s2"]); + run(&["tag", "-d", "v1"]); + + let set = reachable_commit_tag_oids(&bare).unwrap(); + assert!(set.contains(&commit), "the commit must be reachable"); + assert!( + set.contains(&v2), + "the outer tag object (ref tip) must be present" + ); + assert!( + set.contains(&v1), + "the INNER tag object of a tag-of-a-tag must be present (peeled from v2, no ref)" + ); + assert!(set.contains(&s1), "sibling tag s1 must be present"); + assert!(set.contains(&s2), "sibling tag s2 must be present"); + } + + /// #173 (jatmn tag fan-out): the object bound is load-bearing. A repo whose tag + /// count exceeds the bound must FAIL CLOSED (Err), not return a truncated set that + /// would under-withhold a still-reachable tag. Drives `walk_tag_chain` with a tiny + /// injected bound (the public fn uses the real `MAX_TAG_OBJECTS`); with the bound + /// check removed this would collect all tags and return Ok. + #[test] + fn walk_tag_chain_fails_closed_over_object_bound() { + let (_td, bare, _secret, _public) = fixture(); + let run = |args: &[&str]| { + assert!( + Command::new("git") + .args(args) + .current_dir(&bare) + .status() + .unwrap() + .success(), + "git {args:?} failed" + ); + }; + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + let mut seeds = Vec::new(); + for n in 0..5 { + let name = format!("t{n}"); + run(&["tag", "-a", "-m", &name, &name, "HEAD"]); + let oid = Command::new("git") + .args(["rev-parse", &name]) + .current_dir(&bare) + .output() + .unwrap(); + seeds.push(String::from_utf8_lossy(&oid.stdout).trim().to_string()); + } + + // Within a generous bound: the walk succeeds and collects the tags. + let mut ok_set = HashSet::new(); + walk_tag_chain( + &bare, + "git", + seeds.clone(), + &mut ok_set, + 8192, + Instant::now() + WALK_TIMEOUT, + ) + .unwrap(); + assert!( + seeds.iter().all(|s| ok_set.contains(s)), + "all 5 tags collected under a generous bound" + ); + + // Under a bound of 2 with 5 tags: fail closed (Err), not a partial set. + let mut small_set = HashSet::new(); + let result = walk_tag_chain( + &bare, + "git", + seeds, + &mut small_set, + 2, + Instant::now() + WALK_TIMEOUT, + ); + assert!( + result.is_err(), + "a tag count exceeding the object bound must fail closed (Err), not truncate" + ); + } + #[test] fn anonymous_caller_withholds_only_private_blob() { let (_td, bare, secret_oid, public_oid) = fixture(); @@ -735,7 +2448,12 @@ mod tests { String::from_utf8_lossy(&out.stdout).trim().to_string() }; - let all_blobs = crate::git::push_delta::all_blob_oids(&work).unwrap(); + let all_blobs = crate::git::push_delta::all_blob_oids( + &work, + "git", + std::time::Instant::now() + std::time::Duration::from_secs(600), + ) + .unwrap(); assert!( all_blobs.contains(&dangling_oid), "precondition: the dangling blob is in the all-objects universe" diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 3b346190..0a379a92 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -9,6 +9,401 @@ use anyhow::Result; use gitlawb_core::cid::Cid; +use std::time::{Duration, Instant}; + +/// Attempts (including the first) for a transient DB-record retry. +const PIN_RECORD_ATTEMPTS: u32 = 3; +/// Backoff between DB-record retry attempts. +const PIN_RECORD_BACKOFF: Duration = Duration::from_millis(50); + +/// Run an idempotent DB-record operation with a bounded retry so a sub-second +/// transient error does not silently leave the pin-source set permanently +/// incomplete. The resolver treats a nonempty below-cap source set as complete, +/// so a dropped `record_pin_source`/`record_pinned_cid` makes `GET /ipfs/{cid}` +/// 404 a valid public copy. Every wrapped insert is idempotent (`ON CONFLICT DO +/// NOTHING` / provenance-preserving upsert), so re-running is safe. On exhausted +/// attempts the last error is returned and the caller records the durable +/// `pin_sources_incomplete` marker (U3, #173), which is what keeps the resolver's +/// bounded scan fallback available for that object instead of 404ing a public copy. +/// Shared with the `pinata.rs` twin so both pin paths retry identically. Runs +/// inside the already-detached post-push task, so the backoff adds no push latency. +pub(crate) async fn retry_db_record(mut op: F) -> Result<()> +where + F: FnMut() -> Fut, + Fut: std::future::Future>, +{ + let mut attempt = 1; + loop { + match op().await { + Ok(()) => return Ok(()), + Err(e) => { + if attempt >= PIN_RECORD_ATTEMPTS { + return Err(e); + } + tokio::time::sleep(PIN_RECORD_BACKOFF).await; + attempt += 1; + } + } + } +} + +/// Opportunistically repair a legacy provider-CID row on the already-pinned skip +/// path (#173 R8, KTD8). Releases before this branch stored the PROVIDER CID +/// (Kubo dag-pb / Pinata CIDv0) in `pinned_cids.cid`; the `/ipfs` resolver +/// recomputes the raw CID from object bytes and 404s any row whose key does not +/// match, yet `list_pinned_cids` still advertises the stored key — so a client +/// gets a CID the resolver deliberately withholds. When a re-push carries the +/// object again, rewrite the key to the raw CID and stash the old provider value +/// in `legacy_provider_cid`. +/// +/// COST GATE: candidacy is decided from the stored key's codec alone — a +/// CIDv1/raw key is already the resolver key and reads NO bytes, keeping the +/// steady-state skip cost DB-only. Only a legacy-codec row reads the object to +/// recompute. A row whose bytes are gone stays withheld (no destructive rewrite). +async fn repair_legacy_provider_cid( + repo_path: &std::path::Path, + git_bin: &str, + git_timeout: Duration, + sha: &str, + db: &crate::db::Db, +) -> Result { + let stored = match db.cid_for_oid(sha).await? { + Some(c) => c, + None => return Ok(RepairOutcome::Settled), + }; + // Cost gate: a canonical raw CIDv1 key is already correct — never read bytes. + if gitlawb_core::cid::is_raw_cidv1(&stored) { + return Ok(RepairOutcome::Settled); + } + // Legacy-codec row: read the object to recompute. Counted so a test can prove + // the gate above spares non-legacy rows this read. + #[cfg(test)] + note_legacy_repair_read(); + // `read_object_bounded` is SYNCHRONOUS `git cat-file`, and its budget is + // `git_service_timeout_secs` (600 by default), so running it inline parks a tokio + // worker for as long as git takes: one wedged read on the sweep's first pass at boot + // holds a worker for ten minutes, per legacy row. Push it to the blocking pool, the + // same shape `replication_withheld_set` uses in api/repos.rs (#173 round 11, F4). + // Both callers of this function are async, so neither changes shape. The read-counter + // increment above stays on THIS thread so the thread_local cost-gate assertion holds. + let read = { + let repo_path = repo_path.to_path_buf(); + let git_bin = git_bin.to_string(); + let sha = sha.to_string(); + tokio::task::spawn_blocking(move || { + crate::git::store::read_object_bounded(&git_bin, &repo_path, &sha, git_timeout) + }) + .await + }; + let data = match read { + Ok(Ok(Some((_ty, bytes)))) => bytes, + // Bytes gone: the row stays withheld, never destructively rewritten. Nothing a + // later pass changes, so this is a TERMINAL outcome for the sweep's re-walk gate. + Ok(Ok(None)) => return Ok(RepairOutcome::Settled), + // A wedged/D-state `git cat-file` (timeout/infra): the repair is opportunistic + // and best-effort, so skip it and return Ok so the pin task PROCEEDS to + // requeue_or_release rather than hanging the coalescing key until process death + // (grok F2, #173). A later re-push or the deferred sweep retries the repair. + Ok(Err(e)) => { + tracing::warn!(sha = %sha, err = %e, "skipping legacy provider-CID repair: bounded object read failed"); + return Ok(RepairOutcome::Retryable); + } + // The blocking task panicked or was cancelled: same best-effort treatment, and + // worth another walk because it says nothing about the row itself. + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "skipping legacy provider-CID repair: object read task failed"); + return Ok(RepairOutcome::Retryable); + } + }; + let raw = Cid::from_git_object_bytes(&data).to_string(); + if raw == stored { + return Ok(RepairOutcome::Settled); + } + db.repair_legacy_provider_cid(sha, &raw, &stored).await?; + Ok(RepairOutcome::Repaired) +} + +/// What one opportunistic repair did with a row, so the sweep can tell a skip a later +/// run could fix from one nothing will (U4 re-walk, #173 round 11). The push skip path +/// ignores the value: it repairs whatever the push happens to carry and a failure there +/// is already warn-only. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum RepairOutcome { + /// Nothing to do, or nothing a re-walk would change: the stored key is already the + /// raw resolver key, the recomputed key matches it, or the object's bytes are gone. + Settled, + /// The bounded object read failed (a wedged `git cat-file`, an unreadable repo). + /// The bytes may be readable later, so the row is worth walking again. + Retryable, + /// The row's key was rewritten to the raw-content CID. + Repaired, +} + +/// What one sweep pass (or a whole sweep run) did. `scanned` counts `pinned_cids` +/// rows READ, which is the quantity the batch size bounds; `repaired` counts rows +/// whose key was actually rewritten to the raw CID. +#[derive(Debug, Default, PartialEq, Eq)] +pub(crate) struct SweepStats { + pub scanned: usize, + pub repaired: usize, + pub passes: usize, + /// Rows left unrepaired for a reason a LATER run could fix (the source repo is not + /// on this node's local disk, a DB read failed, a bounded object read failed). A + /// nonzero count is what makes the run rewind its cursor instead of parking it at + /// the end of the table forever. Rows that are unrepairable in principle (no + /// provenance, the repo row is gone, the bytes are gone) are NOT counted here. + pub retryable_skips: usize, +} + +/// One bounded pass of the U4 sweep: read at most `batch` `pinned_cids` rows after +/// the persisted cursor, repair the legacy ones, and persist the new cursor. +/// +/// The batch is what bounds the pass. It caps rows READ, not rows repaired, because +/// the legacy predicate is a codec decode SQL cannot express; a table of raw rows +/// therefore costs one indexed range scan per pass and nothing else. +/// +/// The cursor advances to the LAST row read whatever happened to each row, including +/// rows that were skipped as unrepairable. A cursor that only advanced on success +/// would re-read the same unrepairable row on every pass and the sweep would never +/// reach the rows behind it. +async fn sweep_pass( + repos_dir: &std::path::Path, + git_bin: &str, + git_timeout: Duration, + batch: i64, + db: &crate::db::Db, +) -> Result { + let cursor = db.pin_repair_cursor().await?; + let rows = db.pinned_cids_after(&cursor, batch).await?; + let scanned = rows.len(); + let mut repaired = 0usize; + let mut retryable_skips = 0usize; + let mut last = cursor; + + for (sha, stored) in rows { + // Advance FIRST: every path below this line may skip the row, and none of them + // may wedge the walk (scenario 7). + last = sha.clone(); + // Same cost gate as the skip-path repair: a canonical raw CIDv1 key is already + // the resolver key, so it reads no bytes and resolves no repo. + if gitlawb_core::cid::is_raw_cidv1(&stored) { + continue; + } + // Resolve the row's repo from its recorded provenance (first-pinner plus the + // bounded additional source set). An empty set is a pin recorded before + // provenance existed: nothing to read the bytes from, so skip it. + let sources = match db.pin_sources_for_oid(&sha).await { + Ok(s) => s, + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "sweep: failed to read pin sources"); + // A DB read error says nothing about the row, so a later run retries it. + retryable_skips += 1; + continue; + } + }; + // Whether this row ended the source walk repaired, and whether anything it hit + // along the way was a transient obstacle rather than a permanent one. + let mut row_repaired = false; + let mut row_retryable = false; + for repo_id in sources { + let repo = match db.get_repo_by_id(&repo_id).await { + Ok(Some(r)) => r, + // The repo row is gone: a later source may still hold the bytes. A + // deleted repo does not come back, so this is not a retryable skip. + Ok(None) => continue, + Err(e) => { + tracing::warn!(repo_id = %repo_id, err = %e, "sweep: failed to read repo"); + row_retryable = true; + continue; + } + }; + // Derive the LOCAL disk path rather than going through `repo_store.acquire`. + // The sweep is opportunistic background maintenance over every pinned row on + // the node, so it must never pull a cold repo back from remote storage: that + // would turn a repair pass into a bulk restore. A repo that is not on local + // disk simply reads no bytes here and stays withheld, but it IS a retryable + // skip: on a Tigris-backed node the repo is cold now and warm later, and + // without the re-walk that row would never be repaired by anything. + // The path goes through the repo store's VALIDATED resolver (allowlisted + // components, rooted at repos_dir, no ParentDir/CurDir segment), not the raw + // join: the sweep is a second caller of that path logic and gets the same + // barrier the acquire path has (#173 round 11, F3). It is the non-fetching + // variant, so the no-cold-pull property above is untouched. + let repo_path = match crate::git::repo_store::validated_repo_disk_path( + repos_dir, + &repo.owner_did, + &repo.name, + ) { + Ok(p) => p, + // An unsafe name is not something a later run fixes, so it is terminal. + Err(e) => { + tracing::warn!(repo_id = %repo_id, err = %e, "sweep: rejected unsafe repo path"); + continue; + } + }; + if !repo_path.is_dir() { + row_retryable = true; + continue; + } + match repair_legacy_provider_cid(&repo_path, git_bin, git_timeout, &sha, db).await { + Ok(RepairOutcome::Repaired) => { + repaired += 1; + row_repaired = true; + break; + } + // The bytes could not be read from this source right now: try the next + // source, and if none of them works, walk the row again on a later run. + Ok(RepairOutcome::Retryable) => row_retryable = true, + // Nothing to repair from this source and nothing a re-walk changes. + Ok(RepairOutcome::Settled) => {} + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "sweep: legacy provider-CID repair failed"); + row_retryable = true; + } + } + } + if !row_repaired && row_retryable { + retryable_skips += 1; + } + } + + db.set_pin_repair_cursor(&last).await?; + Ok(SweepStats { + scanned, + repaired, + passes: 1, + retryable_skips, + }) +} + +/// Test seam for a single bounded pass (scenarios 4 and 5 drive passes by hand to +/// observe the batch bound and the restart-resumes-from-cursor behavior). +#[cfg_attr(not(test), allow(dead_code))] +pub(crate) async fn sweep_legacy_provider_cids_once( + repos_dir: &std::path::Path, + git_bin: &str, + git_timeout: Duration, + batch: i64, + db: &crate::db::Db, +) -> Result { + sweep_pass(repos_dir, git_bin, git_timeout, batch, db).await +} + +/// U4 (#173): the one-shot legacy provider-CID migration sweep. +/// +/// Releases before this branch stored the PROVIDER CID (Kubo dag-pb / Pinata CIDv0) +/// in `pinned_cids.cid`. This branch's `/ipfs/{cid}` resolver recomputes the raw +/// content CID and withholds any row whose key does not match, so those rows are +/// unresolvable. The opportunistic repair on the already-pinned skip path only fires +/// when a later push re-carries the object, and normal git negotiation omits objects +/// the node already has, so on an upgraded node that push generally never comes. This +/// walks the table instead. +/// +/// Runs until a pass comes back short of a full batch, which is the end of the table. +/// Sleeps `delay` between full batches so it cannot monopolize the DB, and persists +/// its cursor every pass so a restart continues instead of rewinding. Errors reading +/// or repairing an individual row are warn-and-skip; only a failure of the batch query +/// or the cursor write ends the run, and a later run picks up from the stored cursor. +/// +/// A run that skipped at least one RETRYABLE row rewinds the cursor to the start of the +/// table on its way out (#173 round 11). Without that the cursor parked at the maximum +/// `sha256_hex` for good: every later boot read zero rows, so a row skipped for a +/// transient reason (its repo cold on a Tigris-backed node, a DB or object read error) +/// was skipped permanently, unadvertised and unresolvable with nothing left to fix it. +/// The rewind is a per-RUN decision made after the walk has already finished, never +/// mid-walk, so it cannot spin: the cost is one extra ordered scan on the next run, and +/// a row that is unrepairable in principle (bytes gone, provenance gone) does not count +/// as retryable, so a node holding one does not re-walk on every boot forever. +pub(crate) async fn sweep_legacy_provider_cids( + repos_dir: &std::path::Path, + git_bin: &str, + git_timeout: Duration, + batch: i64, + delay: Duration, + db: &crate::db::Db, +) -> SweepStats { + let mut totals = SweepStats::default(); + loop { + let pass = match sweep_pass(repos_dir, git_bin, git_timeout, batch, db).await { + Ok(p) => p, + Err(e) => { + tracing::warn!(err = %e, "legacy provider-CID sweep pass failed; stopping"); + break; + } + }; + totals.scanned += pass.scanned; + totals.repaired += pass.repaired; + totals.retryable_skips += pass.retryable_skips; + totals.passes += 1; + // A short batch means the ordered walk reached the end of the table. Stop here + // rather than after an extra empty pass, and do NOT sleep on the way out. + if (pass.scanned as i64) < batch { + break; + } + tokio::time::sleep(delay).await; + } + if totals.retryable_skips > 0 { + if let Err(e) = db.set_pin_repair_cursor("").await { + tracing::warn!(err = %e, "failed to rewind the legacy provider-CID sweep cursor"); + } + } + totals +} + +// Test-only cost-gate counter (R8, U7): how many times the opportunistic repair +// read an object's bytes on the skip path. The codec gate must spare a CIDv1/raw +// row this read; the counter is the both-ways guard (removing the gate reads the +// raw row and increments it). Same thread_local discipline as the serve-path +// oversize counter — the pin tests await `pin_new_objects` on a current-thread +// runtime, so the increment and the assertion share one thread. +#[cfg(test)] +thread_local! { + static LEGACY_REPAIR_READS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn reset_legacy_repair_reads() { + LEGACY_REPAIR_READS.with(|c| c.set(0)); +} + +#[cfg(test)] +pub(crate) fn legacy_repair_reads() -> usize { + LEGACY_REPAIR_READS.with(|c| c.get()) +} + +#[cfg(test)] +fn note_legacy_repair_read() { + LEGACY_REPAIR_READS.with(|c| c.set(c.get() + 1)); +} + +/// Wall-clock ceiling on one [`pin_new_objects`] batch. +/// +/// The loop runs under a `pin_semaphore` permit and that pool defers rather than +/// sheds, so without a ceiling the hold is O(N) with N (the push's object count) +/// chosen by the pusher. This bounds the drain of a saturated pool instead. +/// +/// 120s is 12x the shared client's 10s whole-request ceiling, so a single large +/// healthy upload that needs more than the client default still has room to +/// finish (the per-request timeout is set to the remainder, not the default), +/// while a batch of them still cannot hold the permit indefinitely. Deliberately +/// a constant and not a config knob: the value only has to be large enough to be +/// uninteresting on a healthy node, and a knob is operator surface that would +/// have to be documented, validated, and kept meaningful. +pub const PIN_BATCH_BUDGET: Duration = Duration::from_secs(120); + +/// The shared outbound client for both IPFS sinks. +/// +/// `pin_new_objects` runs while holding a `pin_semaphore` permit and that pool +/// defers rather than sheds, so an unbounded await here parks the pool. A bare +/// `reqwest::Client::new()` has no timeout, which is exactly that. Built from +/// `crate::build_http_client` rather than a local builder: its docstring forbids +/// hand-rolling an equivalent, so that the redirect and timeout guarantees the +/// node's tests bind stay bound to the client every outbound path actually uses. +fn http_client() -> &'static reqwest::Client { + static CLIENT: std::sync::OnceLock = std::sync::OnceLock::new(); + CLIENT + .get_or_init(|| crate::build_http_client().expect("failed to build production http client")) +} /// Pin a single git object to the local IPFS/Kubo node. /// @@ -16,9 +411,20 @@ use gitlawb_core::cid::Cid; /// If empty the function returns `Ok("")` immediately. /// - `sha256_hex`: the git SHA-256 hex object ID (used only for logging). /// - `data`: raw git object content bytes (same bytes used for CID computation). +/// - `request_timeout`: overrides the shared client's whole-request timeout for +/// THIS request only. `RequestBuilder::timeout` replaces the client-level value +/// per request and leaks nothing to other calls on the same client, so the +/// batch loop can hand each add whatever is left of its budget without +/// loosening or tightening any other outbound path. `None` keeps the client's +/// own ceiling. /// /// Returns the CID string on success, or `""` when IPFS is not configured. -pub async fn pin_git_object(ipfs_api: &str, sha256_hex: &str, data: &[u8]) -> Result { +pub async fn pin_git_object( + ipfs_api: &str, + sha256_hex: &str, + data: &[u8], + request_timeout: Option, +) -> Result { if ipfs_api.is_empty() { return Ok(String::new()); } @@ -37,13 +443,26 @@ pub async fn pin_git_object(ipfs_api: &str, sha256_hex: &str, data: &[u8]) -> Re .mime_str("application/octet-stream")?; let form = reqwest::multipart::Form::new().part("file", part); - let client = reqwest::Client::new(); - let resp = client - .post(&url) - .multipart(form) + let mut req = http_client().post(&url).multipart(form); + if let Some(t) = request_timeout { + req = req.timeout(t); + } + + let resp = req .send() .await - .map_err(|e| anyhow::anyhow!("IPFS add request failed: {e}"))?; + // Keep the `reqwest::Error` as this error's source rather than + // formatting it away. Operators reading a pin failure want the concrete + // transport cause in the logged chain, not a single flattened line, and + // this module's tests downcast to it to prove a silent endpoint really + // surfaces as a timeout rather than as some other failure that happens + // to arrive in time. + // The context keeps the old message verbatim so the callers that log + // this at `%e` (here, `sync.rs`, `encrypted_pin.rs`) read the same. + .map_err(|e| { + let msg = format!("IPFS add request failed: {e}"); + anyhow::Error::new(e).context(msg) + })?; if !resp.status().is_success() { let status = resp.status(); @@ -55,7 +474,10 @@ pub async fn pin_git_object(ipfs_api: &str, sha256_hex: &str, data: &[u8]) -> Re // Kubo returns newline-delimited JSON; we only care about the last object // (there's typically just one for a single-file add). - let body = resp.text().await?; + let body = resp + .text() + .await + .map_err(|e| anyhow::anyhow!("IPFS add response body read failed: {e}"))?; let cid = body .lines() .filter(|l| !l.trim().is_empty()) @@ -76,13 +498,33 @@ pub async fn cat(ipfs_api: &str, cid: &str) -> Result> { return Err(anyhow::anyhow!("IPFS not configured")); } let url = format!("{}/api/v0/cat?arg={}", ipfs_api.trim_end_matches('/'), cid); - let resp = reqwest::Client::new().post(&url).send().await?; + let resp = http_client().post(&url).send().await?; if !resp.status().is_success() { return Err(anyhow::anyhow!("ipfs cat {cid}: {}", resp.status())); } Ok(resp.bytes().await?.to_vec()) } +/// The batch's remaining wall-clock, or `None` once it is spent, after logging +/// the truncation exactly once. +/// +/// Shaped like `api::ipfs`'s `budget_gate` on purpose: the nonzero-ness rides in +/// the returned value, so every call site must consume it as +/// `let Some(x) = ... else { break }` and a zero `Duration` can never reach a +/// request as its timeout. +fn batch_budget_gate(deadline: Instant, pinned: usize, unattempted: usize) -> Option { + let left = deadline.saturating_duration_since(Instant::now()); + if left.is_zero() { + tracing::warn!( + pinned, + unattempted, + "IPFS pin batch deadline reached; the remaining objects are left unpinned" + ); + return None; + } + Some(left) +} + /// Pin any of the given candidate git objects that are not yet recorded in /// `pinned_cids`. /// @@ -90,26 +532,132 @@ pub async fn cat(ipfs_api: &str, cid: &str) -> Result> { /// applies `visibility_pack::replicable_objects` on the delta path or the /// `..._fail_closed` filter on the full-scan path before calling, so this /// function never sees a withheld blob. `repo_path` is still needed to read each -/// object's bytes. The twin in `pinata.rs` mirrors this shape — change both in -/// lockstep. +/// object's bytes. `repo_id` records the pin's provenance so `GET /ipfs/{cid}` +/// resolves straight to this repo instead of scanning every repo (#173). +/// +/// # What `batch_budget` does and does not bound +/// +/// The loop holds a `pin_semaphore` permit and that pool defers rather than +/// sheds, so the hold has to be bounded by something other than the pusher's +/// object count. Two things here are: +/// +/// - this loop's own wall-clock: the deadline is taken once at loop start and +/// checked at the top of every iteration, so no object's work begins with zero +/// budget left. It is a gate, not a hard ceiling, since a started iteration +/// still runs to completion; +/// - each HTTP add: `pin_git_object` is handed the remainder measured at the top +/// of the iteration as its per-request timeout, which is what lets one large +/// healthy upload run past the shared client's 10s default without letting the +/// batch run forever. The DB check and the git read spend a little of that +/// remainder before the request starts, so the add can finish marginally after +/// the deadline. +/// +/// Three things are NOT bounded, and the gate cannot fix any of them: +/// +/// - the git read. `store::read_object` composes two bare +/// `std::process::Command::output()` calls with no timeout and no +/// process-group reaping, and this loop does not run it under +/// `spawn_blocking`, so one hung `git cat-file` overruns the deadline and +/// blocks a runtime worker thread while it does. +/// - the DB round-trips (`is_pinned`, `record_pinned_cid`). +/// - the pool. `api::repos` acquires the same `pin_semaphore` for the Pinata +/// replication task and holds it across a full git re-derivation plus +/// `pinata::pin_new_objects`, neither of which has a deadline. This change +/// bounds this loop's hold, not the semaphore's worst-case queue. +/// +/// # Truncation semantics +/// +/// A batch stopped at the deadline leaves its remaining objects unpinned, and +/// nothing sweeps them up afterwards. There is no reconciliation pass over +/// `pinned_cids`; recovery is opportunistic, happening only if some later push +/// on the repo takes the full-scan fallback (`push_delta::list_all_objects`) and +/// re-derives the whole object set, which then re-offers the skipped OIDs. +/// +/// The twin in `pinata.rs` mirrors this loop's shape but NOT its bound: it has +/// no batch deadline and no per-request override, so the two are no longer at +/// parity here. Everything else about the shape (the skip-if-pinned check, the +/// provenance recording, the warn-and-continue arm, the returned pairs) still +/// changes in lockstep. /// /// Returns a list of `(sha256_hex, cid)` pairs for objects pinned this call. +// Eight because #173's git seam (`git_bin`, `git_timeout`) and pin provenance +// (`repo_id`) sit alongside #174's batch budget. All four callers pass every one, and +// grouping them into a context struct would add a type whose only job is to be +// destructured back into these fields at the top of the loop. +#[allow(clippy::too_many_arguments)] pub async fn pin_new_objects( ipfs_api: &str, repo_path: &std::path::Path, + git_bin: &str, + git_timeout: Duration, object_list: Vec, db: &crate::db::Db, + repo_id: &str, + batch_budget: Duration, ) -> Vec<(String, String)> { if ipfs_api.is_empty() { return vec![]; } + let deadline = Instant::now() + batch_budget; + let total = object_list.len(); let mut pinned = Vec::new(); - for sha in object_list { - // Skip if already pinned + for (attempted, sha) in object_list.into_iter().enumerate() { + // Top of the iteration, before any of this object's work: an object is + // never started with zero budget left. The remainder becomes this add's + // request timeout below. + let Some(budget_left) = batch_budget_gate(deadline, pinned.len(), total - attempted) else { + break; + }; + // Skip if already pinned, but first backfill provenance if the existing + // pin has none. A legacy pin (recorded before repo_id existed, #173, jatmn) + // is skipped here before record_pinned_cid ever runs, so its NULL provenance + // would never resolve to one repo and known CIDs keep hitting the scan. The + // backfill only sets repo_id (AND repo_id IS NULL guard preserves + // first-pinner-owns) and never re-pins the bytes: the object is already on IPFS. match db.is_pinned(&sha).await { - Ok(true) => continue, + Ok(true) => { + match db.provenance_for_oid(&sha).await { + Ok(None) => { + if let Err(e) = db.backfill_pin_provenance(&sha, repo_id).await { + tracing::warn!(sha = %sha, err = %e, "failed to backfill pin provenance"); + } + } + Ok(Some(_)) => {} + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "DB error reading pin provenance"); + } + } + // F1 (#173 round 8): record this repo as an ADDITIONAL source for the + // already-pinned object. This is the load-bearing skip-branch insert — + // a later repo pushing a shared object hits this path (already pinned), + // and without it `GET /ipfs/{cid}` only ever knows the first pinner, so a + // shared object first pinned from a private/quarantined repo 404s even + // when this repo would serve it. Bounded per object (MAX_PIN_SOURCES). + if let Err(e) = retry_db_record(|| db.record_pin_source(&sha, repo_id)).await { + tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); + // U3 (#173): the retries are spent and this repo is NOT in the source + // set, so the set is known incomplete. Persist that, or the resolver + // reads a non-empty below-cap set as COMPLETE and 404s an object this + // repo would serve. Warn-only in turn: if the marker write also fails + // the object degrades to the pre-U3 behavior, never worse. + if let Err(e) = db.mark_pin_sources_incomplete(&sha).await { + tracing::warn!(sha = %sha, err = %e, "failed to mark pin sources incomplete"); + } + } + // R8 (#173 round 10): opportunistically repair a legacy provider-CID + // row (Kubo dag-pb / Pinata) to the raw-content resolver key on this + // re-push. Cost-gated on the stored key's codec — a non-legacy row + // reads no bytes. Warn-only: a failure leaves the row as-is for a + // later re-push or the deferred one-shot sweep. + if let Err(e) = + repair_legacy_provider_cid(repo_path, git_bin, git_timeout, &sha, db).await + { + tracing::warn!(sha = %sha, err = %e, "failed to repair legacy provider CID"); + } + continue; + } Ok(false) => {} Err(e) => { tracing::warn!(sha = %sha, err = %e, "DB error checking pinned status"); @@ -117,22 +665,49 @@ pub async fn pin_new_objects( } } - // Read raw object content - let data = match crate::git::store::read_object(repo_path, &sha) { - Ok(Some((_obj_type, bytes))) => bytes, - Ok(None) => continue, - Err(e) => { - tracing::warn!(sha = %sha, err = %e, "failed to read git object for pinning"); - continue; - } - }; + // Read raw object content under a bounded read so a wedged/D-state `git + // cat-file` (stuck NFS/Tigris backend) is reaped at `git_timeout` instead of + // hanging pin_new_objects forever — which would pin the post-push coalescing + // key until process death (grok F2, #173). On Err the object is simply not + // pinned this pass; a later pass/push retries. + let data = + match crate::git::store::read_object_bounded(git_bin, repo_path, &sha, git_timeout) { + Ok(Some((_obj_type, bytes))) => bytes, + Ok(None) => continue, + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "failed to read git object for pinning"); + continue; + } + }; // Pin to IPFS - match pin_git_object(ipfs_api, &sha, &data).await { + match pin_git_object(ipfs_api, &sha, &data, Some(budget_left)).await { Ok(cid) if !cid.is_empty() => { - if let Err(e) = db.record_pinned_cid(&sha, &cid).await { + // The resolver key (`pinned_cids.cid`) must be the locally-computed + // raw-content CID, never the provider Hash: Kubo returns a dag-pb/UnixFS + // root for objects above its block size, which does not hash the raw + // content, so `GET /ipfs/{provider_cid}` would resolve then fail the F2 + // integrity check (list-then-404). The serve path reads bytes from git and + // verifies them against the requested CID, so the raw CID is the correct + // key. Mirrors the pinata twin, which already records the raw CID. + let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes(&data).to_string(); + // F1 (#173 round 8): the first pinner is recorded in pin_repo_sources too, + // so every source (first and subsequent) is tried uniformly by the + // resolver. U3 (#173): the pin and its source go down in ONE transaction. + // As two independent best-effort calls this path could land the pin while + // dropping its own source, producing a source set silently missing its + // first pinner; atomically there is no such window, and a total failure + // leaves the object unpinned so the next push retries the whole thing. + if let Err(e) = + retry_db_record(|| db.record_pinned_cid_with_source(&sha, &raw_cid, repo_id)) + .await + { tracing::warn!(sha = %sha, err = %e, "failed to record pinned CID in DB"); } + // Return the provider Hash (not the resolver key), mirroring the pinata + // twin's contract: the DB `cid` is the raw resolver key (recorded above), + // the returned value is the provider CID. Here the return is consumed only + // for logging, but keeping the twins structurally identical avoids drift. pinned.push((sha, cid)); } Ok(_) => {} @@ -144,3 +719,479 @@ pub async fn pin_new_objects( pinned } + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::Cell; + + // The retry helper is the load-bearing unit: it converts a sub-second + // transient DB error at the three warn-only record sites into a landed row, + // instead of a permanently incomplete pin-source set. These drive the helper + // directly against a controlled closure (the record sites take a concrete + // `&Db` over a `PgPool`, so a failing-first wrapper cannot slot in without + // changing signatures — see U6 seam note). + + #[tokio::test] + async fn retry_lands_after_transient_failures() { + let calls = Cell::new(0u32); + let result = retry_db_record(|| { + let n = calls.get() + 1; + calls.set(n); + async move { + if n < PIN_RECORD_ATTEMPTS { + Err(anyhow::anyhow!("transient failure on attempt {n}")) + } else { + Ok(()) + } + } + }) + .await; + + assert!( + result.is_ok(), + "retry lands the row after transient failures" + ); + assert_eq!( + calls.get(), + PIN_RECORD_ATTEMPTS, + "op is retried until it succeeds" + ); + } + + #[tokio::test] + async fn retry_returns_last_err_after_exhaustion() { + let calls = Cell::new(0u32); + let result = retry_db_record(|| { + let n = calls.get() + 1; + calls.set(n); + async move { Err::<(), _>(anyhow::anyhow!("attempt {n} failed")) } + }) + .await; + + let err = result.expect_err("all attempts fail so the last error surfaces"); + assert_eq!( + calls.get(), + PIN_RECORD_ATTEMPTS, + "attempts are bounded to the cap" + ); + assert_eq!( + err.to_string(), + "attempt 3 failed", + "the LAST error is returned, not the first" + ); + } + + // Happy path against a real DB: a single-attempt success lands the row, and a + // redundant call is idempotent (`ON CONFLICT DO NOTHING`), so the source set + // holds exactly one row. + #[sqlx::test] + async fn retry_records_pin_source_once(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + let sha = "a".repeat(64); + let repo_id = "repo-retry-1"; + + retry_db_record(|| db.record_pin_source(&sha, repo_id)) + .await + .expect("happy-path record succeeds in one attempt"); + retry_db_record(|| db.record_pin_source(&sha, repo_id)) + .await + .expect("a redundant record is idempotent"); + + let sources = db.pin_sources_for_oid(&sha).await.unwrap(); + assert_eq!( + sources, + vec![repo_id.to_string()], + "exactly one source row lands under ON CONFLICT DO NOTHING" + ); + } + + use std::time::Duration; + + /// Write `n` loose blobs into a fresh bare repo and return their oids. + /// `read_object` shells to `git cat-file`, so the objects must genuinely + /// exist on disk — a fabricated oid would `continue` past the pin call and + /// the loop scenario below would prove nothing. + fn seed_loose_blobs(repo_path: &std::path::Path, n: usize) -> Vec { + crate::git::store::init_bare(repo_path).expect("init bare repo"); + (0..n) + .map(|i| { + let mut cmd = std::process::Command::new("git"); + cmd.args(["hash-object", "-w", "--stdin"]) + .current_dir(repo_path) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()); + let mut child = cmd.spawn().expect("spawn git hash-object"); + { + use std::io::Write; + child + .stdin + .as_mut() + .expect("stdin") + .write_all(format!("pin loop object {i}\n").as_bytes()) + .expect("write stdin"); + } + let out = child.wait_with_output().expect("hash-object output"); + assert!( + out.status.success(), + "git hash-object: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }) + .collect() + } + + /// A live endpoint that answers every add with `500`. Counts the requests it + /// received so a test can tell "the loop kept going" from "the loop stopped", + /// which the returned pin list cannot (it is empty either way). Reads the + /// full request, headers plus the `Content-Length` body, before answering: + /// responding early and closing would surface as a write failure on the + /// client and turn a rejection into something else. + async fn rejecting_endpoint( + requests: std::sync::Arc, + ) -> String { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { + while let Ok((mut sock, _)) = listener.accept().await { + requests.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + tokio::spawn(async move { + let mut acc = Vec::new(); + let mut buf = [0u8; 4096]; + loop { + let n = match sock.read(&mut buf).await { + Ok(0) | Err(_) => break, + Ok(n) => n, + }; + acc.extend_from_slice(&buf[..n]); + // Once the headers are complete, keep reading until the + // declared body has arrived. + if let Some(hdr_end) = + acc.windows(4).position(|w| w == b"\r\n\r\n").map(|p| p + 4) + { + let headers = String::from_utf8_lossy(&acc[..hdr_end]).to_lowercase(); + let len: usize = headers + .lines() + .find_map(|l| l.strip_prefix("content-length:")) + .and_then(|v| v.trim().parse().ok()) + .unwrap_or(0); + if acc.len() >= hdr_end + len { + break; + } + } + } + let _ = sock + .write_all( + b"HTTP/1.1 500 Internal Server Error\r\nContent-Length: 0\r\n\r\n", + ) + .await; + let _ = sock.flush().await; + }); + } + }); + endpoint + } + + /// A sleeping-but-live endpoint. Answers `200` with an empty body after + /// `delays[i]` for the i-th request it accepts (the last entry repeats), so + /// a test can make one add slow and the next fast. Drains the full request, + /// headers plus the declared `Content-Length` body, before sleeping: exactly + /// as in `rejecting_endpoint`, answering early and closing would surface as + /// a write failure on the client and turn a slow-but-healthy add into a + /// different failure shape. + /// + /// An empty body is a successful pin: `pin_git_object` falls back to the CID + /// it computed from the bytes when the response carries no `Hash`. + async fn delaying_endpoint(delays: Vec) -> String { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { + let mut seen = 0usize; + while let Ok((mut sock, _)) = listener.accept().await { + let delay = *delays + .get(seen) + .or_else(|| delays.last()) + .unwrap_or(&Duration::ZERO); + seen += 1; + tokio::spawn(async move { + let mut acc = Vec::new(); + let mut buf = [0u8; 4096]; + loop { + let n = match sock.read(&mut buf).await { + Ok(0) | Err(_) => break, + Ok(n) => n, + }; + acc.extend_from_slice(&buf[..n]); + if let Some(hdr_end) = + acc.windows(4).position(|w| w == b"\r\n\r\n").map(|p| p + 4) + { + let headers = String::from_utf8_lossy(&acc[..hdr_end]).to_lowercase(); + let len: usize = headers + .lines() + .find_map(|l| l.strip_prefix("content-length:")) + .and_then(|v| v.trim().parse().ok()) + .unwrap_or(0); + if acc.len() >= hdr_end + len { + break; + } + } + } + tokio::time::sleep(delay).await; + let _ = sock + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n") + .await; + let _ = sock.flush().await; + }); + } + }); + endpoint + } + + /// A `tracing` sink a test can read back, so the deadline warn can be + /// asserted on rather than assumed. Installed with `set_default`, which is + /// thread-local and scoped to the guard, so it cannot bleed into any other + /// test in the binary. + #[derive(Clone, Default)] + struct CapturedLogs(std::sync::Arc>>); + + impl CapturedLogs { + fn text(&self) -> String { + String::from_utf8_lossy(&self.0.lock().unwrap()).to_string() + } + } + + impl std::io::Write for CapturedLogs { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for CapturedLogs { + type Writer = CapturedLogs; + fn make_writer(&'a self) -> Self::Writer { + self.clone() + } + } + + fn capture_logs() -> (CapturedLogs, tracing::subscriber::DefaultGuard) { + let logs = CapturedLogs::default(); + let subscriber = tracing_subscriber::fmt() + .with_writer(logs.clone()) + .with_max_level(tracing::Level::WARN) + .with_ansi(false) + .finish(); + let guard = tracing::subscriber::set_default(subscriber); + (logs, guard) + } + + /// The add sink must be built from the shared no-redirect client, and the + /// per-request override must actually reach the request. Against a silent + /// endpoint (accept succeeds, no response ever written) a bare + /// `reqwest::Client::new()` blocks forever. With `Some(2s)` the call must + /// come back well inside that, and as a reqwest timeout: the elapsed + /// assertion is the real RED signal, and the outer `tokio::time::timeout` + /// is only a wedge guard so a regression fails the suite instead of hanging + /// it (`cargo test` has no per-test timeout). The old "no elapsed assertion + /// because the timeout is a process-global `OnceLock`" caveat no longer + /// holds now that `request_timeout` overrides it per call. + #[tokio::test] + async fn pin_git_object_against_silent_endpoint_errors_within_its_own_timeout() { + let endpoint = crate::test_support::silent_http_endpoint().await; + let started = std::time::Instant::now(); + let inner = tokio::time::timeout( + Duration::from_secs(30), + pin_git_object( + &endpoint, + "deadbeef", + b"some object bytes\n", + Some(Duration::from_secs(2)), + ), + ) + .await + .expect("wedge guard: pin_git_object must return long before 30s"); + let elapsed = started.elapsed(); + let err = inner.expect_err("a silent endpoint must not surface as a successful pin"); + assert!( + elapsed < Duration::from_secs(5), + "the 2s per-request override must bound this call, not the client's own ceiling (took {elapsed:?})" + ); + assert!( + err.downcast_ref::() + .is_some_and(|e| e.is_timeout()), + "a silent endpoint must surface as a reqwest timeout, preserved as the error's source: {err:#}" + ); + } + + /// The second unhardened sink, reached from `sync.rs`. Same shape as above. + #[tokio::test] + async fn cat_against_silent_endpoint_errors_within_its_own_timeout() { + let endpoint = crate::test_support::silent_http_endpoint().await; + let inner = tokio::time::timeout(Duration::from_secs(30), cat(&endpoint, "bafkqaaa")) + .await + .expect( + "cat must return before the outer 30s timeout — an unbounded client hangs here", + ); + assert!( + inner.is_err(), + "a silent endpoint must surface as a transport error, not successful bytes" + ); + } + + /// The permit-hold bound. `pin_new_objects` runs under a deferring + /// `pin_semaphore`, so without a batch deadline the hold is O(N) with N + /// chosen by the pusher. Five objects against an endpoint that takes 2s + /// each, under a 5.5s budget, must stop partway: only the first two can + /// finish inside the budget, so the batch is truncated and the remainder is + /// left unattempted with one warn naming how many. + /// + /// The windows are deliberately loose. Three pins would need every add to + /// answer in under 1.83s, which the endpoint's own 2s sleep forbids, and one + /// pin needs only the first add to land inside 5.5s, so both bounds hold + /// with more than a second of slack on a loaded box. + #[sqlx::test] + async fn pin_new_objects_stops_the_batch_at_its_deadline(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("slow.git"); + let oids = seed_loose_blobs(&repo_path, 5); + let endpoint = delaying_endpoint(vec![Duration::from_secs(2)]).await; + + let (logs, _guard) = capture_logs(); + let pinned = tokio::time::timeout( + Duration::from_secs(30), + pin_new_objects( + &endpoint, + &repo_path, + "git", + Duration::from_secs(30), + oids, + &db, + "repo-batch-budget", + Duration::from_millis(5500), + ), + ) + .await + .expect("wedge guard: a 5.5s budget cannot take 30s"); + + assert!( + (1..=3).contains(&pinned.len()), + "the batch must stop partway, not pin all five and not stall on the first: pinned {}", + pinned.len() + ); + let text = logs.text(); + let warns: Vec<&str> = text + .lines() + .filter(|l| l.contains("pin batch deadline reached")) + .collect(); + assert_eq!( + warns.len(), + 1, + "the deadline must be reported exactly once for the batch, not per object: {text}" + ); + let unattempted: usize = warns[0] + .split("unattempted=") + .nth(1) + .and_then(|s| { + s.split(|c: char| !c.is_ascii_digit()) + .next() + .and_then(|d| d.parse().ok()) + }) + .unwrap_or_else(|| panic!("the deadline warn must name the unattempted count: {text}")); + assert!( + unattempted >= 1 && unattempted + pinned.len() <= 5, + "unattempted={unattempted} with {} pinned is not a partial batch of five", + pinned.len() + ); + } + + /// The must-not case, and the regression that killed the old transport + /// classifier: an endpoint that is slow but genuinely alive must NOT cost + /// the rest of the batch. The first add takes 13s, past the shared client's + /// 10s ceiling, which is exactly what the classifier used to read as a dead + /// endpoint; the second is immediate. Under a 90s budget both must pin, so + /// this fails if the per-request timeout is left at the client default and + /// fails if any error arm breaks the loop. Two objects, not one, because + /// with one object "did not abandon the rest" would be vacuous. + #[sqlx::test] + async fn pin_new_objects_does_not_abandon_the_batch_on_a_slow_but_alive_endpoint( + pool: sqlx::PgPool, + ) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("slow_alive.git"); + let oids = seed_loose_blobs(&repo_path, 2); + let endpoint = + delaying_endpoint(vec![Duration::from_secs(13), Duration::from_millis(0)]).await; + + let pinned = tokio::time::timeout( + Duration::from_secs(60), + pin_new_objects( + &endpoint, + &repo_path, + "git", + Duration::from_secs(30), + oids, + &db, + "repo-batch-continues", + Duration::from_secs(90), + ), + ) + .await + .expect("wedge guard: a 13s add plus an immediate one cannot take 60s"); + assert_eq!( + pinned.len(), + 2, + "a slow but progressing endpoint must pin both objects: an upload past the client's \ + 10s default is not a dead endpoint" + ); + } + + /// The must-not case for the warn-and-continue arm: a live endpoint + /// rejecting each object with `500` is a per-object failure, so the loop + /// must still warn and continue and every object must be attempted. Without + /// this, a `break` arm could be reintroduced and the deadline test above + /// would not notice. + #[sqlx::test] + async fn pin_new_objects_continues_past_a_per_object_rejection(pool: sqlx::PgPool) { + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.expect("migrations"); + let tmp = tempfile::TempDir::new().unwrap(); + let repo_path = tmp.path().join("rejecting.git"); + let oids = seed_loose_blobs(&repo_path, 4); + let requests = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let endpoint = rejecting_endpoint(std::sync::Arc::clone(&requests)).await; + + let pinned = tokio::time::timeout( + Duration::from_secs(30), + pin_new_objects( + &endpoint, + &repo_path, + "git", + Duration::from_secs(30), + oids, + &db, + "repo-batch-rejects", + Duration::from_secs(60), + ), + ) + .await + .expect("a rejecting endpoint answers immediately, so this cannot take 30s"); + assert!(pinned.is_empty(), "every add was rejected"); + assert_eq!( + requests.load(std::sync::atomic::Ordering::SeqCst), + 4, + "a non-2xx rejection is per-object: all four objects must still be attempted" + ); + } +} diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index aa0483db..2cb6391d 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -58,6 +58,32 @@ struct DbStartupStatus { next_retry_secs: AtomicU64, } +/// Hard ceiling on the advisory-lock pool's `max_connections`. +/// +/// `max_concurrent_git_pushes` is validated all the way up to 1_048_576, and the lock +/// pool used to derive its size straight from that knob, so raising the push cap +/// silently raised the node's Postgres connection ceiling with no CLI error and no +/// relation to the server's own `max_connections` (#173 F4). The node's total budget is +/// now bounded: `db_max_connections` (default 20) + at most this. +const LOCK_POOL_MAX_CONNECTIONS: u32 = 64; + +/// Connections the lock pool keeps above the push cap. Covers the three non-push +/// `acquire_write` callers (`api/issues.rs` x2, `api/pulls.rs`), which hold no +/// concurrency permit, so a push never queues here for a connection where it did not +/// before. +const LOCK_POOL_PUSH_HEADROOM: u8 = 8; + +/// Size the advisory-lock pool for a given push cap: the cap plus +/// [`LOCK_POOL_PUSH_HEADROOM`], clamped to [`LOCK_POOL_MAX_CONNECTIONS`]. Past the +/// clamp a push may wait for a lock-pool connection, which is a bounded wait that sheds +/// a clean 503 (see `LockPoolBusy`), not an unbounded hang. +fn lock_pool_size(max_concurrent_git_pushes: usize) -> u32 { + u32::try_from(max_concurrent_git_pushes) + .unwrap_or(u32::MAX) + .saturating_add(u32::from(LOCK_POOL_PUSH_HEADROOM)) + .min(LOCK_POOL_MAX_CONNECTIONS) +} + #[tokio::main] async fn main() -> Result<()> { tracing_subscriber::fmt() @@ -74,6 +100,13 @@ async fn main() -> Result<()> { // bootstrap peers. Operators can opt out via GITLAWB_BOOTSTRAP_DISABLE_SEEDS. bootstrap::merge_seeds(&mut config); + // Fail fast on config combinations that are individually in-range but jointly + // unsafe — notably a DB pool too small for the concurrent-write cap, which + // would let a push burst starve every other DB path (#174 F1). + config + .validate() + .map_err(|e| anyhow::anyhow!("invalid configuration: {e}"))?; + if !config.public_read { warn!( "GITLAWB_PUBLIC_READ=false is reserved; per-repository private-read enforcement is not wired in alpha" @@ -279,8 +312,17 @@ async fn main() -> Result<()> { None }; - let repo_store = - git::repo_store::RepoStore::new(config.repos_dir.clone(), tigris, db.pool().clone()); + // Repo write locks run on their own pool, never the main query pool: each + // push holds its connection for the whole receive-pack, and + // db_max_connections (20) is below max_concurrent_git_pushes (32), so sharing + // would starve every other query under a push burst. See build_lock_pool for + // the cancellation semantics (#173). + let lock_pool = git::repo_store::build_lock_pool( + db.pool(), + lock_pool_size(config.max_concurrent_git_pushes), + std::time::Duration::from_secs(config.db_acquire_timeout_secs), + ); + let repo_store = git::repo_store::RepoStore::new(config.repos_dir.clone(), tigris, lock_pool); // Per-DID limiter for the creation endpoints. Keyed on the authenticated // DID (attacker-varied), so bound its key set to cap memory. @@ -374,11 +416,100 @@ async fn main() -> Result<()> { rate_limiter, create_ip_rate_limiter, push_rate_limiter, + ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, + // The legacy-probe budget is operator-tunable via GITLAWB_IPFS_MAX_REPOS_WALKED + // (R5); the history-walk ceiling above stays constant (a smaller value false-503s + // a provenanced request). Default 256 preserves the shipped behaviour. + ipfs_max_legacy_probes: AppState::ipfs_legacy_probe_budget(&config), + ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, push_limiter_trust, sync_trigger_rate_limiter, peer_write_rate_limiter, shutdown_tx: shutdown_tx.clone(), + git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_git_ops)), + git_write_semaphore: Arc::new(tokio::sync::Semaphore::new( + config.max_concurrent_git_pushes, + )), + // Anon receive-pack advertisements get their OWN pool, same size as the + // write pool but disjoint, so filling it (which takes many source IPs, each + // capped by git_push_advert_per_caller) never occupies a permit the + // authenticated POST needs (#174). + git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new( + config.max_concurrent_git_pushes, + )), + // Bounds concurrent detached post-push encryption walks, sized from the push + // pool (no separate knob — Q1): completed pushes cannot outnumber active + // encryption walks past this (#174 P1-e). + git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new( + config.max_concurrent_git_pushes, + )), + // Bounds how many post-push pin loops run concurrently across all repos (#174 F6), + // independent of the per-repo encrypt-task coalescing below. Not a bound on the + // MB-scale object-id lists themselves: parked tasks still hold theirs (see the + // field doc on AppState::pin_semaphore). + pin_semaphore: Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_pin_tasks)), + // Coalesces the DETACHED post-push encryption tasks per repo so a rapid pusher + // cannot grow the outstanding parked-waiter set past one task per repo (#174 + // P2-2). No knob: it is a natural cap (one entry per distinct repo), not a + // sized pool. + encrypt_inflight: crate::state::EncryptInflight::new(), + // Per-repo in-process write-lease serializer (#174 U2/F3): supplements the pg + // advisory lock so a disconnected push's still-reaping git group can't be raced + // by a second same-node push. The map is naturally capped (one entry per contended + // repo, freed when unreferenced); the sized knob is how many pushes may PARK on + // one repo, since each parked push holds a fully buffered pack. + repo_write_leases: crate::state::RepoWriteLeases::new(config.repo_lease_max_waiters), + git_read_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( + config.max_concurrent_reads_per_caller, + ), + // Per-source cap on the receive-pack advertisement, sized to an eighth of the + // write pool (min 1): one resolved client key (rate_limit::client_key) can hold + // at most this many slots in the DEDICATED advert pool (git_push_advert_semaphore, + // disjoint from the write pool), so saturating that pool takes ~8 distinct keys + // (#174). That bounds an IPv4 or single-address caller; a caller controlling many + // addresses (an IPv6 /64 is 2^64 keys) still gets one cap per address, since + // client_key uses the full IP with no prefix folding. Narrowing the keying is a + // deferred design call, not something these caps claim to solve. Sized off the + // write pool only because the advert pool is created at the same size; an advert + // flood cannot touch a write permit. + git_push_advert_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( + rate_limit::per_source_push_cap(config.max_concurrent_git_pushes), + ), + // Per-source cap on the authenticated receive-pack POST, sized like the advert + // cap: one resolved client key can hold at most this many write-pool slots, so + // monopolizing the pool takes ~8 distinct keys (#174 P1-d). Same residual as + // above: keys are full IPs, so a caller with many addresses has many caps. + git_write_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( + rate_limit::per_source_push_cap(config.max_concurrent_git_pushes), + ), + // Bounds concurrent /ipfs visibility walks — a distinct public cost center, so + // its own pool + per-source sub-cap + per-IP rate limiter, never a git pool + // (#174 P1-3). The per-source map is bounded (reject-before-insert, INV-15). + git_ipfs_walk_semaphore: Arc::new(tokio::sync::Semaphore::new( + config.max_concurrent_ipfs_walks, + )), + git_ipfs_walk_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( + config.ipfs_walk_per_source, + ), + ipfs_rate_limiter: rate_limit::RateLimiter::new_bounded( + config.ipfs_rate_limit, + std::time::Duration::from_secs(3600), + 200_000, + ), + // Separate WORK-budget bucket for the resolver's per-probe/per-walk charges (R6). + // Its capacity is DERIVED from the route limit (no new knob) and floored at the + // legacy-probe budget, so one full default-config legacy scan never self-throttles + // mid-request while the route brake above stays the pure once-per-request cap. + ipfs_work_rate_limiter: rate_limit::RateLimiter::new_bounded( + AppState::ipfs_work_budget(&config), + std::time::Duration::from_secs(3600), + 200_000, + ), + git_bin: "git".to_string(), }; + if config.ipfs_rate_limit == 0 { + tracing::warn!("GITLAWB_IPFS_RATE_LIMIT=0 — per-IP /ipfs rate limiting disabled"); + } // Periodic peer-count poll for the metrics gauge. If p2p is disabled // we still set the gauge to 0 so dashboards don't show "no data". @@ -408,22 +539,16 @@ async fn main() -> Result<()> { // Periodic cleanup of expired rate limit entries + consumed-proof ledger { - let rl = state.rate_limiter.clone(); - let create_ip_rl = state.create_ip_rate_limiter.clone(); - let push_rl = state.push_rate_limiter.clone(); - let sync_trigger_rl = state.sync_trigger_rate_limiter.clone(); - let peer_write_rl = state.peer_write_rate_limiter.clone(); + let cleanup_state = state.clone(); let db = state.db.clone(); let mut shutdown_rx = state.subscribe_shutdown(); tokio::spawn(async move { loop { tokio::select! { _ = tokio::time::sleep(std::time::Duration::from_secs(300)) => { - rl.cleanup().await; - create_ip_rl.cleanup().await; - push_rl.cleanup().await; - sync_trigger_rl.cleanup().await; - peer_write_rl.cleanup().await; + // Sweep every per-IP/DID limiter (incl. the ipfs walk brake) + // so bounded maps shed stale keys instead of sitting at cap. + cleanup_state.sweep_rate_limiters().await; let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|d| d.as_secs() as i64) @@ -442,6 +567,8 @@ async fn main() -> Result<()> { }); } + let _legacy_cid_sweep = spawn_legacy_cid_sweep(&state, &config); + let router = server::build_router(state.clone()); // Re-register the socket bound at startup — same fd, so there was never a // moment with the port closed between the degraded and full servers. @@ -564,6 +691,49 @@ async fn main() -> Result<()> { Ok(()) } +/// U4 (#173): spawn the one-shot legacy provider-CID repair sweep. Releases before this +/// version stored the PROVIDER CID (Kubo dag-pb / Pinata CIDv0) in `pinned_cids.cid`, +/// and this version's `/ipfs/{cid}` resolver withholds any row whose stored key is not +/// the raw-content CID. The opportunistic repair on the pin path only fires when a push +/// re-carries the object, which normal git negotiation makes it not do, so those rows +/// need a walk. DETACHED, never on the boot path: the caller keeps serving while this +/// runs, and the sweep's own batch bound plus inter-batch delay keep it off the DB's +/// critical path. Its cursor is durable, so a restart mid-walk resumes instead of +/// rewinding. +/// +/// A named function rather than an inline block in `main` so the WIRING has a seam a +/// test can call: that the task is spawned at all, that it reads its batch and delay +/// from the config knobs rather than some other field, that the caller is not blocked +/// on it, and that the shutdown watcher actually ends it mid-walk. The sweep's own +/// behavior is covered elsewhere; this is the boot-path half. +fn spawn_legacy_cid_sweep(state: &AppState, config: &Config) -> tokio::task::JoinHandle<()> { + let db = state.db.clone(); + let repos_dir = config.repos_dir.clone(); + let git_bin = state.git_bin.clone(); + let git_timeout = std::time::Duration::from_secs(config.git_service_timeout_secs); + let batch = config.pin_repair_sweep_batch; + let delay = std::time::Duration::from_secs(config.pin_repair_sweep_delay_secs); + let mut shutdown_rx = state.subscribe_shutdown(); + tokio::spawn(async move { + tokio::select! { + stats = ipfs_pin::sweep_legacy_provider_cids( + &repos_dir, &git_bin, git_timeout, batch, delay, &db, + ) => { + if stats.repaired > 0 { + tracing::info!( + scanned = stats.scanned, + repaired = stats.repaired, + "legacy provider-CID sweep finished" + ); + } + } + // Shutdown mid-walk simply drops the run; the persisted cursor means the + // next boot picks up where this one stopped. + _ = shutdown_rx.changed() => {} + } + }) +} + fn spawn_shutdown_signal(tx: watch::Sender) { tokio::spawn(async move { #[cfg(unix)] @@ -959,6 +1129,52 @@ async fn gossip_task( } } +#[cfg(test)] +mod rate_limiter_sweep_tests { + use crate::rate_limit::RateLimiter; + use std::time::Duration; + + // Every per-key limiter the router mounts must be swept by the periodic + // task, the `/ipfs` one included: a limiter left out keeps expired keys + // until its map fills and the inline capacity sweep fires. Fails on the + // pre-fix sweeper, which skipped `ipfs_rate_limiter`. + #[tokio::test] + async fn sweep_evicts_expired_keys_from_every_limiter() { + let window = Duration::from_millis(30); + let mut state = crate::test_support::test_state_lazy(); + state.rate_limiter = RateLimiter::new(10, window); + state.create_ip_rate_limiter = RateLimiter::new(10, window); + state.push_rate_limiter = RateLimiter::new(10, window); + state.sync_trigger_rate_limiter = RateLimiter::new(10, window); + state.peer_write_rate_limiter = RateLimiter::new(10, window); + state.ipfs_rate_limiter = RateLimiter::new(10, window); + state.ipfs_work_rate_limiter = RateLimiter::new(10, window); + + let limiters = |s: &crate::state::AppState| { + [ + s.rate_limiter.clone(), + s.create_ip_rate_limiter.clone(), + s.push_rate_limiter.clone(), + s.sync_trigger_rate_limiter.clone(), + s.peer_write_rate_limiter.clone(), + s.ipfs_rate_limiter.clone(), + s.ipfs_work_rate_limiter.clone(), + ] + }; + for l in limiters(&state) { + assert!(l.check("1.2.3.4").await); + assert_eq!(l.tracked_keys().await, 1); + } + + tokio::time::sleep(window * 3).await; + state.sweep_rate_limiters().await; + + for (i, l) in limiters(&state).into_iter().enumerate() { + assert_eq!(l.tracked_keys().await, 0, "limiter {i} was not swept"); + } + } +} + /// Build the shared node HTTP client used for every outbound fan-out (sync /// trigger, profile/repo fetches, gossip announce + peer pings). /// @@ -1024,6 +1240,140 @@ fn load_or_create_keypair(config: &Config) -> Result { } } +#[cfg(test)] +mod legacy_cid_sweep_wiring_tests { + use super::spawn_legacy_cid_sweep; + use sqlx::PgPool; + use std::time::Duration; + + /// Seed `count` `pinned_cids` rows whose keys are already canonical raw CIDv1, in a + /// known `sha256_hex` order. The sweep's own cost gate skips a raw-CIDv1 row without + /// reading bytes or resolving a repo, so each row is SCANNED (it advances the cursor) + /// and nothing else. That is what makes the cursor a clean readout of how far the + /// walk got, with no dependency on repos on disk. + async fn seed_scannable_rows(pool: &PgPool, count: usize) -> Vec { + let mut shas = Vec::new(); + for i in 1..=count { + let sha = format!("wire{i:02}"); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(sha.as_bytes()).to_string(); + assert!( + gitlawb_core::cid::is_raw_cidv1(&cid), + "the seeded key must hit the sweep's raw-CIDv1 skip, not a repair attempt" + ); + sqlx::query("INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) VALUES ($1, $2, $3)") + .bind(&sha) + .bind(&cid) + .bind("2020-01-01T00:00:00Z") + .execute(pool) + .await + .unwrap(); + shas.push(sha); + } + shas + } + + /// Poll the persisted sweep cursor until it reaches `want`, or give up. + async fn cursor_reaches(db: &crate::db::Db, want: &str, within: Duration) -> String { + let deadline = std::time::Instant::now() + within; + loop { + let c = db.pin_repair_cursor().await.unwrap(); + if c == want || std::time::Instant::now() >= deadline { + return c; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + } + + /// #173 U4, the BOOT-PATH half. The sweep's own logic (batching, cursor resumption, + /// terminal vs retryable skips) is covered in `test_support`; what this covers is the + /// wiring `main` performs, which nothing else executes: the task is spawned at all, + /// it takes its batch and delay from the two `pin_repair_sweep_*` knobs rather than + /// some other config field, the caller is not blocked on the walk, and the shutdown + /// watcher ends the run mid-walk. + /// + /// Six scannable rows, batch 2, delay 30s. One pass must land the cursor on exactly + /// the second row and the task must then still be alive in its inter-batch sleep, + /// which pins both knobs at once: a different batch stops at a different row, and a + /// delay that did not come from the knob either finishes the table or leaves the task + /// gone. Shutdown must then end it while four rows are still unwalked. + #[sqlx::test] + async fn the_boot_path_spawns_the_sweep_detached_with_its_configured_knobs(pool: PgPool) { + let state = crate::test_support::test_state(pool.clone()).await; + let shas = seed_scannable_rows(&pool, 6).await; + let repos_dir = tempfile::TempDir::new().unwrap(); + + let mut config = (*state.config).clone(); + config.repos_dir = repos_dir.path().to_path_buf(); + config.pin_repair_sweep_batch = 2; + // Far longer than this test runs, so a task still alive after the first pass can + // only be one that is honoring the configured inter-batch delay. + config.pin_repair_sweep_delay_secs = 30; + + let started = std::time::Instant::now(); + let handle = spawn_legacy_cid_sweep(&state, &config); + let spawn_cost = started.elapsed(); + + let cursor = cursor_reaches(&state.db, &shas[1], Duration::from_secs(10)).await; + assert_eq!( + cursor, shas[1], + "the spawned sweep must run and stop its first pass at the CONFIGURED batch \ + bound (2), leaving the cursor on the second row" + ); + assert!( + spawn_cost < Duration::from_secs(1), + "the sweep must be detached, not awaited on the boot path; the spawn took \ + {spawn_cost:?}" + ); + assert!( + !handle.is_finished(), + "with a 30s inter-batch delay the task must still be sleeping between passes, \ + not finished: a finished task means the delay was not the configured one" + ); + + state.shutdown(); + tokio::time::timeout(Duration::from_secs(10), handle) + .await + .expect("the shutdown watcher must end the sweep, and not after its 30s delay") + .expect("the sweep task must not panic"); + + assert_eq!( + state.db.pin_repair_cursor().await.unwrap(), + shas[1], + "shutdown must have ended the run MID-walk, with the remaining rows unwalked" + ); + } +} + +#[cfg(test)] +mod lock_pool_sizing_tests { + use super::{lock_pool_size, LOCK_POOL_MAX_CONNECTIONS, LOCK_POOL_PUSH_HEADROOM}; + + /// The default push cap gets its cap plus headroom, so no push ever queues for a + /// lock-pool connection where it did not before. + #[test] + fn default_push_cap_gets_headroom_over_the_cap() { + assert_eq!(lock_pool_size(32), 32 + u32::from(LOCK_POOL_PUSH_HEADROOM)); + assert_eq!(lock_pool_size(1), 1 + u32::from(LOCK_POOL_PUSH_HEADROOM)); + } + + /// #173 F4: `max_concurrent_git_pushes` is validated all the way to 1_048_576, so an + /// operator raising it used to raise the node's Postgres connection ceiling with it, + /// silently and without bound. The lock pool is CLAMPED instead. + #[test] + fn an_oversized_push_cap_is_clamped_not_propagated() { + assert_eq!(lock_pool_size(1_048_576), LOCK_POOL_MAX_CONNECTIONS); + assert_eq!(lock_pool_size(usize::MAX), LOCK_POOL_MAX_CONNECTIONS); + // The largest cap that still fits under the clamp keeps its full headroom. + let widest = (LOCK_POOL_MAX_CONNECTIONS - u32::from(LOCK_POOL_PUSH_HEADROOM)) as usize; + assert_eq!(lock_pool_size(widest), LOCK_POOL_MAX_CONNECTIONS); + assert_eq!( + lock_pool_size(widest - 1), + LOCK_POOL_MAX_CONNECTIONS - 1, + "values below the clamp must not be rounded up to it" + ); + } +} + #[cfg(test)] mod gossip_ssrf_tests { use super::ping_peer_health; diff --git a/crates/gitlawb-node/src/pinata.rs b/crates/gitlawb-node/src/pinata.rs index 6c9c0bff..e49f9205 100644 --- a/crates/gitlawb-node/src/pinata.rs +++ b/crates/gitlawb-node/src/pinata.rs @@ -74,8 +74,10 @@ pub async fn pin_object( /// `..._fail_closed` filter on the full-scan path before calling. `repo_path` is /// still needed to read each object's bytes. The twin in `ipfs_pin.rs` mirrors /// this shape — change both in lockstep. Objects already recorded with a -/// `pinata_cid` are skipped. Returns `(sha_hex, cid)` pairs for each newly -/// pinned object. +/// `pinata_cid` are skipped. `repo_id` records the pin's provenance (#173). +/// Returns `(sha_hex, provider_cid)` pairs for each newly pinned object: the +/// provider CID is the Pinata gateway CID (used for branch→CID recording and +/// ref-update gossip), NOT the raw resolver-key CID stored in `pinned_cids.cid`. pub async fn pin_new_objects( client: &reqwest::Client, upload_url: &str, @@ -83,6 +85,7 @@ pub async fn pin_new_objects( repo_path: &std::path::Path, object_list: Vec, db: &crate::db::Db, + repo_id: &str, ) -> Vec<(String, String)> { if jwt.is_empty() { return vec![]; @@ -92,7 +95,39 @@ pub async fn pin_new_objects( for sha in object_list { match db.has_pinata_cid(&sha).await { - Ok(true) => continue, + Ok(true) => { + // Backfill NULL first-pinner provenance from a known source, in lockstep + // with the ipfs_pin skip branch: a pinata-only node otherwise leaves + // pre-provenance rows' `pinned_cids.repo_id` NULL forever (grok P2-D). The + // resolver still finds the object via the pin_repo_sources union below, so + // this is a consistency backfill, not a correctness fix. + match db.provenance_for_oid(&sha).await { + Ok(None) => { + if let Err(e) = db.backfill_pin_provenance(&sha, repo_id).await { + tracing::warn!(sha = %sha, err = %e, "failed to backfill pin provenance"); + } + } + Ok(Some(_)) => {} + Err(e) => { + tracing::warn!(sha = %sha, err = %e, "DB error reading pin provenance"); + } + } + // F1 (#173 round 8): record this repo as an additional source for the + // already-pinned object (mirrors the ipfs_pin skip-branch insert) so the + // resolver can serve a shared object from any pin-path source. U3 (#173): + // retried through the SHARED helper (this was a bare call, so a single + // transient error dropped the source outright) and, on exhaustion, marked + // durably so the resolver keeps the bounded scan fallback for the object. + if let Err(e) = + crate::ipfs_pin::retry_db_record(|| db.record_pin_source(&sha, repo_id)).await + { + tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); + if let Err(e) = db.mark_pin_sources_incomplete(&sha).await { + tracing::warn!(sha = %sha, err = %e, "failed to mark pin sources incomplete"); + } + } + continue; + } Ok(false) => {} Err(e) => { tracing::warn!(sha = %sha, err = %e, "DB error checking pinata_cid"); @@ -111,9 +146,32 @@ pub async fn pin_new_objects( match pin_object(client, upload_url, jwt, &sha, &data).await { Ok(cid) if !cid.is_empty() => { - if let Err(e) = db.record_pinata_cid(&sha, &cid).await { + // The resolver key (`pinned_cids.cid`) must be the locally-computed + // raw-content CID, never the provider CID: Pinata wraps the bytes in + // dag-pb/UnixFS, so its returned CID does not hash the raw content and + // must not become an alias `/ipfs/{cid}` serves raw git bytes for (#173). + let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes(&data).to_string(); + // U3 (#173): both records go through the shared retry helper, at parity + // with the ipfs_pin twin. These were bare calls, so one transient DB error + // permanently dropped a pin source. + if let Err(e) = crate::ipfs_pin::retry_db_record(|| { + db.record_pinata_cid(&sha, &raw_cid, &cid, Some(repo_id)) + }) + .await + { tracing::warn!(sha = %sha, err = %e, "failed to record pinata_cid in DB"); } + // F1 (#173 round 8): also record the first pinner in pin_repo_sources. + // U3: an exhausted retry marks the set incomplete so the resolver keeps + // the scan fallback rather than 404ing a copy it could serve. + if let Err(e) = + crate::ipfs_pin::retry_db_record(|| db.record_pin_source(&sha, repo_id)).await + { + tracing::warn!(sha = %sha, err = %e, "failed to record pin source"); + if let Err(e) = db.mark_pin_sources_incomplete(&sha).await { + tracing::warn!(sha = %sha, err = %e, "failed to mark pin sources incomplete"); + } + } pinned.push((sha, cid)); } Ok(_) => {} diff --git a/crates/gitlawb-node/src/rate_limit.rs b/crates/gitlawb-node/src/rate_limit.rs index d40e2691..c3626089 100644 --- a/crates/gitlawb-node/src/rate_limit.rs +++ b/crates/gitlawb-node/src/rate_limit.rs @@ -121,6 +121,28 @@ impl RateLimiter { true } + /// Non-consuming check: is this key ALREADY at its limit for the current window? + /// Unlike [`check`], it records nothing and never inserts a new key — used to shed + /// expensive preparatory work (e.g. the `/ipfs/{cid}` legacy scan's O(repos) DB + /// preload) BEFORE it runs, without perturbing the per-unit budget the consuming + /// `check` maintains (#173, F3). An unknown key or a disabled limiter is not + /// throttled. Prunes the key's expired timestamps as a side effect (keeps state + /// tidy) but adds none, so it cannot itself fill or grow the map. + pub(crate) async fn is_throttled(&self, key: &str) -> bool { + if self.max_requests == 0 { + return false; + } + let now = Instant::now(); + let mut state = self.state.lock().await; + if let Some(window) = state.get_mut(key) { + window + .timestamps + .retain(|t| now.duration_since(*t) < self.window); + return window.timestamps.len() >= self.max_requests; + } + false + } + pub async fn cleanup(&self) { let now = Instant::now(); let mut state = self.state.lock().await; @@ -130,6 +152,118 @@ impl RateLimiter { !w.timestamps.is_empty() }); } + + /// Number of distinct keys currently tracked. Test-only introspection so a + /// cross-module test can assert that a sweep actually evicted expired entries + /// and observe what it reclaimed. There is no production reader. + #[cfg(test)] + pub(crate) async fn tracked_keys(&self) -> usize { + self.state.lock().await.len() + } +} + +/// Per-source concurrency cap derived from the write-pool size: one resolved client +/// key (see [`client_key`]) may hold at most an eighth of the pool, so saturating it +/// takes ~8 distinct keys. Real for an IPv4 or single-address caller; a caller with a +/// routed IPv6 /64 has 2^64 keys, because `client_key` returns the full address with +/// no prefix folding. Narrowing that keying is a deferred decision, so this cap is a +/// bound per key, not a bound per operator. +/// +/// Floored at 1 because the value feeds [`PerCallerConcurrency`], where a cap of 0 +/// would shed EVERY receive-pack advertisement and break all pushes. The floor is +/// load-bearing at the minimum write-pool size (1), which integer-divides to 0. +pub(crate) fn per_source_push_cap(max_concurrent_git_pushes: usize) -> usize { + (max_concurrent_git_pushes / 8).max(1) +} + +/// A bounded per-caller CONCURRENCY limiter — distinct from [`RateLimiter`], which +/// caps request RATE. Each caller key may hold at most `per_caller` in-flight +/// permits at once; beyond that [`try_acquire`](Self::try_acquire) returns `None` +/// and the caller sheds. Used to stop one caller (a single anonymous source-IP or +/// DID) monopolizing the served-git read pool (#174). +/// +/// The key map is self-bounding: a key is removed the instant its in-flight count +/// reaches zero, so it never holds more keys than there are concurrently-active +/// callers (itself bounded by the read semaphore). A `max_keys` reject-before-insert +/// backstop guarantees a key farm can never grow the map even if that invariant +/// weakened — a NEW key at the cap is rejected WITHOUT allocating an entry (INV-15). +/// +/// Uses a `std::sync::Mutex` (not the file's `tokio::sync::Mutex`) because the +/// permit's `Drop` must release synchronously; the critical section holds no await. +#[derive(Clone)] +pub struct PerCallerConcurrency { + state: Arc>>, + per_caller: usize, + max_keys: usize, +} + +/// RAII permit from [`PerCallerConcurrency::try_acquire`]. On drop it decrements +/// the caller's in-flight count and removes the key when it reaches zero. +pub struct PerCallerPermit { + state: Arc>>, + key: String, +} + +impl PerCallerConcurrency { + pub fn new(per_caller: usize, max_keys: usize) -> Self { + Self { + state: Arc::new(std::sync::Mutex::new(HashMap::new())), + per_caller: per_caller.max(1), + max_keys: max_keys.max(1), + } + } + + /// Convenience constructor with the default key bound. + pub fn with_default_max_keys(per_caller: usize) -> Self { + Self::new(per_caller, DEFAULT_MAX_KEYS) + } + + /// `Some(permit)` when the caller is under its cap and the map has room; + /// `None` (shed) otherwise. Reject-before-insert: a new key at `max_keys` is + /// rejected without allocating. + pub fn try_acquire(&self, key: &str) -> Option { + // Recover from a poisoned lock rather than panicking: the critical section + // is pure counter arithmetic and cannot itself panic, so a poisoned mutex + // would only ever come from an unrelated abort, and a slightly-off count + // self-heals as permits drop. A panic here would instead brick the limiter + // for every caller (each subsequent lock re-panics). + let mut map = self.state.lock().unwrap_or_else(|e| e.into_inner()); + match map.get_mut(key) { + Some(count) => { + if *count >= self.per_caller { + return None; + } + *count += 1; + } + None => { + if map.len() >= self.max_keys { + return None; + } + map.insert(key.to_string(), 1); + } + } + Some(PerCallerPermit { + state: self.state.clone(), + key: key.to_string(), + }) + } + + #[cfg(test)] + pub fn tracked_keys(&self) -> usize { + self.state.lock().unwrap().len() + } +} + +impl Drop for PerCallerPermit { + fn drop(&mut self) { + let mut map = self.state.lock().unwrap_or_else(|e| e.into_inner()); + if let Some(count) = map.get_mut(&self.key) { + *count -= 1; + if *count == 0 { + map.remove(&self.key); + } + } + } } pub async fn rate_limit_by_did(request: Request, next: Next) -> Response { @@ -288,6 +422,63 @@ pub async fn rate_limit_by_ip(request: Request, next: Next) -> Response { mod tests { use super::*; + #[test] + fn per_caller_concurrency_caps_one_caller_and_frees_on_drop() { + let lim = PerCallerConcurrency::new(2, 100); + let p1 = lim.try_acquire("did:key:zA").expect("first under cap"); + let p2 = lim.try_acquire("did:key:zA").expect("second under cap"); + assert!( + lim.try_acquire("did:key:zA").is_none(), + "a third in-flight op for the same caller sheds (over the per-caller cap)" + ); + // A DIFFERENT caller is unaffected — the cap is per-caller, not global. + let _other = lim + .try_acquire("did:key:zB") + .expect("a different caller has its own budget"); + drop(p1); + assert!( + lim.try_acquire("did:key:zA").is_some(), + "freeing one in-flight slot lets the same caller back in" + ); + drop(p2); + } + + #[test] + fn per_caller_concurrency_map_is_self_bounding_and_reject_before_insert() { + // Self-bounding: acquire+drop many distinct keys — the map never grows + // because a key is removed the instant its in-flight count hits zero. + let lim = PerCallerConcurrency::new(4, 3); + for i in 0..50 { + let _p = lim.try_acquire(&format!("k{i}")); + } + assert_eq!( + lim.tracked_keys(), + 0, + "keys with zero in-flight ops are removed, so an acquire+drop flood leaves the map empty" + ); + // Reject-before-insert: HOLD max_keys distinct keys, then a new key sheds + // WITHOUT growing the map past the cap (INV-15 — a rejected request never + // allocates an entry). + let held: Vec<_> = (0..3) + .map(|i| lim.try_acquire(&format!("h{i}")).unwrap()) + .collect(); + assert_eq!( + lim.tracked_keys(), + 3, + "three distinct callers held concurrently" + ); + assert!( + lim.try_acquire("h3").is_none(), + "a new key at max_keys is rejected" + ); + assert_eq!( + lim.tracked_keys(), + 3, + "the rejected new key did not allocate an entry (reject-before-insert)" + ); + drop(held); + } + #[tokio::test] async fn allows_within_limit() { let limiter = RateLimiter::new(3, Duration::from_secs(60)); diff --git a/crates/gitlawb-node/src/server.rs b/crates/gitlawb-node/src/server.rs index f4c0d3e3..de61fcbe 100644 --- a/crates/gitlawb-node/src/server.rs +++ b/crates/gitlawb-node/src/server.rs @@ -214,9 +214,20 @@ pub fn build_router(state: AppState) -> Router { // identity and can apply per-repo visibility (#110); anonymous callers stay // anonymous and still read genuinely public content. `/api/v1/ipfs/pins` // stays unsigned — gating the pin index is tracked separately (#121). + // `/ipfs/{cid}` also carries a per-IP flood brake: it is anon-reachable and each + // request can drive a full-history git walk, so the per-IP rate limiter is the + // outermost layer (rejects a flood before the walk-admission work), mirroring the + // push/create routers. The extension MUST be attached or rate_limit_by_ip is a + // silent no-op. `/api/v1/ipfs/pins` (no walk) is merged in unbraked, as before. + let ipfs_limiter = rate_limit::IpRateLimiter { + limiter: state.ipfs_rate_limiter.clone(), + trust: state.push_limiter_trust, + }; let ipfs_routes = Router::new() .route("/ipfs/{cid}", get(ipfs::get_by_cid)) .layer(middleware::from_fn(auth::optional_signature)) + .layer(middleware::from_fn(rate_limit::rate_limit_by_ip)) + .layer(axum::Extension(ipfs_limiter)) .merge(Router::new().route("/api/v1/ipfs/pins", get(ipfs::list_pins))); // ── Arweave permanent anchors ────────────────────────────────────────── diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index d3e53f3a..80dc7e05 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -66,6 +66,45 @@ pub struct AppState { /// brake a push flood from a DID farm (one throwaway DID per repo), so the /// push path throttles on the resolved client IP instead. pub push_rate_limiter: RateLimiter, + /// Per-client-IP ROUTE brake for `GET /ipfs/{cid}`: charged ONCE per request by the + /// `rate_limit_by_ip` middleware (server.rs), never inside the handler. It bounds + /// request RATE (the "requests per hour" contract of `GITLAWB_IPFS_RATE_LIMIT`) on + /// the non-farmable source IP, so an anonymous flood of the public route is capped. + /// The per-probe/per-walk WORK accounting the resolver does WITHIN a request draws + /// from the SEPARATE `ipfs_work_rate_limiter` below — the two cannot share one bucket + /// or a single request that spends a route token and then its own probe token off the + /// same bucket is admitted at the route and falsely shed mid-request (#173 round-10, + /// R6). Keyed by `push_limiter_trust`. + pub ipfs_rate_limiter: RateLimiter, + /// Per-client-IP WORK-budget limiter for the `GET /ipfs/{cid}` resolver's internal + /// fan-out: charged per legacy (NULL-provenance) PROBE (`acquire` + `cat-file`) and + /// per provenance-path WALK, and peeked non-consuming before the O(repos) legacy + /// preload. A legacy CID from the public pins index otherwise lets one request drive + /// O(repos) subprocess spawns and cold Tigris fetches, and repeat requests amplify + /// that across requests with zero limiter contact (INV-10, F3). Charging the work to + /// the non-farmable source IP bounds it. A bucket DISTINCT from the route brake above: + /// one request legitimately spends many work tokens (a full legacy scan is up to + /// `ipfs_max_legacy_probes` probes), so it must not double as the once-per-request + /// route bucket. Capacity is DERIVED from the route limit (`AppState::ipfs_work_budget`, + /// no separate operator knob), floored at the legacy-probe budget so a single default- + /// config deep search never self-throttles mid-scan. Keyed by `push_limiter_trust`. + pub ipfs_work_rate_limiter: RateLimiter, + /// Per-request ceiling on full-history reachability walks the CID resolver + /// may spawn (default `api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST`). A field, + /// not a bare const, so tests can shrink it to exercise the cap cheaply; + /// production keeps the const default. + pub ipfs_max_history_walks: u32, + /// Per-request ceiling on legacy (NULL-provenance) repo probes in the CID + /// resolver's scan fallback (default `api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST`). + /// Bounds the anonymous `acquire` + `cat-file` fan-out across the node (#173, + /// INV-10); a field for the same test-seam reason as `ipfs_max_history_walks`. + pub ipfs_max_legacy_probes: u32, + /// Hard ceiling on the byte size of an object `GET /ipfs/{cid}` will buffer and + /// serve (default `api::ipfs::MAX_SERVED_OBJECT_BYTES`). The serve reads via a + /// blocking `git cat-file` and buffers the whole object; without a bound a large + /// public blob could exhaust memory or block a runtime worker (#173, F6, INV-10). + /// A field for the same test-seam reason as the sibling caps. + pub ipfs_max_served_object_bytes: u64, /// Which forwarded header (if any) the edge is trusted to set, for /// resolving the push limiter's client-IP key. See `GITLAWB_TRUSTED_PROXY`. /// Node-wide; also keys the two peer-sync limiters below. @@ -89,6 +128,134 @@ pub struct AppState { /// * the libp2p swarm task /// * the gossip, sync, operator heartbeat, and rate-limit cleanup loops pub shutdown_tx: tokio::sync::watch::Sender, + /// Bounds concurrent served git READ operations (upload-pack + both info/refs + /// advertisements). A read handler acquires a permit before spawning git and + /// holds it for the op; when none are free the request is shed with a 503. + /// Writes draw from `git_write_semaphore` so a read flood cannot shed an + /// authenticated push at admission (#174). + pub git_read_semaphore: Arc, + /// Bounds concurrent `git-receive-pack` (push) operations, a pool separate + /// from `git_read_semaphore` so an anonymous READ flood can never shed an + /// authenticated push (#174). Sized by `max_concurrent_git_pushes`. Drawn from + /// by the `git-receive-pack` POST (owner-gated) ONLY. The anon-reachable + /// receive-pack `info/refs` advertisement draws from the SEPARATE + /// `git_push_advert_semaphore` below, never this pool, so a multi-source flood + /// of push-handshake advertisements can never occupy a permit an authenticated + /// POST needs at admission (#174). + pub git_write_semaphore: Arc, + /// Bounds concurrent anon-reachable `git-receive-pack` `info/refs` + /// advertisements — a pool SEPARATE from `git_write_semaphore` so adverts (which + /// hold a permit across `acquire_fresh` + `info/refs`) can never consume a slot + /// the authenticated POST relies on. A per-source flood can at worst exhaust this + /// advert pool (each source also capped by `git_push_advert_per_caller` and the + /// per-IP push rate limiter), and the reserved POST pool is untouched (#174). + pub git_push_advert_semaphore: Arc, + /// Bounds concurrent post-receive git scans. Each successful push releases its + /// handler write permit the moment receive-pack's git group is reaped, then runs + /// up to four scans over the repo: the anonymous withheld walk + /// (`replication_withheld_set`), the pin-candidate scan + /// (`resolve_candidates_for_push`), the fail-closed full scan + /// (`fail_closed_full_scan_objects`), and the DETACHED encrypt-then-pin walk + /// (`withheld_blob_recipients_bounded`). Without a cap, N fast pushes spawn N + /// concurrent full-history git walks past `max_concurrent_git_pushes` (which only + /// bounds the in-handler receive-pack phase) — #174 P1-e closed the detached walk, + /// F4 closed the other three. Each scan acquires ONE permit here per walk and + /// DEFERS (blocks) when the pool is full rather than shedding — dropping the work + /// would lose the recovery copy or silently under-pin the push. No-walk fast + /// paths (not announceable, no path-scoped rule, deletion-only push) never touch + /// the pool. A pool of its own, not `git_write_semaphore`: a long background + /// walk must not hold a foreground write slot, and a handler already holding a + /// write permit that needed a second would self-deadlock at pool size 1. + pub git_encrypt_semaphore: Arc, + /// Bounds concurrent post-push pin loops (`ipfs_pin` / `pinata` `pin_new_objects`) + /// across all repos (#174 F6). `encrypt_inflight` caps the pin-task COUNT to one + /// per repo, but each pin loop holds a full per-push object-id list while walking + /// it, so N distinct repos could hold N such MB-scale lists at once. This caps how + /// many run concurrently; a loop DEFERS (waits) when the pool is full, never drops. + /// + /// It does NOT bound the lists held by tasks PARKED on it. The local IPFS path + /// materializes its list before acquiring, so a parked task still holds one, and the + /// parked-task count is capped only per repo by `encrypt_inflight`. Cross-repo + /// retained memory is therefore not bounded by this pool. The Pinata twin acquires + /// before it derives and does not carry that residual. + pub pin_semaphore: Arc, + /// Bounds the outstanding post-push encryption-task set to at most one PER REPO by + /// coalescing (#174 P2-2). This is NOT a global cap: N distinct repos still admit N + /// tasks; the cross-repo residual (an authenticated actor pushing to many repos + /// leaves many parked tasks) is throttled by auth plus the per-IP/per-DID rate + /// limits. Its real cost, the MB-scale per-push object-id list each parked task + /// holds, is NOT bounded by `pin_semaphore` either; see that field's doc above for + /// why. Nothing currently bounds this memory across repos. `git_encrypt_semaphore` caps + /// *active* walks; this caps duplicate SPAWNS per repo. Before spawning a per-push + /// encryption task, the receive-pack handler consults this set: if the repo already + /// has a task in flight it coalesces (skips the duplicate spawn) rather than parking + /// a new waiter, and its tip pairs are recorded for that task's drain loop (#174 F5). + /// Coalescing only delays the coalesced push's walk — it never drops the withheld-blob + /// recovery copy, which `2a54c15` deliberately kept fail-closed (there is no + /// reconciliation sweep to re-derive a dropped copy). See [`EncryptInflight`]. + pub encrypt_inflight: EncryptInflight, + /// Per-repo in-process write serializer that SUPPLEMENTS the cluster-wide pg + /// advisory lock on the receive-pack path (#174 U2/F3). On a client disconnect + /// mid-`receive-pack`, `RepoWriteGuard::Drop` releases the pg advisory lock at the + /// disconnect instant, but the disconnected push's git process GROUP is still + /// being torn down by `KillGroupOnDrop`'s detached reaper (~4s TERM/grace/KILL/reap) + /// over the shared LOCAL objects/ dir — so a second SAME-NODE push could acquire + /// the repo and race the still-writing group into a torn snapshot. This lease is + /// held by the write-path `AdmissionGuard`, which rides that reaper, so a second + /// same-repo push blocks until the first group is reaped. It is per-NODE (the + /// corruption is same-node: shared local objects/ + in-process reaper, and the + /// disconnect path uploads nothing to Tigris), so it needs no cross-node counterpart + /// and does NOT replace the pg lock (which stays the genuine cluster-wide serializer). + /// See [`RepoWriteLeases`]. + pub repo_write_leases: RepoWriteLeases, + /// Per-caller concurrency sub-cap on the read pool: each caller (keyed on the + /// resolved source IP, #174 U1) may hold at most `max_concurrent_reads_per_caller` + /// in-flight read ops, so one caller cannot monopolize `git_read_semaphore` + /// (#174). Applied by `git_upload_pack` and the upload-pack `info/refs` + /// advertisement. + pub git_read_per_caller: crate::rate_limit::PerCallerConcurrency, + /// Per-source concurrency sub-cap on the anon-reachable receive-pack `info/refs` + /// advertisement: each source IP may hold at most a small share of the DEDICATED + /// advert pool (`git_push_advert_semaphore`), so a multi-source flood of + /// push-handshake advertisements cannot saturate that pool and shed other sources' + /// advertisements (#174). An advert flood cannot reach `git_write_semaphore` at + /// all, since the two pools are disjoint. Sized as a fraction of + /// `max_concurrent_git_pushes` because the advert pool is created at the same size, + /// so filling it takes many distinct source IPs (each also braked by the per-IP + /// push rate limiter). + pub git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency, + /// Per-source concurrency sub-cap on the authenticated `git-receive-pack` POST: + /// each source IP may hold at most a small share of `git_write_semaphore`, so one + /// host minting disposable `did:key` identities cannot open enough slow pushes to + /// monopolize the write pool and 503 every other source's push (#174 P1-d). Keyed + /// on the resolved source IP (never the DID — a DID farm defeats a DID key). Sized + /// like `git_push_advert_per_caller`, a fraction of `max_concurrent_git_pushes`. + pub git_write_per_caller: crate::rate_limit::PerCallerConcurrency, + /// Bounds concurrent `GET /ipfs/{cid}` visibility-walk requests. The public + /// `/ipfs/{cid}` route runs `allowed_blob_set_for_caller_bounded` in + /// `spawn_blocking` (a full-history git walk) with NO served-git admission of its + /// own; without this a permissionless caller fans out concurrent walks past every + /// git pool, exhausting the blocking pool + PIDs (#174 P1-3). A request acquires a + /// permit before the repo loop and holds it for the whole request (across every + /// `spawn_blocking` walk), so the slot reflects real thread occupancy — a tokio + /// walk-timeout cannot free it while the blocking work still runs. A pool of its + /// own (`max_concurrent_ipfs_walks`), NOT a git pool: distinct cost center + public + /// surface, so anonymous /ipfs traffic can never shed an authenticated git op. + pub git_ipfs_walk_semaphore: Arc, + /// Per-source concurrency sub-cap on the `/ipfs/{cid}` walk pool: each source + /// (keyed on the resolved source IP, never the DID — `/ipfs` admits any `did:key` + /// unthrottled, so a DID key would be free to mint around) may hold at most + /// `ipfs_walk_per_source` in-flight walk slots, so one source cannot monopolize + /// `git_ipfs_walk_semaphore` (#174 P1-3). A request with no resolvable key is + /// bounded by the global pool only, never this sub-cap. The key map is bounded + /// (`with_default_max_keys`, reject-before-insert) so a source-key farm cannot grow + /// it (INV-15). + pub git_ipfs_walk_per_caller: crate::rate_limit::PerCallerConcurrency, + /// The `git` executable the served-git withheld-blob walk spawns. Production is + /// `"git"` (resolved via PATH); injectable so a fake `git` can drive the walk's + /// process-group teardown in handler tests without mutating the process-global + /// PATH (#174). + pub git_bin: String, } impl AppState { @@ -98,6 +265,21 @@ impl AppState { self.shutdown_tx.subscribe() } + /// Sweep expired entries from every per-IP/DID rate limiter. Driven by the + /// periodic cleanup task so a bounded limiter's key map sheds stale entries + /// instead of sitting near its cap until an inline capacity sweep reclaims + /// them. Every limiter on the state is swept here; adding a new limiter means + /// adding it to this list. + pub(crate) async fn sweep_rate_limiters(&self) { + self.rate_limiter.cleanup().await; + self.create_ip_rate_limiter.cleanup().await; + self.push_rate_limiter.cleanup().await; + self.ipfs_rate_limiter.cleanup().await; + self.ipfs_work_rate_limiter.cleanup().await; + self.sync_trigger_rate_limiter.cleanup().await; + self.peer_write_rate_limiter.cleanup().await; + } + /// Trigger graceful shutdown. Idempotent — calling more than once /// has no effect. Returns `true` if this call was the one that /// flipped the signal. @@ -118,4 +300,1007 @@ impl AppState { pub fn is_shutting_down(&self) -> bool { *self.shutdown_tx.borrow() } + + /// Legacy-probe budget wired from the `GITLAWB_IPFS_MAX_LEGACY_PROBES` operator + /// knob. The knob seeds `ipfs_max_legacy_probes` at construction so it controls the + /// per-request legacy (NULL-provenance) probe fan-out it advertises. It deliberately + /// does NOT feed the history-walk ceiling: that is governed by + /// `ipfs_max_repos_walked` under a `MAX_PIN_SOURCES + 1` floor, because a value + /// below the floor truncates a provenanced request with a full source set into a + /// false 503. The knob is `usize`, the field `u32`; the range cap (1_048_576) keeps + /// the cast lossless. + pub(crate) fn ipfs_legacy_probe_budget(config: &crate::config::Config) -> u32 { + config.ipfs_max_legacy_probes as u32 + } + + /// Work-budget capacity for [`ipfs_work_rate_limiter`](Self#structfield.ipfs_work_rate_limiter) + /// (R6, KTD6), DERIVED from the route limit rather than a new operator knob. The route + /// limiter (`ipfs_rate_limiter`) charges once per request; this separate bucket absorbs + /// the resolver's per-probe/per-walk work charges so both the route "requests per hour" + /// contract and the amplification bound hold. Floor: at least one full legacy search per + /// window, the effective `ipfs_max_legacy_probes` (the `GITLAWB_IPFS_MAX_LEGACY_PROBES` + /// knob), so a single default-config deep search cannot self-throttle mid-scan and + /// recreate the admit-then-429 for a legitimate caller. `GITLAWB_IPFS_RATE_LIMIT=0` + /// disables the route brake and this derived bucket alike (a 0-capacity limiter admits + /// everything). + /// + /// The floor is the LEGACY-PROBE knob, not `ipfs_max_repos_walked`. Those were one + /// field before the walk cap and the probe budget were split apart, and reading the + /// walk cap here would silently size this bucket at 64 instead of 256. + pub(crate) fn ipfs_work_budget(config: &crate::config::Config) -> usize { + if config.ipfs_rate_limit == 0 { + return 0; + } + config.ipfs_rate_limit.max(config.ipfs_max_legacy_probes) + } +} + +/// Bounds the OUTSTANDING post-push encryption-task set by per-repo coalescing +/// (#174 P2-2). Each successful path-scoped push `tokio::spawn`s a DETACHED task that +/// parks on `git_encrypt_semaphore.acquire_owned().await` (which DEFERS when the pool +/// is full rather than shedding — `2a54c15` kept it fail-closed so the withheld-blob +/// recovery copy is never dropped). The semaphore caps *active* walks, but nothing +/// capped how many detached tasks *spawn and park* on that await: N rapid pushes to a +/// repo spawn N parked tasks, each holding cloned object lists/rules/paths/keys — an +/// unbounded outstanding set. +/// +/// This tracks the repo keys with an in-flight encryption task. Before spawning, the +/// handler calls [`try_begin`](Self::try_begin) with the push's (old, new) tip pairs: +/// if no task is in-flight the push is [`Admitted`](BeginOutcome::Admitted) and spawns +/// one; if a task IS in-flight the push [`Coalesces`](BeginOutcome::Coalesced) — no +/// duplicate spawn — and its tip pairs are merged into the in-flight key's pending +/// slot in the SAME critical section as the presence check. The in-flight task pins +/// only its own pre-spawn object-list snapshot, so the merge is what keeps coalescing +/// lossless (#174 F5): the task loop-drains the pending slot via +/// [`EncryptInflightGuard::finish_or_take_pending`] before releasing the key, so a +/// coalesced push's pins and recovery copies are delayed, never dropped (there is no +/// reconciliation sweep, so a *dropped* job would be lost forever). Check-then-record +/// as two lock acquisitions would race the task's final pending check — hence one +/// critical section for both. +/// +/// The returned [`EncryptInflightGuard`] is moved into the detached task. On normal +/// exit the key is removed (and the guard disarmed) inside `finish_or_take_pending`'s +/// empty-pending critical section; the guard's Drop is the PANIC backstop (Drop runs +/// on unwind), so one crashed walk can never permanently lock a repo out of future +/// recovery copies. +#[derive(Clone, Default)] +pub struct EncryptInflight { + // std::sync::Mutex: only ever held for O(1)-ish map ops (insert/remove/merge — + // the merge is an O(pairs) Vec extend bounded by MAX_PENDING_TIP_PAIRS) in a + // sync context, never across an await, so a std Mutex is correct and cheaper + // than a tokio one. Key present == task in flight; the value is the work + // recorded by pushes that coalesced against it. + repos: Arc>>, +} + +/// Cap on the accumulated coalesced tip pairs per repo. Past it the pending slot +/// degrades to [`PendingWork::FullScan`], so a hostile pusher cannot grow the slot +/// without bound while a walk is in flight; the drain then costs one full-repo +/// enumeration instead (the same already-tested fallback the push path uses). +const MAX_PENDING_TIP_PAIRS: usize = 1024; + +/// Work recorded by pushes that coalesced against an in-flight encryption task, +/// drained by that task one batch per loop iteration. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PendingWork { + /// The coalesced pushes' raw (old_sha, new_sha) ref-update pairs, zeros + /// included — the drain strips the create/delete sentinels exactly like the + /// handler tail does. An EMPTY vec is "nothing pending", never a work item. + Tips(Vec<(String, String)>), + /// The pair bound overflowed: drain with a FORCED full-repo scan. This must be + /// signalled explicitly (the `force_full_scan` flag on + /// `resolve_candidates_for_push`), never encoded as an empty tip list — empty + /// tips resolve to an empty delta and would pin nothing (the F5 loss again). + FullScan, +} + +/// Outcome of [`EncryptInflight::try_begin`]. +pub enum BeginOutcome { + /// No task was in flight: the caller spawns one, moving the guard into it. The + /// push's own tip pairs are NOT recorded — the caller's pre-spawn snapshot + /// covers them; the pending slot starts empty. + Admitted(EncryptInflightGuard), + /// A task is in flight; this push's tip pairs were merged into its pending + /// slot (same critical section as the presence check). The in-flight task's + /// drain loop will process them. + Coalesced, +} + +/// Outcome of [`EncryptInflightGuard::finish_or_take_pending`]. +pub enum FinishOutcome { + /// Coalesced work was pending: it is handed back with the still-armed guard + /// (the repo key is retained) and the task must run another drain iteration. + Pending(EncryptInflightGuard, PendingWork), + /// Nothing was pending: the repo key was removed AND the guard disarmed in one + /// critical section, so dropping the returned guard is inert. The task exits. + /// Remove-then-drop as two steps would double-remove: a successor task admitted + /// between them would have ITS key deleted by the late Drop. The disarmed guard + /// is handed back rather than dropped internally so that remove→drop window is + /// real and the disarm is testable; production just lets it fall out of scope + /// (hence the allow). + Finished(#[allow(dead_code)] EncryptInflightGuard), +} + +impl EncryptInflight { + pub fn new() -> Self { + Self::default() + } + + /// Begin-or-coalesce an encryption task for `repo_id`, in one critical section. + /// `tip_pairs` is this push's raw (old_sha, new_sha) ref-update list; it is + /// merged into the pending slot only on the [`Coalesced`](BeginOutcome::Coalesced) + /// arm (an admitted caller's own snapshot already covers its pairs). + pub fn try_begin(&self, repo_id: &str, tip_pairs: Vec<(String, String)>) -> BeginOutcome { + let mut map = self.repos.lock().expect("encrypt_inflight mutex poisoned"); + match map.entry(repo_id.to_string()) { + std::collections::hash_map::Entry::Vacant(slot) => { + slot.insert(PendingWork::Tips(Vec::new())); + BeginOutcome::Admitted(EncryptInflightGuard { + repos: Arc::clone(&self.repos), + repo_id: repo_id.to_string(), + armed: true, + }) + } + std::collections::hash_map::Entry::Occupied(mut slot) => { + merge_pending(slot.get_mut(), tip_pairs); + BeginOutcome::Coalesced + } + } + } + + /// Number of repos with an in-flight encryption task. Test/metrics observability; + /// the bound under saturation is `len() <= number of distinct repos`, i.e. at most + /// one task per repo. + #[allow(dead_code)] + pub fn len(&self) -> usize { + self.repos + .lock() + .expect("encrypt_inflight mutex poisoned") + .len() + } + + #[allow(dead_code)] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// The work currently queued against a repo's in-flight task, or `None` when + /// no task holds the key. Test-only observability: it lets a test assert that + /// a coalesced push's tip pairs were actually recorded, rather than that the + /// outcome merely was `Coalesced`. + #[cfg(test)] + pub fn pending_for(&self, repo_id: &str) -> Option { + self.repos + .lock() + .expect("encrypt_inflight mutex poisoned") + .get(repo_id) + .cloned() + } +} + +/// Merge a coalesced push's tip pairs into a repo's pending slot. FullScan absorbs +/// everything; a Tips slot that would exceed [`MAX_PENDING_TIP_PAIRS`] degrades to +/// FullScan rather than growing without bound. +fn merge_pending(slot: &mut PendingWork, pairs: Vec<(String, String)>) { + match slot { + PendingWork::FullScan => {} + PendingWork::Tips(acc) => { + if acc.len().saturating_add(pairs.len()) > MAX_PENDING_TIP_PAIRS { + *slot = PendingWork::FullScan; + } else { + acc.extend(pairs); + } + } + } +} + +/// Guard owned by the detached encryption task for its repo key. Move-only — there +/// is no reason to clone a guard, and cloning would double-remove. Normal exit goes +/// through [`finish_or_take_pending`](Self::finish_or_take_pending); Drop is the +/// panic-path backstop only. +pub struct EncryptInflightGuard { + repos: Arc>>, + repo_id: String, + /// True until the normal-exit path removes the key. A disarmed guard's Drop is + /// a no-op: the key slot may already belong to a successor task admitted after + /// our removal, and removing THAT key would break at-most-one-task-per-repo. + armed: bool, +} + +impl EncryptInflightGuard { + /// The task's end-of-iteration step, one critical section: if coalesced work is + /// pending, take it and hand the still-armed guard back (key retained — iterate); + /// if nothing is pending, remove the key and disarm the guard (task exits; the + /// returned guard's Drop is inert). The atomicity is load-bearing both ways: a + /// push landing before this call is merged and therefore drained here; a push + /// landing after it finds the key gone and is admitted as a fresh task. No + /// interleaving can lose the work or admit two tasks for one repo. + pub fn finish_or_take_pending(mut self) -> FinishOutcome { + let mut map = self.repos.lock().expect("encrypt_inflight mutex poisoned"); + match map.get_mut(&self.repo_id) { + Some(PendingWork::Tips(acc)) if acc.is_empty() => { + map.remove(&self.repo_id); + self.armed = false; + drop(map); + FinishOutcome::Finished(self) + } + Some(slot) => { + let work = std::mem::replace(slot, PendingWork::Tips(Vec::new())); + drop(map); + FinishOutcome::Pending(self, work) + } + None => { + // Unreachable while armed (only this method removes a live key), + // but never panic in the release path: treat as finished. + self.armed = false; + drop(map); + FinishOutcome::Finished(self) + } + } + } +} + +impl Drop for EncryptInflightGuard { + fn drop(&mut self) { + // Normal exit disarmed us inside finish_or_take_pending's critical section; + // an armed drop means the task ended abnormally (panic-unwind, or a future + // code path that returns without finishing). Release the key so the repo is + // not permanently locked out, and log any pending work this loses — there + // is no sweep, so it stays lost until a later push re-walks the repo. + if !self.armed { + return; + } + // A poisoned lock is not expected (the critical sections above are small + // and panic-free); remove best-effort. + if let Ok(mut map) = self.repos.lock() { + match map.remove(&self.repo_id) { + Some(PendingWork::Tips(acc)) if !acc.is_empty() => tracing::warn!( + repo = %self.repo_id, + lost_tip_pairs = acc.len(), + "encryption task ended abnormally with coalesced pushes pending; \ + their pins/recovery copies are lost until a later push" + ), + Some(PendingWork::FullScan) => tracing::warn!( + repo = %self.repo_id, + "encryption task ended abnormally with a pending full-scan drain; \ + it is lost until a later push" + ), + _ => {} + } + } + } +} + +/// Per-repo in-process write-lease serializer (#174 U2/F3). Keyed by the repo's DB +/// id (1:1 with the pg advisory lock's owner/name key), each entry is a one-permit +/// semaphore: the receive-pack handler takes it BEFORE `acquire_write` (see the acquire +/// order note on [`acquire`](Self::acquire)) and a second same-repo push BLOCKS on it — +/// block-and-wait, NOT coalesce. It mirrors [`EncryptInflight`]'s keyed-map + guard + +/// Drop-frees-key STRUCTURE; the semantics differ (block-and-wait, so there is no +/// lossy-coalesce degradation to fall back on). +#[derive(Clone)] +pub struct RepoWriteLeases { + // std::sync::Mutex: held only for O(1) map ops (get-or-create + refcount) in a sync + // context, never across an await — the semaphore wait happens OUTSIDE this lock. + repos: Arc>>, + /// Most handlers allowed to be PARKED on one repo's lease at once + /// (`GITLAWB_REPO_LEASE_MAX_WAITERS`). Past it `acquire` sheds instead of queueing. + max_waiters: usize, +} + +/// The stable per-repo identity that [`RepoWriteLeases`] and [`EncryptInflight`] +/// key on (#174 U2). +/// +/// NOT `record.id`. A repo deleted and recreated under the same owner/name gets a +/// new row id while the bare repo on disk is reused, so an id-keyed serializer +/// stops serializing across that rotation and lets two writers onto one +/// `objects/` directory. `RepoStore::local_path` and the pg advisory lock both +/// key on the sanitized owner slug plus repo name, and this reproduces exactly +/// that identity so the in-process serializers agree with them. +/// +/// Three details are load-bearing: +/// +/// - The sanitization is [`crate::git::store::repo_disk_path`]'s +/// (`replace([':', '/'], "_")`), NOT `db::normalize_owner_key`, which strips a +/// `did:key:` prefix instead and would map the same input to a different +/// string. The disk path is authoritative because the `objects/` directory is +/// the resource being serialized. +/// - The separator is `/`, which cannot occur in the owner slug by construction +/// (`replace([':', '/'], "_")` removes it) and mirrors the shape of the disk +/// path this key exists to reproduce. A plain join would collide (owner `a` + +/// name `bc` against owner `ab` + name `c`, both `abc`), letting one repo's +/// push park another's. It must stay PRINTABLE: this key is logged as the +/// `repo` field on the lease shed and steal-bound warnings below, and an unprintable +/// separator (a NUL, a unit separator) truncates at a NUL-hostile log sink and +/// renders two different repos' warnings identically. +/// - Callers pass `record.owner_did` / `record.name`, never the request's path +/// segments: `db::get_repo` normalizes DID aliases, so a caller could otherwise +/// mint two keys for one directory just by varying the DID spelling. +/// +/// The sanitization is not injective (`did:web:example.com:alice` and +/// `did:web:example.com/alice` fold to one slug), which would be a cross-tenant +/// hazard for the coalescing map if it were reachable. It is not: `repos.disk_path` +/// is `NOT NULL UNIQUE` and holds exactly this derivation, so two rows that fold +/// to one key cannot coexist. Do not "fix" this by making the key injective — +/// that is precisely what would let two owners sharing one `objects/` directory +/// push concurrently. +pub fn repo_identity_key(owner_did: &str, repo_name: &str) -> String { + let owner_slug = owner_did.replace([':', '/'], "_"); + format!("{owner_slug}/{repo_name}") +} + +/// A per-repo lease entry: the one-permit semaphore, a refcount of the handlers +/// currently referencing it (holding or waiting), and a count of the ones actually +/// PARKED. While `refs > 0` every acquirer shares the SAME semaphore, so mutual exclusion +/// holds; the entry is removed only when `refs` hits 0 (no one references it), so a fresh +/// entry can never split serialization. +/// +/// `waiters` and `refs` are deliberately different counts, and the shed cap is on +/// `waiters`. `refs` includes the HOLDER, and a holder whose Drop never runs (task abort, +/// runtime teardown without unwind, `mem::forget`: precisely the leak `steal_after` exists +/// to survive) keeps its ref forever. Capping `refs` would let that leaked ref +/// permanently occupy a slot and wedge the repo, reintroducing the permanent wedge the +/// steal backstop was written to prevent. A waiter, by contrast, always leaves: it either +/// gets the permit, steals at `steal_after`, is shed, or is cancelled, and every one of +/// those paths drops the RAII waiter guard. +struct LeaseSlot { + sem: Arc, + refs: usize, + waiters: usize, +} + +impl RepoWriteLeases { + /// `max_waiters` is the per-repo live-waiter cap (see [`acquire`](Self::acquire)). + pub fn new(max_waiters: usize) -> Self { + Self { + repos: Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())), + max_waiters: max_waiters.max(1), + } + } + + /// Acquire the per-repo write lease, blocking until it is free (a second same-repo + /// writer waits). `steal_after` bounds that wait: past it the acquirer STEALS + /// (proceeds permit-less) rather than block forever. Returns `None` when this repo + /// already has `max_waiters` handlers parked, so the caller sheds (503) instead of + /// adding to an unbounded queue: `git_receive_pack` reaches here with the whole pack + /// already buffered by axum, and the park runs to `steal_after` (1260s at defaults), + /// so unbounded parking is unbounded buffered bytes. The cap is per repo and counts + /// only live waiters, so shedding is confined to the contended repo; a push to any + /// other repo, from any source, is unaffected. + /// + /// Why a bounded steal: block-and-wait has no degradation of its own (unlike the + /// coalescing [`EncryptInflight`], whose lost key merely delays a best-effort copy), + /// and unlike the pg advisory lock (60s stale reclaim) an in-process waiter has no + /// reclaim — so a leaked/never-run Drop (runtime teardown without unwind, task abort, + /// `mem::forget`) would otherwise wedge the repo permanently. A stealer takes NO + /// permit and touches NO count, so a merely-slow holder that later drops can never + /// leave the semaphore over-counted; the caller must therefore set `steal_after` + /// safely ABOVE any legitimate hold (a full receive-pack under + /// `git_service_timeout_secs` + the ~4s reaper cap + the Tigris upload). + /// + /// Acquire order (one consistent order everywhere, so no inversion self-hang): the + /// lease is taken BEFORE the pg advisory lock (`acquire_write`) and released AFTER + /// it. Nothing anywhere takes the pg lock before this lease, so the two serializers + /// can never deadlock; taking the lease first also means a blocked second writer + /// pins no pooled pg connection while it waits. + pub async fn acquire( + &self, + repo_id: &str, + steal_after: std::time::Duration, + ) -> Option { + // Take the entry refcount BEFORE the await, so the entry cannot be GC'd out from + // under a waiter (a fresh entry for a new acquirer would split serialization). + let sem = { + let mut map = self.repos.lock().expect("repo_write_leases mutex poisoned"); + let slot = map.entry(repo_id.to_string()).or_insert_with(|| LeaseSlot { + sem: Arc::new(tokio::sync::Semaphore::new(1)), + refs: 0, + waiters: 0, + }); + slot.refs += 1; + Arc::clone(&slot.sem) + }; + // Cancellation-safe refcount: hold a reservation across the (cancellable) wait so + // that if this acquire future is DROPPED mid-wait — a client disconnect while a + // second same-repo push is blocked here — the reservation's Drop still decrements + // the ref it just took, rather than stranding it (which would defeat the + // Drop-frees-key GC). On success the reservation is `forget`-transferred into the + // returned guard, which then owns the single decrement. + let reservation = RefReservation { + repos: Arc::clone(&self.repos), + repo_id: repo_id.to_string(), + }; + + // Uncontended fast path: take the free permit without spending waiter budget, so + // the cap bounds only handlers that actually park. `try_acquire_owned` on tokio's + // semaphore does NOT barge a queued FIFO waiter (it fails while anyone is queued), + // so the fast path is not a fairness hole; probed at 2000 rounds with a queued + // waiter present, 0 barges. + let permit = match Arc::clone(&sem).try_acquire_owned() { + Ok(p) => Some(p), + Err(_) => { + // Contended: park, holding a waiter slot for exactly the cancellable wait. + // Claimed in ONE critical section (check and increment together), so a + // burst of concurrent acquirers cannot all read an under-cap count and + // then all park. Past the cap, drop the reservation (freeing the ref, and + // the entry with it if nobody else holds one) and shed. + let waiter = match WaiterSlot::claim(&self.repos, repo_id, self.max_waiters) { + Some(w) => w, + None => { + tracing::warn!( + repo = %repo_id, + max_waiters = self.max_waiters, + "repo write-lease waiter cap reached; shedding this acquirer" + ); + drop(reservation); + return None; + } + }; + let parked = + tokio::time::timeout(steal_after, Arc::clone(&sem).acquire_owned()).await; + // Released HERE, at the end of the wait, not at the end of the push: past + // this point the handler is a holder, counted by `refs`, and a waiter slot + // it kept would be budget no parked request could ever use. + drop(waiter); + match parked { + Ok(Ok(p)) => Some(p), + // The semaphore is never closed; treat the (unreachable) closed case + // as a steal so acquire always makes forward progress. + Ok(Err(_closed)) => None, + Err(_elapsed) => { + tracing::warn!( + repo = %repo_id, + steal_after_secs = steal_after.as_secs(), + "repo write-lease wait exceeded the steal bound; presuming a \ + leaked lease and proceeding permit-less (in-process \ + serializer reclaim)" + ); + None + } + } + } + }; + // Transfer the ref from the reservation to the guard: forget the reservation (so + // it does NOT decrement) and let the guard own the single decrement on its Drop. + std::mem::forget(reservation); + Some(RepoWriteLease(Arc::new(LeaseGuardInner { + repos: Arc::clone(&self.repos), + repo_id: repo_id.to_string(), + _permit: permit, + }))) + } + + /// Number of repos with a live lease entry. Test/metrics observability. + #[allow(dead_code)] + pub fn len(&self) -> usize { + self.repos + .lock() + .expect("repo_write_leases mutex poisoned") + .len() + } + + #[allow(dead_code)] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// How many handlers currently reference this repo's lease entry: the holder plus + /// every waiter parked in [`acquire`](Self::acquire). The ref is taken SYNCHRONOUSLY, + /// before the (cancellable) semaphore wait, so a waiter is visible here the moment it + /// reaches the lease. Tests use it to observe "this push parked" as a state rather + /// than inferring it from a deadline that never fired. + #[cfg(test)] + pub fn refs_for(&self, repo_id: &str) -> usize { + self.repos + .lock() + .expect("repo_write_leases mutex poisoned") + .get(repo_id) + .map(|slot| slot.refs) + .unwrap_or(0) + } + + /// How many handlers are PARKED on this repo's lease right now (the count the shed + /// cap is enforced against, holder excluded). Tests use it to observe "this push + /// parked" and "the waiter slot was released" as state. + #[cfg(test)] + pub fn waiters_for(&self, repo_id: &str) -> usize { + self.repos + .lock() + .expect("repo_write_leases mutex poisoned") + .get(repo_id) + .map(|slot| slot.waiters) + .unwrap_or(0) + } +} + +/// Shared-ownership handle to a held repo write lease (#174 U2/F3). `Clone` hands a +/// second holder a handle to the SAME inner guard; the lease (permit + map refcount) +/// frees only when the LAST clone drops. The receive-pack handler makes two: +/// * clone (a) rides the write-path [`AdmissionGuard`] into `KillGroupOnDrop`'s +/// detached reaper, so on a client disconnect it drops only AFTER the git group is +/// reaped (this is the F3 fix — a lease tied to `RepoWriteGuard` would instead drop +/// at the disconnect instant, reopening the race); +/// * clone (b) is held by the handler across `guard.release()`, so on the clean path +/// it spans the success-only Tigris upload that runs inside `release`, AFTER +/// `receive_pack` has already dropped clone (a) inside `run_git_service`. +/// +/// `Send + 'static` with NO pg connection (just an `Arc`), so it can ride the reaper. +#[derive(Clone)] +pub struct RepoWriteLease(#[allow(dead_code)] Arc); + +struct LeaseGuardInner { + repos: Arc>>, + repo_id: String, + // `None` only on the steal path (the bounded wait elapsed). Dropping `None` releases + // no permit, so a stealer never corrupts the semaphore's permit count. + _permit: Option, +} + +impl Drop for LeaseGuardInner { + fn drop(&mut self) { + // Runs exactly ONCE per handler acquisition — when the last `RepoWriteLease` + // clone drops (the Arc strong count hits 0) — so the refcount decrements once, + // however many clones existed. `_permit` drops after this body, releasing the + // semaphore permit so a waiting acquirer proceeds. + release_lease_ref(&self.repos, &self.repo_id); + } +} + +/// Holds the entry refcount across the cancellable wait inside +/// [`RepoWriteLeases::acquire`]. If that acquire future is dropped mid-wait, this Drop +/// decrements the ref it took; on success `acquire` `forget`s it and hands the ref to +/// the returned [`LeaseGuardInner`], so the ref is decremented exactly once either way. +struct RefReservation { + repos: Arc>>, + repo_id: String, +} + +impl Drop for RefReservation { + fn drop(&mut self) { + release_lease_ref(&self.repos, &self.repo_id); + } +} + +/// Holds one LIVE-WAITER slot on a lease entry for exactly the duration of the +/// cancellable park inside [`RepoWriteLeases::acquire`]. Every way out of that park +/// (permit acquired, steal on timeout, shed, or the acquire future being dropped on a +/// client disconnect) drops this guard, so the count can only reflect handlers that are +/// parked right now. That is why the shed cap is on this count and not on `refs`. +struct WaiterSlot { + repos: Arc>>, + repo_id: String, +} + +impl WaiterSlot { + /// Claim a waiter slot, or `None` when the repo already has `max_waiters` live + /// waiters. The entry always exists here: the caller took its `refs` reference in an + /// earlier critical section and still holds it. Check and increment share one + /// critical section, so concurrent acquirers cannot overshoot the cap. + fn claim( + repos: &Arc>>, + repo_id: &str, + max_waiters: usize, + ) -> Option { + if let Ok(mut map) = repos.lock() { + if let Some(slot) = map.get_mut(repo_id) { + if slot.waiters >= max_waiters { + return None; + } + slot.waiters += 1; + } + } + Some(Self { + repos: Arc::clone(repos), + repo_id: repo_id.to_string(), + }) + } +} + +impl Drop for WaiterSlot { + fn drop(&mut self) { + if let Ok(mut map) = self.repos.lock() { + if let Some(slot) = map.get_mut(&self.repo_id) { + slot.waiters = slot.waiters.saturating_sub(1); + } + } + } +} + +/// Decrement a lease entry's refcount and remove it once no handler references it, so +/// the map cannot grow without bound (Drop-frees-key, like `EncryptInflight`). Safe +/// under block-and-wait: while `refs > 0` every acquirer shares the SAME semaphore, and +/// a fresh entry is created only after `refs` hits 0, when no one references the old one. +fn release_lease_ref( + repos: &Arc>>, + repo_id: &str, +) { + if let Ok(mut map) = repos.lock() { + if let Some(slot) = map.get_mut(repo_id) { + slot.refs = slot.refs.saturating_sub(1); + if slot.refs == 0 { + map.remove(repo_id); + } + } + } +} + +/// Admit a post-receive git scan to the shared `git_encrypt_semaphore` pool +/// (#174 F4): DEFER (await), never shed — a dropped scan would lose the push's +/// recovery copy or silently under-pin it. The returned permit must move into +/// the blocking closure so a started scan always completes holding it (a +/// disconnect cannot cancel `spawn_blocking` or leak the permit mid-walk). +/// Accepted residual, stated once for every caller: the park wait is queue-depth +/// multiplied — post-receive tails are no longer admission-bounded once the write +/// permit is released, so N landed pushes can queue N scans and the last waits N +/// scan-durations. A client-timeout disconnect no longer loses the work (#174 F2): +/// the whole post-receive replication tail runs in an independently owned task, so +/// dropping the request future cannot drop this parked scan — the park no longer +/// precedes any durable-record gate in a cancellable future. +pub async fn acquire_scan_permit( + scan_sem: Arc, + repo: &std::path::Path, + stage: &'static str, +) -> tokio::sync::OwnedSemaphorePermit { + let parked = std::time::Instant::now(); + let permit = scan_sem + .acquire_owned() + .await + .expect("git_encrypt_semaphore is never closed"); + tracing::debug!( + repo = %repo.display(), + stage, + queue_wait_ms = parked.elapsed().as_millis() as u64, + "post-receive scan admitted to the scan pool" + ); + permit +} + +#[cfg(test)] +mod repo_identity_key_tests { + use super::repo_identity_key; + + /// The key must reproduce `repo_disk_path`'s slug derivation exactly, because + /// the whole point is to agree with the on-disk identity the store and the pg + /// advisory lock already use. + #[test] + fn matches_repo_disk_paths_sanitization() { + let owner = "did:key:z6Mkfoo"; + let name = "r"; + let expected_slug = owner.replace([':', '/'], "_"); + assert_eq!( + repo_identity_key(owner, name), + format!("{expected_slug}/{name}") + ); + + // The disk path for the same pair must carry the same slug component. + let disk = crate::git::store::repo_disk_path(std::path::Path::new("/srv"), owner, name); + assert!( + disk.to_string_lossy().contains(&expected_slug), + "the key's slug must be the one repo_disk_path puts on disk: {disk:?}" + ); + } + + /// Stability across the rotation is the entire reason this key exists: the row + /// id changes on delete+recreate, the identity does not. + #[test] + fn is_stable_across_a_row_id_rotation() { + assert_eq!( + repo_identity_key("did:key:z6Mkfoo", "r"), + repo_identity_key("did:key:z6Mkfoo", "r"), + ); + } + + /// The `/` separator is what stops one repo's push parking another's. These + /// two pairs both concatenate to `abc`, so with a plain join they would + /// produce a single key. + #[test] + fn separator_prevents_the_owner_name_boundary_collision() { + assert_ne!( + repo_identity_key("a", "bc"), + repo_identity_key("ab", "c"), + "owner/name boundary must not be ambiguous" + ); + } + + /// The key is logged as the `repo` field on the lease waiter-cap shed and the + /// steal-bound warning, so it must contain no control characters. A NUL (or a + /// unit separator) truncates at a NUL-hostile log sink, which would render two + /// different repos' shed warnings identically — an observability lie on + /// exactly the messages an operator reads to find a contended repo. + #[test] + fn key_is_printable_so_the_lease_warnings_stay_readable() { + let k = repo_identity_key("did:web:example.com:alice", "my-repo.git"); + assert!( + !k.chars().any(|c| c.is_control()), + "the identity key is logged; it must carry no control characters: {k:?}" + ); + assert_eq!(k, "did_web_example.com_alice/my-repo.git"); + } + + /// Distinct repos and distinct owners never share a key. + #[test] + fn distinct_repos_and_owners_do_not_share_a_key() { + assert_ne!( + repo_identity_key("did:key:z6A", "r"), + repo_identity_key("did:key:z6A", "s") + ); + assert_ne!( + repo_identity_key("did:key:z6A", "r"), + repo_identity_key("did:key:z6B", "r") + ); + } + + /// Documents the ONE collision the sanitization admits, and why it is safe + /// rather than fixed: `repos.disk_path` is UNIQUE and holds this derivation, + /// so two rows folding to one key cannot coexist. If this assertion ever + /// flips to `assert_ne!`, the key became injective and two owners sharing one + /// `objects/` directory could push concurrently — the defect U2 closes. + #[test] + fn folds_did_web_alias_spellings_together_which_the_unique_disk_path_makes_unreachable() { + assert_eq!( + repo_identity_key("did:web:example.com:alice", "r"), + repo_identity_key("did:web:example.com/alice", "r"), + ); + } +} + +#[cfg(test)] +mod repo_write_lease_tests { + use super::RepoWriteLeases; + use std::time::Duration; + + /// #174 U2/F3 lease mechanics: block-and-wait serialization on the same repo, + /// no serialization across distinct repos, Drop-frees-key GC, and the bounded-wait + /// steal reclaim so a leaked (never-run Drop) holder cannot wedge the repo forever. + #[tokio::test] + async fn serializes_same_repo_frees_key_and_steals_on_leak() { + let leases = RepoWriteLeases::new(8); + let big = Duration::from_secs(3600); + + // Block-and-wait: a second same-repo acquire waits while the first is held. + let a = leases.acquire("repo1", big).await.expect("uncontended"); + let blocked = + tokio::time::timeout(Duration::from_millis(200), leases.acquire("repo1", big)).await; + assert!( + blocked.is_err(), + "a second same-repo acquire must block while the first lease is held" + ); + // ... and proceeds once the first frees. + drop(a); + let b = tokio::time::timeout(Duration::from_millis(500), leases.acquire("repo1", big)) + .await + .expect("the second acquire must proceed once the first lease frees") + .expect("under the waiter cap, so it must not shed"); + drop(b); + + // Drop-frees-key: with no holders the entry is removed (bounded map growth). + assert!( + leases.is_empty(), + "the lease entry must be removed once no handler references it" + ); + + // Distinct repos never serialize against each other. + let x = leases.acquire("repoX", big).await.expect("uncontended"); + let _y = tokio::time::timeout(Duration::from_millis(200), leases.acquire("repoY", big)) + .await + .expect("distinct repos must not serialize") + .expect("a distinct repo has its own waiter budget"); + drop(x); + drop(_y); + } + + /// Scenario 4: the steal backstop survives the waiter cap. A holder whose Drop never + /// runs (task abort, runtime teardown, `mem::forget`) keeps its entry `refs` forever; + /// the next acquirer must still proceed permit-less at `steal_after` and must not be + /// shed on the way in. This is why the cap counts LIVE WAITERS and never `refs`: + /// capping `refs` (which includes the holder) lets one leaked lease pin a slot + /// permanently and wedge the repo, the exact permanent wedge the steal exists to + /// prevent. `max_waiters` is 1 here, so a `refs`-based cap has no room at all. + #[tokio::test] + async fn steal_on_leaked_lease_still_works_under_the_waiter_cap() { + let leases = RepoWriteLeases::new(1); + let big = Duration::from_secs(3600); + + let leaked = leases.acquire("repoZ", big).await.expect("uncontended"); + std::mem::forget(leaked); + assert_eq!( + leases.refs_for("repoZ"), + 1, + "the leaked holder keeps its entry reference forever (that is the leak)" + ); + + let stolen = tokio::time::timeout( + Duration::from_secs(5), + leases.acquire("repoZ", Duration::from_millis(150)), + ) + .await + .expect("a leaked lease must be reclaimed by the bounded-wait steal, not hang forever") + .expect( + "the waiter cap must not shed the stealer: it counts live waiters, and a leaked \ + HOLDER is not a waiter. Capping refs instead wedges the repo permanently", + ); + assert_eq!( + leases.waiters_for("repoZ"), + 0, + "the stealer must return its waiter slot when its wait ends" + ); + drop(stolen); + } + + /// Scenario 5: a shed leaves no residue. Past the cap `acquire` returns `None` + /// without keeping either count, so the entry still GCs once the real holder and the + /// real waiter finish. A shed that stranded a ref would leak the map entry forever; + /// one that stranded a waiter slot would shrink the repo's budget permanently. + #[tokio::test] + async fn shed_waiter_leaves_no_ref_or_waiter_residue() { + let leases = RepoWriteLeases::new(1); + let big = Duration::from_secs(3600); + + let holder = leases.acquire("repoS", big).await.expect("uncontended"); + let waiting = leases.clone(); + let parked = tokio::spawn(async move { waiting.acquire("repoS", big).await }); + assert!( + wait_for(Duration::from_secs(5), || leases.waiters_for("repoS") == 1).await, + "the second acquire must park and be counted as a live waiter" + ); + + // At the cap (1 live waiter): the next acquire sheds instead of queueing. + let shed = tokio::time::timeout(Duration::from_secs(5), leases.acquire("repoS", big)) + .await + .expect("a shed must return immediately, not park"); + assert!( + shed.is_none(), + "past max_waiters the acquire must shed (None), not join the queue" + ); + assert_eq!( + leases.refs_for("repoS"), + 2, + "the shed must leave no refcount residue: only the holder and the real waiter" + ); + assert_eq!( + leases.waiters_for("repoS"), + 1, + "the shed must leave no waiter-count residue: only the real waiter" + ); + + // The entry still GCs once the real handlers finish. + drop(holder); + let promoted = parked + .await + .expect("the parked acquire task must not panic") + .expect("the parked acquire must be served once the holder frees"); + drop(promoted); + assert!( + wait_for(Duration::from_secs(5), || leases.is_empty()).await, + "the lease entry must still GC after a shed (no stranded ref)" + ); + } + + /// Cancellation safety: dropping an acquire future while it is BLOCKED waiting for + /// the lease (a client disconnect on a second same-repo push) must not strand the + /// entry refcount — after the holder frees and the waiter is cancelled, the key GCs. + #[tokio::test] + async fn cancelled_waiter_does_not_strand_the_refcount() { + let leases = RepoWriteLeases::new(8); + let big = Duration::from_secs(3600); + + let a = leases.acquire("repoC", big).await.expect("uncontended"); + // A waiter blocks, then is cancelled (its acquire future dropped) mid-wait. + let cancelled = + tokio::time::timeout(Duration::from_millis(150), leases.acquire("repoC", big)).await; + assert!( + cancelled.is_err(), + "the waiter must be blocked, then cancelled" + ); + + // Release the holder. If the cancelled waiter had stranded its ref, the entry + // would never GC; assert it does once the holder frees. + drop(a); + // Let any pending Drop bookkeeping settle. + tokio::task::yield_now().await; + assert!( + leases.is_empty(), + "a cancelled waiter must not strand the entry refcount (key must GC)" + ); + } + + /// Scenario 6: a cancelled waiter returns its WAITER slot too, not just its ref. A + /// client that disconnects while parked is the common case, so a slot stranded here + /// would shrink the repo's waiter budget on every disconnect until the repo shed + /// every push. Observed as state: the count drops back to 0, and a later acquire on + /// the still-held lease parks rather than shedding, with the cap at 1. + #[tokio::test] + async fn cancelled_waiter_releases_its_waiter_slot() { + let leases = RepoWriteLeases::new(1); + let big = Duration::from_secs(3600); + + let holder = leases.acquire("repoX2", big).await.expect("uncontended"); + let cancelled = + tokio::time::timeout(Duration::from_millis(200), leases.acquire("repoX2", big)).await; + assert!( + cancelled.is_err(), + "the waiter must be parked, then cancelled mid-wait" + ); + assert_eq!( + leases.waiters_for("repoX2"), + 0, + "a cancelled waiter must release its waiter slot" + ); + + // The freed budget is usable: the next acquire parks (times out) instead of + // shedding (returning None immediately). + let next = + tokio::time::timeout(Duration::from_millis(300), leases.acquire("repoX2", big)).await; + assert!( + next.is_err(), + "the next acquire must be able to park on the freed waiter slot; it was shed \ + instead, so the cancelled waiter's slot was stranded" + ); + drop(holder); + } + + /// Scenario 7: an uncontended acquire spends no waiter budget, and a promoted waiter + /// hands its slot back at the END OF ITS WAIT rather than holding it for the whole + /// push. With `max_waiters` at 1: the uncontended holder leaves the count at 0, the + /// one waiter is counted while parked, and once that waiter is promoted to holder the + /// count returns to 0 so the NEXT push can still park. Holding the slot for the push + /// instead would let one waiter permanently occupy the only slot, shedding every + /// same-repo push behind it. + #[tokio::test] + async fn uncontended_acquire_spends_no_waiter_budget() { + let leases = RepoWriteLeases::new(1); + let big = Duration::from_secs(3600); + + // Fast path: the lease is free, so this takes it without parking. + let holder = leases.acquire("repoF", big).await.expect("uncontended"); + assert_eq!( + leases.waiters_for("repoF"), + 0, + "an uncontended acquire must take the free permit without spending waiter budget" + ); + + // One waiter parks behind it, spending the single slot. + let waiting = leases.clone(); + let parked = tokio::spawn(async move { waiting.acquire("repoF", big).await }); + assert!( + wait_for(Duration::from_secs(5), || leases.waiters_for("repoF") == 1).await, + "the second acquire must park and be counted" + ); + + // Promote it: the slot must come back at the end of its WAIT. + drop(holder); + let promoted = parked + .await + .expect("the parked acquire task must not panic") + .expect("the parked acquire must be served once the holder frees"); + assert!( + wait_for(Duration::from_secs(5), || leases.waiters_for("repoF") == 0).await, + "a promoted waiter must release its waiter slot when its wait ends, not when \ + its push finishes" + ); + + // ... so a third acquire can still park behind the new holder. + let third = + tokio::time::timeout(Duration::from_millis(300), leases.acquire("repoF", big)).await; + assert!( + third.is_err(), + "the freed waiter slot must be usable: this acquire was shed instead of parking" + ); + drop(promoted); + } + + /// Poll `cond` until it holds, yielding so spawned tasks progress. Returns false if + /// `cap` elapses first; callers assert on the state the loop settled into. + async fn wait_for(cap: Duration, mut cond: impl FnMut() -> bool) -> bool { + let deadline = std::time::Instant::now() + cap; + loop { + if cond() { + return true; + } + if std::time::Instant::now() >= deadline { + return false; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + } } diff --git a/crates/gitlawb-node/src/sync.rs b/crates/gitlawb-node/src/sync.rs index c838178b..0ed4a9f9 100644 --- a/crates/gitlawb-node/src/sync.rs +++ b/crates/gitlawb-node/src/sync.rs @@ -605,7 +605,7 @@ async fn replicate_encrypted_blobs( continue; } }; - match crate::ipfs_pin::pin_git_object(ipfs_api, &blob.oid, &envelope).await { + match crate::ipfs_pin::pin_git_object(ipfs_api, &blob.oid, &envelope, None).await { Ok(cid) if !cid.is_empty() => { if cid != blob.cid { warn!(oid = %blob.oid, expected = %blob.cid, got = %cid, "replicated envelope CID mismatch; skipping record"); diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index be6fb7b8..7da4e269 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -78,10 +78,34 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { rate_limiter: RateLimiter::new(100, Duration::from_secs(60)), create_ip_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), push_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_work_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, + ipfs_max_legacy_probes: crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, + ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, push_limiter_trust: crate::rate_limit::TrustedProxy::None, sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), shutdown_tx: tokio::sync::watch::channel(false).0, + // Generous — no test drives the handler-level shed (git_permit is unit-tested). + git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + pin_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + encrypt_inflight: crate::state::EncryptInflight::new(), + repo_write_leases: crate::state::RepoWriteLeases::new(8), + git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), + git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( + 8, + ), + git_write_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), + // Generous — a test that drives the /ipfs walk shed overrides these directly. + git_ipfs_walk_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_ipfs_walk_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( + 16, + ), + git_bin: "git".to_string(), } } @@ -109,6 +133,25 @@ pub(crate) fn signed_request_as(did: &str, method: Method, uri: &str, body: Body .expect("request builder") } +/// A local endpoint whose TCP accept succeeds instantly but that never writes an +/// HTTP response, so any request against it stalls deterministically until the +/// caller's own timeout. (A non-routable address hangs only if the network +/// blackholes the SYN — a fast RST would end the stall early and make a timeout +/// test pass for the wrong reason.) The accepted sockets are parked in the +/// spawned task, which dies with the test's runtime, so the peer never sees a +/// close mid-test. +pub(crate) async fn silent_http_endpoint() -> String { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { + let mut held = Vec::new(); + while let Ok((sock, _)) = listener.accept().await { + held.push(sock); + } + }); + endpoint +} + #[cfg(test)] mod tests { use super::*; @@ -187,6 +230,207 @@ mod tests { ); } + /// PR3 (#62): the served-git concurrency cap sheds at the HTTP layer before the + /// DB. The held `git_permit` acquire now sits after the per-source cap, so the + /// cheap early shed is carried by an explicit `available_permits() == 0` check at + /// the top of the handler (the held permit remains the authoritative bound further + /// down). That check is a permit-less snapshot: it spares a request's DB work once + /// the pool is ALREADY saturated, which is the case this test drives, and it does + /// not bound the DB window in general. DB-free here because an exhausted semaphore + /// sheds before any DB/disk access, so a lazy state works. Remove the early-shed block + /// from git_info_refs and this goes red (the request falls through to the DB and + /// returns something other than 503). + #[tokio::test] + async fn git_info_refs_sheds_with_503_when_semaphore_exhausted() { + let mut state = test_state_lazy(); + state.git_read_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + + let router = Router::new() + .route( + "/{owner}/{repo}/info/refs", + axum::routing::get(crate::api::repos::git_info_refs), + ) + .with_state(state); + let resp = router + .oneshot(anon_get( + "/alice/repo.git/info/refs?service=git-upload-pack", + )) + .await + .unwrap(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted git semaphore must shed info/refs with 503 before touching the DB" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + + /// PR3 (#62) sibling of the info/refs shed test: git-upload-pack carries the same + /// explicit `available_permits() == 0` early check at the top, so an ALREADY + /// exhausted semaphore must shed the request with a 503 before its DB/disk work. + /// That is the case the permit-less snapshot does deliver; it is not an admission + /// bound on the DB window. Anonymous-reachable, so no auth injection is needed. + /// Remove the early-shed block from git_upload_pack and this goes red. + #[tokio::test] + async fn git_upload_pack_sheds_with_503_when_semaphore_exhausted() { + let mut state = test_state_lazy(); + state.git_read_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + + let router = Router::new() + .route( + "/{owner}/{repo}/git-upload-pack", + axum::routing::post(crate::api::repos::git_upload_pack), + ) + .with_state(state); + let req = Request::builder() + .method(Method::POST) + .uri("/alice/repo.git/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + let resp = router.oneshot(req).await.unwrap(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted git semaphore must shed git-upload-pack with 503 before touching the DB" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + + /// PR3 (#62) receive-pack sibling of the info/refs shed test: the early shed + /// selects the dedicated ADVERT pool for a git-receive-pack advertisement (#174), + /// so an ALREADY exhausted advert pool sheds the advert with 503 before its DB/disk + /// work (the case the permit-less snapshot delivers, not an admission bound on the + /// DB window), while the write pool (reserved for authenticated POSTs) is left + /// free here. + /// Flip the pool selection back to the write pool, or remove the early-shed + /// block, and this goes red. + #[tokio::test] + async fn git_info_refs_receive_pack_sheds_with_503_when_advert_pool_exhausted() { + let mut state = test_state_lazy(); + state.git_push_advert_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + + let router = Router::new() + .route( + "/{owner}/{repo}/info/refs", + axum::routing::get(crate::api::repos::git_info_refs), + ) + .with_state(state); + let resp = router + .oneshot(anon_get( + "/alice/repo.git/info/refs?service=git-receive-pack", + )) + .await + .unwrap(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted ADVERT pool must shed the receive-pack advertisement with 503 before touching the DB" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + + /// PR3 (#62) sibling for the push path: git-receive-pack requires an + /// AuthenticatedDid extension (production: require_signature injects it), so the + /// request carries one via signed_request_as — without it the Extension + /// extractor 500s before the handler body reaches the shed. What sits at the top of + /// the handler is a permit-less `available_permits() == 0` peek, NOT the permit + /// itself: the authoritative held acquire is taken after the per-repo lease, so a + /// lease-blocked waiter pins no write slot. An ALREADY exhausted pool is the case + /// the peek delivers, so the request sheds 503 before its DB work here. Remove the + /// early-shed block from git_receive_pack and this goes red. + #[tokio::test] + async fn git_receive_pack_sheds_with_503_when_semaphore_exhausted() { + let mut state = test_state_lazy(); + state.git_write_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + + let router = Router::new() + .route( + "/{owner}/{repo}/git-receive-pack", + axum::routing::post(crate::api::repos::git_receive_pack), + ) + .with_state(state); + let owner = "did:key:zRECVSHEDOWNERAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let resp = router + .oneshot(signed_request_as( + owner, + Method::POST, + "/alice/repo.git/git-receive-pack", + Body::from(&b"0000"[..]), + )) + .await + .unwrap(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted write pool must shed git-receive-pack with 503 before touching the DB" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + + /// #174 (SC1, load-bearing): a saturated READ pool must NOT shed an + /// authenticated push — the write pool is a separate budget. Read pool at zero, + /// write pool with capacity: the push proceeds PAST admission (it then errors on + /// the placeholder DB, but crucially it is not a 503). Route git-receive-pack + /// back to the read pool and this goes red — that is the isolation proof. + #[tokio::test] + async fn git_receive_pack_not_shed_by_exhausted_read_pool() { + let mut state = test_state_lazy(); + // Read pool exhausted as if a flood of anonymous clones held every slot. + state.git_read_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + // Write pool keeps its default capacity from test_state_lazy. + + let router = Router::new() + .route( + "/{owner}/{repo}/git-receive-pack", + axum::routing::post(crate::api::repos::git_receive_pack), + ) + .with_state(state); + let owner = "did:key:zRECVCROSSBOUNDARYAAAAAAAAAAAAAAAAAAAAA"; + let resp = router + .oneshot(signed_request_as( + owner, + Method::POST, + "/alice/repo.git/git-receive-pack", + Body::from(&b"0000"[..]), + )) + .await + .unwrap(); + + assert_ne!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted READ pool must not shed a push — the write pool is a separate budget (#174)" + ); + } + /// N7: merge_pr is owner-only. A non-owner is rejected by require_repo_owner /// before any git work (so no on-disk repo is needed for the rejection). #[sqlx::test] @@ -2896,13 +3140,19 @@ mod tests { /// Seed a SHA-256 source repo (public/a.txt + secret/b.txt), bare-clone it /// into each `/tmp//.git` path, and return guards + oids. - /// SHA-256 object format is required: `get_by_cid` resolves a CID whose - /// multihash digest IS the git object id, which only matches in sha256 repos. + /// SHA-256 object format matches production (`--object-format=sha256`) so the + /// oids are 64-hex. A real CID digests the raw object CONTENT (not the git + /// oid), so tests build the request CID with `pin_cid_for` — mirroring the pin + /// path — and `get_by_cid` maps it back to the oid via `pinned_cids` (#173). struct CidFixture { _guards: Vec, secret_oid: String, public_oid: String, secret_tree_oid: String, + public_tree_oid: String, + root_tree_oid: String, + commit_oid: String, + tag_oid: String, } impl Drop for CidFixture { fn drop(&mut self) { @@ -2936,6 +3186,8 @@ mod tests { run(&["config", "user.name", "t"], &src); run(&["add", "."], &src); run(&["commit", "-qm", "seed"], &src); + // Annotated tag of the commit — exercises the "tags stay served" guard. + run(&["tag", "-a", "-m", "annotated", "v1", "HEAD"], &src); let oid = |rev: &str| { let out = Command::new("git") .args(["rev-parse", rev]) @@ -2948,6 +3200,10 @@ mod tests { let secret_oid = oid("HEAD:secret/b.txt"); let public_oid = oid("HEAD:public/a.txt"); let secret_tree_oid = oid("HEAD:secret"); + let public_tree_oid = oid("HEAD:public"); + let root_tree_oid = oid("HEAD^{tree}"); + let commit_oid = oid("HEAD"); + let tag_oid = oid("refs/tags/v1"); let mut guards = vec![src.clone()]; for name in bare_names { let bare = std::path::PathBuf::from("/tmp") @@ -2965,6 +3221,12 @@ mod tests { ], &src, ); + // `git clone --bare` does NOT copy the source repo's local identity, so + // fixtures that create objects directly in the bare repo (`commit-tree`, + // `git tag -a`) abort with "identity unknown" on a CI runner that has no + // ambient/global git identity. Set it explicitly so the suite is portable. + run(&["config", "user.email", "t@t"], &bare); + run(&["config", "user.name", "t"], &bare); } // One guard for the whole /tmp/ tree covers every bare clone. guards.push(std::path::PathBuf::from("/tmp").join(slug)); @@ -2973,120 +3235,4943 @@ mod tests { secret_oid, public_oid, secret_tree_oid, + public_tree_oid, + root_tree_oid, + commit_oid, + tag_oid, } } - /// CID whose sha2-256 multihash digest equals the given 64-hex git oid, so - /// `get_by_cid` decodes it back to that oid and `git cat-file`s it. - fn cid_for_oid(oid_hex: &str) -> String { - use gitlawb_core::cid::Cid; - let bytes = hex::decode(oid_hex).expect("hex oid"); - let arr: [u8; 32] = bytes.as_slice().try_into().expect("32-byte sha256 oid"); - Cid::from_sha256_bytes(&arr).to_string() + /// Record a pin exactly as the production pin path does — read the object's + /// raw bytes (`git cat-file `, no framing), CID them with + /// `Cid::from_git_object_bytes`, and store the `(oid, cid)` row — then return + /// the CID string the node advertises (`gl ipfs list`) and a client sends to + /// `GET /ipfs/{cid}`. Building the CID from the oid instead (the old + /// `cid_for_oid`) produced an identifier that never occurs in production and + /// made the gate assertions vacuous: a real pin CID digests the raw content, + /// not the git oid, so `get_by_cid` resolves it through `pinned_cids` (#173). + async fn pin_cid_for(bare_repo: &std::path::Path, oid: &str, db: &crate::db::Db) -> String { + let (_ty, raw) = crate::git::store::read_object(bare_repo, oid) + .expect("read object bytes") + .expect("object exists in repo"); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(&raw).to_string(); + // Legacy-style pin (no provenance) so existing CID tests exercise the + // resolver's scan fallback; provenance-path tests pin via `pin_cid_for_repo`. + db.record_pinned_cid(oid, &cid, None) + .await + .expect("record pinned cid"); + cid } - fn cid_router(state: &AppState) -> Router { - Router::new() - .route( - "/ipfs/{cid}", - axum::routing::get(crate::api::ipfs::get_by_cid), - ) - .layer(axum::middleware::from_fn(crate::auth::optional_signature)) - .with_state(state.clone()) + /// Like [`pin_cid_for`] but records the pin's provenance (`repo_id`), so the + /// resolver resolves the CID straight to `repo_id` instead of scanning (#173). + #[allow(dead_code)] // used by the provenance-path resolver tests (P-U3) + async fn pin_cid_for_repo( + bare_repo: &std::path::Path, + oid: &str, + db: &crate::db::Db, + repo_id: &str, + ) -> String { + let (_ty, raw) = crate::git::store::read_object(bare_repo, oid) + .expect("read object bytes") + .expect("object exists in repo"); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(&raw).to_string(); + db.record_pinned_cid(oid, &cid, Some(repo_id)) + .await + .expect("record pinned cid with provenance"); + cid } - async fn cid_parts(resp: axum::response::Response) -> (StatusCode, String) { - let st = resp.status(); - let b = axum::body::to_bytes(resp.into_body(), usize::MAX) + + /// INV-7 upgrade path for the pin-provenance column (#173, jatmn round 2): a node + /// already past v11 gets `pinned_cids.repo_id` from the NEW v19 migration, and a + /// legacy pin recorded before the column existed survives with NULL provenance (so + /// it falls back to the repo scan). Simulate the pre-v19 node by dropping the + /// column and un-applying v12, seed a legacy row, then re-migrate. RED before the + /// v19 migration exists (the column is never re-added → the SELECT errors); GREEN + /// after. + #[sqlx::test] + async fn pinned_cids_repo_provenance_upgrade_path(pool: PgPool) { + let state = test_state(pool.clone()).await; + + // Pre-v19 shape: drop the provenance column and forget v19 was applied. + sqlx::query("ALTER TABLE pinned_cids DROP COLUMN IF EXISTS repo_id") + .execute(&pool) .await .unwrap(); - (st, String::from_utf8_lossy(&b).to_string()) - } - fn cid_anon(cid: &str) -> Request { - Request::builder() - .method(Method::GET) - .uri(format!("/ipfs/{cid}")) - .body(Body::empty()) - .unwrap() + sqlx::query("DELETE FROM schema_migrations WHERE version = 19") + .execute(&pool) + .await + .unwrap(); + + // A legacy pin recorded before provenance existed. + sqlx::query("INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) VALUES ($1, $2, $3)") + .bind("legacyoid") + .bind("legacycid") + .bind("2020-01-01T00:00:00Z") + .execute(&pool) + .await + .unwrap(); + + // Upgrade: re-run migrations → v19 re-adds the column. + state.db.run_migrations().await.expect("migrate to v12"); + + // The legacy pin survives with NULL provenance. + let legacy: Option = + sqlx::query_scalar("SELECT repo_id FROM pinned_cids WHERE sha256_hex = 'legacyoid'") + .fetch_one(&pool) + .await + .expect("legacy pin row survives the upgrade"); + assert!( + legacy.is_none(), + "a pin recorded before v19 must keep NULL provenance (it falls back to the scan)" + ); + + // A new pin can carry provenance. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + ) + .bind("newoid") + .bind("newcid") + .bind("2026-01-01T00:00:00Z") + .bind("repo-abc") + .execute(&pool) + .await + .unwrap(); + let prov: Option = + sqlx::query_scalar("SELECT repo_id FROM pinned_cids WHERE sha256_hex = 'newoid'") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + prov.as_deref(), + Some("repo-abc"), + "a pin recorded after v19 carries its source repo_id" + ); } - fn cid_signed(kp: &gitlawb_core::identity::Keypair, cid: &str) -> Request { - let path = format!("/ipfs/{cid}"); - let s = gitlawb_core::http_sig::sign_request(kp, "GET", &path, b""); - Request::builder() - .method(Method::GET) - .uri(&path) - .header("content-digest", s.content_digest) - .header("signature-input", s.signature_input) - .header("signature", s.signature) - .body(Body::empty()) - .unwrap() + + /// #173: a pin records the repository it came from; `provenance_for_oid` reads it + /// back; a legacy pin (no repo) reads back None; and first-pinner-owns holds — a + /// second push of the same oid does NOT rewrite provenance (ON CONFLICT DO + /// NOTHING). This is what lets the resolver gate a CID against its ONE source repo. + #[sqlx::test] + async fn record_pinned_cid_stores_and_reads_provenance(pool: PgPool) { + let state = test_state(pool).await; + + state + .db + .record_pinned_cid("oidA", "cidA", Some("repo-xyz")) + .await + .unwrap(); + assert_eq!( + state + .db + .provenance_for_oid("oidA") + .await + .unwrap() + .as_deref(), + Some("repo-xyz"), + "a provenanced pin reads back its source repo_id" + ); + + state + .db + .record_pinned_cid("oidB", "cidB", None) + .await + .unwrap(); + assert_eq!( + state.db.provenance_for_oid("oidB").await.unwrap(), + None, + "a legacy pin (no repo) has NULL provenance" + ); + + // First-pinner-owns: a later push of the same oid must not rewrite provenance. + state + .db + .record_pinned_cid("oidA", "cidA", Some("repo-OTHER")) + .await + .unwrap(); + assert_eq!( + state + .db + .provenance_for_oid("oidA") + .await + .unwrap() + .as_deref(), + Some("repo-xyz"), + "ON CONFLICT DO NOTHING keeps the first repo's provenance" + ); + + // An unpinned oid has no provenance. + assert_eq!( + state.db.provenance_for_oid("never-pinned").await.unwrap(), + None + ); } - /// #110: `GET /ipfs/{cid}` must gate a withheld blob by per-caller visibility. - /// RED before U2 (the current handler serves the secret to anon). + /// #173 (provenance, happy path): a CID pinned with provenance resolves straight + /// to its ONE source repo and serves an authorized reader — no repo scan. #[sqlx::test] - async fn ipfs_cid_gate_withholds_blob_from_unauthorized(pool: PgPool) { - use crate::db::VisibilityMode; + async fn ipfs_cid_provenance_serves_from_pinning_repo(pool: PgPool) { use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let _fx = seed_cid_repos(&slug, &short, &["provserve"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("provserve.git"); + let fx = &_fx; + + // Build the repo FIRST so the pin can carry its id as provenance. + let repo = seed_repo(&owner_did, "provserve"); // public + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo.id).await; + + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "a provenanced public CID serves its content" + ); + assert!( + body.contains("public bytes"), + "the pinning repo's object is served" + ); + } + /// #173 (provenance, THE load-bearing one — #124 flip + bounded fan-out): a CID + /// pinned from a PRIVATE repo must gate against that pinning repo (404), NOT serve + /// from a byte-identical PUBLIC copy in another repo. Provenance is strictly more + /// restrictive than the old scan (which served the public copy). RED before the + /// rework (the scan serves the public copy → 200 + leaks the secret bytes); GREEN + /// after (provenance → the private repo → 404, no leak). + #[sqlx::test] + async fn ipfs_cid_provenance_private_denies_despite_public_copy(pool: PgPool) { + use gitlawb_core::identity::Keypair; let owner = Keypair::generate(); let owner_did = owner.did().to_string(); - let reader = Keypair::generate(); - let reader_did = reader.did().to_string(); - let stranger = Keypair::generate(); let slug = owner_did.replace([':', '/'], "_"); let short = owner_did.split(':').next_back().unwrap().to_string(); let state = test_state(pool).await; - let fx = seed_cid_repos(&slug, &short, &["withhold"]); - let secret_cid = cid_for_oid(&fx.secret_oid); - let tree_cid = cid_for_oid(&fx.secret_tree_oid); - let public_cid = cid_for_oid(&fx.public_oid); + let fx = seed_cid_repos(&slug, &short, &["privsrc", "pubcopy"]); + let priv_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("privsrc.git"); + // Private source repo, built first so the pin carries its id as provenance. + let mut priv_repo = seed_repo(&owner_did, "privsrc"); + priv_repo.is_public = false; state .db - .create_repo(&seed_repo(&owner_did, "withhold")) - .await - .expect("seed repo"); - let rec = state - .db - .get_repo(&owner_did, "withhold") + .create_repo(&priv_repo) .await - .unwrap() - .unwrap(); + .expect("seed private repo"); + let cid = pin_cid_for_repo(&priv_bare, &fx.secret_oid, &state.db, &priv_repo.id).await; + + // A PUBLIC repo holds the SAME object (the old scan would serve it). + let pub_repo = seed_repo(&owner_did, "pubcopy"); // public, no rule state .db - .set_visibility_rule( - &rec.id, - "/secret/**", - VisibilityMode::B, - std::slice::from_ref(&reader_did), - &owner_did, - ) + .create_repo(&pub_repo) .await - .expect("deny rule"); + .expect("seed public copy"); - // anon → withheld blob: must 404, must not leak content. (RED on current handler.) - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_anon(&secret_cid)) - .await - .unwrap(), - ) - .await; + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; assert_eq!( st, StatusCode::NOT_FOUND, - "anon must not read the withheld blob" + "a provenanced private CID must 404, not serve from a public copy elsewhere (#124 flip)" + ); + assert!( + !body.contains("TOP SECRET"), + "the 404 body must not leak the withheld object" + ); + } + + /// #173 (jatmn round 8, F1 — load-bearing): a shared object first pinned from a + /// PRIVATE repo, then pushed again from a PUBLIC repo through the real pin path, + /// must serve by CID to an anonymous caller from the public source. First-pinner- + /// only provenance 404s it (only the private source is known); recording EVERY + /// pin-path source fixes it. The second push hits the already-pinned skip branch, + /// so this proves the skip-branch source insert fires (and does NOT re-pin: /add + /// expect(0)). RED before U1 (anon 404); GREEN after. + #[sqlx::test] + async fn ipfs_cid_multi_source_serves_from_later_public_pinner(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["privfirst", "pubsecond"]); + let priv_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("privfirst.git"); + let pub_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("pubsecond.git"); + + // Private repo pins the object FIRST — it owns the first-pinner provenance. + let mut priv_repo = seed_repo(&owner_did, "privfirst"); + priv_repo.is_public = false; + state + .db + .create_repo(&priv_repo) + .await + .expect("seed private first-pinner"); + let cid = pin_cid_for_repo(&priv_bare, &fx.public_oid, &state.db, &priv_repo.id).await; + + // A PUBLIC repo pushes the SAME object through the real pin path. The object is + // already pinned, so this hits the already-pinned skip branch, which must record + // the public repo as an additional source without re-pinning (/add expect 0). + let pub_repo = seed_repo(&owner_did, "pubsecond"); // public, no rule + state + .db + .create_repo(&pub_repo) + .await + .expect("seed public second-pinner"); + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyshouldnothappen"}"#) + .expect(0) + .create_async() + .await; + crate::ipfs_pin::pin_new_objects( + &server.url(), + &pub_bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + vec![fx.public_oid.clone()], + &state.db, + &pub_repo.id, + crate::ipfs_pin::PIN_BATCH_BUDGET, + ) + .await; + m.assert_async().await; // asserts /add was NOT called (already pinned) + + // Anonymous CID fetch: the private first source denies, the public second + // source serves → 200. Before F1 only the private source is known → 404. + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "a shared object must serve by CID from a later public pin-path source (F1)" + ); + assert!( + body.contains("public bytes"), + "the served body is the public object's bytes" + ); + } + + /// U1 (grok round-4 P1): `pin_sources_at_cap` flips exactly at `MAX_PIN_SOURCES`. + /// It is the signal `get_by_cid` uses to decide a provenance miss may be hiding a + /// dropped servable source and must fall back to the bounded scan. + #[sqlx::test] + async fn pin_sources_at_cap_flips_at_max(pool: PgPool) { + let state = test_state(pool).await; + let cap = crate::db::MAX_PIN_SOURCES; + assert!( + !state.db.pin_sources_at_cap("atcapoid").await.unwrap(), + "an oid with no pin_repo_sources rows is not at cap" + ); + for i in 0..(cap - 1) { + state + .db + .record_pin_source("atcapoid", &format!("r-{i:02}")) + .await + .unwrap(); + } + assert!( + !state.db.pin_sources_at_cap("atcapoid").await.unwrap(), + "one below MAX_PIN_SOURCES is not at cap" + ); + state + .db + .record_pin_source("atcapoid", "r-last") + .await + .unwrap(); + assert!( + state.db.pin_sources_at_cap("atcapoid").await.unwrap(), + "exactly MAX_PIN_SOURCES rows is at cap" + ); + } + + /// U2 (grok round-4 P1, load-bearing): the pin-source GRIEFING hole. A private + /// first-pinner denies anon; an attacker fills the whole `MAX_PIN_SOURCES` source + /// window with deny-anon sources BEFORE a legitimate public repo pins the same + /// object, so the public repo's `record_pin_source` no-ops (cap full) and it is + /// buried — present in NO provenance record. The resolver's provenance set is then + /// {private + 16 attacker}, all deny anon. Because the set is at_cap (may hide a + /// dropped source), the handler falls back to the bounded legacy scan, which gates + /// every repo through the real gate and finds the buried PUBLIC copy → 200. + /// MUTATION (RED): remove the `at_cap` fallback edge in `get_by_cid` and the buried + /// public object 404s forever. + #[sqlx::test] + async fn ipfs_cid_buried_public_source_still_serves_via_scan_fallback(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["privfirst", "pubburied"]); + let priv_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("privfirst.git"); + let pub_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("pubburied.git"); + + // Private repo pins FIRST — owns the first-pinner provenance, denies anon. + let mut priv_repo = seed_repo(&owner_did, "privfirst"); + priv_repo.is_public = false; + state + .db + .create_repo(&priv_repo) + .await + .expect("seed private first-pinner"); + let cid = pin_cid_for_repo(&priv_bare, &fx.public_oid, &state.db, &priv_repo.id).await; + + // Attacker fills the ENTIRE MAX_PIN_SOURCES window with deny-anon (non-existent) + // sources BEFORE the public repo registers, so the cap is full. + let cap = crate::db::MAX_PIN_SOURCES; + for i in 0..cap { + state + .db + .record_pin_source(&fx.public_oid, &format!("00-attacker-{i:02}")) + .await + .expect("attacker source"); + } + + // A PUBLIC repo pushes the SAME object through the real pin path. Already pinned + // (skip branch), so it only tries record_pin_source — which NO-OPS because the + // cap is full. The public repo is thus buried: not the first-pinner, not in + // pin_repo_sources. + let pub_repo = seed_repo(&owner_did, "pubburied"); // public, no rule + state + .db + .create_repo(&pub_repo) + .await + .expect("seed public buried source"); + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyshouldnothappen"}"#) + .expect(0) + .create_async() + .await; + crate::ipfs_pin::pin_new_objects( + &server.url(), + &pub_bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + vec![fx.public_oid.clone()], + &state.db, + &pub_repo.id, + crate::ipfs_pin::PIN_BATCH_BUDGET, + ) + .await; + m.assert_async().await; // /add NOT called (already pinned) + + // The buried public object must STILL serve: the provenance set is at_cap and + // all-deny, so the handler falls back to the bounded scan, which finds pubburied. + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "a public source buried by a full attacker source window must still serve via the bounded scan fallback (F1)" + ); + assert!( + body.contains("public bytes"), + "the served body is the buried public object's bytes" + ); + } + + /// #173 (jatmn round 8, F1 — bound, R2): the per-object source set is capped at + /// `MAX_PIN_SOURCES` so an adversary pushing one object from many repos cannot make + /// resolution O(repos). Recording the same oid from `MAX_PIN_SOURCES + 3` distinct + /// repos leaves exactly `MAX_PIN_SOURCES` rows. + #[sqlx::test] + async fn ipfs_cid_pin_sources_capped_at_max(pool: PgPool) { + let state = test_state(pool).await; + let cap = crate::db::MAX_PIN_SOURCES; + for i in 0..(cap + 3) { + state + .db + .record_pin_source("capoid", &format!("repo-{i}")) + .await + .expect("record source"); + } + let sources = state.db.pin_sources_for_oid("capoid").await.unwrap(); + assert_eq!( + sources.len() as i64, + cap, + "the per-object source set is capped at MAX_PIN_SOURCES" + ); + } + + /// #173 (jatmn round 8, F1 — availability, grok-4.5 adversarial catch): the resolver's + /// per-object source cap must NEVER evict the first-pinner. A legacy public pin keeps + /// its source in `pinned_cids.repo_id` but not in `pin_repo_sources` (pre-v20 pins, or + /// a pin whose best-effort `record_pin_source` missed). If the cap `LIMIT` were applied + /// to the whole union with a lexicographic order, an attacker could push the same + /// object from `MAX_PIN_SOURCES` repos whose grindable ids sort before the public + /// source and evict it from the window — turning a public CID that served 200 into a + /// 404. This drives exactly that: a legacy public first-pinner plus `MAX_PIN_SOURCES` + /// lower-sorting attacker sources must STILL serve the public object. RED with a + /// whole-union LIMIT (the first-pinner is dropped → 404); GREEN once the first-pinner + /// is always included and the LIMIT caps only the additional sources. + #[sqlx::test] + async fn ipfs_cid_first_pinner_never_evicted_by_lower_sorting_sources(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["pubfirst"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("pubfirst.git"); + // Public repo whose id sorts AFTER every attacker id below. Legacy shape: the + // source lives in pinned_cids.repo_id only (pin_cid_for_repo records no + // pin_repo_sources row), exactly like a pin from before v13. + let mut pub_repo = seed_repo(&owner_did, "pubfirst"); // public, no rule + pub_repo.id = "zzzzzzzz-pubfirst".to_string(); + state + .db + .create_repo(&pub_repo) + .await + .expect("seed public first-pinner"); + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &pub_repo.id).await; + + // Attacker fills the whole MAX_PIN_SOURCES window with lower-sorting source ids + // (non-existent repos — their mere presence would evict the first-pinner under a + // whole-union LIMIT). + let cap = crate::db::MAX_PIN_SOURCES; + for i in 0..cap { + state + .db + .record_pin_source(&fx.public_oid, &format!("00-attacker-{i:02}")) + .await + .expect("attacker source"); + } + + // The public first-pinner must still serve — never evicted by the cap window. + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "the first-pinner public source must never be evicted by lower-sorting attacker sources (F1 availability)" + ); + assert!( + body.contains("public bytes"), + "the public object is served from the first-pinner" + ); + } + + /// INV-7 upgrade path for the F1 `pin_repo_sources` table (#173, jatmn round 8): a + /// node already past v19 gets the table from the NEW v20 migration. Simulate the + /// pre-v20 node by dropping the table and un-applying v13, then re-migrate and + /// assert a source row round-trips. RED before the v20 migration exists. + #[sqlx::test] + async fn pin_repo_sources_upgrade_path(pool: PgPool) { + let state = test_state(pool.clone()).await; + sqlx::query("DROP TABLE IF EXISTS pin_repo_sources") + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version = 20") + .execute(&pool) + .await + .unwrap(); + state.db.run_migrations().await.expect("re-migrate"); + state + .db + .record_pin_source("upgradeoid", "repo-upg") + .await + .expect("record after re-migrate"); + assert_eq!( + state.db.pin_sources_for_oid("upgradeoid").await.unwrap(), + vec!["repo-upg".to_string()], + "the v20 pin_repo_sources table is present after upgrade" + ); + } + + // ── U3 (#173): durable pin-source incompleteness marker ────────────────── + // + // `record_pin_source` is best effort at every call site, so a non-empty, + // below-cap source set is NOT proof of completeness: an object first pinned + // from a PRIVATE repo and later pushed from a PUBLIC repo whose record failed + // has a set that names only the private source. The resolver used to treat + // that set as complete and 404 an object the public repo would serve. The + // pinned_cids.pin_sources_incomplete marker records the miss durably so the + // bounded scan fallback still runs. These tests drive both arms: the marker + // set (fallback runs, object serves, denial still denies) and the marker + // clear (ordinary denials stay off the O(repos) path, INV-10). + + /// Make `record_pin_source` fail for the duration of `body` by moving the + /// `pin_repo_sources` table out from under it, the closest honest stand-in for + /// the transient DB error the retry wrapper is there to absorb. Every other + /// pin-path query keeps working, so only the source record (and its retries) + /// fails, which is exactly the partial-record shape the finding turns on. + async fn with_pin_sources_broken(pool: &PgPool, body: F) -> T + where + F: FnOnce() -> Fut, + Fut: std::future::Future, + { + sqlx::query("ALTER TABLE pin_repo_sources RENAME TO pin_repo_sources_hidden") + .execute(pool) + .await + .expect("hide pin_repo_sources"); + let out = body().await; + sqlx::query("ALTER TABLE pin_repo_sources_hidden RENAME TO pin_repo_sources") + .execute(pool) + .await + .expect("restore pin_repo_sources"); + out + } + + /// Pin `oid` from `repo_id` through the real ipfs_pin path with a mock Kubo that + /// must NOT be called (the object is already pinned, so this drives the + /// skip-branch `record_pin_source` and nothing else). + async fn repin_via_skip_branch( + state: &AppState, + bare: &std::path::Path, + oid: &str, + repo_id: &str, + ) { + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyshouldnothappen"}"#) + .expect(0) + .create_async() + .await; + crate::ipfs_pin::pin_new_objects( + &server.url(), + bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + vec![oid.to_string()], + &state.db, + repo_id, + crate::ipfs_pin::PIN_BATCH_BUDGET, + ) + .await; + m.assert_async().await; + } + + /// U3 scenario 1 (#173, the finding's exact case): an object first pinned from a + /// PRIVATE repo, then pushed from a PUBLIC repo whose `record_pin_source` + /// exhausts its retries. The source set is non-empty and below cap, so the old + /// gate called it COMPLETE and 404'd an object the public repo would happily + /// serve. With the durable marker the bounded scan fallback still runs and the + /// public copy serves. RED before the marker (404); GREEN after (200). + #[sqlx::test] + async fn ipfs_cid_incomplete_source_set_falls_back_to_scan(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["u3priv", "u3pub"]); + let priv_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3priv.git"); + let pub_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3pub.git"); + + // Private first-pinner owns the only recorded source. + let mut priv_repo = seed_repo(&owner_did, "u3priv"); + priv_repo.is_public = false; + state + .db + .create_repo(&priv_repo) + .await + .expect("seed private"); + let cid = pin_cid_for_repo(&priv_bare, &fx.public_oid, &state.db, &priv_repo.id).await; + + // The PUBLIC repo holds the same object, but its source record never lands. + let pub_repo = seed_repo(&owner_did, "u3pub"); // public, no rule + state.db.create_repo(&pub_repo).await.expect("seed public"); + with_pin_sources_broken(&pool, || { + repin_via_skip_branch(&state, &pub_bare, &fx.public_oid, &pub_repo.id) + }) + .await; + + // The recorded set still names only the private repo, and it is below cap. + assert_eq!( + state.db.pin_sources_for_oid(&fx.public_oid).await.unwrap(), + vec![priv_repo.id.clone()], + "the public source really did fail to record" + ); + assert!( + !state.db.pin_sources_at_cap(&fx.public_oid).await.unwrap(), + "the set is below cap, so at_cap cannot be what triggers the fallback" + ); + + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "a KNOWN-incomplete source set must keep the scan fallback so the public copy serves" + ); + assert!( + body.contains("public bytes"), + "the served body is the public object's bytes" + ); + } + + /// U3 scenario 2 (#173, INV-10 guard): the marker must not turn ORDINARY denials + /// into an O(repos) fan-out. With the marker false, a non-empty below-cap source + /// set and a provenance miss, the request must 404 WITHOUT the scan preload ever + /// running. The preload counter is the both-ways proof: forcing the marker true + /// unconditionally turns this red (count 1), which is what keeps the assertion + /// from being vacuous. + #[sqlx::test] + async fn ipfs_cid_complete_source_set_never_preloads(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["u3only"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3only.git"); + + // One PRIVATE source, recorded cleanly: the set is complete and below cap. + let mut priv_repo = seed_repo(&owner_did, "u3only"); + priv_repo.is_public = false; + state + .db + .create_repo(&priv_repo) + .await + .expect("seed private"); + let cid = pin_cid_for_repo(&bare, &fx.secret_oid, &state.db, &priv_repo.id).await; + state + .db + .record_pin_source(&fx.secret_oid, &priv_repo.id) + .await + .expect("record source"); + assert!( + !state + .db + .pin_sources_incomplete(&fx.secret_oid) + .await + .unwrap(), + "a clean record leaves the set marked complete" + ); + assert!( + !state.db.pin_sources_at_cap(&fx.secret_oid).await.unwrap(), + "the set is below cap, so at_cap cannot be what drives the gate" + ); + + crate::api::ipfs::reset_preload_queries(); + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "an anonymous caller denied by the only recorded source gets the opaque 404" + ); + assert!( + !body.contains("TOP SECRET"), + "the 404 body must not leak the withheld object" + ); + assert_eq!( + crate::api::ipfs::preload_queries(), + 0, + "an ordinary denial against a COMPLETE source set must never run the O(repos) preload (INV-10)" + ); + } + + /// U3 scenario 3 (#173): the marker is not permanent. Once a later + /// `record_pin_source` for the object succeeds, nothing is missing, so the marker + /// clears and the scan stops being triggered. BOTH sources here are private, so the + /// provenance walk MISSES and the request actually reaches the `needs_scan` gate: + /// with a marker left stuck the gate arms the O(repos) preload for an ordinary + /// denial forever. Drop the clear and both halves go red (marker still true, preload + /// 1). A public second source would make the preload half vacuous, because the + /// provenance path serves and returns before the gate is ever evaluated. + #[sqlx::test] + async fn ipfs_cid_marker_clears_on_a_later_successful_record(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["u3cfirst", "u3csecond"]); + let first_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3cfirst.git"); + let second_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3csecond.git"); + + let mut first_repo = seed_repo(&owner_did, "u3cfirst"); + first_repo.is_public = false; + state.db.create_repo(&first_repo).await.expect("seed first"); + let cid = pin_cid_for_repo(&first_bare, &fx.secret_oid, &state.db, &first_repo.id).await; + let mut second_repo = seed_repo(&owner_did, "u3csecond"); + second_repo.is_public = false; + state + .db + .create_repo(&second_repo) + .await + .expect("seed second"); + + // First push from the second repo: the source record fails, so the set is marked. + with_pin_sources_broken(&pool, || { + repin_via_skip_branch(&state, &second_bare, &fx.secret_oid, &second_repo.id) + }) + .await; + assert!( + state + .db + .pin_sources_incomplete(&fx.secret_oid) + .await + .unwrap(), + "the exhausted record marked the set incomplete" + ); + + // A later push from the same repo records cleanly, so nothing is missing. + repin_via_skip_branch(&state, &second_bare, &fx.secret_oid, &second_repo.id).await; + assert!( + !state + .db + .pin_sources_incomplete(&fx.secret_oid) + .await + .unwrap(), + "a successful record clears the marker" + ); + assert_eq!( + state + .db + .pin_sources_for_oid(&fx.secret_oid) + .await + .unwrap() + .len(), + 2, + "the repaired set really does name both sources" + ); + + crate::api::ipfs::reset_preload_queries(); + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "both sources are private, so the anonymous caller is denied" + ); + assert!( + !body.contains("TOP SECRET"), + "the 404 body must not leak the withheld object" + ); + assert_eq!( + crate::api::ipfs::preload_queries(), + 0, + "a repaired source set stops triggering the scan: the denial is back off the O(repos) path" + ); + } + + /// F5 (#173 round 11): the work-budget peek sheds an already-throttled caller BEFORE + /// the two marker queries, so a spent-budget source stops paying two lookups per + /// request for a scan it will never be allowed to run. The source set here is + /// non-empty and complete, which is the case that used to reach the queries anyway. + /// The counter is the both-ways guard: moving the peek back below the pair reads 1. + #[sqlx::test] + async fn ipfs_cid_throttled_caller_sheds_before_the_marker_queries(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + // One PRIVATE source, recorded cleanly: the set is non-empty, below cap and + // unmarked, so nothing but the peek can keep the request off the queries. + let fx = seed_cid_repos(&slug, &short, &["f5only"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("f5only.git"); + let mut repo = seed_repo(&owner_did, "f5only"); + repo.is_public = false; + state.db.create_repo(&repo).await.expect("seed private"); + let cid = pin_cid_for_repo(&bare, &fx.secret_oid, &state.db, &repo.id).await; + state + .db + .record_pin_source(&fx.secret_oid, &repo.id) + .await + .expect("record source"); + + // Spend the caller's whole work budget before the request. + assert!( + state.ipfs_work_rate_limiter.check("9.9.9.9").await, + "the budget starts with room" + ); + + crate::api::ipfs::reset_marker_queries(); + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&cid, "9.9.9.9")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::TOO_MANY_REQUESTS, + "a spent-budget caller is shed at the peek" + ); + assert!( + !body.contains("TOP SECRET"), + "the shed response must not leak the withheld object" + ); + assert_eq!( + crate::api::ipfs::marker_queries(), + 0, + "a shed caller pays neither marker query" + ); + } + + /// U3 scenario 6 (#173, regression): a record that inserts NOTHING must not clear + /// the marker. `record_pin_source` is called for EVERY already-pinned object on the + /// skip path, and on a requeue pass that is the whole-repo enumeration, so the next + /// coalesced push from a repo ALREADY in the source set re-runs the insert as a + /// no-op. Clearing on that no-op re-hides the hole a different repo's failed record + /// recorded: the public copy stops being scanned for and 404s again. The assertion + /// is the SERVE outcome, not the column, so it still bites if the resolver ever + /// stops consulting the marker. RED before the rows_affected gate (404); GREEN after. + #[sqlx::test] + async fn ipfs_cid_noop_record_must_not_clear_the_marker(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["u3npriv", "u3npub"]); + let priv_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3npriv.git"); + let pub_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3npub.git"); + + // Repo A (private) is the first pinner AND is already recorded as a source, so a + // later record from A is a pure no-op insert. + let mut priv_repo = seed_repo(&owner_did, "u3npriv"); + priv_repo.is_public = false; + state + .db + .create_repo(&priv_repo) + .await + .expect("seed private"); + let cid = pin_cid_for_repo(&priv_bare, &fx.public_oid, &state.db, &priv_repo.id).await; + state + .db + .record_pin_source(&fx.public_oid, &priv_repo.id) + .await + .expect("record the first pinner as a source"); + + // Repo B (public) holds the same object, but its source record never lands, so + // the node marks the set known-incomplete. + let pub_repo = seed_repo(&owner_did, "u3npub"); + state.db.create_repo(&pub_repo).await.expect("seed public"); + with_pin_sources_broken(&pool, || { + repin_via_skip_branch(&state, &pub_bare, &fx.public_oid, &pub_repo.id) + }) + .await; + assert!( + state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "the exhausted record marked the set incomplete" + ); + + // A pushes again. The insert affects zero rows (A is already a source), so it + // recorded nothing and must not claim the set is complete. + repin_via_skip_branch(&state, &priv_bare, &fx.public_oid, &priv_repo.id).await; + assert_eq!( + state.db.pin_sources_for_oid(&fx.public_oid).await.unwrap(), + vec![priv_repo.id.clone()], + "the re-push really did add no source" + ); + + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "a no-op record must not clear the marker: the public copy still has to serve" + ); + assert!( + body.contains("public bytes"), + "the served body is the public object's bytes" + ); + } + + /// U3 scenario 4 (#173): the marker tracks the record's OUTCOME, not the attempt. + /// An exhausted retry sets it; a first-attempt success never does. Without the + /// second arm the first could be satisfied by marking unconditionally. + #[sqlx::test] + async fn pin_sources_incomplete_marks_only_exhausted_records(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["u3mark"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3mark.git"); + let repo = seed_repo(&owner_did, "u3mark"); + state.db.create_repo(&repo).await.expect("seed repo"); + let _ = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo.id).await; + + // Arm A: a first-attempt success must leave the marker alone. + repin_via_skip_branch(&state, &bare, &fx.public_oid, &repo.id).await; + assert!( + !state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "a record that lands on the first attempt never marks the set incomplete" + ); + + // Arm B: an exhausted retry marks it. + with_pin_sources_broken(&pool, || { + repin_via_skip_branch(&state, &bare, &fx.public_oid, &repo.id) + }) + .await; + assert!( + state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "an exhausted record marks the set incomplete" + ); + + // An unpinned oid has no row and must read as complete, never as missing. + assert!( + !state + .db + .pin_sources_incomplete(&"f".repeat(64)) + .await + .unwrap(), + "an unpinned oid reads complete, so an unknown CID cannot arm the fallback" + ); + } + + /// U3 scenario 5 (#173): the Pinata pin path had BARE `record_pin_source` calls, so + /// one transient DB error dropped a source permanently. It now shares the ipfs_pin + /// retry helper and marks/clears the same marker. The elapsed-time assertion is the + /// retry proof: a bare call returns immediately, whereas the wrapper sleeps + /// `PIN_RECORD_BACKOFF` between each of `PIN_RECORD_ATTEMPTS` tries. + #[sqlx::test] + async fn pinata_pin_path_retries_and_marks_incomplete(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["u3pinata"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3pinata.git"); + let repo = seed_repo(&owner_did, "u3pinata"); + state.db.create_repo(&repo).await.expect("seed repo"); + + // Already carries a pinata_cid, so pin_new_objects takes the skip branch and the + // only DB write under test is the source record. + let (_ty, raw) = crate::git::store::read_object(&bare, &fx.public_oid) + .unwrap() + .expect("object readable"); + let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes(&raw).to_string(); + state + .db + .record_pinata_cid(&fx.public_oid, &raw_cid, "QmProvider", Some(&repo.id)) + .await + .expect("seed pinata pin"); + + let client = reqwest::Client::new(); + let run = |db_broken: bool| { + let client = client.clone(); + let bare = bare.clone(); + let oid = fx.public_oid.clone(); + let repo_id = repo.id.clone(); + let state = &state; + async move { + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Any) + .with_status(200) + .with_body(r#"{"data":{"cid":"QmShouldNotHappen"}}"#) + .expect(0) + .create_async() + .await; + let started = std::time::Instant::now(); + crate::pinata::pin_new_objects( + &client, + &server.url(), + "test-jwt", + &bare, + vec![oid], + &state.db, + &repo_id, + ) + .await; + m.assert_async().await; // the upload is skipped: DB-only path + let _ = db_broken; + started.elapsed() + } + }; + + // Failing arm: retried (so it sleeps the full backoff horizon) and marked. + let elapsed = with_pin_sources_broken(&pool, || run(true)).await; + assert!( + elapsed >= std::time::Duration::from_millis(100), + "the pinata source record now RETRIES (bare call returns at once, got {elapsed:?})" + ); + assert!( + state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "an exhausted pinata record marks the set incomplete, same as the ipfs_pin path" + ); + + // Recovery arm: a later successful pinata record clears it, same as ipfs_pin. + run(false).await; + assert!( + !state + .db + .pin_sources_incomplete(&fx.public_oid) + .await + .unwrap(), + "a successful pinata record clears the marker" + ); + assert_eq!( + state.db.pin_sources_for_oid(&fx.public_oid).await.unwrap(), + vec![repo.id.clone()], + "the recovered record actually landed the source row" + ); + } + + /// U3 scenario 6 (#173, authorization): the marker arms a FALLBACK, never a bypass. + /// With the set marked incomplete and the object living only in a repo the caller + /// may not read, the scan gates every repo through the same per-caller gate, so the + /// caller is still denied and no bytes leak. + #[sqlx::test] + async fn ipfs_cid_marked_incomplete_still_denies_unauthorized_caller(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["u3deny"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("u3deny.git"); + let mut priv_repo = seed_repo(&owner_did, "u3deny"); + priv_repo.is_public = false; + state + .db + .create_repo(&priv_repo) + .await + .expect("seed private"); + let cid = pin_cid_for_repo(&bare, &fx.secret_oid, &state.db, &priv_repo.id).await; + + // The set is marked incomplete, so the fallback scan definitely runs. + with_pin_sources_broken(&pool, || { + repin_via_skip_branch(&state, &bare, &fx.secret_oid, &priv_repo.id) + }) + .await; + assert!( + state + .db + .pin_sources_incomplete(&fx.secret_oid) + .await + .unwrap(), + "the marker is set, so the scan fallback is armed for this object" + ); + + crate::api::ipfs::reset_preload_queries(); + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + crate::api::ipfs::preload_queries(), + 1, + "the fallback really did run (otherwise the denial below proves nothing)" + ); + assert_eq!( + st, + StatusCode::NOT_FOUND, + "the fallback scan gates every repo, so an unauthorized caller is still denied" + ); + assert!( + !body.contains("TOP SECRET"), + "the denial must not leak the withheld object's bytes" + ); + } + + /// U3 scenario 7 (#173, INV-7 upgrade path): a node already past v21 gets + /// `pinned_cids.pin_sources_incomplete` from the NEW v22 migration, re-running the + /// migrations is idempotent, and a row written before the column existed reads as + /// COMPLETE (so an upgrade cannot arm the O(repos) fallback for every legacy pin). + #[sqlx::test] + async fn pinned_cids_sources_incomplete_upgrade_path(pool: PgPool) { + let state = test_state(pool.clone()).await; + + // Pre-v22 shape: drop the column and forget v22 was applied. + sqlx::query("ALTER TABLE pinned_cids DROP COLUMN IF EXISTS pin_sources_incomplete") + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version = 22") + .execute(&pool) + .await + .unwrap(); + sqlx::query("INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) VALUES ($1, $2, $3)") + .bind("preu3oid") + .bind("preu3cid") + .bind(chrono::Utc::now().to_rfc3339()) + .execute(&pool) + .await + .unwrap(); + + state.db.run_migrations().await.expect("re-migrate"); + state + .db + .run_migrations() + .await + .expect("migrations are idempotent: a second run succeeds"); + + assert!( + !state.db.pin_sources_incomplete("preu3oid").await.unwrap(), + "a row predating the column reads COMPLETE, so the upgrade arms no fallback" + ); + state + .db + .mark_pin_sources_incomplete("preu3oid") + .await + .expect("mark after upgrade"); + assert!( + state.db.pin_sources_incomplete("preu3oid").await.unwrap(), + "the v22 column is present and writable after the upgrade" + ); + } + + /// #173 (jatmn round 8, F2 — load-bearing): a legacy `pinned_cids` row keyed on a + /// PROVIDER CID (Pinata/Kubo dag-pb — every release before this branch stored the + /// provider CID as the resolver key, not the raw-content CID) must NOT serve raw git + /// bytes that do not hash to the requested CID. `get_by_cid` recomputes the CID over + /// the served bytes and refuses to serve on mismatch. Seeded with a RAW SQL INSERT + /// because the current helpers store the raw CID, so a helper-seeded row is already + /// correct-shape and the RED assertion would be vacuous (INV-21). RED before U2 + /// (serves the git bytes → 200); GREEN after (not served, no bytes egress). + #[sqlx::test] + async fn ipfs_cid_legacy_provider_cid_row_not_served(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["provsrc"]); + let repo = seed_repo(&owner_did, "provsrc"); // public, no rule + state.db.create_repo(&repo).await.expect("seed repo"); + + // A valid sha2-256 CID whose digest is NOT the object's raw-content digest — + // stands in for a Pinata/Kubo dag-pb provider CID (the legacy resolver key). + let provider_cid = gitlawb_core::cid::Cid::from_git_object_bytes( + b"a decoy object whose CID is not the served object's CID", + ) + .to_string(); + + // Legacy-shape row: cid = the PROVIDER CID (raw SQL — the helpers now store the + // raw CID and cannot reproduce this shape). The object itself is public+servable. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + ) + .bind(&fx.public_oid) + .bind(&provider_cid) + .bind("2020-01-01T00:00:00Z") + .bind(&repo.id) + .execute(&pool) + .await + .unwrap(); + + // Requesting the provider CID resolves the row and passes the repo gate, but the + // served bytes hash to a DIFFERENT CID, so the integrity check must withhold them. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&provider_cid)) + .await + .unwrap(), + ) + .await; + assert_ne!( + st, + StatusCode::OK, + "a provider-CID legacy row must not serve raw git bytes (F2)" + ); + assert!( + !body.contains("public bytes"), + "the mismatched bytes must not egress" + ); + } + + /// #173 (jatmn round 8, F6 — INV-10 cost guard): the serve path buffers the object via + /// a blocking `cat-file`; an object larger than `ipfs_max_served_object_bytes` must be + /// WITHHELD (rejected by the size precheck, never buffered), with zero body bytes + /// egressed. Under the cap it serves unchanged. The oversize-reject counter guards it + /// both ways: a removed size precheck serves the object and leaves the counter at 0. + #[sqlx::test] + async fn ipfs_cid_f6_oversized_object_withheld(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["big"]); + let bare = std::path::PathBuf::from("/tmp").join(&slug).join("big.git"); + let repo = seed_repo(&owner_did, "big"); // public, no rule + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo.id).await; + + // Cap below the object size ("public bytes\n" = 13 bytes) → withheld. + state.ipfs_max_served_object_bytes = 5; + crate::api::ipfs::reset_oversize_rejects(); + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_ne!( + st, + StatusCode::OK, + "an object over the size cap must not serve (F6)" + ); + assert!( + !body.contains("public bytes"), + "no object bytes egress for an over-cap object" + ); + assert_eq!( + crate::api::ipfs::oversize_rejects(), + 1, + "the oversized object was rejected by the size precheck" + ); + + // Control: raise the cap above the object size → serves unchanged. + state.ipfs_max_served_object_bytes = crate::api::ipfs::MAX_SERVED_OBJECT_BYTES; + crate::api::ipfs::reset_oversize_rejects(); + let (st2, body2) = + cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st2, + StatusCode::OK, + "under the cap the object serves normally" + ); + assert!( + body2.contains("public bytes"), + "the served body is the object's bytes" + ); + assert_eq!( + crate::api::ipfs::oversize_rejects(), + 0, + "no oversize reject under the cap" + ); + } + + /// #173 (provenance, INV-11): a quarantined pinning repo must 404 by CID even for + /// its own owner — quarantine hard-drops before the visibility gate on the + /// provenance path too. The owner-signed 404 is the load-bearing negative (a + /// visibility-only gate would Allow the owner). + #[sqlx::test] + async fn ipfs_cid_provenance_quarantined_repo_404_even_owner(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["quarsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("quarsrc.git"); + let repo = seed_repo(&owner_did, "quarsrc"); // public + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo.id).await; + + // Baseline: before quarantine the provenanced CID serves (proves the path works). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&owner, &cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "provenanced CID serves before quarantine" + ); + + state + .db + .set_repo_quarantine(&repo.id, true) + .await + .expect("quarantine"); + + for req in [cid_anon(&cid), cid_signed(&owner, &cid)] { + let (st, body) = cid_parts(cid_router(&state).oneshot(req).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "a quarantined pinning repo must 404 by CID (anon + owner)" + ); + assert!( + !body.contains("public bytes"), + "the 404 body must not leak quarantined content" + ); + } + } + + /// #173 (provenance, bounded — must NOT fall back to the scan): a CID whose + /// provenance points at a repo that no longer exists must 404 rather than scan + /// every repo and serve a byte-identical public copy. Falling back to the scan + /// would reopen the O(repos) anonymous fan-out the provenance rework closes. RED + /// before the rework (the scan serves the public copy → 200); GREEN after. + #[sqlx::test] + async fn ipfs_cid_provenance_missing_repo_404_no_scan_fallback(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["gonesrc", "pubcopy2"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("gonesrc.git"); + + // Pin with provenance = a repo_id that is never created (deleted/absent). + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, "nonexistent-repo-id").await; + + // A public repo holds the SAME object (the old scan would serve it). + let pub_repo = seed_repo(&owner_did, "pubcopy2"); + state + .db + .create_repo(&pub_repo) + .await + .expect("seed public copy"); + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "a provenance pointing at a missing repo must 404, not fall back to the scan" + ); + } + + /// #173 (provenance, path-scoped WALK gate): the #135/#173 per-object gates must + /// run on the NEW provenance path, not only the legacy scan. A provenanced pin from + /// a repo under a `/secret/**` rule runs `allowed_blob_set_for_caller` via the shared + /// gate: a withheld secret blob 404s to anon (no byte leak); the allowed reader gets + /// it. Exercises the walk gate on the provenance path in BOTH directions. + #[sqlx::test] + async fn ipfs_cid_provenance_path_scoped_walk_gates_withheld_blob(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let reader = Keypair::generate(); + let reader_did = reader.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["provwalk"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("provwalk.git"); + let repo = seed_repo(&owner_did, "provwalk"); // public at "/" + state.db.create_repo(&repo).await.expect("seed repo"); + // /secret/** Mode B with the reader allowed → the secret blob walk gates by caller. + state + .db + .set_visibility_rule( + &repo.id, + "/secret/**", + VisibilityMode::B, + std::slice::from_ref(&reader_did), + &owner_did, + ) + .await + .expect("path rule"); + let cid = pin_cid_for_repo(&bare, &fx.secret_oid, &state.db, &repo.id).await; + + // Anon: the walk denies the secret blob → 404, no leak. + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "a withheld secret blob 404s to anon on the provenance path (walk gate runs)" + ); + assert!( + !body.contains("TOP SECRET"), + "the 404 body must not leak the withheld blob" + ); + + // Allowed reader: the walk includes the secret blob → 200 with content. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&reader, &cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "an allowed reader gets the secret blob via the provenance walk gate" + ); + assert!( + body.contains("TOP SECRET"), + "the allowed reader receives the content" + ); + } + + /// #173: the pinata pin path stores the locally-computed raw CID in the + /// resolver-key `cid` column and the provider CID in `pinata_cid`, and its ON + /// CONFLICT COALESCE fills a NULL provenance without overwriting an existing one + /// (first-pinner-owns). On conflict `cid` is left untouched so a prior local pin's + /// raw CID is never clobbered by a provider CID. + #[sqlx::test] + async fn record_pinata_cid_stores_and_coalesces_provenance(pool: PgPool) { + let state = test_state(pool).await; + + // Real raw-CIDv1 resolver keys, as the pin paths write them: `list_pinned_cids` + // withholds any row keyed on a non-raw (legacy provider) value (U4, #173), so a + // placeholder string here would be filtered out and make the assertions vacuous. + let raw1 = gitlawb_core::cid::Cid::from_git_object_bytes(b"pinata raw 1").to_string(); + let local2 = gitlawb_core::cid::Cid::from_git_object_bytes(b"local raw 2").to_string(); + + // A new row created via the pinata path carries provenance, and stores the + // raw CID in `cid` with the provider CID in `pinata_cid`. + state + .db + .record_pinata_cid("po1", &raw1, "pcid1", Some("repoA")) + .await + .unwrap(); + assert_eq!( + state.db.provenance_for_oid("po1").await.unwrap().as_deref(), + Some("repoA") + ); + let po1 = state + .db + .list_pinned_cids() + .await + .unwrap() + .into_iter() + .find(|r| r.sha256_hex == "po1") + .expect("po1 row exists"); + assert_eq!(po1.cid, raw1, "resolver-key cid is the raw CID"); + assert_eq!( + po1.pinata_cid.as_deref(), + Some("pcid1"), + "the provider CID is kept in pinata_cid" + ); + + // An existing NULL-provenance row: the pinata COALESCE fills it, and the + // prior local pin's `cid` is left untouched (not overwritten by the raw arg). + state + .db + .record_pinned_cid("po2", &local2, None) + .await + .unwrap(); + state + .db + .record_pinata_cid("po2", "rawcid2", "pcid2", Some("repoB")) + .await + .unwrap(); + assert_eq!( + state.db.provenance_for_oid("po2").await.unwrap().as_deref(), + Some("repoB"), + "pinata fills a NULL provenance" + ); + let po2 = state + .db + .list_pinned_cids() + .await + .unwrap() + .into_iter() + .find(|r| r.sha256_hex == "po2") + .expect("po2 row exists"); + assert_eq!( + po2.cid, local2, + "on conflict the prior local pin's cid is left untouched" + ); + + // An existing provenance: the pinata COALESCE must NOT overwrite it. + state + .db + .record_pinned_cid("po3", "cid3", Some("repoX")) + .await + .unwrap(); + state + .db + .record_pinata_cid("po3", "rawcid3", "pcid3", Some("repoY")) + .await + .unwrap(); + assert_eq!( + state.db.provenance_for_oid("po3").await.unwrap().as_deref(), + Some("repoX"), + "pinata COALESCE keeps the first-pinner's provenance" + ); + } + + /// #173 (jatmn, F4, load-bearing security): a Pinata-first pin (no prior local pin) + /// must make the resolver key (`pinned_cids.cid`) the locally-computed raw CID, NOT + /// the provider CID. Pinata wraps the bytes in dag-pb/UnixFS, so its returned CID + /// does not hash the raw content; if it became the resolver key, `/ipfs/{provider_cid}` + /// would serve raw git bytes that do not hash to it, breaking raw content-addressing. + /// Assert `oids_for_cid(raw_cid)` finds the sha AND `oids_for_cid(provider_cid)` does NOT. + #[sqlx::test] + async fn record_pinata_cid_resolver_key_is_raw_not_provider(pool: PgPool) { + let state = test_state(pool).await; + + let bytes = b"raw git object content for pinata-first pin"; + let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes(bytes).to_string(); + // A distinct provider CID (a dag-pb wrapper CID Pinata would return). + let provider_cid = "QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG"; + assert_ne!( + raw_cid, provider_cid, + "the provider CID must differ from the raw CID for this test to be meaningful" + ); + + // Pinata-first: no prior local pin, so this INSERT creates the row. + state + .db + .record_pinata_cid("pfsha", &raw_cid, provider_cid, Some("repoP")) + .await + .unwrap(); + + // The raw CID resolves to the sha. + assert_eq!( + state.db.oids_for_cid(&raw_cid).await.unwrap(), + vec!["pfsha".to_string()], + "the locally-computed raw CID is the resolver key" + ); + // The provider (dag-pb) CID must NOT resolve raw bytes. + assert!( + state + .db + .oids_for_cid(provider_cid) + .await + .unwrap() + .is_empty(), + "the provider dag-pb CID must never resolve raw git bytes" + ); + } + + /// #173 (end-to-end pin wiring): `pin_new_objects` records the repo_id it is given + /// as the pin's provenance. Drives the real pin path against a mocked IPFS `/add` + /// endpoint (so `pin_git_object` succeeds) and asserts `provenance_for_oid` returns + /// the repo — closing the gap between the push handler's threading and the DB write. + #[sqlx::test] + async fn pin_new_objects_records_provenance(pool: PgPool) { + let state = test_state(pool).await; + + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovtest"}"#) + .expect_at_least(1) + .create_async() + .await; + + let fx = seed_cid_repos("provpin_e2e", "ppe2e", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("provpin_e2e") + .join("pinsrc.git"); + + let pinned = crate::ipfs_pin::pin_new_objects( + &server.url(), + &bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + vec![fx.public_oid.clone()], + &state.db, + "repoZ", + crate::ipfs_pin::PIN_BATCH_BUDGET, + ) + .await; + assert!( + !pinned.is_empty(), + "the object was pinned via the real pin path" + ); + m.assert_async().await; + assert_eq!( + state + .db + .provenance_for_oid(&fx.public_oid) + .await + .unwrap() + .as_deref(), + Some("repoZ"), + "pin_new_objects records the repo_id it was given as the pin's provenance" + ); + } + + /// #173 (grok F2): the post-push pin read is BOUNDED, so a wedged/D-state + /// `git cat-file` (stuck NFS/Tigris backend) is reaped at `git_timeout` and + /// `pin_new_objects` RETURNS — reaching `requeue_or_release` in production — + /// instead of hanging forever and pinning the per-repo coalescing key until + /// process death. A fake `git` whose `cat-file` records its pid then sleeps far + /// past a SHORT 1s timeout stands in for the wedged backend; the `run_bounded_git` + /// watchdog (SIGTERM -> grace -> SIGKILL of the process group) must reap it well + /// before its 8s natural exit, and the call must return with nothing pinned. + /// + /// REVERT PROOF (RED): swap `read_object_bounded` back to the bare + /// `store::read_object` at the pin read and the wedged child is STILL RUNNING at + /// the mid-flight liveness poll below (unbounded `Command::output` cannot be + /// reaped at the deadline) — the reap assertion fails. + #[cfg(unix)] + #[sqlx::test] + async fn pin_new_objects_reaps_wedged_read_at_deadline(pool: PgPool) { + use std::time::Duration; + let db = crate::db::Db::for_testing(pool); + db.run_migrations().await.unwrap(); + + // Fake `git`: `cat-file` records its own pid then sleeps 8s (>> the 1s + // deadline) so the read is genuinely wedged; the watchdog is what must end it. + let tmp = tempfile::TempDir::new().unwrap(); + let pidfile = tmp.path().join("catfile.pid"); + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + cat-file) echo $$ > \"{}\"; sleep 8 ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + pidfile.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + let repo = tmp.path().to_path_buf(); + let git = git_path.to_str().unwrap().to_string(); + // A never-pinned OID so the call reaches the object-read stage (not the + // already-pinned skip path). + let oid = "f".repeat(64); + // Non-empty ipfs_api so `pin_new_objects` does not early-return; the wedged + // read is reaped and the OID skipped before any `/add`, so this URL is unused. + let ipfs_api = "http://127.0.0.1:1".to_string(); + + // `pin_new_objects` must run on THIS runtime so its `is_pinned` DB call keeps + // the sqlx pool on its home runtime. The bounded read is a synchronous blocking + // call, so the reap poll runs on a separate OS thread (independent of tokio): it + // captures the wedged child's pid, waits past the deadline, records whether it + // was reaped, then SIGKILLs defensively so even a true infinite hang cannot leak + // an orphan or stall the awaited call. + let pidfile_poll = pidfile.clone(); + let poll = std::thread::spawn(move || -> (Option, bool) { + let alive = |pid: i32| unsafe { libc::kill(pid, 0) == 0 }; + let mut pid = None; + for _ in 0..500 { + if let Some(p) = std::fs::read_to_string(&pidfile_poll) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + pid = Some(p); + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + let pid = match pid { + Some(p) => p, + None => return (None, false), + }; + // Past the 1s deadline + SIGTERM grace but well before the 8s natural exit: + // the bounded read must already have reaped the wedged group. The unbounded + // `store::read_object` leaves it running here — the load-bearing RED. + std::thread::sleep(Duration::from_secs(3)); + let reaped = !alive(pid); + unsafe { + libc::kill(pid, libc::SIGKILL); + } + (Some(pid), reaped) + }); + + // The call must RETURN — reaching `requeue_or_release` in production — rather + // than hang on the 8s sleep. The poll thread's defensive SIGKILL guarantees the + // read completes even in the unbounded RED case, so this observes a bounded + // return either way; the reap assertion below is what separates RED from GREEN. + let pinned = tokio::time::timeout( + Duration::from_secs(6), + crate::ipfs_pin::pin_new_objects( + &ipfs_api, + &repo, + &git, + Duration::from_secs(1), + vec![oid], + &db, + "repoWedge", + crate::ipfs_pin::PIN_BATCH_BUDGET, + ), + ) + .await + .expect("pin_new_objects must return within the bound, not hang on the wedged read"); + + let (pid, reaped) = poll.join().expect("poll thread joins"); + pid.expect("the fake cat-file must have spawned and recorded its pid"); + assert!( + reaped, + "the post-push pin read must reap the wedged cat-file child at the deadline, \ + not leave it running (which would pin the coalescing key until process death)" + ); + assert!( + pinned.is_empty(), + "a wedged read pins nothing this pass; a later pass/push retries" + ); + } + + /// #173 (jatmn, F2): a legacy pin with NULL provenance backfills its source + /// via `backfill_pin_provenance`, and the `AND repo_id IS NULL` guard preserves + /// first-pinner-owns (a non-NULL provenance is left untouched). + #[sqlx::test] + async fn backfill_pin_provenance_fills_null_keeps_existing(pool: PgPool) { + let state = test_state(pool).await; + + // A legacy pin: no provenance recorded. + state + .db + .record_pinned_cid("legacy_oid", "legacy_cid", None) + .await + .unwrap(); + assert_eq!( + state.db.provenance_for_oid("legacy_oid").await.unwrap(), + None, + "a legacy pin starts with NULL provenance" + ); + + // Backfill sets the NULL provenance. + state + .db + .backfill_pin_provenance("legacy_oid", "repo-src") + .await + .unwrap(); + assert_eq!( + state + .db + .provenance_for_oid("legacy_oid") + .await + .unwrap() + .as_deref(), + Some("repo-src"), + "backfill fills a NULL provenance from the known source" + ); + + // A pin that already has provenance: backfill must NOT overwrite it. + state + .db + .record_pinned_cid("owned_oid", "owned_cid", Some("repo-first")) + .await + .unwrap(); + state + .db + .backfill_pin_provenance("owned_oid", "repo-second") + .await + .unwrap(); + assert_eq!( + state + .db + .provenance_for_oid("owned_oid") + .await + .unwrap() + .as_deref(), + Some("repo-first"), + "the AND repo_id IS NULL guard keeps the first-pinner's provenance" + ); + } + + /// #173 (jatmn, F2, load-bearing): an object already pinned with NULL provenance + /// (a pre-provenance legacy pin) acquires its source when `pin_new_objects` sees + /// it again. The already-pinned skip path must backfill rather than leave the + /// object stuck on the O(repos) scan fallback — and it must NOT re-pin the bytes + /// (no IPFS `/add` call, the object is already on IPFS). + #[sqlx::test] + async fn pin_new_objects_backfills_legacy_null_provenance(pool: PgPool) { + let state = test_state(pool).await; + + let fx = seed_cid_repos("provpin_backfill", "ppbf", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("provpin_backfill") + .join("pinsrc.git"); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes( + &crate::git::store::read_object(&bare, &fx.public_oid) + .expect("read object bytes") + .expect("object exists") + .1, + ) + .to_string(); + + // Legacy pin: the object is already recorded with NULL provenance. + state + .db + .record_pinned_cid(&fx.public_oid, &cid, None) + .await + .unwrap(); + assert_eq!( + state.db.provenance_for_oid(&fx.public_oid).await.unwrap(), + None, + "the object starts as a legacy pin with NULL provenance" + ); + + // Mock IPFS `/add` and require it is NOT called: the already-pinned object + // must be backfilled, never re-pinned. + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyshouldnothappen"}"#) + .expect(0) + .create_async() + .await; + + let pinned = crate::ipfs_pin::pin_new_objects( + &server.url(), + &bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + vec![fx.public_oid.clone()], + &state.db, + "repoBF", + crate::ipfs_pin::PIN_BATCH_BUDGET, + ) + .await; + + assert!( + pinned.is_empty(), + "an already-pinned object is not re-pinned (no bytes returned)" + ); + m.assert_async().await; // asserts /add was called 0 times + assert_eq!( + state + .db + .provenance_for_oid(&fx.public_oid) + .await + .unwrap() + .as_deref(), + Some("repoBF"), + "pin_new_objects backfills the legacy pin's NULL provenance" + ); + } + + /// Build a legacy provider CID (CIDv1 dag-pb — the Kubo above-block-size root + /// shape, and codec-equivalent to the Pinata CIDv0 legacy key for the cost + /// gate) over the object's own multihash. Non-raw codec, so `is_raw_cidv1` + /// flags it a repair candidate, and a different string from the raw key, so a + /// repair rewrites it. The existing `ipfs_cid_legacy_provider_cid_row_not_served` + /// fixture seeds a raw-codec decoy (an integrity negative the cost gate treats + /// as non-legacy on purpose); this produces the genuine dag-pb legacy shape the + /// repair path targets. Uses only the `cid` crate (already a node dep). + fn legacy_dagpb_cid(raw_cid: &str) -> String { + const DAG_PB: u64 = 0x70; + let parsed = raw_cid + .parse::>() + .expect("the raw CID parses"); + cid::CidGeneric::<64>::new_v1(DAG_PB, *parsed.hash()).to_string() + } + + /// #173 R8 (jatmn round 10, U7 — load-bearing): a legacy row keyed on a PROVIDER + /// CID (Kubo dag-pb / Pinata) is opportunistically rewritten to the raw-content + /// key on a re-push whose pack carries the object, stashing the old value in + /// `legacy_provider_cid`. The advertised key 404s while the row is legacy (the + /// resolver recomputes the raw CID and the stored key does not match) and serves + /// after repair. RED before the skip-branch repair lands (the raw key 404s post + /// pin). Also asserts the repair leaves `pinata_cid` NULL (scenario 3) and that + /// the retired provider CID still refuses to serve (scenario 6, integrity). + #[sqlx::test] + async fn ipfs_cid_legacy_provider_cid_repaired_on_repush(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["provsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("provsrc.git"); + let repo = seed_repo(&owner_did, "provsrc"); // public, no rule + state.db.create_repo(&repo).await.expect("seed repo"); + + // The canonical raw key the resolver accepts once the row is repaired. + let raw_cid = gitlawb_core::cid::Cid::from_git_object_bytes( + &crate::git::store::read_object(&bare, &fx.public_oid) + .unwrap() + .unwrap() + .1, + ) + .to_string(); + // The key stored today: a genuine legacy dag-pb provider CID. + let provider_cid = legacy_dagpb_cid(&raw_cid); + assert_ne!( + provider_cid, raw_cid, + "the provider CID differs from the raw resolver key" + ); + + // Legacy-shape row: cid = the PROVIDER CID (raw SQL — the helpers store the + // raw CID). The object itself is public and servable. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + ) + .bind(&fx.public_oid) + .bind(&provider_cid) + .bind("2020-01-01T00:00:00Z") + .bind(&repo.id) + .execute(&pool) + .await + .unwrap(); + + // RED baseline: the raw key a correct client sends 404s while the row is legacy. + let (st_before, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&raw_cid)) + .await + .unwrap(), + ) + .await; + assert_ne!( + st_before, + StatusCode::OK, + "the raw key 404s while the row is keyed on the provider CID" + ); + + // Re-push carries the object again: `pin_new_objects` hits the already-pinned + // skip branch and repairs the row. The `/add` mock must NOT fire — the object + // is already on IPFS, never re-pinned. + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyshouldnothappen"}"#) + .expect(0) + .create_async() + .await; + crate::ipfs_pin::pin_new_objects( + &server.url(), + &bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + vec![fx.public_oid.clone()], + &state.db, + &repo.id, + crate::ipfs_pin::PIN_BATCH_BUDGET, + ) + .await; + m.assert_async().await; + + // GREEN: the key is repaired to the raw CID and the old value is stashed. + let (stored_cid, stashed): (String, Option) = sqlx::query_as( + "SELECT cid, legacy_provider_cid FROM pinned_cids WHERE sha256_hex = $1", + ) + .bind(&fx.public_oid) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + stored_cid, raw_cid, + "the key is repaired to the raw-content CID" + ); + assert_eq!( + stashed.as_deref(), + Some(provider_cid.as_str()), + "the old provider CID is stashed in legacy_provider_cid" + ); + + // The advertised (raw) key now serves 200. + let (st_after, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&raw_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st_after, + StatusCode::OK, + "the repaired raw key serves after the re-push" + ); + assert!(body.contains("public bytes"), "the object's bytes serve"); + + // Scenario 3: repair never wrote `pinata_cid`, so the Pinata pin-skip gate + // (`has_pinata_cid`) is untouched and Pinata still pins the object. + assert!( + !state.db.has_pinata_cid(&fx.public_oid).await.unwrap(), + "repair leaves pinata_cid NULL" + ); + + // Scenario 6 (integrity negative): the retired provider CID still 404s — no + // serve-path alias for a CID the bytes do not hash to. + let (st_old, body_old) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&provider_cid)) + .await + .unwrap(), + ) + .await; + assert_ne!( + st_old, + StatusCode::OK, + "the retired provider CID must not serve after repair" + ); + assert!( + !body_old.contains("public bytes"), + "no bytes egress under the retired provider CID" + ); + } + + /// #173 R8 (U7 cost gate): a well-formed CIDv1/raw already-pinned row triggers NO + /// object read on the skip path — the codec check decides candidacy from the + /// stored string alone, so a non-legacy row keeps the DB-only skip cost. Also + /// covers the small-object equivalence: a small legacy object Kubo pins under the + /// raw key (raw-leaves) is already CIDv1/raw and needs no repair. The read counter + /// is the both-ways guard: removing the codec gate reads the raw row and trips it. + #[sqlx::test] + async fn ipfs_cid_repair_codec_gate_skips_raw_row(pool: PgPool) { + let state = test_state(pool).await; + let fx = seed_cid_repos("codecgate", "cg", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("codecgate") + .join("pinsrc.git"); + + // A correct raw-CID row (steady state), recorded via the production helper. + let raw_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + assert!( + gitlawb_core::cid::is_raw_cidv1(&raw_cid), + "the helper records a CIDv1/raw key" + ); + + let mut server = mockito::Server::new_async().await; + let m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"x"}"#) + .expect(0) + .create_async() + .await; + + crate::ipfs_pin::reset_legacy_repair_reads(); + crate::ipfs_pin::pin_new_objects( + &server.url(), + &bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + vec![fx.public_oid.clone()], + &state.db, + "repoCG", + crate::ipfs_pin::PIN_BATCH_BUDGET, + ) + .await; + m.assert_async().await; + + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 0, + "a CIDv1/raw row triggers no object read on the skip path (cost gate)" + ); + assert_eq!( + state + .db + .cid_for_oid(&fx.public_oid) + .await + .unwrap() + .as_deref(), + Some(raw_cid.as_str()), + "the raw row is left as-is" + ); + } + + /// #173 R8 (U7): a legacy row whose object bytes are gone stays withheld — the + /// repair never destructively rewrites it, so the row is preserved for a future + /// re-push or the deferred one-shot sweep. + #[sqlx::test] + async fn ipfs_cid_repair_unrepairable_row_stays_withheld(pool: PgPool) { + let state = test_state(pool.clone()).await; + let _fx = seed_cid_repos("unrep", "ur", &["pinsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join("unrep") + .join("pinsrc.git"); + + // A legacy dag-pb row for an oid whose bytes are NOT in this bare repo. + let phantom_oid = "b".repeat(64); + let raw_cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"bytes that live nowhere").to_string(); + let provider_cid = legacy_dagpb_cid(&raw_cid); + sqlx::query("INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) VALUES ($1, $2, $3)") + .bind(&phantom_oid) + .bind(&provider_cid) + .bind("2020-01-01T00:00:00Z") + .execute(&pool) + .await + .unwrap(); + + let mut server = mockito::Server::new_async().await; + server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"x"}"#) + .expect(0) + .create_async() + .await; + + // Skip-branch runs (is_pinned true) but read_object returns None (bytes gone), + // so the repair returns without touching the row. + crate::ipfs_pin::pin_new_objects( + &server.url(), + &bare, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + vec![phantom_oid.clone()], + &state.db, + "repoUR", + crate::ipfs_pin::PIN_BATCH_BUDGET, + ) + .await; + + let (stored, stashed): (String, Option) = sqlx::query_as( + "SELECT cid, legacy_provider_cid FROM pinned_cids WHERE sha256_hex = $1", + ) + .bind(&phantom_oid) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + stored, provider_cid, + "an unrepairable row keeps its provider CID (no destructive rewrite)" + ); + assert_eq!( + stashed, None, + "no legacy_provider_cid is stashed when the bytes are gone" + ); + } + + /// #173 R8 (U7, INV-7 upgrade path): a node already at the prior-max schema (v13) + /// gets `pinned_cids.legacy_provider_cid` from the NEW v21 migration. Simulate the + /// pre-v21 node by dropping the column and un-applying v14, then re-migrate and + /// assert a repair round-trips through the column. RED before the v21 migration + /// exists (the column is never re-added → the repair UPDATE errors). + #[sqlx::test] + async fn pinned_cids_legacy_provider_cid_upgrade_path(pool: PgPool) { + let state = test_state(pool.clone()).await; + + // Pre-v21 shape: drop the column and forget v21 was applied. + sqlx::query("ALTER TABLE pinned_cids DROP COLUMN IF EXISTS legacy_provider_cid") + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version = 21") + .execute(&pool) + .await + .unwrap(); + + // Upgrade: re-run migrations → v21 re-adds the column. + state.db.run_migrations().await.expect("migrate to v14"); + + // A repair round-trips through the v21 column. + state + .db + .record_pinned_cid("upg_oid", "QmProviderLegacy", None) + .await + .unwrap(); + state + .db + .repair_legacy_provider_cid("upg_oid", "bRawContentKey", "QmProviderLegacy") + .await + .unwrap(); + let (cid, stashed): (String, Option) = sqlx::query_as( + "SELECT cid, legacy_provider_cid FROM pinned_cids WHERE sha256_hex = 'upg_oid'", + ) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(cid, "bRawContentKey", "v21 lets the repair rewrite the key"); + assert_eq!( + stashed.as_deref(), + Some("QmProviderLegacy"), + "the v21 legacy_provider_cid column is present after upgrade" + ); + } + + // ---- #173 U4: legacy provider-CID migration sweep ---- + + /// Seed a legacy PROVIDER-CID `pinned_cids` row for `oid` (the pre-branch shape: + /// `cid` holds the Kubo dag-pb / Pinata key, not the raw-content resolver key). + /// Returns `(raw_cid, provider_cid)`. Raw SQL because every production helper + /// stores the already-correct raw key. + async fn seed_legacy_pin( + pool: &PgPool, + bare: &std::path::Path, + oid: &str, + repo_id: Option<&str>, + ) -> (String, String) { + let (_ty, bytes) = crate::git::store::read_object(bare, oid) + .expect("read object bytes") + .expect("object exists in the bare repo"); + let raw = gitlawb_core::cid::Cid::from_git_object_bytes(&bytes).to_string(); + let provider = legacy_dagpb_cid(&raw); + assert_ne!(provider, raw, "the legacy key differs from the raw key"); + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + ) + .bind(oid) + .bind(&provider) + .bind("2020-01-01T00:00:00Z") + .bind(repo_id) + .execute(pool) + .await + .unwrap(); + (raw, provider) + } + + /// The `pinned_cids.cid` currently stored for an oid, unfiltered (unlike + /// `list_pinned_cids`, which withholds unrepaired legacy rows). + async fn stored_pin(pool: &PgPool, oid: &str) -> (String, Option) { + sqlx::query_as("SELECT cid, legacy_provider_cid FROM pinned_cids WHERE sha256_hex = $1") + .bind(oid) + .fetch_one(pool) + .await + .unwrap() + } + + /// U4 (#173, INV-7 upgrade path): a node already at the prior-max schema (v15) gets + /// the `pin_repair_sweep` cursor table from the NEW v23 migration. Simulate the + /// pre-v23 node by dropping the table and un-applying v16, then re-migrate and + /// assert the cursor round-trips. RED before the v23 migration exists (the table is + /// never recreated, so the cursor read errors). + #[sqlx::test] + async fn pin_repair_sweep_cursor_upgrade_path(pool: PgPool) { + let state = test_state(pool.clone()).await; + + // Pre-v23 shape: drop the table and forget v23 was applied. + sqlx::query("DROP TABLE IF EXISTS pin_repair_sweep") + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version = 23") + .execute(&pool) + .await + .unwrap(); + + state.db.run_migrations().await.expect("migrate to v16"); + + // Absent row reads as the "never swept" start, and a write round-trips. + assert_eq!( + state.db.pin_repair_cursor().await.unwrap(), + "", + "a node that has never swept starts at the beginning of the table" + ); + state.db.set_pin_repair_cursor("abc").await.unwrap(); + state.db.set_pin_repair_cursor("def").await.unwrap(); + assert_eq!( + state.db.pin_repair_cursor().await.unwrap(), + "def", + "the v23 cursor table persists the walk position across writes" + ); + let rows: i64 = sqlx::query_scalar("SELECT count(*) FROM pin_repair_sweep") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(rows, 1, "the cursor is a single row, not an append log"); + } + + /// U4 scenario 1 (#173): a legacy provider-CID row with intact object bytes is + /// repaired to the raw-content resolver key by the SWEEP alone, with the old value + /// stashed in `legacy_provider_cid`. No push, no re-pin: this is the whole point of + /// U4, because normal git negotiation omits objects the node already has, so the + /// skip-branch repair's re-push trigger generally never fires on an upgraded node. + /// RED before the sweep is implemented (the row keeps its provider key). + #[sqlx::test] + async fn sweep_repairs_legacy_row_without_a_push(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["swsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("swsrc.git"); + let repo = seed_repo(&owner_did, "swsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + let (raw_cid, provider_cid) = + seed_legacy_pin(&pool, &bare, &fx.public_oid, Some(&repo.id)).await; + + let stats = crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + ) + .await; + assert_eq!(stats.repaired, 1, "the sweep repairs the one legacy row"); + + let (stored, stashed) = stored_pin(&pool, &fx.public_oid).await; + assert_eq!( + stored, raw_cid, + "the key is rewritten to the raw-content CID" + ); + assert_eq!( + stashed.as_deref(), + Some(provider_cid.as_str()), + "the old provider CID is stashed in legacy_provider_cid" + ); + + // End to end: the repaired key is now advertised AND serves. + assert!( + state + .db + .list_pinned_cids() + .await + .unwrap() + .iter() + .any(|r| r.cid == raw_cid), + "the repaired row is advertised" + ); + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&raw_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "the repaired raw key serves"); + assert!(body.contains("public bytes"), "the object's bytes serve"); + } + + /// U4 scenario 2 (#173): a legacy row whose object bytes are gone is left exactly + /// as it is by the sweep: never rewritten, never deleted. The row stays withheld + /// until the bytes come back, which is the non-destructive contract the skip-branch + /// repair already holds. + #[sqlx::test] + async fn sweep_leaves_a_bytes_gone_row_untouched(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let _fx = seed_cid_repos(&slug, &short, &["gonesrc"]); + let repo = seed_repo(&owner_did, "gonesrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + // An oid whose bytes are NOT in the repo, but whose provenance resolves fine. + let phantom_oid = "d".repeat(64); + let raw_cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"bytes that live nowhere").to_string(); + let provider_cid = legacy_dagpb_cid(&raw_cid); + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + ) + .bind(&phantom_oid) + .bind(&provider_cid) + .bind("2020-01-01T00:00:00Z") + .bind(&repo.id) + .execute(&pool) + .await + .unwrap(); + + let stats = crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + ) + .await; + assert_eq!(stats.repaired, 0, "an unrepairable row is not repaired"); + + let (stored, stashed) = stored_pin(&pool, &phantom_oid).await; + assert_eq!( + stored, provider_cid, + "the bytes-gone row keeps its provider CID (no destructive rewrite)" + ); + assert_eq!(stashed, None, "nothing is stashed when the bytes are gone"); + let count: i64 = sqlx::query_scalar("SELECT count(*) FROM pinned_cids") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(count, 1, "the row is not deleted"); + } + + /// U4 scenario 3 (#173): the sweep inherits `repair_legacy_provider_cid`'s cost + /// gate, so a row already keyed on a raw CIDv1 is NEVER read for bytes. The + /// test-only `legacy_repair_reads` counter is the both-ways guard: dropping the + /// codec gate reads the raw row and trips it off zero. + #[sqlx::test] + async fn sweep_never_reads_bytes_for_a_raw_cidv1_row(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["rawsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("rawsrc.git"); + let repo = seed_repo(&owner_did, "rawsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + let raw_cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo.id).await; + assert!( + gitlawb_core::cid::is_raw_cidv1(&raw_cid), + "the seeded row is already the canonical resolver key" + ); + + crate::ipfs_pin::reset_legacy_repair_reads(); + let stats = crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + ) + .await; + assert_eq!(stats.scanned, 1, "the sweep walked the row"); + assert_eq!(stats.repaired, 0, "a raw row needs no repair"); + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 0, + "a raw-CIDv1 row is never read for bytes (cost gate)" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + raw_cid, + "the raw row is left as-is" + ); + } + + /// U4 scenario 4 (#173, BOUND): one pass reads at most `batch` rows, so it repairs + /// at most `batch` of them. The exact count is asserted, so raising or removing the + /// bound fails. This is what keeps the sweep from monopolizing the DB on a node + /// with a large `pinned_cids` table. + #[sqlx::test] + async fn sweep_one_pass_is_bounded_by_the_batch_size(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["batchsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("batchsrc.git"); + let repo = seed_repo(&owner_did, "batchsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + // Five legacy rows, batch of two. + for oid in [ + &fx.public_oid, + &fx.secret_oid, + &fx.public_tree_oid, + &fx.secret_tree_oid, + &fx.commit_oid, + ] { + seed_legacy_pin(&pool, &bare, oid, Some(&repo.id)).await; + } + + let stats = crate::ipfs_pin::sweep_legacy_provider_cids_once( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 2, + &state.db, + ) + .await + .expect("one pass runs"); + assert_eq!(stats.scanned, 2, "one pass reads exactly the batch size"); + assert_eq!(stats.repaired, 2, "one pass repairs at most the batch size"); + + let repaired: i64 = sqlx::query_scalar( + "SELECT count(*) FROM pinned_cids WHERE legacy_provider_cid IS NOT NULL", + ) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(repaired, 2, "exactly two of the five rows were rewritten"); + } + + /// U4 scenario 5 (#173, RESUMPTION): the walk cursor persists, so a sweep + /// interrupted mid-table continues from where it stopped instead of restarting. + /// Two bounded passes are driven by hand (the restart), and the second pass is + /// asserted to repair the NEXT two rows in cursor order, not the first two again. + /// The read counter proves the already-repaired rows are not re-read. + #[sqlx::test] + async fn sweep_resumes_from_the_persisted_cursor(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["resumesrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("resumesrc.git"); + let repo = seed_repo(&owner_did, "resumesrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + let mut oids = vec![ + fx.public_oid.clone(), + fx.secret_oid.clone(), + fx.public_tree_oid.clone(), + fx.secret_tree_oid.clone(), + ]; + for oid in &oids { + seed_legacy_pin(&pool, &bare, oid, Some(&repo.id)).await; + } + // The cursor is an ordered walk over the `pinned_cids` primary key. + oids.sort(); + + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let pass1 = crate::ipfs_pin::sweep_legacy_provider_cids_once( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 2, + &state.db, + ) + .await + .expect("pass 1 runs"); + assert_eq!(pass1.repaired, 2, "pass 1 repairs the first two rows"); + + // The restart: a second pass over the SAME state must continue, not rewind. + crate::ipfs_pin::reset_legacy_repair_reads(); + let pass2 = crate::ipfs_pin::sweep_legacy_provider_cids_once( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 2, + &state.db, + ) + .await + .expect("pass 2 runs"); + assert_eq!(pass2.repaired, 2, "pass 2 repairs the NEXT two rows"); + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 2, + "pass 2 reads bytes only for the two rows it repaired; the already-repaired \ + rows are not re-read" + ); + for oid in &oids { + let (_cid, stashed) = stored_pin(&pool, oid).await; + assert!( + stashed.is_some(), + "every row is repaired after two resumed passes" + ); + } + } + + /// U4 scenario 7 (#173, cursor liveness): a row that cannot be repaired (NULL + /// provenance, or a provenance whose repo row is gone) is skipped AND the cursor + /// still advances past it. With `batch = 1` the two unrepairable rows sort first, + /// so a cursor that failed to advance would re-read the same row forever and never + /// reach the repairable row behind them. The outer timeout turns that into a + /// FAILURE rather than a hung suite. + #[sqlx::test] + async fn sweep_advances_past_unrepairable_rows(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["skipsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("skipsrc.git"); + let repo = seed_repo(&owner_did, "skipsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + + // Two blockers that sort ahead of any real 64-hex oid: one with NULL + // provenance, one naming a repo row that no longer exists. + let null_prov_oid = "0".repeat(64); + let ghost_repo_oid = format!("{}1", "0".repeat(63)); + seed_legacy_pin(&pool, &bare, &fx.public_oid, Some(&repo.id)).await; + for (oid, prov) in [ + (&null_prov_oid, None), + (&ghost_repo_oid, Some("repo-that-is-gone")), + ] { + let raw = gitlawb_core::cid::Cid::from_git_object_bytes(oid.as_bytes()).to_string(); + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + ) + .bind(oid) + .bind(legacy_dagpb_cid(&raw)) + .bind("2020-01-01T00:00:00Z") + .bind(prov) + .execute(&pool) + .await + .unwrap(); + } + assert!( + null_prov_oid < fx.public_oid && ghost_repo_oid < fx.public_oid, + "the blockers really do sort ahead of the repairable row" + ); + + let stats = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 1, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the sweep terminates instead of looping on an unrepairable row"); + + assert_eq!( + stats.repaired, 1, + "the sweep advanced past both blockers and repaired the row behind them" + ); + assert!( + stored_pin(&pool, &fx.public_oid).await.1.is_some(), + "the row behind the blockers is the one that got repaired" + ); + for oid in [&null_prov_oid, &ghost_repo_oid] { + assert_eq!( + stored_pin(&pool, oid).await.1, + None, + "an unrepairable row is left untouched" + ); + } + } + + /// U4 scenario 9 (#173, regression): a row skipped for a TRANSIENT reason is + /// retried by a later run. The sweep never pulls a cold repo back from remote + /// storage, so on a Tigris-backed node a repo that is not on local disk at boot + /// contributes nothing to the pass. With the cursor parked at the end of the table + /// that row was skipped FOREVER: every later boot read zero rows and the row stayed + /// unadvertised and unresolvable with nothing left to repair it. Here the repo is + /// off disk for the first run and back for the second, so only a re-walk repairs it. + /// RED before the transient-skip cursor reset (the second run scans nothing). + #[sqlx::test] + async fn sweep_rewalks_after_a_transient_skip(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + + let fx = seed_cid_repos(&slug, &short, &["coldsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("coldsrc.git"); + let repo = seed_repo(&owner_did, "coldsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + let (raw_cid, _provider) = + seed_legacy_pin(&pool, &bare, &fx.public_oid, Some(&repo.id)).await; + + // The repo is COLD: its provenance resolves, but the bytes are not on this + // node's disk right now, exactly the state the sweep refuses to fix by pulling. + let stashed_away = bare.with_extension("git.away"); + let _ = std::fs::remove_dir_all(&stashed_away); + std::fs::rename(&bare, &stashed_away).expect("take the repo off local disk"); + + let first = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the first run terminates"); + assert_eq!( + (first.scanned, first.repaired), + (1, 0), + "the cold repo's row is walked but cannot be repaired yet" + ); + + // The repo is warm again (a later boot, a fetch, an operator restore). + std::fs::rename(&stashed_away, &bare).expect("put the repo back on local disk"); + + let second = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the second run terminates"); + assert_eq!( + second.repaired, 1, + "a later run re-walks the transiently skipped row and repairs it" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + raw_cid, + "the row now carries the raw-content resolver key" + ); + assert!( + state + .db + .list_pinned_cids() + .await + .unwrap() + .iter() + .any(|r| r.cid == raw_cid), + "the repaired row is advertised again" + ); + } + + /// U4 scenario 10 (#173, the other arm of scenario 9): a PERMANENTLY unrepairable + /// row must not make the sweep re-walk forever. Bytes that are genuinely gone are a + /// terminal skip, so the cursor stays parked and a later run reads nothing. Without + /// that split the transient-skip reset of scenario 9 turns every boot on such a node + /// into a full table walk. Both runs are timeout-bounded, so a hot loop FAILS here. + #[sqlx::test] + async fn sweep_does_not_rewalk_for_a_terminal_skip(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + + // The repo IS on local disk; the object's bytes are not in it and never will be. + let _fx = seed_cid_repos(&slug, &short, &["termsrc"]); + let repo = seed_repo(&owner_did, "termsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + let phantom_oid = "e".repeat(64); + let raw_cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"bytes that live nowhere").to_string(); + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo_id) VALUES ($1, $2, $3, $4)", + ) + .bind(&phantom_oid) + .bind(legacy_dagpb_cid(&raw_cid)) + .bind("2020-01-01T00:00:00Z") + .bind(&repo.id) + .execute(&pool) + .await + .unwrap(); + + let first = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the first run terminates"); + assert_eq!( + (first.scanned, first.repaired), + (1, 0), + "the row is walked and cannot be repaired" + ); + + let second = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 16, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the second run terminates"); + assert_eq!( + (second.scanned, second.repaired, second.passes), + (0, 0, 1), + "a terminal skip leaves the cursor parked: the next run re-reads nothing" + ); + } + + /// U4 scenario 11 (#173, path barrier): the sweep resolves a source repo's disk path + /// through the SAME validated logic the repo store uses, so a repo row whose name + /// carries `..` reads nothing. Names are validated at creation today, so this is a + /// defence-in-depth barrier on a second caller of the raw path helper rather than a + /// live exploit. The escapee repo really does hold the object's bytes, so before the + /// barrier the sweep happily read them from outside `repos_dir` and repaired the row. + /// RED before routing through the validated path (repaired 1). + #[sqlx::test] + async fn sweep_refuses_a_source_path_that_escapes_repos_dir(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + // The bytes live at /tmp/{slug}/escapee.git, OUTSIDE the repos_dir below. + let fx = seed_cid_repos(&slug, &short, &["escapee"]); + let escapee_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("escapee.git"); + let repos_dir = std::path::PathBuf::from("/tmp").join(&slug).join("root"); + std::fs::create_dir_all(repos_dir.join(&slug)).expect("create the repos_dir tree"); + + // A repo row whose name walks back out of repos_dir: repos_dir/{slug}/../../escapee.git + let mut repo = seed_repo(&owner_did, "../../escapee"); + repo.disk_path = escapee_bare.display().to_string(); + state.db.create_repo(&repo).await.expect("seed repo"); + let (_raw_cid, provider_cid) = + seed_legacy_pin(&pool, &escapee_bare, &fx.public_oid, Some(&repo.id)).await; + assert!( + crate::git::store::repo_disk_path(&repos_dir, &owner_did, &repo.name).exists(), + "the unvalidated helper really does resolve to the escapee repo" + ); + + let stats = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + &repos_dir, + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the sweep terminates"); + + assert_eq!( + stats.repaired, 0, + "a repo path that escapes repos_dir must never be read" + ); + assert_eq!( + stored_pin(&pool, &fx.public_oid).await.0, + provider_cid, + "the row is untouched because its bytes were never read" + ); + } + + /// U4 scenario 12 (#173, F4): the repair's object read is SYNCHRONOUS `git cat-file`, + /// so running it inline parks the async worker for as long as git takes, up to the + /// whole `git_service_timeout_secs` budget on a wedged read, and the sweep does this + /// per legacy row starting at boot. A slow git stand-in makes that observable: a + /// concurrent 20ms ticker cannot tick at all while the only worker thread is blocked, + /// and ticks freely once the read is on the blocking pool. RED before the + /// `spawn_blocking` (0 ticks). + #[sqlx::test] + async fn repair_object_read_does_not_block_the_async_worker(pool: PgPool) { + use gitlawb_core::identity::Keypair; + use std::sync::atomic::{AtomicUsize, Ordering}; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["slowsrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("slowsrc.git"); + let repo = seed_repo(&owner_did, "slowsrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + seed_legacy_pin(&pool, &bare, &fx.public_oid, Some(&repo.id)).await; + + // A git that takes 300ms per invocation (the read makes two: type, then content). + let slow_git = std::env::temp_dir().join(format!("gl-slow-git-{short}")); + std::fs::write(&slow_git, "#!/bin/sh\nsleep 0.3\nexec git \"$@\"\n").expect("write shim"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&slow_git, std::fs::Permissions::from_mode(0o755)) + .expect("chmod shim"); + } + + let ticks = std::sync::Arc::new(AtomicUsize::new(0)); + let ticker = { + let ticks = ticks.clone(); + tokio::spawn(async move { + loop { + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + ticks.fetch_add(1, Ordering::Relaxed); + } + }) + }; + + let stats = crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + slow_git.to_str().unwrap(), + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 16, + std::time::Duration::ZERO, + &state.db, + ) + .await; + ticker.abort(); + + assert_eq!(stats.repaired, 1, "the slow git still repairs the row"); + assert!( + ticks.load(Ordering::Relaxed) >= 5, + "the runtime kept running other tasks during the blocking git read (ticks: {})", + ticks.load(Ordering::Relaxed) + ); + } + + /// U4 scenario 8 (#173, degenerate states): an empty `pinned_cids` table and a + /// table with zero legacy rows both complete cleanly, with no repair and no read. + #[sqlx::test] + async fn sweep_completes_on_degenerate_tables(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + + // Empty table. + crate::ipfs_pin::reset_legacy_repair_reads(); + let empty = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 4, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the sweep terminates on an empty table"); + assert_eq!( + (empty.scanned, empty.repaired), + (0, 0), + "an empty table is a clean no-op" + ); + + // Zero legacy rows: every row already carries the canonical raw key. + let fx = seed_cid_repos(&slug, &short, &["degensrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("degensrc.git"); + let repo = seed_repo(&owner_did, "degensrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + for oid in [&fx.public_oid, &fx.secret_oid, &fx.commit_oid] { + pin_cid_for_repo(&bare, oid, &state.db, &repo.id).await; + } + + let clean = tokio::time::timeout( + std::time::Duration::from_secs(30), + crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + git_timeout, + 4, + std::time::Duration::ZERO, + &state.db, + ), + ) + .await + .expect("the sweep terminates on a table with no legacy rows"); + assert_eq!(clean.scanned, 3, "every row is walked"); + assert_eq!(clean.repaired, 0, "nothing needs repair"); + assert_eq!( + crate::ipfs_pin::legacy_repair_reads(), + 0, + "no object bytes are read when no row is legacy" + ); + } + + /// U4 (#173, BOUND): the inter-batch delay is real, observed by wall clock. Five + /// rows at a batch of two means two full batches and a trailing partial one, so the + /// run sleeps twice. Without the sleep the whole run is sub-millisecond DB work and + /// a node's `pinned_cids` table gets walked as fast as Postgres will answer. + #[sqlx::test] + async fn sweep_sleeps_between_batches(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool.clone()).await; + + let fx = seed_cid_repos(&slug, &short, &["delaysrc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("delaysrc.git"); + let repo = seed_repo(&owner_did, "delaysrc"); + state.db.create_repo(&repo).await.expect("seed repo"); + for oid in [ + &fx.public_oid, + &fx.secret_oid, + &fx.public_tree_oid, + &fx.secret_tree_oid, + &fx.commit_oid, + ] { + pin_cid_for_repo(&bare, oid, &state.db, &repo.id).await; + } + + let delay = std::time::Duration::from_millis(150); + let started = std::time::Instant::now(); + let stats = crate::ipfs_pin::sweep_legacy_provider_cids( + std::path::Path::new("/tmp"), + &state.git_bin, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + 2, + delay, + &state.db, + ) + .await; + let elapsed = started.elapsed(); + + assert_eq!( + stats.passes, 3, + "five rows at a batch of two is three passes" + ); + assert!( + elapsed >= delay * 2, + "the run sleeps once between each pair of full batches: {elapsed:?} < {:?}", + delay * 2 + ); + } + + /// U4 scenario 6 (#173): `list_pinned_cids` never advertises a key the `/ipfs` + /// resolver would withhold. The resolver recomputes the raw CIDv1 from the object + /// bytes and 404s any row keyed on a legacy PROVIDER CID, so advertising that key + /// hands clients a CID this node deliberately refuses. Both states of ONE row are + /// asserted (omitted while legacy, present once repaired) so the test cannot pass + /// by accident. RED before the `is_raw_cidv1` filter lands: the legacy row is + /// advertised. + #[sqlx::test] + async fn list_pinned_cids_omits_unrepaired_legacy_row(pool: PgPool) { + let state = test_state(pool).await; + + let raw_cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"u4 advertise bytes").to_string(); + let provider_cid = legacy_dagpb_cid(&raw_cid); + let oid = "c".repeat(64); + state + .db + .record_pinned_cid(&oid, &provider_cid, None) + .await + .unwrap(); + + let listed = state.db.list_pinned_cids().await.unwrap(); + assert!( + !listed.iter().any(|r| r.sha256_hex == oid), + "an unrepaired legacy provider-CID row is not advertised" + ); + + // Same row, repaired: it comes back, keyed on the raw CID the resolver serves. + state + .db + .repair_legacy_provider_cid(&oid, &raw_cid, &provider_cid) + .await + .unwrap(); + let listed = state.db.list_pinned_cids().await.unwrap(); + let rec = listed + .iter() + .find(|r| r.sha256_hex == oid) + .expect("the repaired row is advertised again"); + assert_eq!( + rec.cid, raw_cid, + "the advertised key is the raw-content resolver key" + ); + } + + /// #173 (provenance-path throttle): a walk-requiring provenanced candidate whose + /// per-IP walk quota is spent returns 429 (the provenance arm's Throttled outcome, + /// then the fall-through). quota=1, keyed on XFF. The first reader request runs the + /// walk and spends the token; the second from the same IP is throttled → 429. + #[sqlx::test] + async fn ipfs_cid_provenance_walk_throttle_returns_429(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let reader = Keypair::generate(); + let reader_did = reader.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let fx = seed_cid_repos(&slug, &short, &["provthrottle"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("provthrottle.git"); + let repo = seed_repo(&owner_did, "provthrottle"); + state.db.create_repo(&repo).await.expect("seed repo"); + state + .db + .set_visibility_rule( + &repo.id, + "/secret/**", + VisibilityMode::B, + std::slice::from_ref(&reader_did), + &owner_did, + ) + .await + .expect("path rule"); + let cid = pin_cid_for_repo(&bare, &fx.secret_oid, &state.db, &repo.id).await; + + // 1st reader request runs the walk (reader is allowed) and spends the token. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_signed_xff(&reader, &cid, "1.2.3.4")) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "1st provenance walk from the IP serves"); + + // 2nd request from the same IP: the walk is throttled → 429 (provenance path). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_signed_xff(&reader, &cid, "1.2.3.4")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::TOO_MANY_REQUESTS, + "a throttled provenance walk returns 429" + ); + } + + /// #173 (multi-oid dispatch, mixed provenance + legacy): one CID mapping to a + /// provenanced-then-denied oid AND a legacy (NULL-provenance) oid must still resolve + /// to the legacy-servable copy — the provenance arm's skip does not abort the loop. + #[sqlx::test] + async fn ipfs_cid_mixed_provenance_and_legacy_serves_legacy(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["mixpriv", "mixpub"]); + + // Private repo holds secret_oid, pinned with provenance = itself (denies anon). + let mut priv_repo = seed_repo(&owner_did, "mixpriv"); + priv_repo.is_public = false; + state + .db + .create_repo(&priv_repo) + .await + .expect("seed private"); + // Public repo holds public_oid, legacy pin (NULL provenance -> scan serves it). + let pub_repo = seed_repo(&owner_did, "mixpub"); + state.db.create_repo(&pub_repo).await.expect("seed public"); + + // One REAL CID (the non-unique cid index) maps to BOTH oids: the public oid as a + // legacy (NULL) pin, and the secret oid provenanced to the private repo. + let pub_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("mixpub.git"); + let shared_cid = pin_cid_for(&pub_bare, &fx.public_oid, &state.db).await; + state + .db + .record_pinned_cid(&fx.secret_oid, &shared_cid, Some(&priv_repo.id)) + .await + .unwrap(); + + // Anon: secret_oid (provenance -> private -> denied), public_oid (legacy -> scan + // -> public -> served). Resolves to the public copy regardless of oid order. + let resp = cid_router(&state) + .oneshot(cid_anon(&shared_cid)) + .await + .unwrap(); + let served = resp + .headers() + .get("x-git-hash") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + let (st, body) = cid_parts(resp).await; + assert_eq!( + st, + StatusCode::OK, + "a CID mixing a provenanced-denied oid and a legacy-servable oid resolves" + ); + assert_eq!( + served.as_deref(), + Some(fx.public_oid.as_str()), + "the served object is the legacy public oid" + ); + assert!( + body.contains("public bytes"), + "the public content is served" + ); + } + + // ---- #173 round 3: legacy (NULL-provenance) scan bound + 503-on-truncation ---- + // The provenance path targets one repo and is already bounded. These cover the + // legacy scan fallback, where an anonymous request could otherwise fan out to + // O(repos) `acquire` + `cat-file` probes (F1) and a walk-cap truncation could + // false-404 an object that may be readable (F2). The bound is a per-request probe + // BUDGET, not a per-IP brake: a walk-free public fetch stays un-rate-limited + // (ipfs_walk_rate_limited_per_source), while the expensive walk keeps its IP brake. + + /// T1 (F1): the probe budget gates BEFORE `acquire`/`cat-file`, so it genuinely + /// bounds the fan-out — a repo past the budget is never probed, even one that + /// WOULD serve. With the budget at 0, a PUBLIC legacy copy that would otherwise + /// serve 200 is not probed at all → 503 truncated (absence unproven). RED before + /// the budget check (the repo is probed and serves 200). + #[sqlx::test] + async fn ipfs_cid_legacy_probe_budget_gates_before_serving(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_max_legacy_probes = 0; // probe nothing → any legacy candidate truncates + + let fx = seed_cid_repos(&slug, &short, &["pubprobe"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("pubprobe.git"); + let repo = seed_repo(&owner_did, "pubprobe"); // public, no path rule → would serve + state.db.create_repo(&repo).await.expect("seed repo"); + // Legacy pin (NULL provenance) → resolver takes the scan fallback. + let cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::SERVICE_UNAVAILABLE, + "the probe budget gates before the probe: a servable copy past the budget is not reached → 503" + ); + } + + /// T7 (F1/F3 pre-limit): EVERY legacy probe is braked on the source IP from the + /// FIRST one, so a hostile caller cannot repeatedly force the whole-node `acquire` + /// fan-out across requests (each cold `acquire` is a Tigris round-trip, INV-10). + /// Since #173-F3 (jatmn) there is no free budget: a single-repo legacy scan is + /// itself charged. quota=1 keyed on XFF, one PUBLIC legacy copy that serves + /// walk-free (never touches the walk brake), so the second same-IP request can only + /// be shed by the probe brake: req1 serves and spends the token, req2 → 429. RED + /// before the probe brake (req2 serves 200). The cross-request bound this proves is + /// exactly the amplification F3 closes. + #[sqlx::test] + async fn ipfs_cid_legacy_fanout_braked_on_ip_past_free_budget(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let fx = seed_cid_repos(&slug, &short, &["fanout"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("fanout.git"); + let repo = seed_repo(&owner_did, "fanout"); // public, no path rule → walk-free serve + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; // legacy pin + + let (st1, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&cid, "1.2.3.4")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st1, + StatusCode::OK, + "1st legacy fan-out probe from the IP serves" + ); + + let (st2, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&cid, "1.2.3.4")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st2, + StatusCode::TOO_MANY_REQUESTS, + "with no free budget, a repeat fan-out from the same IP is braked at the first probe" + ); + } + + /// F3 (jatmn, across-request amplification): the pre-fix free-probe budget was + /// PER REQUEST, so a caller could repeat a known NULL-provenance CID and force a + /// fresh batch of `acquire` + `cat-file` probes every request with zero limiter + /// contact, unbounded anonymous amplification against Tigris. Charging every + /// legacy probe from the first one makes those probes accumulate against the + /// per-IP `ipfs_work_rate_limiter` ACROSS requests. Four repos, none holding the CID, + /// so a full scan probes all four; the per-IP budget is sized to exactly ONE such + /// scan (4 tokens). req1 (a genuine absence) fully scans and 404s, spending the + /// budget; req2 from the SAME IP is shed at the first probe → 429 (it never + /// re-runs the four `acquire` probes). RED with the old free carve-out restored: + /// req2 re-scans un-braked and 404s again (the amplification stays open). This is + /// the load-bearing across-request bound F3 asks for. + #[sqlx::test] + async fn ipfs_cid_legacy_fanout_bounded_across_requests(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + // Budget = one full scan of the four seeded repos. A repeat scan from the same + // IP then finds it spent. Keyed on XFF so `oneshot` can choose the source IP. + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(4, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let names = ["a0", "a1", "a2", "a3"]; + let _fx = seed_cid_repos(&slug, &short, &names); + for n in names { + let repo = seed_repo(&owner_did, n); + state.db.create_repo(&repo).await.expect("seed repo"); + } + // A legacy pin whose oid is absent from every repo → each probed repo misses, + // so req1 scans all four (spending the four-token budget) and 404s cleanly. + let bogus_oid = "0".repeat(64); + let cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"absent-across-requests").to_string(); + state + .db + .record_pinned_cid(&bogus_oid, &cid, None) + .await + .expect("record legacy pin"); + + let (st1, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&cid, "9.9.9.9")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st1, + StatusCode::NOT_FOUND, + "1st scan completes under budget: a genuine absence is a definitive 404" + ); + + let (st2, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&cid, "9.9.9.9")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st2, + StatusCode::TOO_MANY_REQUESTS, + "2nd same-IP scan is shed at the first probe (429), not re-run un-braked: the across-request amplification is closed" + ); + } + + /// #173 (jatmn round 8, F3 — INV-10 cost guard): an already-throttled source's + /// legacy NULL-provenance request must be shed by the non-consuming admission peek + /// BEFORE the O(repos) `scan_ctx` preload runs — not after, where the per-probe + /// brake sits. The preload-query counter proves it both ways: 0 for the throttled + /// replay, 1 for an unthrottled source. RED if the peek is removed (the preload runs + /// while throttled → count 1). The two existing `_fanout_` tests confirm the per- + /// probe consuming charge is untouched (no double-charge, no under-charge). + #[sqlx::test] + async fn ipfs_cid_f3_throttled_source_skips_preload(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + // Budget 1, keyed on XFF so `oneshot` can choose the source IP. + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let _fx = seed_cid_repos(&slug, &short, &["r0"]); + state + .db + .create_repo(&seed_repo(&owner_did, "r0")) + .await + .expect("seed repo"); + // A legacy pin absent from every repo → the scan probes and 404s (spending the + // one token on the first probe). + let bogus_oid = "0".repeat(64); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"f3-absent").to_string(); + state + .db + .record_pinned_cid(&bogus_oid, &cid, None) + .await + .expect("legacy pin"); + + // Req1 from 9.9.9.9 spends the one token (and runs the preload once). + let _ = cid_router(&state) + .oneshot(cid_anon_xff(&cid, "9.9.9.9")) + .await + .unwrap(); + + // Measure the throttled replay: the peek must shed it before the preload runs. + crate::api::ipfs::reset_preload_queries(); + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon_xff(&cid, "9.9.9.9")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::TOO_MANY_REQUESTS, + "an already-throttled legacy replay is 429" + ); + assert_eq!( + crate::api::ipfs::preload_queries(), + 0, + "a throttled source must NOT run the O(repos) preload (F3): shed before scan_ctx" + ); + + // Control: an unthrottled source (a different IP) still runs the preload once — + // the peek must not over-block. + crate::api::ipfs::reset_preload_queries(); + let _ = cid_router(&state) + .oneshot(cid_anon_xff(&cid, "8.8.8.8")) + .await + .unwrap(); + assert_eq!( + crate::api::ipfs::preload_queries(), + 1, + "an unthrottled source runs the preload once (the peek must not over-block)" + ); + } + + /// T2 (F1): the legacy scan is bounded per request. With the probe ceiling shrunk + /// to 2 and 3 candidate repos none of which hold the object, the 3rd repo is never + /// probed and the search is reported truncated → 503, not an unbounded fan-out. + /// RED before the probe cap (all 3 probe, none serve, definitive 404). + #[sqlx::test] + async fn ipfs_cid_legacy_scan_probe_cap_truncates_to_503(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_max_legacy_probes = 2; + + let _fx = seed_cid_repos(&slug, &short, &["r0", "r1", "r2"]); + for n in ["r0", "r1", "r2"] { + let repo = seed_repo(&owner_did, n); + state.db.create_repo(&repo).await.expect("seed repo"); + } + // A legacy pin whose oid is absent from every repo: each probed repo misses, + // so the cap (not a hit) decides the outcome. + let bogus_oid = "0".repeat(64); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"absent-marker-t2").to_string(); + state + .db + .record_pinned_cid(&bogus_oid, &cid, None) + .await + .expect("record legacy pin"); + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::SERVICE_UNAVAILABLE, + "a scan truncated by the probe cap is a retryable 503, not a definitive 404" + ); + } + + /// T2b (R5, KTD5): the `GITLAWB_IPFS_MAX_REPOS_WALKED` knob drives the legacy-probe + /// budget end to end. With the knob at 1 (fed through the same production helper the + /// state seeding uses) and two candidate repos that miss, the first repo spends the + /// single probe and the second is skipped at the cap → truncated → 503. If the knob + /// budget were not honoured (unbounded), both would probe, both miss, and the request + /// would be a definitive 404. Proves the wired knob=1 → exactly one probe path. + #[sqlx::test] + async fn ipfs_cid_repos_walked_knob_caps_legacy_probes(pool: PgPool) { + use clap::Parser; + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + // Seed the legacy-probe budget the way production does: from the operator knob. + // The legacy-probe budget knob, renamed in the merge: #174 already owned + // `--ipfs-max-repos-walked` for its expensive-walk cap, so #173's identically + // named knob became `--ipfs-max-legacy-probes`. + let cfg = + crate::config::Config::parse_from(["gitlawb-node", "--ipfs-max-legacy-probes", "1"]); + state.ipfs_max_legacy_probes = AppState::ipfs_legacy_probe_budget(&cfg); + assert_eq!(state.ipfs_max_legacy_probes, 1, "knob=1 → one-probe budget"); + // The knob must not touch the history-walk ceiling (must stay MAX_PIN_SOURCES + 1). + assert_eq!( + state.ipfs_max_history_walks, + crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, + "the repos-walked knob leaves the history-walk ceiling untouched" + ); + + let _fx = seed_cid_repos(&slug, &short, &["k0", "k1"]); + for n in ["k0", "k1"] { + let repo = seed_repo(&owner_did, n); + state.db.create_repo(&repo).await.expect("seed repo"); + } + // A legacy pin whose oid is absent from every repo: the cap, not a hit, decides. + let bogus_oid = "0".repeat(64); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"absent-marker-knob").to_string(); + state + .db + .record_pinned_cid(&bogus_oid, &cid, None) + .await + .expect("record legacy pin"); + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::SERVICE_UNAVAILABLE, + "knob=1 caps the scan at one probe → incomplete search → retryable 503" + ); + } + + /// T3 (F2): a walk-cap truncation must not false-404. Walk ceiling shrunk to 1; + /// two public repos each carry a path-scoped rule over the object and deny anon. + /// The 1st spends the single walk (deny), the 2nd is skipped at the cap — the + /// resolver did NOT prove the object unreadable everywhere, so 503, not 404. + /// RED before the walk-cap `truncated` flag (returns the opaque 404). + #[sqlx::test] + async fn ipfs_cid_legacy_walk_cap_truncates_to_503(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let reader = Keypair::generate(); + let reader_did = reader.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_max_history_walks = 1; + + let fx = seed_cid_repos(&slug, &short, &["wa", "wb"]); + for n in ["wa", "wb"] { + let repo = seed_repo(&owner_did, n); + state.db.create_repo(&repo).await.expect("seed repo"); + state + .db + .set_visibility_rule( + &repo.id, + "/secret/**", + VisibilityMode::B, + std::slice::from_ref(&reader_did), + &owner_did, + ) + .await + .expect("path rule"); + } + // Legacy pin of the path-scoped secret blob (present in both repos, denies anon). + let bare_wa = std::path::PathBuf::from("/tmp").join(&slug).join("wa.git"); + let cid = pin_cid_for(&bare_wa, &fx.secret_oid, &state.db).await; + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::SERVICE_UNAVAILABLE, + "the walk cap truncated the scan, so absence is unproven → 503, not a false 404" + ); + } + + /// T4 (must-not over-fire): a legacy CID genuinely absent from every repo on a + /// node UNDER the probe cap still returns the definitive 404 — the 503 fires only + /// on real truncation, never as a blanket replacement for not-found. + #[sqlx::test] + async fn ipfs_cid_legacy_true_absence_stays_404(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_max_legacy_probes = 8; // well above the single repo → no truncation + + let _fx = seed_cid_repos(&slug, &short, &["only"]); + let repo = seed_repo(&owner_did, "only"); + state.db.create_repo(&repo).await.expect("seed repo"); + let bogus_oid = "0".repeat(64); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"absent-marker-t4").to_string(); + state + .db + .record_pinned_cid(&bogus_oid, &cid, None) + .await + .expect("record legacy pin"); + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "a fully-scanned genuine absence is a definitive 404, not a 503" + ); + } + + /// T5 (provenance path untouched): the probe cap governs ONLY the legacy scan. + /// With the cap set to 0 (which would truncate any legacy probe immediately) a + /// PROVENANCED pin still resolves to its one repo and serves 200 — proving the + /// `legacy_scan=false` guard exempts the provenance path. RED if the guard were + /// dropped (provenance would truncate to 503). + #[sqlx::test] + async fn ipfs_cid_provenance_serves_despite_zero_probe_cap(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_max_legacy_probes = 0; // would truncate every LEGACY probe + + let fx = seed_cid_repos(&slug, &short, &["provonly"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("provonly.git"); + let repo = seed_repo(&owner_did, "provonly"); // public, no path rule + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for_repo(&bare, &fx.public_oid, &state.db, &repo.id).await; + + let (st, _) = cid_parts(cid_router(&state).oneshot(cid_anon(&cid)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "the provenance path ignores the legacy probe cap and serves" + ); + } + + fn cid_router(state: &AppState) -> Router { + Router::new() + .route( + "/ipfs/{cid}", + axum::routing::get(crate::api::ipfs::get_by_cid), + ) + .layer(axum::middleware::from_fn(crate::auth::optional_signature)) + .with_state(state.clone()) + } + async fn cid_parts(resp: axum::response::Response) -> (StatusCode, String) { + let st = resp.status(); + let b = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + (st, String::from_utf8_lossy(&b).to_string()) + } + /// Raw body bytes (NOT lossy-decoded). A git tree body stores each child oid + /// as 32 RAW bytes that `from_utf8_lossy` mangles to U+FFFD, so a hex + /// `contains` check on `cid_parts`'s String is vacuous. #135 deny tests must + /// witness the leak on these raw bytes. + async fn cid_bytes(resp: axum::response::Response) -> (StatusCode, Vec) { + let st = resp.status(); + let b = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + (st, b.to_vec()) + } + /// True if `needle` appears as a contiguous byte subsequence of `haystack`. + fn bytes_contain(haystack: &[u8], needle: &[u8]) -> bool { + !needle.is_empty() && haystack.windows(needle.len()).any(|w| w == needle) + } + fn cid_anon(cid: &str) -> Request { + Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap() + } + /// Anonymous CID request carrying `x-forwarded-for: ` — an anon caller with a + /// resolvable source IP, so the per-IP walk brake keys on it (the walk still + /// denies anon at a path rule). + fn cid_anon_xff(cid: &str, xff_ip: &str) -> Request { + Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .header("x-forwarded-for", xff_ip) + .body(Body::empty()) + .unwrap() + } + fn cid_signed(kp: &gitlawb_core::identity::Keypair, cid: &str) -> Request { + let path = format!("/ipfs/{cid}"); + let s = gitlawb_core::http_sig::sign_request(kp, "GET", &path, b""); + Request::builder() + .method(Method::GET) + .uri(&path) + .header("content-digest", s.content_digest) + .header("signature-input", s.signature_input) + .header("signature", s.signature) + .body(Body::empty()) + .unwrap() + } + /// Signed CID request carrying `x-forwarded-for: `. Used by the walk + /// rate-limit test to key the per-IP limiter off a chosen source under + /// `TrustedProxy::XForwardedFor` (the request goes through `oneshot`, which + /// leaves no socket peer, so the header is the only key source). + fn cid_signed_xff( + kp: &gitlawb_core::identity::Keypair, + cid: &str, + xff_ip: &str, + ) -> Request { + let path = format!("/ipfs/{cid}"); + let s = gitlawb_core::http_sig::sign_request(kp, "GET", &path, b""); + Request::builder() + .method(Method::GET) + .uri(&path) + .header("content-digest", s.content_digest) + .header("signature-input", s.signature_input) + .header("signature", s.signature) + .header("x-forwarded-for", xff_ip) + .body(Body::empty()) + .unwrap() + } + + /// #110: `GET /ipfs/{cid}` must gate a withheld blob by per-caller visibility. + /// RED before U2 (the current handler serves the secret to anon). + #[sqlx::test] + async fn ipfs_cid_gate_withholds_blob_from_unauthorized(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let reader = Keypair::generate(); + let reader_did = reader.did().to_string(); + let stranger = Keypair::generate(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["withhold"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("withhold.git"); + // Request CIDs are the production pin CIDs (content-hash), recorded in + // pinned_cids so get_by_cid resolves each back to its oid (#173). + let secret_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; + let tree_cid = pin_cid_for(&bare, &fx.secret_tree_oid, &state.db).await; + let public_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + let root_tree_cid = pin_cid_for(&bare, &fx.root_tree_oid, &state.db).await; + let public_tree_cid = pin_cid_for(&bare, &fx.public_tree_oid, &state.db).await; + let commit_cid = pin_cid_for(&bare, &fx.commit_oid, &state.db).await; + let tag_cid = pin_cid_for(&bare, &fx.tag_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "withhold")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "withhold") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + VisibilityMode::B, + std::slice::from_ref(&reader_did), + &owner_did, + ) + .await + .expect("deny rule"); + + // anon → withheld blob: must 404, must not leak content. (RED on current handler.) + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "anon must not read the withheld blob" ); assert!( !body.contains("TOP SECRET"), "404 body must not leak the secret" ); - // signed non-reader → 404. - let (st, body) = cid_parts( + // signed non-reader → 404. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&stranger, &secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "non-reader must not read the withheld blob" + ); + assert!(!body.contains("TOP SECRET")); + + // owner (signed) → 200 + secret bytes. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&owner, &secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "owner reads the withheld blob"); + assert!(body.contains("TOP SECRET"), "owner gets the content"); + + // listed reader (signed) → 200. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&reader, &secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "listed reader reads the blob"); + assert!(body.contains("TOP SECRET")); + + // #135: anon tree CID under withheld /secret → 404. The 404 body is an opaque + // error string (never the object), so status is the load-bearing deny check; + // the real leak witness is the CONTRAST with the reader below, who DOES get a + // 200 carrying the child structure that anon is denied. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "withheld subtree tree must not be served to anon (#135)" + ); + + // Over-denial guard + positive leak witness: the listed reader (signed) DOES + // read the withheld subtree's tree, and its body carries the exact child + // structure anon was denied — the child filename plus the child oid as the 32 + // RAW bytes a git tree stores (witnessed on raw bytes, since cid_parts's lossy + // decode would mangle them). This proves b.txt / secret_raw are the real leak + // markers and that the anon 404 above actually withheld them. + let secret_raw = hex::decode(&fx.secret_oid).expect("hex oid"); + let (st, body) = cid_bytes( + cid_router(&state) + .oneshot(cid_signed(&reader, &tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "listed reader reads the withheld subtree tree" + ); + assert!( + bytes_contain(&body, b"b.txt") && bytes_contain(&body, &secret_raw), + "reader's tree body carries the child filename and raw child oid" + ); + + // Root tree (path "/") stays served to anon who passes the "/" gate. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&root_tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "root tree stays served (must-serve)"); + + // /public subtree tree stays served to anon (allowed path). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&public_tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "public subtree tree stays served"); + + // Commit and annotated tag objects stay served (unchanged by #135). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&commit_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "commit object stays served"); + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&tag_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "tag object stays served"); + + // R3: public blob anon → 200 (non-withheld content not affected). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&public_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "public blob stays served"); + + // R5: a genuine unknown CID also 404, uniform with the withheld 404. A + // well-formed pin-style CID that was never recorded in pinned_cids, so the + // oid_for_cid resolve misses (the production not-found path). + let absent_cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"never pinned to this node").to_string(); + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&absent_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "absent CID 404 (uniform with withheld)" + ); + + // malformed CID → 400 (unchanged). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon("not-a-cid")) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::BAD_REQUEST, "malformed CID still 400"); + } + + /// R4: the same object withheld in one repo but public in another is still + /// served from the public copy; the withholding repo is iterated first. + #[sqlx::test] + async fn ipfs_cid_served_from_public_copy_when_withheld_elsewhere(pool: PgPool) { + use crate::db::VisibilityMode; + use chrono::Utc; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["withhold", "pubcopy"]); + // Same content in both clones -> same oid/CID; read from either. + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("withhold.git"); + let secret_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; + + // Withholding repo, iterated FIRST (later updated_at; list_all_repos is DESC). + let mut withhold = seed_repo(&owner_did, "withhold"); + withhold.updated_at = Utc::now(); + state + .db + .create_repo(&withhold) + .await + .expect("withhold repo"); + state + .db + .set_visibility_rule( + &withhold.id, + "/secret/**", + VisibilityMode::B, + &[], + &owner_did, + ) + .await + .expect("deny rule"); + + // Public copy, no rules, iterated AFTER. + let mut pubcopy = seed_repo(&owner_did, "pubcopy"); + pubcopy.updated_at = Utc::now() - chrono::Duration::seconds(60); + state.db.create_repo(&pubcopy).await.expect("pubcopy repo"); + + // anon: denied at the withholding repo (continue), served from the public copy. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "served from the public copy despite the other deny" + ); + assert!( + body.contains("TOP SECRET"), + "the public copy serves the content" + ); + } + + /// Repo-level "/" gate (KTD2a, first continue branch): a fully private repo + /// (is_public=false, no rules) denies anon before any per-blob check; the + /// owner still reads. The path-scoped tests pass the "/" gate and deny at the + /// per-blob stage, so this exercises the coarser repo-level deny separately. + #[sqlx::test] + async fn ipfs_cid_private_repo_denies_anon_at_repo_gate(pool: PgPool) { + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["priv"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("priv.git"); + let blob_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + let mut rec = seed_repo(&owner_did, "priv"); + rec.is_public = false; + state.db.create_repo(&rec).await.expect("private repo"); + + // anon → repo-level deny → 404, no content leaked. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&blob_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "anon denied at a private repo's / gate" + ); + assert!(!body.contains("public bytes"), "404 must not leak content"); + + // owner-signed → 200. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&owner, &blob_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "owner reads their private repo's object" + ); + assert!(body.contains("public bytes"), "owner gets the content"); + } + + /// Fail-closed walk-error arm: if `withheld_blob_oids` errors (here, a ref + /// pointing at a non-tree-ish blob, which `git ls-tree -r` cannot traverse — + /// the same induction as `visibility_pack::fails_closed_when_a_ref_cannot_be_traversed`), + /// the handler skips the whole repo rather than serving. Asserts no leak of the + /// withheld blob AND that even the *public* blob in that repo is withheld — the + /// latter distinguishes fail-closed-skip from normal per-blob withholding and + /// would serve 200 if the error arm wrongly proceeded. The skip carries no + /// VERDICT (F2), so the response is the retryable truncation 503, not a 404 + /// claiming the object is absent — never-serve-unproven and never-404-unproven + /// hold together. + #[sqlx::test] + async fn ipfs_cid_walk_error_fails_closed(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["withhold"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("withhold.git"); + // Recorded pins so get_by_cid resolves each CID to its oid and reaches the + // walk; the 404s below are then the fail-closed skip, not a table miss. + let secret_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; + let public_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + // Force the withheld walk to fail closed: a ref pointing at a blob (not + // tree-ish) makes `git ls-tree -r` error, which `withheld_blob_oids` + // propagates as Err → the handler's `Ok(Err)` arm skips the repo. + std::fs::write( + bare.join("refs/heads/blobref"), + format!("{}\n", fx.secret_oid), + ) + .unwrap(); + + state + .db + .create_repo(&seed_repo(&owner_did, "withhold")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "withhold") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("deny rule"); + + // Withheld secret CID under a walk error → the repo is skipped without a + // verdict, so the scan is truncated (503), and nothing leaks. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&secret_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::SERVICE_UNAVAILABLE, + "walk error must not serve the withheld blob — the unproven skip sheds 503" + ); + assert!( + !body.contains("TOP SECRET"), + "walk-error 503 must not leak the secret" + ); + + // The PUBLIC blob in the same repo is also not served: the walk error fails + // closed by skipping the whole repo. Without the fail-closed arm this would + // serve 200, so this assertion is the load-bearing discriminator. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&public_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::SERVICE_UNAVAILABLE, + "walk error fails closed: repo skipped without a verdict, even the public \ + blob is not served and the scan sheds 503" + ); + } + + /// #173 review (F2): the commit/tag reachability walk must FAIL CLOSED on a git + /// error, exactly like the blob/tree walk. A ref pointing at a nonexistent object + /// makes `rev-list --all` fail, so `reachable_commit_tag_oids` returns Err, which + /// the handler's shared `Ok(Err) => continue` arm turns into a repo skip. The + /// load-bearing discriminator is that the PUBLIC commit is ALSO 404: if the arm + /// fail-OPENed (served on error) it would 200. Drives the commit/tag branch of + /// the shared fail-closed arm specifically (the sibling test covers blob/tree). + #[sqlx::test] + async fn ipfs_cid_commit_tag_walk_error_fails_closed(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["cterr"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("cterr.git"); + // A reachable commit CID — would serve 200 if the walk succeeded. + let commit_cid = pin_cid_for(&bare, &fx.commit_oid, &state.db).await; + + // A ref to a NONEXISTENT object: `git rev-list --all` fails ("bad object"), + // so reachable_commit_tag_oids bails → the walk arm skips the repo. + std::fs::write( + bare.join("refs/heads/broken"), + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef\n", + ) + .unwrap(); + + state + .db + .create_repo(&seed_repo(&owner_did, "cterr")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "cterr") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + + // Fail-closed: a walk error skips the repo, so even the otherwise-reachable + // public commit is NOT served. A fail-OPEN arm would 200 here. + // + // The skip is a truncation, not an absence verdict (#174 F2): the walk failed, + // so nothing was proven about whether this caller may read the object, and the + // tail sheds a retryable 503 rather than the definitive 404 this asserted + // before the merge. Withholding is the property under test either way; what + // changed is that the response no longer claims the object is absent. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&commit_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::SERVICE_UNAVAILABLE, + "a commit/tag walk error must fail closed (repo skipped), never serve" + ); + } + + /// #126: a dangling blob (written via `git hash-object -w`, never referenced + /// by any commit/tree) must 404 through `GET /ipfs/{cid}` under path-scoped + /// rules — for anon AND the owner. The pre-#126 deny-set was fail-open by + /// construction: dangling oids were absent from the reachable enumeration + /// and thus absent from the deny-set, so the handler served 200. The + /// allowed-set is fail-closed: dangling oids are absent from the reachable + /// allowed-set, so the handler 404s (per team memory: the owner shift to + /// 404 is the accepted fail-closed default — owners can still + /// `git cat-file` directly). + #[sqlx::test] + async fn ipfs_cid_dangling_blob_fails_closed_under_path_rules(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + // Seed a normal repo with `secret/b.txt` reachable from HEAD, so the + // path-scoped rule has something to match — without this the rule has + // no anchor and we'd be testing nothing. + let _fx = seed_cid_repos(&slug, &short, &["dangling"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("dangling.git"); + + // Write a dangling blob: `git hash-object -w --stdin` adds it to the + // object DB but nothing references it, so the reachable walk never + // enumerates it. + let mut cmd = std::process::Command::new("git"); + cmd.args(["hash-object", "-w", "--stdin"]) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()); + let mut child = cmd.spawn().expect("spawn git hash-object"); + { + use std::io::Write; + let stdin = child.stdin.as_mut().expect("stdin"); + stdin.write_all(b"DANGLING SECRET\n").expect("write stdin"); + } + let out = child.wait_with_output().expect("hash-object output"); + assert!( + out.status.success(), + "git hash-object: {}", + String::from_utf8_lossy(&out.stderr) + ); + let dangling_oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); + // Sanity: must be a 64-hex sha256 oid, since the repo is sha256-format. + assert_eq!( + dangling_oid.len(), + 64, + "expected sha256 oid: {dangling_oid}" + ); + // Record the pin so oid_for_cid resolves it — the 404 must then come from + // the allowed-set gate excluding the dangling oid, not from a table miss. + let dangling_cid = pin_cid_for(&bare, &dangling_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "dangling")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "dangling") + .await + .unwrap() + .unwrap(); + // Path-scoped rule triggers the per-blob allowed-set gate (KTD4). + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("deny rule"); + + // anon: the dangling blob is absent from the reachable allowed-set → + // 404, no leak. Pre-#126 (deny-set) would serve 200. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&dangling_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "dangling blob must 404 under path-scoped rules" + ); + assert!( + !body.contains("DANGLING SECRET"), + "404 body must not leak the dangling content" + ); + + // owner (signed): same 404. The dangling blob has no path, so it's + // never visibility-checked → never in the allowed set, even for the + // owner. This is the accepted fail-closed shift documented in the PR. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed(&owner, &dangling_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "owner also 404s on dangling blobs under path-scoped rules (fail-closed default)" + ); + assert!(!body.contains("DANGLING SECRET")); + } + + /// #135: a DANGLING tree (in the ODB, referenced by no commit) 404s under + /// path-scoped rules for anon AND owner — the reachable-only allowed-tree-set + /// never enumerates it. Handler-level companion to the helper test + /// `allowed_tree_set_excludes_dangling_tree`, proving the `get_by_cid` tree arm + /// (memo insert + `!in_allowed` continue) fails closed on the dangling case. + #[sqlx::test] + async fn ipfs_cid_dangling_tree_fails_closed_under_path_rules(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["dangtree"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("dangtree.git"); + + // Dangling tree via `git mktree`: a UNIQUE entry name so its oid is + // content-distinct from every reachable tree (a content-identical tree would + // dedup to a reachable oid — that is T2, not danglingness). + let mut child = std::process::Command::new("git") + .args(["mktree"]) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .expect("spawn git mktree"); + { + use std::io::Write; + writeln!( + child.stdin.as_mut().unwrap(), + "100644 blob {}\tdangling-only-unreferenced.txt", + fx.secret_oid + ) + .unwrap(); + } + let out = child.wait_with_output().expect("mktree output"); + assert!( + out.status.success(), + "git mktree: {}", + String::from_utf8_lossy(&out.stderr) + ); + let dangling_tree_oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); + assert_eq!(dangling_tree_oid.len(), 64, "expected sha256 oid"); + // Record the pin so the 404 is the allowed-tree-set gate excluding the + // dangling tree, not a table miss. + let dangling_cid = pin_cid_for(&bare, &dangling_tree_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "dangtree")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "dangtree") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("deny rule"); + + for req in [cid_anon(&dangling_cid), cid_signed(&owner, &dangling_cid)] { + let (st, _) = cid_parts(cid_router(&state).oneshot(req).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "dangling tree must 404 under path-scoped rules (anon + owner)" + ); + } + } + + /// #173 (F1): a QUARANTINED repo must not serve a pinned object by CID, to anon + /// OR to the mirror's own owner — quarantine is "hidden from serve/clone/listings, + /// owner included" (authorize_repo_read / feed_quarantined_mirror_withheld_from_owner). + /// The repo is PUBLIC with no path-scoped rule, so the "/" visibility gate ALLOWS + /// it and quarantine is the sole possible denier: RED before the fix (the loop + /// never checks quarantine → serves 200 + bytes), GREEN after the quarantine skip. + /// The owner-signed 404 is the load-bearing negative — a visibility-only gate + /// would Allow the owner and miss this. + #[sqlx::test] + async fn ipfs_cid_quarantined_repo_withheld_from_anon_and_owner(pool: PgPool) { + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["quar"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("quar.git"); + // Pin a ROOT-readable object (public/a.txt) — no path-scoped rule, so only + // quarantine can deny it. + let public_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "quar")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "quar") + .await + .unwrap() + .unwrap(); + + // Baseline: before quarantine the object serves 200 (proves the CID resolves + // and the object is otherwise servable, so the 404 below is quarantine's doing). + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&public_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "public root object serves before quarantine" + ); + assert!(body.contains("public bytes"), "baseline serves the content"); + + // Quarantine it. + state + .db + .set_repo_quarantine(&rec.id, true) + .await + .expect("quarantine"); + + // anon AND owner-signed must both 404 with no content leak. + for req in [cid_anon(&public_cid), cid_signed(&owner, &public_cid)] { + let (st, body) = cid_parts(cid_router(&state).oneshot(req).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "quarantined repo must not serve by CID (anon + owner)" + ); + assert!( + !body.contains("public bytes"), + "404 body must not leak quarantined content" + ); + } + } + + /// #173 (F2): a DANGLING commit or annotated tag (in the ODB, referenced by no + /// ref) must 404 under path-scoped rules for anon AND owner. The resolver proved + /// reachability only for blobs/trees, so a dangling commit/tag fell through to + /// serve, leaking its message/metadata. RED before the fix (serves 200 + + /// sentinel), GREEN after (the reachable commit/tag set excludes them). The + /// reachable-commit/tag serve path is covered by + /// ipfs_cid_gate_withholds_blob_from_unauthorized (commit + annotated tag → 200). + #[sqlx::test] + async fn ipfs_cid_dangling_commit_and_tag_fail_closed_under_path_rules(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["dangct"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("dangct.git"); + + // Run a git plumbing command that reads from stdin and prints an oid. + let oid_from_stdin = |args: &[&str], input: &[u8]| -> String { + use std::io::Write; + let mut child = std::process::Command::new("git") + .args(args) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("spawn git"); + child.stdin.as_mut().unwrap().write_all(input).unwrap(); + let out = child.wait_with_output().expect("git output"); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + + // Dangling commit: commit-tree with a sentinel message, NO ref update. + let dangling_commit_oid = oid_from_stdin( + &["commit-tree", &fx.root_tree_oid], + b"DANGLING COMMIT SECRET\n", + ); + assert_eq!(dangling_commit_oid.len(), 64, "expected sha256 commit oid"); + // Dangling annotated tag: mktag of the dangling commit, NO ref. + let tag_body = format!( + "object {dangling_commit_oid}\ntype commit\ntag dang\ntagger t 0 +0000\n\nDANGLING TAG SECRET\n" + ); + let dangling_tag_oid = oid_from_stdin(&["mktag"], tag_body.as_bytes()); + assert_eq!(dangling_tag_oid.len(), 64, "expected sha256 tag oid"); + + let commit_cid = pin_cid_for(&bare, &dangling_commit_oid, &state.db).await; + let tag_cid = pin_cid_for(&bare, &dangling_tag_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "dangct")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "dangct") + .await + .unwrap() + .unwrap(); + // Path-scoped rule triggers the per-object gate (KTD4). + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("deny rule"); + + for (cid, sentinel) in [ + (&commit_cid, "DANGLING COMMIT SECRET"), + (&tag_cid, "DANGLING TAG SECRET"), + ] { + for req in [cid_anon(cid), cid_signed(&owner, cid)] { + let (st, body) = cid_parts(cid_router(&state).oneshot(req).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "dangling commit/tag must 404 under path-scoped rules (anon + owner)" + ); + assert!( + !body.contains(sentinel), + "404 body must not leak the dangling message: {sentinel}" + ); + } + } + } + + /// #173 review (F2 hardening): a REACHABLE commit must still serve under a + /// path-scoped rule even when the repo carries a pushable non-commit ref (an + /// annotated tag of a tree, accepted by receive-pack). `reachable_commit_tag_oids` + /// must NOT route through `assert_all_refs_are_commits` (which bails on such a + /// ref and would fail-closed 404 every reachable commit/tag CID in the repo). + /// RED before the decoupling (the guard bails → 404), GREEN after. + #[sqlx::test] + async fn ipfs_cid_reachable_commit_served_despite_non_commit_ref(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["weirdref"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("weirdref.git"); + + // A pushable non-commit ref: an annotated tag pointing at a TREE. `git tag -a` + // in the bare repo creates refs/tags/treetag -> tag object -> tree, which + // peels to a non-commit and makes assert_all_refs_are_commits bail. + let out = std::process::Command::new("git") + .args([ + "tag", + "-a", + "treetag", + &fx.root_tree_oid, + "-m", + "tag of a tree", + ]) + .current_dir(&bare) + .output() + .expect("git tag -a"); + assert!( + out.status.success(), + "git tag -a: {}", + String::from_utf8_lossy(&out.stderr) + ); + + // Pin the REACHABLE root commit. + let commit_cid = pin_cid_for(&bare, &fx.commit_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "weirdref")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "weirdref") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + + // The reachable commit must still serve — the non-commit ref must not + // fail-closed the whole repo's commit/tag CID retrieval. + let resp = cid_router(&state) + .oneshot(cid_anon(&commit_cid)) + .await + .unwrap(); + let served_hash = resp + .headers() + .get("x-git-hash") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + let (st, _body) = cid_parts(resp).await; + assert_eq!( + st, + StatusCode::OK, + "a reachable commit must serve despite a pushable non-commit ref in the repo" + ); + assert_eq!( + served_hash.as_deref(), + Some(fx.commit_oid.as_str()), + "the served object is the reachable root commit" + ); + } + + /// #173 review (F-F): an annotated tag pointing at a TREE is pushable through + /// receive-pack, and the TREE allowed-set path + /// (`allowed_tree_set_for_caller` -> `tree_paths` -> `reachable_commits`) runs + /// `assert_all_refs_are_commits`, which bails on that ref and fail-closes the + /// whole repo — 404-ing EVERY tree CID (root + public subtrees) for its owner + /// and readers, not just the offending tag. The tree allowed-set feeds ONLY the + /// CID gate (absence = fail-closed 404), so `tree_paths` uses the lenient + /// reachable-commit enumeration: commit-reachable trees still serve, while a + /// tree reachable only via such a tag stays excluded. `blob_paths` keeps the + /// strict guard (it also feeds serve/replication, where a miss under-withholds). + /// RED before the decoupling (whole-repo bail -> 404 on the root/public tree), + /// GREEN after; the withheld-subtree 404 is the load-bearing must-not. + #[sqlx::test] + async fn ipfs_cid_tree_served_despite_non_commit_ref(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["treeweird"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("treeweird.git"); + + // Pushable non-commit ref: an annotated tag pointing at the ROOT TREE. + let out = std::process::Command::new("git") + .args([ + "tag", + "-a", + "treetag", + &fx.root_tree_oid, + "-m", + "tag of a tree", + ]) + .current_dir(&bare) + .output() + .expect("git tag -a"); + assert!( + out.status.success(), + "git tag -a: {}", + String::from_utf8_lossy(&out.stderr) + ); + + // Pin the reachable root tree and public subtree (both at ALLOWED paths), + // plus the secret subtree (a DENIED path — the fail-closed negative). + let root_tree_cid = pin_cid_for(&bare, &fx.root_tree_oid, &state.db).await; + let public_tree_cid = pin_cid_for(&bare, &fx.public_tree_oid, &state.db).await; + let secret_tree_cid = pin_cid_for(&bare, &fx.secret_tree_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "treeweird")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "treeweird") + .await + .unwrap() + .unwrap(); + // Path-scoped rule triggers the per-object tree gate (KTD4). + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + + // Reachable trees at ALLOWED paths must still serve despite the tag-of-tree. + for (cid, want_oid, label) in [ + (&root_tree_cid, &fx.root_tree_oid, "root tree"), + (&public_tree_cid, &fx.public_tree_oid, "public subtree"), + ] { + let resp = cid_router(&state).oneshot(cid_anon(cid)).await.unwrap(); + let served = resp + .headers() + .get("x-git-hash") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + let (st, _) = cid_parts(resp).await; + assert_eq!( + st, + StatusCode::OK, + "{label} CID must serve despite a pushable tag-of-tree in the repo" + ); + assert_eq!( + served.as_deref(), + Some(want_oid.as_str()), + "{label}: the served object is the reachable tree" + ); + } + + // Fail-closed preserved: the DENIED subtree's CID is still withheld — the + // lenient walk must not under-withhold a path the caller cannot read. + let (st, _) = cid_parts( cid_router(&state) - .oneshot(cid_signed(&stranger, &secret_cid)) + .oneshot(cid_anon(&secret_tree_cid)) .await .unwrap(), ) @@ -3094,82 +8179,524 @@ mod tests { assert_eq!( st, StatusCode::NOT_FOUND, - "non-reader must not read the withheld blob" + "a withheld subtree's tree CID stays 404 (lenient walk must not under-withhold)" ); - assert!(!body.contains("TOP SECRET")); + } - // owner (signed) → 200 + secret bytes. - let (st, body) = cid_parts( + /// #173 review (F2 hardening): the INNER tag object of a nested tag-of-a-tag is + /// reachable (via the outer ref tag) and pinnable, so its CID must serve under a + /// path rule. `reachable_commit_tag_oids` peels tag chains to include it. RED + /// before the peel loop (the inner tag is not a ref tip and rev-list dereferences + /// to the commit, so it is absent → 404), GREEN after. + #[sqlx::test] + async fn ipfs_cid_nested_tag_inner_object_served(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["nested"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("nested.git"); + + let git_stdin = |args: &[&str], input: &[u8]| -> String { + use std::io::Write; + let mut child = std::process::Command::new("git") + .args(args) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("spawn git"); + child.stdin.as_mut().unwrap().write_all(input).unwrap(); + let out = child.wait_with_output().expect("git output"); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + + // Inner annotated tag of the reachable commit (no ref of its own). + let inner_body = format!( + "object {}\ntype commit\ntag inner\ntagger t 0 +0000\n\ninner\n", + fx.commit_oid + ); + let inner_tag_oid = git_stdin(&["mktag"], inner_body.as_bytes()); + // Outer annotated tag of the inner tag, then a ref to the outer tag. The + // inner tag is reachable only THROUGH the outer, not as a ref tip. + let outer_body = format!( + "object {inner_tag_oid}\ntype tag\ntag outer\ntagger t 0 +0000\n\nouter\n" + ); + let outer_tag_oid = git_stdin(&["mktag"], outer_body.as_bytes()); + let out = std::process::Command::new("git") + .args(["update-ref", "refs/tags/nested", &outer_tag_oid]) + .current_dir(&bare) + .output() + .expect("update-ref"); + assert!( + out.status.success(), + "update-ref: {}", + String::from_utf8_lossy(&out.stderr) + ); + + let inner_cid = pin_cid_for(&bare, &inner_tag_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "nested")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "nested") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + + let (st, _) = cid_parts( cid_router(&state) - .oneshot(cid_signed(&owner, &secret_cid)) + .oneshot(cid_anon(&inner_cid)) .await .unwrap(), ) .await; - assert_eq!(st, StatusCode::OK, "owner reads the withheld blob"); - assert!(body.contains("TOP SECRET"), "owner gets the content"); + assert_eq!( + st, + StatusCode::OK, + "the inner tag of a nested tag-of-a-tag is reachable and must serve" + ); + } + + /// #135: with NO path-scoped rule the per-object gate is skipped, so a tree CID + /// is served (the `"/"` gate is the whole story). Guards against over-gating + /// trees — the tree analog of the blob skip-walk branch. + #[sqlx::test] + async fn ipfs_cid_tree_served_when_no_path_scoped_rule(pool: PgPool) { + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["nopathrule"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("nopathrule.git"); + let tree_cid = pin_cid_for(&bare, &fx.secret_tree_oid, &state.db).await; + + // Public repo, no visibility rules → has_path_scoped_rule is false. + state + .db + .create_repo(&seed_repo(&owner_did, "nopathrule")) + .await + .expect("seed repo"); + + let (st, body) = cid_bytes( + cid_router(&state) + .oneshot(cid_anon(&tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "tree served to anon when no path-scoped rule exists" + ); + assert!( + bytes_contain(&body, b"b.txt"), + "served tree carries its child structure" + ); + } + + /// #173 (Fix 1): the pinned_cids lookup must use the canonical base32 CID, not + /// the raw request spelling. A pin is stored under `cid.to_string()` (canonical + /// base32); a request carrying the SAME CID re-encoded to a different multibase + /// (base58btc) parses and passes the sha2-256 check but, on the pre-fix handler, + /// misses the lookup key → false 404. Public repo, no path-scoped rule, so no + /// walk — this isolates the lookup-key canonicalization. + #[sqlx::test] + async fn ipfs_alt_encoding_cid_resolves(pool: PgPool) { + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["altenc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("altenc.git"); + // Canonical base32 CID as stored by the pin path. + let public_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + // Public repo, no visibility rules (no path-scoped walk). + state + .db + .create_repo(&seed_repo(&owner_did, "altenc")) + .await + .expect("seed repo"); + + // Re-encode the SAME CID to base58btc — a different, equally-valid spelling + // that is NOT the stored key. The `cid` crate re-exports `multibase`. + let alt = public_cid + .parse::>() + .unwrap() + .to_string_of_base(cid::multibase::Base::Base58Btc) + .unwrap(); + assert_ne!(alt, public_cid, "alt encoding must differ from canonical"); + + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&alt)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "alt-multibase spelling of a pinned CID must resolve (canonicalized lookup)" + ); + assert!( + body.contains("public bytes"), + "resolved object serves its content" + ); + } + + /// #173 (Fix 2a, db-level): `oids_for_cid` returns EVERY oid recorded under a + /// CID, not an arbitrary one. `record_pinned_cid` is unique on the git oid and + /// non-unique on cid, so two distinct oids can share one content-CID. Old + /// `oid_for_cid` did `LIMIT 1`; the new plural method must surface both. + #[sqlx::test] + async fn oids_for_cid_returns_all_duplicates(pool: PgPool) { + let state = test_state(pool).await; + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"shared content cid").to_string(); + let oid_a = "a".repeat(64); + let oid_b = "b".repeat(64); + state + .db + .record_pinned_cid(&oid_a, &cid, None) + .await + .unwrap(); + state + .db + .record_pinned_cid(&oid_b, &cid, None) + .await + .unwrap(); + + let mut oids = state.db.oids_for_cid(&cid).await.unwrap(); + oids.sort(); + assert_eq!( + oids, + vec![oid_a, oid_b], + "oids_for_cid must return every oid recorded under the shared CID" + ); + } + + /// #173 (Fix 2b, handler-level): when two oids collide on one CID and the + /// first-recorded is absent from every repo while the second is a readable + /// public object, the handler must try both and serve the readable one. The + /// pre-fix handler resolved a single oid (LIMIT 1 → first-inserted for equal + /// keys) and 404'd. Ordering caveat: this relies on `oids_for_cid` returning + /// the absent oid before the readable one (heap/insert order for equal keys); + /// if that ordering ever changes, `oids_for_cid_returns_all_duplicates` remains + /// the load-bearing, deterministic driver for Fix 2. + #[sqlx::test] + async fn ipfs_cid_collision_serves_readable_duplicate(pool: PgPool) { + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["collision"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("collision.git"); + + // A GENUINE content collision: the shared CID is the readable object's REAL + // content CID, and a second (absent) oid is recorded under the SAME cid. The + // handler must try every oid and serve the one whose bytes hash to the CID. + // (F2, #173: the served bytes must match the requested content address, so the + // shared cid has to be the object's real cid — an arbitrary seed would now be + // withheld by the integrity check as an unverifiable provider-CID-style row.) + let (_ty, raw) = crate::git::store::read_object(&bare, &fx.public_oid) + .unwrap() + .unwrap(); + let shared_cid = gitlawb_core::cid::Cid::from_git_object_bytes(&raw).to_string(); + let absent_oid = "c".repeat(64); + state + .db + .record_pinned_cid(&absent_oid, &shared_cid, None) + .await + .expect("record absent oid first"); + state + .db + .record_pinned_cid(&fx.public_oid, &shared_cid, None) + .await + .expect("record readable oid second"); + + // Public repo, no rules → the readable public object is served if reached. + state + .db + .create_repo(&seed_repo(&owner_did, "collision")) + .await + .expect("seed repo"); - // listed reader (signed) → 200. let (st, body) = cid_parts( cid_router(&state) - .oneshot(cid_signed(&reader, &secret_cid)) + .oneshot(cid_anon(&shared_cid)) .await .unwrap(), ) .await; - assert_eq!(st, StatusCode::OK, "listed reader reads the blob"); - assert!(body.contains("TOP SECRET")); + assert_eq!( + st, + StatusCode::OK, + "handler must try every oid under the CID and serve the readable duplicate" + ); + assert!( + body.contains("public bytes"), + "the readable duplicate's content is served" + ); + } + + /// #173 (Fix 3/F3, INV-10): the expensive legacy fan-out is rate-limited per + /// source IP. A valid tree CID makes the object-type pre-check pass, so each + /// repeat request pays a fresh walk (request-scoped memo only) — unbounded + /// amplification. Since #173-F3 (jatmn) the source charge sits on the LEGACY + /// PROBE (`acquire` + `cat-file`), which precedes the walk, so every legacy + /// candidate is charged to the non-farmable source IP from the first probe; a + /// second identical request from the same IP is shed with 429, but a targeted + /// PROVENANCE fetch (no scan) and a request from a different IP are unaffected. + /// The limiter is sized to admit one full scan of the two seeded repos (2 probes) + /// so the first request serves; the repeat then finds the bucket spent. + #[sqlx::test] + async fn ipfs_walk_rate_limited_per_source(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let reader = Keypair::generate(); + let reader_did = reader.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + + let mut state = test_state(pool).await; + // The scan probes both seeded repos (walklimit + walkpublic) per request, so + // size the per-IP budget to admit exactly one full scan (2 probes). A repeat + // scan from the same IP then finds the bucket spent. Keyed on the rightmost + // X-Forwarded-For hop so the test can choose a source IP under `oneshot`. + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(2, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let fx = seed_cid_repos(&slug, &short, &["walklimit"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("walklimit.git"); + // The tree CID drives a path-scoped walk (the load-bearing amplification + // surface). The reader is allowed under /secret so the walk returns 200. + let secret_tree_cid = pin_cid_for(&bare, &fx.secret_tree_oid, &state.db).await; + + // Oldest `updated_at` → `list_all_repos` (ORDER BY updated_at DESC) probes + // this serving repo LAST, so a scan deterministically charges the walk-free + // `walkpublic` miss first then this serve: exactly 2 probes per scan. + let mut walklimit = seed_repo(&owner_did, "walklimit"); + walklimit.updated_at = chrono::Utc::now() - chrono::Duration::seconds(60); + state.db.create_repo(&walklimit).await.expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "walklimit") + .await + .unwrap() + .unwrap(); + // Mode B path rule over /secret with the reader allowed → the reader's + // secret-tree fetch runs the allowed-tree walk and returns 200. + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + VisibilityMode::B, + std::slice::from_ref(&reader_did), + &owner_did, + ) + .await + .expect("path rule"); + + // The MUST-NOT object must be a genuinely CHEAP fetch: an object served + // from a repo with NO path-scoped rule takes the no-walk path, so the WALK + // brake never rate-limits it. It has to live in a repo that carries no path + // rule AND whose object graph does not overlap `walklimit` (a blob shared + // with the path-scoped repo would still walk there), so we seed a second bare + // repo with UNIQUE content. `acquire(owner, "walkpublic")` resolves to + // `/tmp//walkpublic.git`. This copy is PROVENANCED (`pin_cid_for_repo`) + // so it resolves straight to its repo and skips the legacy probe brake: the + // point here is the WALK brake, and post-#173-F3 a walk-free LEGACY fetch is + // itself source-charged at the probe, so a legacy pin would (correctly) be + // shed from the exhausted IP and no longer isolate the walk brake. + let pub_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("walkpublic.git"); + { + use std::process::Command; + let run = |args: &[&str], cwd: &std::path::Path| { + let out = Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + }; + let src = std::env::temp_dir().join(format!("gl-cid-pub-{short}")); + let _ = std::fs::remove_dir_all(&src); + std::fs::create_dir_all(&src).unwrap(); + std::fs::write(src.join("cheap.txt"), b"cheap public bytes\n").unwrap(); + run(&["init", "-q", "--object-format=sha256"], &src); + run(&["config", "user.email", "t@t"], &src); + run(&["config", "user.name", "t"], &src); + run(&["add", "."], &src); + run(&["commit", "-qm", "cheap"], &src); + let _ = std::fs::remove_dir_all(&pub_bare); + run( + &[ + "clone", + "--bare", + "-q", + src.to_str().unwrap(), + pub_bare.to_str().unwrap(), + ], + &src, + ); + let _ = std::fs::remove_dir_all(&src); + } + let cheap_oid = { + use std::process::Command; + let out = Command::new("git") + .args(["rev-parse", "HEAD:cheap.txt"]) + .current_dir(&pub_bare) + .output() + .unwrap(); + assert!(out.status.success(), "rev-parse cheap.txt"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + // Public repo, NO visibility rules → the cheap object takes the no-walk path. + state + .db + .create_repo(&seed_repo(&owner_did, "walkpublic")) + .await + .expect("seed public repo"); + let pub_rec = state + .db + .get_repo(&owner_did, "walkpublic") + .await + .unwrap() + .unwrap(); + let public_cid = pin_cid_for_repo(&pub_bare, &cheap_oid, &state.db, &pub_rec.id).await; - // KTD3: anon tree CID under /secret → 200 (trees/commits are not withheld). + // 1st legacy scan from 1.2.3.4 → 200 (its two probes fit the budget; the + // walk ran, reader allowed). let (st, _) = cid_parts( cid_router(&state) - .oneshot(cid_anon(&tree_cid)) + .oneshot(cid_signed_xff(&reader, &secret_tree_cid, "1.2.3.4")) .await .unwrap(), ) .await; - assert_eq!(st, StatusCode::OK, "tree object is served to anon (KTD3)"); + assert_eq!( + st, + StatusCode::OK, + "1st legacy scan from a source IP is served" + ); - // R3: public blob anon → 200 (non-withheld content not affected). + // 2nd identical scan from the SAME IP → 429 (per-IP probe budget spent). let (st, _) = cid_parts( cid_router(&state) - .oneshot(cid_anon(&public_cid)) + .oneshot(cid_signed_xff(&reader, &secret_tree_cid, "1.2.3.4")) .await .unwrap(), ) .await; - assert_eq!(st, StatusCode::OK, "public blob stays served"); + assert_eq!( + st, + StatusCode::TOO_MANY_REQUESTS, + "2nd legacy scan from the same source IP is shed with 429" + ); - // R5: a genuine unknown CID also 404, uniform with the withheld 404. - let absent_cid = cid_for_oid(&"ab".repeat(32)); - let (st, _) = cid_parts( + // MUST-NOT: a targeted PROVENANCE fetch (no scan, no probe brake) from the + // SAME limited IP, even after the 429, is served: the brake is on the legacy + // scan, not the route. + let (st, body) = cid_parts( cid_router(&state) - .oneshot(cid_anon(&absent_cid)) + .oneshot(cid_signed_xff(&reader, &public_cid, "1.2.3.4")) .await .unwrap(), ) .await; assert_eq!( st, - StatusCode::NOT_FOUND, - "absent CID 404 (uniform with withheld)" + StatusCode::OK, + "a provenance (non-scan) fetch is never rate-limited, even from the exhausted IP" + ); + assert!( + body.contains("cheap public bytes"), + "the cheap fetch serves content" ); - // malformed CID → 400 (unchanged). + // PER-SOURCE isolation: the same tree-CID scan from a DIFFERENT IP → 200. let (st, _) = cid_parts( cid_router(&state) - .oneshot(cid_anon("not-a-cid")) + .oneshot(cid_signed_xff(&reader, &secret_tree_cid, "5.6.7.8")) .await .unwrap(), ) .await; - assert_eq!(st, StatusCode::BAD_REQUEST, "malformed CID still 400"); + assert_eq!( + st, + StatusCode::OK, + "one source's exhaustion must not shed another source's walk" + ); } - /// R4: the same object withheld in one repo but public in another is still - /// served from the public copy; the withholding repo is iterated first. + /// #173 review (F-C): a SKIPPED legacy candidate (a walk-and-deny denier, OR a + /// probe-throttled repo since #173-F3) must not end the whole request: the scan + /// keeps going so a later walk-free copy still serves, and a spent probe budget is + /// a clean 429, never a false 404/503. Otherwise a public CID would 404/429 solely + /// because a newer path-scoped duplicate sorts ahead of an older no-rule copy under + /// `updated_at DESC`. Two same-oid legacy copies: a NEWER `/secret`-scoped denier + /// and an OLDER no-rule public copy. + /// + /// Two requests from the SAME IP, budget = 2 (one full scan of both copies): + /// req1 probes the denier (charged), its allowed-blob walk denies anon → skip and + /// keep scanning, then probes+serves the walk-free public copy → 200. That proves + /// the denier skip is non-fatal (`continue`, not `break`). req2 from the same IP + /// finds the probe budget spent, so the denier's probe throttles → skip-continue, + /// the public copy's probe throttles too → nothing servable → a clean 429 (not a + /// truncation 503 nor a false 404), proving the throttle is likewise non-fatal but + /// correctly shed. RED before `continue` (a `break` on the skipped denier 404s + /// req1 outright). #[sqlx::test] - async fn ipfs_cid_served_from_public_copy_when_withheld_elsewhere(pool: PgPool) { + async fn ipfs_walk_quota_skips_denier_and_serves_public_copy(pool: PgPool) { use crate::db::VisibilityMode; use chrono::Utc; use gitlawb_core::identity::Keypair; @@ -3178,61 +8705,109 @@ mod tests { let owner_did = owner.did().to_string(); let slug = owner_did.replace([':', '/'], "_"); let short = owner_did.split(':').next_back().unwrap().to_string(); - let state = test_state(pool).await; - - let fx = seed_cid_repos(&slug, &short, &["withhold", "pubcopy"]); - let secret_cid = cid_for_oid(&fx.secret_oid); - - // Withholding repo, iterated FIRST (later updated_at; list_all_repos is DESC). - let mut withhold = seed_repo(&owner_did, "withhold"); - withhold.updated_at = Utc::now(); + let mut state = test_state(pool).await; + // Budget = one full two-repo scan (2 probes), keyed on the rightmost XFF hop + // so `oneshot` can choose a source IP (no socket peer). A repeat scan from the + // same IP then finds the budget spent. + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(2, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + // Identical secret-blob content in both bare clones → one CID resolves to + // `secret_oid` in each. A NEWER path-scoped denier (walk-and-deny anon) and an + // OLDER no-rule public copy (walk-free serve). + let fx = seed_cid_repos(&slug, &short, &["scopeddenier", "publiccopy"]); + let denier_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("scopeddenier.git"); + let secret_cid = pin_cid_for(&denier_bare, &fx.secret_oid, &state.db).await; + + // Newer denier: public at "/", `/secret/**` Mode B empty readers → an anon + // blob fetch clears "/", runs the allowed-blob walk, is denied → continue. + let mut denier = seed_repo(&owner_did, "scopeddenier"); + denier.updated_at = Utc::now(); + state.db.create_repo(&denier).await.expect("seed denier"); state .db - .create_repo(&withhold) + .set_visibility_rule(&denier.id, "/secret/**", VisibilityMode::B, &[], &owner_did) .await - .expect("withhold repo"); + .expect("path rule"); + + // Older public copy — NO rule → the secret blob serves via the no-walk path. + let mut public = seed_repo(&owner_did, "publiccopy"); + public.updated_at = Utc::now() - chrono::Duration::seconds(60); state .db - .set_visibility_rule( - &withhold.id, - "/secret/**", - VisibilityMode::B, - &[], - &owner_did, - ) + .create_repo(&public) .await - .expect("deny rule"); + .expect("seed public copy"); - // Public copy, no rules, iterated AFTER. - let mut pubcopy = seed_repo(&owner_did, "pubcopy"); - pubcopy.updated_at = Utc::now() - chrono::Duration::seconds(60); - state.db.create_repo(&pubcopy).await.expect("pubcopy repo"); + // req1 from 1.2.3.4: the denier is skipped (walk denies anon) and the scan + // keeps going to serve the older walk-free public copy. Both probes fit the + // budget, so this leaves the IP bucket spent. + let resp = cid_router(&state) + .oneshot(cid_anon_xff(&secret_cid, "1.2.3.4")) + .await + .unwrap(); + let served_hash = resp + .headers() + .get("x-git-hash") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + let (st, _body) = cid_parts(resp).await; + assert_eq!( + st, + StatusCode::OK, + "a skipped walk-requiring denier must not end the scan: the later walk-free public copy still serves" + ); + assert_eq!( + served_hash.as_deref(), + Some(fx.secret_oid.as_str()), + "the served object is the secret blob from the no-rule public copy" + ); - // anon: denied at the withholding repo (continue), served from the public copy. - let (st, body) = cid_parts( + // req2 from the SAME exhausted IP: every legacy probe is now throttled. The + // throttle is non-fatal (skip and keep scanning), but nothing is servable, so + // it resolves to a clean 429, not a truncation 503, not a false 404. + let (st, _) = cid_parts( cid_router(&state) - .oneshot(cid_anon(&secret_cid)) + .oneshot(cid_anon_xff(&secret_cid, "1.2.3.4")) .await .unwrap(), ) .await; assert_eq!( st, - StatusCode::OK, - "served from the public copy despite the other deny" - ); - assert!( - body.contains("TOP SECRET"), - "the public copy serves the content" + StatusCode::TOO_MANY_REQUESTS, + "with the probe budget spent, the repeat legacy scan is shed with a clean 429" ); } - /// Repo-level "/" gate (KTD2a, first continue branch): a fully private repo - /// (is_public=false, no rules) denies anon before any per-blob check; the - /// owner still reads. The path-scoped tests pass the "/" gate and deny at the - /// per-blob stage, so this exercises the coarser repo-level deny separately. + /// INV-10 amplification bound: a single `GET /ipfs/{cid}` must not fan out an + /// unbounded number of full-history walks. The route brake (`ipfs_rate_limiter`) + /// fires once per request and the per-walk `ipfs_work_rate_limiter` charge bounds + /// walk work across requests, but within ONE request the same object can exist under + /// path-scoped rules in many repos, each paying its own walk. + /// `MAX_HISTORY_WALKS_PER_REQUEST` caps that fan-out. + /// + /// Load-bearing witness (#173, F4): a readable public copy (no path rule → + /// served via the no-walk path, exactly like + /// `ipfs_cid_served_from_public_copy_when_withheld_elsewhere`) is given the + /// OLDEST `updated_at` so `list_all_repos` (ORDER BY updated_at DESC) iterates it + /// LAST. Ahead of it sit `cap + 1` path-scoped deniers, each forcing an + /// allowed-blob walk that denies anon. The cap bounds SPAWNED walks to `cap`, but + /// hitting it must `continue` (skip only the walk-requiring denier), NOT `break` + /// the whole repo loop: the walk-free public copy needs no walk, so it is still + /// reached and served (200, `x-git-hash` = the blob oid). The old `break` + /// wrongly 404'd this publicly-readable content. Reverting `continue`→`break` + /// turns this 200 back into a 404: the RED proof that the loop keeps scanning for + /// a cheap readable copy after the cap. The `cap` walk ceiling still holds — only + /// `cap` walks are spawned across the deniers regardless (the amplification bound + /// is proven separately by `ipfs_walk_cap_still_serves_walk_free_candidate`). #[sqlx::test] - async fn ipfs_cid_private_repo_denies_anon_at_repo_gate(pool: PgPool) { + async fn ipfs_walk_fanout_capped_per_request(pool: PgPool) { + use crate::db::VisibilityMode; + use chrono::Utc; use gitlawb_core::identity::Keypair; let owner = Keypair::generate(); @@ -3241,240 +8816,396 @@ mod tests { let short = owner_did.split(':').next_back().unwrap().to_string(); let state = test_state(pool).await; - let fx = seed_cid_repos(&slug, &short, &["priv"]); - let blob_cid = cid_for_oid(&fx.public_oid); + let cap = crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST as usize; - let mut rec = seed_repo(&owner_did, "priv"); - rec.is_public = false; - state.db.create_repo(&rec).await.expect("private repo"); + // `cap + 1` deniers guarantee the fan-out crosses the ceiling before the + // readable copy (iterated last) is reached. All bare clones share identical + // content, so the one secret-BLOB CID resolves to `secret_oid` in every repo. + let denier_names: Vec = (0..=cap).map(|i| format!("denier{i}")).collect(); + let mut names: Vec<&str> = vec!["readable"]; + names.extend(denier_names.iter().map(|s| s.as_str())); + let fx = seed_cid_repos(&slug, &short, &names); - // anon → repo-level deny → 404, no content leaked. - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_anon(&blob_cid)) + let readable_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("readable.git"); + // The secret BLOB CID drives the path-scoped allowed-blob walk in every + // denier (the amplification surface) and is served cheaply from the + // no-rule public copy — the proven serve path. + let secret_cid = pin_cid_for(&readable_bare, &fx.secret_oid, &state.db).await; + + // 1) Readable public copy — OLDEST updated_at → iterated LAST. Public with + // NO visibility rule, so the blob serves via the no-walk path. This is + // the copy an uncapped fan-out would eventually reach and serve. + let mut readable = seed_repo(&owner_did, "readable"); + readable.updated_at = Utc::now() - chrono::Duration::seconds(60); + state + .db + .create_repo(&readable) + .await + .expect("seed readable copy"); + + // 2) cap+1 deniers with NEWER updated_at → iterated before the copy. Public + // at "/", but a `/secret/**` Mode B rule with an EMPTY reader list, so an + // anon blob fetch clears the "/" gate, runs the allowed-blob walk, and is + // denied (the secret blob is in no one's set) → continue. Each distinct + // repo.id is its own walk (the memo only dedups the same repo). + for name in &denier_names { + let mut denier = seed_repo(&owner_did, name); + denier.updated_at = Utc::now(); + state.db.create_repo(&denier).await.expect("seed denier"); + state + .db + .set_visibility_rule(&denier.id, "/secret/**", VisibilityMode::B, &[], &owner_did) .await - .unwrap(), - ) - .await; + .expect("path rule"); + } + + // Anon (no peer, no XFF → the IP brake is skipped, so the walk cap is the + // only thing in play). After the cap, `continue` skips only the + // walk-requiring deniers and keeps scanning, reaching the walk-free public + // copy (iterated last) → served 200. The served object is the secret blob + // from the no-rule public copy, which is legitimately public THERE. + let resp = cid_router(&state) + .oneshot(cid_anon(&secret_cid)) + .await + .unwrap(); + let served_hash = resp + .headers() + .get("x-git-hash") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + let (st, _body) = cid_parts(resp).await; assert_eq!( st, - StatusCode::NOT_FOUND, - "anon denied at a private repo's / gate" + StatusCode::OK, + "hitting the walk cap must skip only the walk-requiring candidate, not abandon the walk-free readable copy" ); - assert!(!body.contains("public bytes"), "404 must not leak content"); + assert_eq!( + served_hash.as_deref(), + Some(fx.secret_oid.as_str()), + "the served object is the blob from the no-rule public copy reached after the cap" + ); + } - // owner-signed → 200. + /// Multi-oid companion to `ipfs_walk_fanout_capped_per_request`: exercises the + /// outer oid loop and proves the per-request walk budget PERSISTS across oid + /// candidates, so a commit/tag candidate cannot re-open the fan-out. Since #173 + /// (F2) a `commit`/`tag` under a path-scoped rule is itself walk-gated (its + /// reachability is proven by a `rev-list` walk via `reachable_commit_tag_oids`), + /// so it is NOT walk-free — it draws from the same budget as the blob/tree walks. + /// + /// One CID → TWO oids (the non-unique cid index, #173): a withheld `/secret` + /// blob (walk-triggering, denied to anon in every denier) recorded FIRST so a + /// seq scan tries it first and burns the whole walk budget across the deniers; + /// the reachable root commit is second. Because the budget is already spent, the + /// commit candidate's reachability walk is also capped in every denier, so the + /// request 404s — proving commit/tag walks (F2) respect the fan-out ceiling and + /// cannot be used to bypass it (R6/F3). A reachable commit served with budget to + /// spare is covered by `ipfs_cid_gate_withholds_blob_from_unauthorized`. The + /// withheld blob must not leak. Since #173 F2 a scan the walk cap truncated + /// returns 503 (absence unproven), not the old opaque 404. + #[sqlx::test] + async fn ipfs_walk_commit_tag_candidate_respects_the_walk_cap(pool: PgPool) { + use crate::db::VisibilityMode; + use chrono::Utc; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let cap = crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST as usize; + + // cap+1 path-scoped deniers, all carrying identical content (same oids). + let denier_names: Vec = (0..=cap).map(|i| format!("m{i}")).collect(); + let names: Vec<&str> = denier_names.iter().map(|s| s.as_str()).collect(); + let fx = seed_cid_repos(&slug, &short, &names); + let bare = std::path::PathBuf::from("/tmp").join(&slug).join("m0.git"); + + // ONE cid → TWO oids. The withheld blob is recorded first (seq scan lists it + // first → tried first → burns the budget); the reachable commit is second. + let multi_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; + state + .db + .record_pinned_cid(&fx.commit_oid, &multi_cid, None) + .await + .expect("co-locate the commit oid under the same cid"); + + for name in &denier_names { + let mut d = seed_repo(&owner_did, name); + d.updated_at = Utc::now(); + state.db.create_repo(&d).await.expect("seed denier"); + state + .db + .set_visibility_rule(&d.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + } + + // Anon: the blob candidate is denied in every denier (a walk each, spending + // the budget); the commit candidate's reachability walk is then also capped + // in every denier — so no candidate is served AND the walk cap truncated the + // scan, leaving absence unproven → 503 (not the old false 404, #173 F2). + // Either way commit/tag walks respect the ceiling and cannot re-open the + // fan-out (R6/F3). The withheld blob must not leak in the body. let (st, body) = cid_parts( cid_router(&state) - .oneshot(cid_signed(&owner, &blob_cid)) + .oneshot(cid_anon(&multi_cid)) .await .unwrap(), ) .await; assert_eq!( st, - StatusCode::OK, - "owner reads their private repo's object" + StatusCode::SERVICE_UNAVAILABLE, + "a commit/tag reachability walk respects the per-request cap; a truncated scan is 503, not a false 404" + ); + assert!( + !body.contains("TOP SECRET"), + "the withheld blob must not leak in the truncation response" ); - assert!(body.contains("public bytes"), "owner gets the content"); } - /// Fail-closed walk-error arm: if `withheld_blob_oids` errors (here, a ref - /// pointing at a non-tree-ish blob, which `git ls-tree -r` cannot traverse — - /// the same induction as `visibility_pack::fails_closed_when_a_ref_cannot_be_traversed`), - /// the handler skips the whole repo rather than serving. Asserts no leak of the - /// withheld blob AND that even the *public* blob in that repo is withheld — the - /// latter distinguishes fail-closed-skip from normal per-blob withholding and - /// would serve 200 if the error arm wrongly proceeded. + /// #173 (F3, INV-15): the per-IP quota debits ONE token per expensive legacy + /// candidate, not once per request, so one IP cannot drive an unbounded fan-out. + /// With quota=1 and two path-scoped deniers holding one CID, a SINGLE request is + /// shed at 429: since #173-F3 (jatmn) each legacy PROBE (`acquire` + `cat-file`, + /// which precedes the walk) debits, so the first denier probes+walks+denies on + /// token 1 and the second denier's probe finds no token → 429. (Before F3 the + /// debit sat on the walk; the outcome is unchanged, the charge point moved earlier + /// to also bound walk-free probes.) Defeating the per-candidate debit let one IP + /// drive up to MAX_HISTORY_WALKS_PER_REQUEST × quota expensive ops/hour. #[sqlx::test] - async fn ipfs_cid_walk_error_fails_closed(pool: PgPool) { + async fn ipfs_walk_quota_debited_per_walk(pool: PgPool) { use crate::db::VisibilityMode; use gitlawb_core::identity::Keypair; let owner = Keypair::generate(); let owner_did = owner.did().to_string(); + // Signed but NOT a reader → cleared at "/", denied at /secret → forces a walk. + let stranger = Keypair::generate(); let slug = owner_did.replace([':', '/'], "_"); let short = owner_did.split(':').next_back().unwrap().to_string(); - let state = test_state(pool).await; - - let fx = seed_cid_repos(&slug, &short, &["withhold"]); - let secret_cid = cid_for_oid(&fx.secret_oid); - let public_cid = cid_for_oid(&fx.public_oid); - - // Force the withheld walk to fail closed: a ref pointing at a blob (not - // tree-ish) makes `git ls-tree -r` error, which `withheld_blob_oids` - // propagates as Err → the handler's `Ok(Err)` arm skips the repo. - let bare = std::path::PathBuf::from("/tmp") - .join(&slug) - .join("withhold.git"); - std::fs::write( - bare.join("refs/heads/blobref"), - format!("{}\n", fx.secret_oid), - ) - .unwrap(); - state - .db - .create_repo(&seed_repo(&owner_did, "withhold")) - .await - .expect("seed repo"); - let rec = state - .db - .get_repo(&owner_did, "withhold") - .await - .unwrap() - .unwrap(); - state - .db - .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) - .await - .expect("deny rule"); + let mut state = test_state(pool).await; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let fx = seed_cid_repos(&slug, &short, &["w0", "w1"]); + let bare = std::path::PathBuf::from("/tmp").join(&slug).join("w0.git"); + // The secret BLOB CID forces a path-scoped allowed-blob walk in each denier. + let secret_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; + + // Two path-scoped deniers (Mode B /secret, empty readers): each forces a + // walk that denies the signed stranger, so ONE request spawns two walks. + for name in ["w0", "w1"] { + let d = seed_repo(&owner_did, name); + state.db.create_repo(&d).await.expect("seed denier"); + state + .db + .set_visibility_rule(&d.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("path rule"); + } - // Withheld secret CID under a walk error → 404, no leak. - let (st, body) = cid_parts( + // ONE request, quota 1: walk 1 debits the token, walk 2 has none → 429. + let (st, _) = cid_parts( cid_router(&state) - .oneshot(cid_anon(&secret_cid)) + .oneshot(cid_signed_xff(&stranger, &secret_cid, "1.2.3.4")) .await .unwrap(), ) .await; assert_eq!( st, - StatusCode::NOT_FOUND, - "walk error must not serve the withheld blob" + StatusCode::TOO_MANY_REQUESTS, + "the second full-history walk in one request must be shed with 429 (per-walk debit)" ); + } + + /// The periodic cleanup task must sweep the ipfs walk limiter, not only its + /// five siblings. Drives `AppState::sweep_rate_limiters` — the exact method the + /// 300s loop calls — and asserts the ipfs limiter's expired entry is evicted. + /// Dropping `ipfs_rate_limiter.cleanup()` from that method leaves the entry in + /// place (`tracked_keys` stays 1): the RED proof that the sweep covers it. + #[sqlx::test] + async fn sweep_rate_limiters_includes_ipfs_limiter(pool: PgPool) { + let mut state = test_state(pool).await; + // Short window so a single recorded hit is already expired at sweep time. + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(5, Duration::from_millis(50)); + assert!( - !body.contains("TOP SECRET"), - "walk-error 404 must not leak the secret" + state.ipfs_rate_limiter.check("1.2.3.4").await, + "record a hit on the ipfs limiter" + ); + assert_eq!( + state.ipfs_rate_limiter.tracked_keys().await, + 1, + "the source-IP key is tracked before the sweep" ); - // The PUBLIC blob in the same repo is also 404: the walk error fails closed - // by skipping the whole repo, not by serving. Without the fail-closed arm - // this would serve 200, so this assertion is the load-bearing discriminator. - let (st, _) = cid_parts( - cid_router(&state) - .oneshot(cid_anon(&public_cid)) - .await - .unwrap(), - ) - .await; + // Expire the entry (still mapped — cleanup hasn't run), then sweep. + tokio::time::sleep(Duration::from_millis(60)).await; + state.sweep_rate_limiters().await; + assert_eq!( - st, - StatusCode::NOT_FOUND, - "walk error fails closed: repo skipped, even the public blob is not served" + state.ipfs_rate_limiter.tracked_keys().await, + 0, + "the periodic sweep must evict the ipfs limiter's expired entries" ); } - /// #126: a dangling blob (written via `git hash-object -w`, never referenced - /// by any commit/tree) must 404 through `GET /ipfs/{cid}` under path-scoped - /// rules — for anon AND the owner. The pre-#126 deny-set was fail-open by - /// construction: dangling oids were absent from the reachable enumeration - /// and thus absent from the deny-set, so the handler served 200. The - /// allowed-set is fail-closed: dangling oids are absent from the reachable - /// allowed-set, so the handler 404s (per team memory: the owner shift to - /// 404 is the accepted fail-closed default — owners can still - /// `git cat-file` directly). + /// U5 (R6, KTD6), the observed defect: the `/ipfs` route rate limit and the + /// resolver's per-probe WORK budget are SEPARATE buckets, so a single request with + /// one probe COMPLETES even at route limit = 1. Through the production router the + /// `rate_limit_by_ip` middleware charges `ipfs_rate_limiter` once (its 1-slot bucket + /// is now full); the handler's legacy pre-scan peek and per-probe charge then draw + /// from `ipfs_work_rate_limiter`, a different bucket, so the walk-free public copy + /// still serves 200. RED before the split (both charges on `ipfs_rate_limiter`): the + /// middleware fills the one slot, the pre-scan peek reads it throttled, nothing is + /// servable → 429 on the FIRST request. Trust None so the middleware and the handler + /// resolve the same `ConnectInfo` peer IP. #[sqlx::test] - async fn ipfs_cid_dangling_blob_fails_closed_under_path_rules(pool: PgPool) { - use crate::db::VisibilityMode; + async fn ipfs_route_limit_1_still_serves_one_probe(pool: PgPool) { use gitlawb_core::identity::Keypair; - let owner = Keypair::generate(); let owner_did = owner.did().to_string(); let slug = owner_did.replace([':', '/'], "_"); let short = owner_did.split(':').next_back().unwrap().to_string(); - let state = test_state(pool).await; - - // Seed a normal repo with `secret/b.txt` reachable from HEAD, so the - // path-scoped rule has something to match — without this the rule has - // no anchor and we'd be testing nothing. - let _fx = seed_cid_repos(&slug, &short, &["dangling"]); + let mut state = test_state(pool).await; + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(600, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // Public, no-rule legacy pin (NULL provenance) → the resolver takes the scan + // fallback and serves walk-free (exactly one probe). + let fx = seed_cid_repos(&slug, &short, &["routeone"]); let bare = std::path::PathBuf::from("/tmp") .join(&slug) - .join("dangling.git"); + .join("routeone.git"); + let repo = seed_repo(&owner_did, "routeone"); + state.db.create_repo(&repo).await.expect("seed repo"); + let cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; - // Write a dangling blob: `git hash-object -w --stdin` adds it to the - // object DB but nothing references it, so the reachable walk never - // enumerates it. - let mut cmd = std::process::Command::new("git"); - cmd.args(["hash-object", "-w", "--stdin"]) - .current_dir(&bare) - .stdin(std::process::Stdio::piped()) - .stdout(std::process::Stdio::piped()); - let mut child = cmd.spawn().expect("spawn git hash-object"); - { - use std::io::Write; - let stdin = child.stdin.as_mut().expect("stdin"); - stdin.write_all(b"DANGLING SECRET\n").expect("write stdin"); - } - let out = child.wait_with_output().expect("hash-object output"); - assert!( - out.status.success(), - "git hash-object: {}", - String::from_utf8_lossy(&out.stderr) - ); - let dangling_oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); - // Sanity: must be a 64-hex sha256 oid, since the repo is sha256-format. + let router = crate::server::build_router(state); + let peer: std::net::SocketAddr = "203.0.113.7:5000".parse().unwrap(); + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut() + .insert(axum::extract::ConnectInfo(peer)); + let resp = router.oneshot(req).await.unwrap(); assert_eq!( - dangling_oid.len(), - 64, - "expected sha256 oid: {dangling_oid}" + resp.status(), + StatusCode::OK, + "a single /ipfs request with one probe must serve even at route limit = 1 \ + (the route brake and the resolver's work budget are separate buckets)" ); - let dangling_cid = cid_for_oid(&dangling_oid); + } + /// U5 (R6): the two buckets are independent — the WORK budget can be exhausted + /// (429) WITHOUT draining the ROUTE bucket. Through the production router, route + /// generous (5) but work tight (1): one request drives two legacy probes, so the + /// second probe finds the work bucket spent → 429 (the route middleware admitted it). + /// The route bucket, charged once by the middleware, still has room afterward — the + /// work charges never touched it, so it admits four more direct checks. + #[sqlx::test] + async fn ipfs_work_exhaustion_leaves_route_bucket_intact(pool: PgPool) { + use gitlawb_core::identity::Keypair; + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let mut state = test_state(pool).await; + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(5, Duration::from_secs(3600)); + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // A legacy pin absent from every repo so the scan probes both seeded repos: two + // probes, work budget 1 → the second probe is shed → 429. + let names = ["we0", "we1"]; + let _fx = seed_cid_repos(&slug, &short, &names); + for n in names { + state + .db + .create_repo(&seed_repo(&owner_did, n)) + .await + .expect("seed repo"); + } + let bogus_oid = "0".repeat(64); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"work-exhaustion").to_string(); state .db - .create_repo(&seed_repo(&owner_did, "dangling")) - .await - .expect("seed repo"); - let rec = state - .db - .get_repo(&owner_did, "dangling") - .await - .unwrap() - .unwrap(); - // Path-scoped rule triggers the per-blob allowed-set gate (KTD4). - state - .db - .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .record_pinned_cid(&bogus_oid, &cid, None) .await - .expect("deny rule"); + .expect("legacy pin"); - // anon: the dangling blob is absent from the reachable allowed-set → - // 404, no leak. Pre-#126 (deny-set) would serve 200. - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_anon(&dangling_cid)) - .await - .unwrap(), - ) - .await; + let route_bucket = state.ipfs_rate_limiter.clone(); + let peer_ip = "203.0.113.8"; + let peer: std::net::SocketAddr = format!("{peer_ip}:5000").parse().unwrap(); + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut() + .insert(axum::extract::ConnectInfo(peer)); + let resp = router.oneshot(req).await.unwrap(); assert_eq!( - st, - StatusCode::NOT_FOUND, - "dangling blob must 404 under path-scoped rules" + resp.status(), + StatusCode::TOO_MANY_REQUESTS, + "a request whose probes exceed the work budget is shed 429 (work bucket), \ + not blocked at the route (route bucket generous)" ); + // The route bucket recorded only the single request the middleware charged; the + // work charges did not drain it. Sized 5, one used by the request → four left. + for i in 0..4 { + assert!( + route_bucket.check(peer_ip).await, + "route check {i} must still admit — work charges never drained the route bucket" + ); + } + } + + /// U5 (R6): the periodic cleanup task sweeps the NEW work-budget limiter too, not + /// only the route limiter and its siblings. Mirrors + /// `sweep_rate_limiters_includes_ipfs_limiter`: drive `sweep_rate_limiters` and + /// assert the work limiter's expired entry is evicted. Dropping the + /// `ipfs_work_rate_limiter.cleanup()` call from that method leaves the entry in place + /// (`tracked_keys` stays 1): the RED proof the sweep covers it. + #[sqlx::test] + async fn sweep_rate_limiters_includes_ipfs_work_limiter(pool: PgPool) { + let mut state = test_state(pool).await; + state.ipfs_work_rate_limiter = + crate::rate_limit::RateLimiter::new(5, Duration::from_millis(50)); + assert!( - !body.contains("DANGLING SECRET"), - "404 body must not leak the dangling content" + state.ipfs_work_rate_limiter.check("1.2.3.4").await, + "record a hit on the work limiter" + ); + assert_eq!( + state.ipfs_work_rate_limiter.tracked_keys().await, + 1, + "the source-IP key is tracked before the sweep" ); - // owner (signed): same 404. The dangling blob has no path, so it's - // never visibility-checked → never in the allowed set, even for the - // owner. This is the accepted fail-closed shift documented in the PR. - let (st, body) = cid_parts( - cid_router(&state) - .oneshot(cid_signed(&owner, &dangling_cid)) - .await - .unwrap(), - ) - .await; + tokio::time::sleep(Duration::from_millis(60)).await; + state.sweep_rate_limiters().await; + assert_eq!( - st, - StatusCode::NOT_FOUND, - "owner also 404s on dangling blobs under path-scoped rules (fail-closed default)" + state.ipfs_work_rate_limiter.tracked_keys().await, + 0, + "the periodic sweep must evict the work limiter's expired entries" ); - assert!(!body.contains("DANGLING SECRET")); } // --------------------------------------------------------------------------- @@ -5183,4 +10914,1039 @@ mod tests { "result includes the deep cert matching the prefix" ); } + + /// Coalesced-drain behavior of the detached post-push encrypt/pin task. + /// + /// A push arriving while a task is in flight does not spawn a second task; its + /// (old_sha, new_sha) tip pairs are merged into the in-flight key's pending slot + /// and the task loop-drains them before releasing the key. These tests drive the + /// real task through `run_encrypt_pin_task_for_test` and assert on the WORK + /// PERFORMED (what is pinned, what is sealed, whether the key is released), not + /// on control flow. The drain re-reads repo state FRESH, so a rule tightened + /// between the coalesced push and its drain must be honored, fail closed. + mod u3_requeue { + use super::*; + use crate::db::VisibilityMode; + use crate::state::{BeginOutcome, PendingWork}; + use std::path::{Path, PathBuf}; + use std::process::Command; + + fn git(args: &[&str], dir: &Path) { + let ok = Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(); + assert!(ok, "git {args:?} failed"); + } + fn oid(rev: &str, dir: &Path) -> String { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(dir) + .output() + .unwrap(); + assert!(out.status.success(), "rev-parse {rev}: {out:?}"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + struct Repo { + _td: tempfile::TempDir, + path: PathBuf, + } + fn init_repo() -> Repo { + let td = tempfile::TempDir::new().unwrap(); + let path = td.path().to_path_buf(); + git(&["init", "-q"], &path); + git(&["config", "user.email", "t@t"], &path); + git(&["config", "user.name", "t"], &path); + Repo { _td: td, path } + } + /// Commit `content` at `rel`, return the blob oid. + fn commit(repo: &Path, rel: &str, content: &str) -> String { + let full = repo.join(rel); + std::fs::create_dir_all(full.parent().unwrap()).unwrap(); + std::fs::write(&full, content).unwrap(); + git(&["add", "."], repo); + git(&["commit", "-qm", rel], repo); + oid(&format!("HEAD:{rel}"), repo) + } + /// Write a loose, UNREACHABLE blob (dangling object). + fn write_dangling_blob(repo: &Path, content: &str) -> String { + let out = Command::new("git") + .args(["hash-object", "-w", "--stdin"]) + .current_dir(repo) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap(); + use std::io::Write; + out.stdin + .as_ref() + .unwrap() + .write_all(content.as_bytes()) + .unwrap(); + let o = out.wait_with_output().unwrap(); + assert!(o.status.success()); + String::from_utf8_lossy(&o.stdout).trim().to_string() + } + fn new_did() -> String { + Keypair::generate().did().to_string() + } + /// Admit push A on the in-flight key, or fail the test. + fn admit(state: &AppState, key: &str) -> crate::state::EncryptInflightGuard { + match state.encrypt_inflight.try_begin(key, Vec::new()) { + BeginOutcome::Admitted(g) => g, + BeginOutcome::Coalesced => panic!("push A must be admitted, nothing is in flight"), + } + } + /// Coalesce push B's tip pairs into the in-flight key, or fail the test. + fn coalesce(state: &AppState, key: &str, pairs: Vec<(String, String)>) { + match state.encrypt_inflight.try_begin(key, pairs) { + BeginOutcome::Coalesced => {} + BeginOutcome::Admitted(_) => { + panic!("push B must coalesce, a task is already in flight") + } + } + } + + /// SCENARIO 2 + 5 (pin half, TAIL-PLACEMENT guard). A coalesced push on a PUBLIC + /// repo with NO path-scoped rule must still drain its pin half: the second + /// push's new object is pinned after the task. RED without the drain (the stale + /// spawn object_list never lists obj2), and RED if the drain sits inside the + /// `has_path_scoped_rule` block (a rules-free repo would never reach it). + #[sqlx::test] + async fn u3_rules_free_public_repo_requeues_pin_half(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u3-pin"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let obj1 = commit(&git_repo.path, "a.txt", "one\n"); + let tip_a = oid("HEAD", &git_repo.path); + // The coalesced push B adds obj2 (present at drain time, NOT in the stale + // push-A spawn object_list). + let obj2 = commit(&git_repo.path, "b.txt", "two\n"); + let tip_b = oid("HEAD", &git_repo.path); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // Push A admits (guard); push B coalesces its tip pair into the slot. + let guard = admit(&state, &key); + coalesce(&state, &key, vec![(tip_a, tip_b)]); + + // Spawn-time (push A) captures are STALE: object_list lists only obj1, no rule. + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![obj1.clone()], + Some(vec![]), + true, + ) + .await; + + assert!( + state.db.is_pinned(&obj1).await.unwrap(), + "push A's object is pinned on the first pass" + ); + assert!( + state.db.is_pinned(&obj2).await.unwrap(), + "the coalesced push's new object is pinned by the DRAIN lap (RED without \ + the drain, or if the drain sits inside the encrypt gate)" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is released once the task exits clean" + ); + } + + /// SCENARIO 1 + 3 (encrypt half, FRESH re-read). A coalesced push adds a + /// path-scoped rule withholding a blob. The task must re-read rules FRESH on + /// the drain lap and seal the newly-withheld blob's recovery copy. RED without + /// the fresh read (pass one's stale empty rule set seals nothing). + #[sqlx::test] + async fn u3_requeue_seals_blob_withheld_by_coalesced_rule_change(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let reader = new_did(); + let repo = seed_repo(&owner, "u3-enc"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let _pub_oid = commit(&git_repo.path, "public/a.txt", "public\n"); + let tip_a = oid("HEAD", &git_repo.path); + let secret_oid = commit(&git_repo.path, "secret/b.txt", "TOP SECRET\n"); + let tip_b = oid("HEAD", &git_repo.path); + + // Coalesced push B changes .gitlawb: withhold /secret/** from anon, grant reader. + state + .db + .set_visibility_rule(&repo.id, "/secret/**", VisibilityMode::B, &[reader], &owner) + .await + .expect("set rule"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + let guard = admit(&state, &key); + coalesce(&state, &key, vec![(tip_a, tip_b)]); + + // Push A captures are STALE: no rule, public repo. + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![], + Some(vec![]), + true, + ) + .await; + + assert!( + state + .db + .encrypted_blob_recipients_tag(&repo.id, &secret_oid) + .await + .unwrap() + .is_some(), + "the coalesced push's newly-withheld blob is sealed after the DRAIN re-read \ + (RED without the fresh read: pass one's stale empty rules seal nothing)" + ); + assert!(state.encrypt_inflight.is_empty(), "guard key released"); + } + + /// SCENARIO 4 (visibility-leak negative). The drain's full scan must feed + /// `list_all_objects` through the fail-closed filter, never pin it bare: a + /// withheld secret blob and a dangling blob must NOT land in the public pin set. + /// + /// The full scan is forced through the public API: one coalescing push carrying + /// more than the pending tip-pair cap degrades the slot to `PendingWork::FullScan`, + /// which is also the overflow path itself. + #[sqlx::test] + async fn u3_requeue_full_scan_does_not_publicly_pin_withheld_or_dangling(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let reader = new_did(); + let repo = seed_repo(&owner, "u3-leak"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let pub_oid = commit(&git_repo.path, "public/a.txt", "public\n"); + let secret_oid = commit(&git_repo.path, "secret/b.txt", "TOP SECRET\n"); + state + .db + .set_visibility_rule(&repo.id, "/secret/**", VisibilityMode::B, &[reader], &owner) + .await + .expect("set rule"); + // Coalesced push adds a new public object and a dangling blob. + let new_pub_oid = commit(&git_repo.path, "public/c.txt", "more public\n"); + let tip = oid("HEAD", &git_repo.path); + let dangling_oid = write_dangling_blob(&git_repo.path, "orphan bytes\n"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + let rules = state.db.list_visibility_rules(&repo.id).await.unwrap(); + + let guard = admit(&state, &key); + // 1025 pairs is one past the pending cap, so the slot degrades to FullScan. + coalesce(&state, &key, vec![(tip.clone(), tip.clone()); 1025]); + assert_eq!( + state.encrypt_inflight.pending_for(&key), + Some(PendingWork::FullScan), + "an overflowing coalesce degrades the pending slot to a forced full scan" + ); + + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![pub_oid.clone()], + Some(rules), + true, + ) + .await; + + assert!( + state.db.is_pinned(&new_pub_oid).await.unwrap(), + "the coalesced push's new PUBLIC object is pinned by the drain full scan" + ); + assert!( + !state.db.is_pinned(&secret_oid).await.unwrap(), + "a WITHHELD blob is never publicly pinned by the drain enumeration (leak guard)" + ); + assert!( + !state.db.is_pinned(&dangling_oid).await.unwrap(), + "a DANGLING blob is never publicly pinned by the drain enumeration (leak guard)" + ); + // The withheld blob still gets its ENCRYPTED recovery copy (not a public pin). + assert!( + state + .db + .encrypted_blob_recipients_tag(&repo.id, &secret_oid) + .await + .unwrap() + .is_some(), + "withheld blob is sealed as an encrypted recovery copy, not pinned in the clear" + ); + } + + /// SCENARIO 8 (no-coalesce happy path). A single push with no coalesced follower + /// runs exactly one pass, pins its object, and releases the key. No drain lap. + #[sqlx::test] + async fn u3_no_coalesce_single_pass_pins_and_releases(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u3-happy"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let obj1 = commit(&git_repo.path, "a.txt", "one\n"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // No second try_begin: nothing is ever merged into the pending slot. + let guard = admit(&state, &key); + assert_eq!( + state.encrypt_inflight.pending_for(&key), + Some(PendingWork::Tips(vec![])), + "clean, no coalesce" + ); + + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![obj1.clone()], + Some(vec![]), + true, + ) + .await; + + assert!( + state.db.is_pinned(&obj1).await.unwrap(), + "the single push's object is pinned" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the key is released after one pass" + ); + } + + mod u2_reread_retry { + use super::*; + use crate::api::repos::drain_faults; + + /// Process-wide tracing capture so a test can assert the give-up is logged at + /// ERROR. A global default subscriber can only be installed once per process, + /// so it is shared by every test here and assertions filter on the repo id, + /// which is a fresh uuid per test. + mod logcap { + use std::sync::{Arc, Mutex, OnceLock}; + use tracing::{Event, Level, Subscriber}; + use tracing_subscriber::layer::{Context, Layer}; + use tracing_subscriber::prelude::*; + + type Lines = Arc>>; + + fn lines() -> &'static Lines { + static LINES: OnceLock = OnceLock::new(); + LINES.get_or_init(|| Arc::new(Mutex::new(Vec::new()))) + } + + struct Capture; + impl Layer for Capture { + fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) { + struct V(String); + impl tracing::field::Visit for V { + fn record_debug( + &mut self, + field: &tracing::field::Field, + value: &dyn std::fmt::Debug, + ) { + self.0.push_str(&format!(" {}={:?}", field.name(), value)); + } + } + let mut v = V(String::new()); + event.record(&mut v); + lines() + .lock() + .unwrap() + .push((*event.metadata().level(), v.0)); + } + } + + pub(super) fn install() { + static ONCE: OnceLock<()> = OnceLock::new(); + ONCE.get_or_init(|| { + let _ = tracing::subscriber::set_global_default( + tracing_subscriber::registry().with(Capture), + ); + }); + } + + pub(super) fn errors_containing(needle: &str) -> Vec { + lines() + .lock() + .unwrap() + .iter() + .filter(|(lvl, msg)| *lvl == Level::ERROR && msg.contains(needle)) + .map(|(_, msg)| msg.clone()) + .collect() + } + } + + /// SCENARIO 1. The repo re-read fails once, then succeeds: the drain lap + /// must still RUN, under the refreshed state, and pin the coalesced push's + /// object. RED before the fix (the single `Err` returned `None`, the lap + /// pinned nothing, and `finish_or_take_pending` had already taken the + /// pending work out of the slot, so it was gone for good). + #[sqlx::test] + async fn u2_transient_repo_reread_failure_is_retried_and_work_lands(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u2-retry"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let obj1 = commit(&git_repo.path, "a.txt", "one\n"); + let tip_a = oid("HEAD", &git_repo.path); + // The coalesced push B adds obj2, absent from push A's spawn captures. + let obj2 = commit(&git_repo.path, "b.txt", "two\n"); + let tip_b = oid("HEAD", &git_repo.path); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // One transient repo re-read failure, then the real DB answers. + drain_faults::inject(&repo.id, 1, 0); + + let guard = admit(&state, &key); + coalesce(&state, &key, vec![(tip_a, tip_b)]); + + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![obj1.clone()], + Some(vec![]), + true, + ) + .await; + + assert!( + state.db.is_pinned(&obj2).await.unwrap(), + "the coalesced push's object is pinned after the retried re-read (RED \ + before this unit: the Err arm dropped the lap and the work with it)" + ); + let c = drain_faults::counters(&repo.id); + assert_eq!( + c.repo_read_attempts, 2, + "the failed re-read is retried exactly once before it succeeds" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is released once the task exits" + ); + } + + /// SCENARIO 2. Every re-read attempt fails: the loop must give up on a BOUND + /// (asserted as a literal, so raising or removing the bound goes RED) and log + /// the give-up at ERROR so the residual loss is observable rather than silent. + #[sqlx::test] + async fn u2_sustained_repo_reread_failure_is_bounded_and_logged(pool: PgPool) { + logcap::install(); + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u2-bounded"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let obj1 = commit(&git_repo.path, "a.txt", "one\n"); + let tip_a = oid("HEAD", &git_repo.path); + let obj2 = commit(&git_repo.path, "b.txt", "two\n"); + let tip_b = oid("HEAD", &git_repo.path); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // Far more failures than the bound allows: the outage never clears. + drain_faults::inject(&repo.id, 10_000, 0); + + let guard = admit(&state, &key); + coalesce(&state, &key, vec![(tip_a, tip_b)]); + + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![obj1.clone()], + Some(vec![]), + true, + ) + .await; + + let c = drain_faults::counters(&repo.id); + assert_eq!( + c.repo_read_attempts, 3, + "the re-read is bounded at 3 attempts; unbounded retry or a raised \ + bound must fail here" + ); + assert!( + !state.db.is_pinned(&obj2).await.unwrap(), + "with the read never succeeding there is nothing fresh to act on" + ); + let errs = logcap::errors_containing(&repo.id); + assert!( + !errs.is_empty(), + "the exhausted drain re-read is logged at ERROR with the repo id, so \ + the residual work loss is observable; captured: {errs:?}" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is still released on the give-up path" + ); + } + + /// SCENARIO 3. `Ok(None)` (the repo was deleted during the in-flight window) + /// is NOT a transient failure: it must release immediately without burning the + /// retry budget. The repo row is never created, so the re-read legitimately + /// returns `Ok(None)`. + #[sqlx::test] + async fn u2_repo_gone_releases_without_consuming_retries(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let missing_id = uuid::Uuid::new_v4().to_string(); + let missing_name = "u2-gone".to_string(); + let key = crate::state::repo_identity_key(&owner, &missing_name); + let git_repo = init_repo(); + let _obj1 = commit(&git_repo.path, "a.txt", "one\n"); + let tip_a = oid("HEAD", &git_repo.path); + let _obj2 = commit(&git_repo.path, "b.txt", "two\n"); + let tip_b = oid("HEAD", &git_repo.path); + + let server = mockito::Server::new_async().await; + + drain_faults::inject(&missing_id, 0, 0); + + let guard = admit(&state, &key); + // A real pair, so a drain lap actually runs and reaches the re-read. + coalesce(&state, &key, vec![(tip_a, tip_b)]); + + // Empty object list: pass one touches no pin rows for a repo that is gone. + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + missing_id.clone(), + owner.clone(), + missing_name.clone(), + server.url(), + vec![], + Some(vec![]), + true, + ) + .await; + + let c = drain_faults::counters(&missing_id); + assert_eq!( + c.repo_read_attempts, 1, + "a deleted repo is a terminal answer, never retried" + ); + assert_eq!( + c.rules_read_attempts, 0, + "no rules read is attempted once the repo row is gone" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is released cleanly" + ); + } + + /// SCENARIO 4. A failed visibility-rule read is transient, never an empty + /// policy. RED before the fix, where `.ok()` made "the rules read failed" and + /// "this repo has no rules" the same value: the withheld blob was then neither + /// sealed nor covered, because a `None` rule set skips the entire lap. + #[sqlx::test] + async fn u2_transient_rules_read_failure_is_retried_not_read_as_empty(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let reader = new_did(); + let repo = seed_repo(&owner, "u2-rules"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let pub_oid = commit(&git_repo.path, "public/a.txt", "public\n"); + let tip_a = oid("HEAD", &git_repo.path); + let secret_oid = commit(&git_repo.path, "secret/b.txt", "TOP SECRET\n"); + let tip_b = oid("HEAD", &git_repo.path); + + // The coalesced push B is what added the path-scoped rule. + state + .db + .set_visibility_rule( + &repo.id, + "/secret/**", + VisibilityMode::B, + &[reader], + &owner, + ) + .await + .expect("set rule"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // The repo row reads fine; the RULES read is the one that blips. + drain_faults::inject(&repo.id, 0, 1); + + let guard = admit(&state, &key); + coalesce(&state, &key, vec![(tip_a, tip_b)]); + + // Push A's captures are stale: no rule, nothing withheld. + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![pub_oid.clone()], + Some(vec![]), + true, + ) + .await; + + assert!( + state + .db + .encrypted_blob_recipients_tag(&repo.id, &secret_oid) + .await + .unwrap() + .is_some(), + "the withheld blob is sealed under the RETRIED rule set (RED with \ + list_visibility_rules(..).ok(): an empty policy seals nothing)" + ); + let c = drain_faults::counters(&repo.id); + assert_eq!( + c.rules_read_attempts, 2, + "the failed rules read is retried, not collapsed into an empty rule set" + ); + assert!( + !state.db.is_pinned(&secret_oid).await.unwrap(), + "the withheld blob is never pinned in the clear by the drain" + ); + } + + /// SCENARIO 5. The fault-free control for scenario 4: the rules applied by the + /// drain are the COALESCED push's fresh ones, never the spawn-time capture, + /// and the retry path does not perturb that (exactly one read of each). + #[sqlx::test] + async fn u2_requeue_applies_fresh_rules_not_spawn_captures(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let reader = new_did(); + let repo = seed_repo(&owner, "u2-fresh"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let pub_oid = commit(&git_repo.path, "public/a.txt", "public\n"); + let tip_a = oid("HEAD", &git_repo.path); + let secret_oid = commit(&git_repo.path, "secret/b.txt", "TOP SECRET\n"); + let tip_b = oid("HEAD", &git_repo.path); + state + .db + .set_visibility_rule( + &repo.id, + "/secret/**", + VisibilityMode::B, + &[reader], + &owner, + ) + .await + .expect("set rule"); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + drain_faults::inject(&repo.id, 0, 0); + + let guard = admit(&state, &key); + coalesce(&state, &key, vec![(tip_a, tip_b)]); + + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![pub_oid.clone()], + Some(vec![]), + true, + ) + .await; + + let c = drain_faults::counters(&repo.id); + assert_eq!( + (c.repo_read_attempts, c.rules_read_attempts), + (1, 1), + "a healthy DB is read exactly once per drain lap" + ); + assert!( + state.db.is_pinned(&pub_oid).await.unwrap(), + "the visible object is pinned under the fresh rules" + ); + assert!( + !state.db.is_pinned(&secret_oid).await.unwrap(), + "the freshly-read rule withholds the secret blob (the spawn-time \ + capture had no rules at all)" + ); + } + + /// SCENARIO 6. Regression guard on the property the fix must not disturb: the + /// finish-or-take critical section is atomic, so a push coalescing during it is + /// still covered by exactly one more lap, and the key is released after. + #[sqlx::test] + async fn u2_coalesced_push_still_covered_by_exactly_one_requeue_pass(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u2-coalesce"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let obj1 = commit(&git_repo.path, "a.txt", "one\n"); + let tip_a = oid("HEAD", &git_repo.path); + let obj2 = commit(&git_repo.path, "b.txt", "two\n"); + let tip_b = oid("HEAD", &git_repo.path); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + drain_faults::inject(&repo.id, 0, 0); + + let guard = admit(&state, &key); + // Push B lands during the in-flight window: its tip pair is merged. + coalesce(&state, &key, vec![(tip_a.clone(), tip_b.clone())]); + assert_eq!( + state.encrypt_inflight.pending_for(&key), + Some(PendingWork::Tips(vec![(tip_a, tip_b)])), + "the coalesced push recorded its work in the pending slot" + ); + + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![obj1.clone()], + Some(vec![]), + true, + ) + .await; + + assert_eq!( + drain_faults::counters(&repo.id).repo_read_attempts, + 1, + "one coalesced push means exactly one drain lap, no re-spin" + ); + assert!( + state.db.is_pinned(&obj1).await.unwrap(), + "push A's object is pinned" + ); + assert!( + state.db.is_pinned(&obj2).await.unwrap(), + "the coalesced push's object is covered by the drain lap" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the key is released once the task is clean" + ); + } + + /// Wait for `finish_or_take_pending` to take the pending work out of the slot + /// (`Tips(nonempty)` -> `Tips(empty)`), which is the exact instant the task + /// enters `drain_refresh_state`'s retry window. Deterministic, so the + /// coalescing push below lands INSIDE that window rather than on a sleep + /// guess. `None` means the key is already gone (the task exited), which the + /// caller reports as its own failure. + async fn wait_for_drain_window( + inflight: &crate::state::EncryptInflight, + key: &str, + ) -> bool { + for _ in 0..5_000 { + match inflight.pending_for(key) { + Some(PendingWork::Tips(acc)) if acc.is_empty() => return true, + None => return false, + Some(_) => {} + } + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + } + false + } + + /// SCENARIO 7 (RED-before/GREEN-after). A push that coalesces WHILE the + /// re-read is retrying must not be thrown away when that re-read finally + /// gives up. Breaking the drain loop on the give-up would let + /// `EncryptInflightGuard::drop` remove the key with push C's work still + /// recorded, and push C's lap would never run: a silent drop with no + /// reconciliation sweep behind it. + /// + /// Exactly `DRAIN_REREAD_MAX_ATTEMPTS` injected repo-read faults, so the + /// first refresh exhausts its budget and the DB is healthy for the next one. + /// Push C coalesces inside that window. + #[sqlx::test] + async fn u2_failed_reread_keeps_a_push_that_coalesced_during_the_window(pool: PgPool) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u2-window"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let obj_a = commit(&git_repo.path, "a.txt", "one\n"); + let tip_a = oid("HEAD", &git_repo.path); + let _obj_b = commit(&git_repo.path, "b.txt", "two\n"); + let tip_b = oid("HEAD", &git_repo.path); + let obj_c = commit(&git_repo.path, "c.txt", "three\n"); + let tip_c = oid("HEAD", &git_repo.path); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // Exactly the bound: the FIRST refresh burns all three attempts and gives + // up; every later refresh sees a healthy DB. + drain_faults::inject(&repo.id, 3, 0); + + let guard = admit(&state, &key); + coalesce(&state, &key, vec![(tip_a.clone(), tip_b.clone())]); + + // Push C lands during the retry window, after the loop already took push + // B's pending work out of the slot. It MUST carry a real tip pair: an + // empty merge leaves the slot empty and no extra lap runs at all. + let inflight = state.encrypt_inflight.clone(); + let watch_key = key.clone(); + let coalesced = tokio::spawn(async move { + if !wait_for_drain_window(&inflight, &watch_key).await { + return false; + } + matches!( + inflight.try_begin(&watch_key, vec![(tip_b, tip_c)]), + BeginOutcome::Coalesced + ) + }); + + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![obj_a.clone()], + Some(vec![]), + true, + ) + .await; + + assert!( + coalesced.await.expect("coalescing task"), + "push C must have coalesced inside the retry window for this test to \ + mean anything" + ); + assert!( + state.db.is_pinned(&obj_c).await.unwrap(), + "the push that coalesced during the retry window must still get a lap \ + once the DB recovers (RED if the give-up breaks the loop: the pending \ + work was already taken, so the lap was dropped with nothing to \ + re-derive it)" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is released once the task exits" + ); + } + + /// SCENARIO 8 (the sustained-outage guard on the fall-through). Continuing the + /// loop on a give-up means `finish_or_take_pending` runs again, so a DB that + /// never recovers must still TERMINATE rather than spin. It does: an extra lap + /// only happens when a push actually coalesced, and each lap pays a full + /// bounded re-read (3 attempts with backoff). One coalescing push during the + /// window buys exactly one extra lap: 6 repo-read attempts, then exit. + #[sqlx::test] + async fn u2_sustained_failure_with_a_coalesce_terminates_after_one_more_lap( + pool: PgPool, + ) { + let state = test_state(pool).await; + let owner = new_did(); + let repo = seed_repo(&owner, "u2-sustained-window"); + state.db.create_repo(&repo).await.expect("seed repo"); + let key = crate::state::repo_identity_key(&owner, &repo.name); + let git_repo = init_repo(); + let obj_a = commit(&git_repo.path, "a.txt", "one\n"); + let tip_a = oid("HEAD", &git_repo.path); + let _obj_b = commit(&git_repo.path, "b.txt", "two\n"); + let tip_b = oid("HEAD", &git_repo.path); + let _obj_c = commit(&git_repo.path, "c.txt", "three\n"); + let tip_c = oid("HEAD", &git_repo.path); + + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", mockito::Matcher::Regex(r"^/api/v0/add".to_string())) + .with_status(200) + .with_body(r#"{"Hash":"bafyprovider"}"#) + .expect_at_least(1) + .create_async() + .await; + + // The outage never clears. + drain_faults::inject(&repo.id, 10_000, 0); + + let guard = admit(&state, &key); + coalesce(&state, &key, vec![(tip_a.clone(), tip_b.clone())]); + + let inflight = state.encrypt_inflight.clone(); + let watch_key = key.clone(); + let coalesced = tokio::spawn(async move { + if !wait_for_drain_window(&inflight, &watch_key).await { + return false; + } + matches!( + inflight.try_begin(&watch_key, vec![(tip_b, tip_c)]), + BeginOutcome::Coalesced + ) + }); + + // The watchdog is the real assertion: a loop that re-spins without the + // pending gate would never return here. + tokio::time::timeout( + std::time::Duration::from_secs(60), + crate::api::repos::run_encrypt_pin_task_for_test( + &state, + guard, + git_repo.path.clone(), + repo.id.clone(), + owner.clone(), + repo.name.clone(), + server.url(), + vec![obj_a.clone()], + Some(vec![]), + true, + ), + ) + .await + .expect( + "the task must terminate under a sustained outage; a fall-through that \ + does not gate on the pending slot spins forever", + ); + + assert!( + coalesced.await.expect("coalescing task"), + "push C must have coalesced inside the retry window" + ); + assert_eq!( + drain_faults::counters(&repo.id).repo_read_attempts, + 6, + "one coalescing push buys exactly one more bounded re-read lap \ + (3 + 3 attempts), never an unbounded retry" + ); + assert!( + state.encrypt_inflight.is_empty(), + "the guard key is released on the give-up path" + ); + } + } + } } diff --git a/crates/gitlawb-node/src/visibility.rs b/crates/gitlawb-node/src/visibility.rs index 56616872..a8d7ddba 100644 --- a/crates/gitlawb-node/src/visibility.rs +++ b/crates/gitlawb-node/src/visibility.rs @@ -437,6 +437,31 @@ mod tests { ); } + // #135 T1: a Mode-B rule on `/secret/**` must DENY the withheld directory's + // OWN path `/secret` (the `path == prefix` arm), not just strict descendants — + // otherwise get_by_cid's tree gate would serve the /secret tree object and leak + // its children. Pins parity with get_tree, which denies the /secret path. + #[test] + fn subtree_rule_denies_the_withheld_directory_itself() { + let reader = "did:key:z6MkReader"; + let rules = [rule("/secret/**", VisibilityMode::B, &[reader])]; + assert_eq!( + visibility_check(&rules, true, OWNER, None, "/secret"), + Decision::Deny, + "anon denied at the withheld directory's OWN path /secret" + ); + assert_eq!( + visibility_check(&rules, true, OWNER, None, "/public"), + Decision::Allow, + "anon allowed at a sibling path outside the withheld subtree" + ); + assert_eq!( + visibility_check(&rules, true, OWNER, Some(reader), "/secret"), + Decision::Allow, + "listed reader allowed at the withheld directory (caller-aware)" + ); + } + // #153 regression: cross-method DID must still be denied even when the // trailing segment collides with a bare owner key. #[test] diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs new file mode 100644 index 00000000..33815022 --- /dev/null +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -0,0 +1,573 @@ +//! #174 U6 — INV-22 completeness guard (rung-raising). +//! +//! INV-22: a permit held per op recovers only if every path that holds it is also +//! duration-bounded and reaps the process group before releasing admission, and every +//! detached git/blocking task carries its own admission. PR #174 fixed five paths +//! (U1-U5) that violated this. Each fix has a per-unit RED/GREEN regression; together +//! those form the five-revert matrix. This guard adds the missing piece: a source-scan +//! tripwire that fails when a NEW site reintroduces the class, or when one of the five +//! gates is removed. +//! +//! It lives in `tests/` (a separate crate) on purpose: a guard that scanned the same +//! file it lives in would match its own identifier literals and pass vacuously. Here +//! the scanned `src/` files never contain this file's literals, so each check is +//! load-bearing — reverting the named gate turns the assertion red. +//! +//! These are deliberately coarse structural checks, not a parser. They cannot prove a +//! gate is *correct* (the per-unit tests do that); they prove a gate is *present and +//! not bypassed*, which is what stops the class from silently regressing. + +use std::path::Path; + +fn src(rel: &str) -> String { + let p = Path::new(env!("CARGO_MANIFEST_DIR")).join("src").join(rel); + std::fs::read_to_string(&p).unwrap_or_else(|e| panic!("read {}: {e}", p.display())) +} + +#[test] +fn inv22_concurrency_gates_present_and_not_bypassed() { + let repos = src("api/repos.rs"); + let smart_http = src("git/smart_http.rs"); + let vis = src("git/visibility_pack.rs"); + let ipfs = src("api/ipfs.rs"); + + // U1 / P1-a: run_bounded_git stands the watchdog down only after confirming the + // child actually terminated (WNOWAIT), not on the raw stdout-drain EOF — otherwise + // a child that closes stdout then hangs pins the permit past the deadline. The + // probe is defined and called, so >= 2 occurrences; reverting the fix removes both. + assert!( + vis.matches("child_terminated_without_reaping").count() >= 2, + "U1/P1-a gate missing: run_bounded_git must confirm child exit via \ + child_terminated_without_reaping before signalling the watchdog" + ); + + // U2 / P1-c: on client disconnect KillGroupOnDrop must launch a detached reaper + // that runs the full TERM/grace/KILL/reap, not a lone SIGTERM. The reaper is spawned + // via a runtime handle in Drop; `Handle::try_current` is unique to that launch (the + // timeout path already has an async context and never calls it). + assert!( + smart_http.contains("Handle::try_current"), + "U2/P1-c gate missing: KillGroupOnDrop::drop must launch the full \ + TERM/grace/KILL reaper on disconnect (Handle::try_current), not a lone SIGTERM" + ); + + // U4 / P1-d: git_receive_pack must acquire the per-source write sub-cap before the + // global write permit. The acquire reads `state.git_write_per_caller`; comments name + // the field without the `state.` prefix, so this targets the real acquire site. + assert!( + repos.contains("state.git_write_per_caller"), + "U4/P1-d gate missing: git_receive_pack must acquire the per-source write cap \ + (state.git_write_per_caller) before the global write permit" + ); + + // U5 / P1-e: the detached post-push encryption walk must run through the + // admission-gated helper, which is wired to the shared encrypt pool. + assert!( + repos.contains("fn withheld_recipients_gated") + && repos.contains("state.git_encrypt_semaphore"), + "U5/P1-e gate missing: the encryption walk must run through \ + withheld_recipients_gated, which acquires git_encrypt_semaphore" + ); + + // U1 / R2 (#173 round-10): the path-scoped filtered-pack serve must thread the + // caller's AdmissionGuard through BOTH git stages so read + per-caller admission is + // held until the pack-objects group is reaped on disconnect, closing the cap bypass + // the plain upload_pack path already fixed. Two load-bearing markers: rev-list hands + // the disarmed guard back (its tuple return type), and build_filtered_pack forwards + // that guard into the pack-objects stage (the `admission` arg after the + // "pack-objects" label). Reverting either — dropping the guard between stages, or + // passing `None` to pack-objects — trips this. + assert!( + smart_http.contains("Result<(Vec, Option)>") + && smart_http.contains("\"pack-objects\",\n admission,"), + "U1/R2 gate missing: build_filtered_pack must thread the AdmissionGuard through \ + rev-list -> pack-objects so the permits are held until the pack-objects group \ + is reaped on disconnect (the path-scoped half of #174 P1-a)" + ); + + // U2 / R1: a cancelled or timed-out `GET /ipfs/{cid}` must release admission only + // after the blocking work it admitted has finished, not the instant the handler + // future drops (the /ipfs half of #174 P1-a). The mechanism is the shared + // `Arc`: both walk permits are moved into it once per request and a + // clone rides every `spawn_blocking`, so the permits release when the LAST holder + // drops. Reverting to handler-local permits trips this; the per-site clones are + // bound separately by `inv22_ipfs_walk_admission_reaches_every_blocking_site`. + // + // This previously required the permits to be owned by a detached `tokio::spawn` + // running the whole pipeline. That closed the same bypass but kept an abandoned + // request's full legacy scan running against a held slot, so the merge of #173 and + // #174 kept the Arc and dropped the detached task. + assert!( + ipfs.contains("struct WalkAdmission") + && ipfs.contains("let admission = std::sync::Arc::new(WalkAdmission {"), + "U2/R1 gate missing: get_by_cid must move both /ipfs admission permits into a \ + shared Arc whose clones ride the blocking work, so admission is \ + released only once that work completes (the /ipfs half of #174 P1-a)" + ); + + // U2 / KTD2 (#173 round-10): the probe/read children on the /ipfs path must be the + // duration-bounded twins (process-group teardown via run_bounded_git), not the bare + // `store::object_type` / `read_object_content` (or an unbounded `cat-file -s`) a tokio + // timeout cannot cancel — otherwise a wedged cat-file lingers and pins the held + // admission past the deadline. Reverting any twin call site back to a bare read trips + // this. + assert!( + ipfs.contains("object_type_bounded(") + && ipfs.contains("object_size_bounded(") + && ipfs.contains("read_object_content_bounded("), + "U2/KTD2 gate missing: the /ipfs probe+read must call the run_bounded_git-backed \ + *_bounded twins so a wedged cat-file is reaped at the deadline, not left to pin \ + the held /ipfs walk admission" + ); + + // P1-e non-bypass tripwire: the bounded recipients walk is spawn_blocking'd nowhere + // but inside withheld_recipients_gated. A second call site (count > 1) is a new + // detached git walk that skips the admission gate — exactly the class U5 closed. + assert_eq!( + repos.matches("withheld_blob_recipients_bounded").count(), + 1, + "P1-e bypass: the bounded recipients walk must be invoked only inside \ + withheld_recipients_gated; a new call site bypasses the encrypt-walk admission cap" + ); + + // U4 / P2-2: the detached post-push encryption task must be gated by the per-repo + // coalescing set (`encrypt_inflight.try_begin`) so the OUTSTANDING parked-task set is + // bounded to <=1 per repo. Removing the gate lets N rapid pushes spawn N parked + // waiters (the unbounded set U4 closed); the semaphore only caps active walks. + assert!( + repos.contains("state.encrypt_inflight.try_begin"), + "U4/P2-2 gate missing: the detached post-push encryption spawn must consult \ + encrypt_inflight.try_begin to coalesce per repo (bound the outstanding-task set)" + ); + + // F5: coalescing must REQUEUE, not shed. The in-flight task pins only its own + // pre-spawn snapshot, so a coalesced push's tip pairs are recorded on its key and + // the task must loop-drain them (`finish_or_take_pending`) before releasing it. + // Removing the drain reverts to the silent loss: a coalesced push's pins and + // recovery copies are absent until an unrelated later push. Scan only the + // production half of the file — the u5 tests in its `mod tests` also name the + // drain call, and matching them would make this check vacuous. + // Split at the TEST MODULE, not at the first `#[cfg(test)]`. api/repos.rs carries + // test-only items (the drain fault seam, the task entry point) ABOVE the production + // code these gates scan for, so splitting on the attribute truncated the production + // half above every line being checked and the gate went vacuously blind. + let repos_production = repos + .split("\nmod tests {") + .next() + .expect("split always yields a first chunk"); + assert!( + repos_production.contains("finish_or_take_pending"), + "F5 gate missing: the post-push encryption task must loop-drain coalesced \ + pushes via finish_or_take_pending before releasing its repo key" + ); + + // F4: every post-receive scan helper admits itself to the shared scan pool via + // `crate::state::acquire_scan_permit` BEFORE its spawn_blocking git walk, so a + // push burst cannot accumulate unbounded concurrent scans once the write permit + // is released. Two halves, both load-bearing: the helper body must actually + // acquire the pool (state.rs sits the helper at the end of the file, so the + // definition tail contains no other `acquire_owned` to match vacuously), and + // within each scan helper the first qualified call precedes the first + // `spawn_blocking`. Severing a call site pushes the next occurrence past the + // helper's own `spawn_blocking` (or off the end of the file), turning the + // assertion red; comments name the helper without the `crate::state::` prefix, + // so this targets the real call sites. + let state_rs = src("state.rs"); + let helper_def = state_rs + .find("fn acquire_scan_permit") + .expect("F4 gate missing: state.rs no longer defines acquire_scan_permit"); + assert!( + state_rs[helper_def..].contains("acquire_owned"), + "F4 gate gutted: acquire_scan_permit must acquire the scan pool via acquire_owned" + ); + let push_delta = src("git/push_delta.rs"); + for (file_src, file, helper) in [ + (&repos, "api/repos.rs", "fn replication_withheld_set"), + (&repos, "api/repos.rs", "fn fail_closed_full_scan_objects"), + ( + &push_delta, + "git/push_delta.rs", + "fn resolve_candidates_for_push", + ), + ] { + let start = file_src + .find(helper) + .unwrap_or_else(|| panic!("{file}: `{helper}` not found")); + let tail = &file_src[start..]; + let acquire = tail + .find("crate::state::acquire_scan_permit(") + .unwrap_or_else(|| { + panic!("F4 gate missing: {file} `{helper}` no longer acquires a scan permit") + }); + let spawn = tail.find("spawn_blocking").unwrap_or_else(|| { + panic!("{file}: `{helper}` lost its spawn_blocking walk — update this guard") + }); + assert!( + acquire < spawn, + "F4 gate bypassed: {file} `{helper}` must acquire its git_encrypt_semaphore \ + permit BEFORE dispatching the blocking git scan" + ); + } +} + +/// F4 (repo_store advisory-unlock cancellation safety): `RepoWriteGuard::release` +/// must await `pg_advisory_unlock` while `self` still owns the pooled connection, +/// and must not mark itself `released` until that await resolves. Either shape, +/// reintroduced, re-opens the mid-unlock cancellation leak: taking the connection +/// early leaves `Drop` with `conn == None`, and setting `released = true` early +/// leaves the `Drop` backstop inert — both strand the session lock on cancellation. +/// +/// Scoped to the `release` fn body: the `Drop` impl legitimately takes the +/// connection and unlocks, so a whole-file scan would match it and read as a false +/// pass. Reverting either ordering turns this red (proven load-bearing). +#[test] +fn f4_release_keeps_conn_owned_until_unlock_resolves() { + let repo_store = src("git/repo_store.rs"); + + let rel_start = repo_store + .find("pub async fn release(mut self") + .expect("F4 gate: repo_store.rs no longer defines RepoWriteGuard::release"); + let rel_end = repo_store[rel_start..] + .find("impl Drop for RepoWriteGuard") + .map(|off| rel_start + off) + .expect("F4 gate: release fn / Drop impl markers moved — update this guard"); + let release_body = &repo_store[rel_start..rel_end]; + + let unlock = release_body + .find("pg_advisory_unlock") + .expect("F4 gate: release must still issue pg_advisory_unlock"); + let before_unlock = &release_body[..unlock]; + + // (a) the connection must still be owned by `self` at the unlock await. + assert!( + !before_unlock.contains("self.conn.take()"), + "F4 regression: RepoWriteGuard::release takes self.conn BEFORE awaiting \ + pg_advisory_unlock. A cancellation during the unlock await then strands the \ + session advisory lock (Drop sees conn == None and skips its backstop). \ + Unlock through the still-owned connection instead." + ); + // (b) `released` must not be set before the unlock await — the other + // reintroduction shape a single-reorder check on (a) alone is blind to. + assert!( + !before_unlock.contains("released = true"), + "F4 regression: RepoWriteGuard::release sets `released = true` BEFORE awaiting \ + pg_advisory_unlock. A cancellation during the await then leaves the Drop \ + backstop inert (it early-returns on released). Set released only AFTER the \ + unlock await resolves." + ); +} + +/// F6/KTD-5 (initial IPFS metadata queries deadline-wrapped): `get_by_cid` acquires +/// the scarce walk permits (RAII, held for the whole request) BEFORE its two initial +/// metadata queries, and the per-repo loop's first budget gate runs only later. So +/// both `list_all_repos` and `list_visibility_rules_for_repos` must be clamped to the +/// remaining request budget — otherwise a query blocked in Postgres pins the walk slot +/// for the whole stall, past the budget. This scans the PRODUCTION half of `api/ipfs.rs` +/// (the `mod tests` half names the same calls in its own harness and would make the +/// check vacuous) and asserts each query call is immediately preceded by a +/// `tokio::time::timeout(` wrapper. Removing either wrapper turns this red (proven +/// load-bearing). +#[test] +fn f6_ipfs_metadata_queries_are_deadline_wrapped() { + let ipfs = src("api/ipfs.rs"); + // Split at the TEST MODULE, not at the first `#[cfg(test)]`: the handler carries + // in-line `#[cfg(test)]` query counters, so splitting on the attribute cut the + // production half off above the calls this guard exists to check and the scan went + // vacuously quiet (found 0 of each rather than failing on a missing wrapper). + let production = ipfs + .split("\nmod tests {") + .next() + .expect("split always yields a first chunk"); + + // EVERY occurrence must be wrapped, not just the first. The handler now reaches + // these queries from two places (the provenance fast path, one repo at a time, and + // the legacy scan's preload), and an exact-count check would have to be relaxed + // every time a call site is added, which is how a guard quietly stops covering the + // site that matters. Requiring all of them scales with the code instead. + for call in [ + ".list_all_repos()", + ".list_visibility_rules_for_repos(", + ".get_repo_by_id(", + ".is_repo_quarantined(", + ] { + let occurrences = production.matches(call).count(); + assert!( + occurrences >= 1, + "F6 guard stale: api/ipfs.rs no longer calls `{call}` — update this guard" + ); + for (n, (idx, _)) in production.match_indices(call).enumerate() { + // The wrapper opens a few lines above the call (match tokio::time::timeout( + // ... remaining budget ..., )); a 240-char lookback covers it + // without reaching the previous statement. + let window = &production[idx.saturating_sub(240)..idx]; + assert!( + window.contains("tokio::time::timeout("), + "F6 gate missing: occurrence {n} of `{call}` (of {occurrences}) is not \ + wrapped in tokio::time::timeout(...) clamped to the remaining request \ + budget. An unwrapped await pins the held walk permit for the whole DB \ + stall, past GITLAWB_IPFS_REQUEST_BUDGET_SECS." + ); + } + } +} + +/// #174 F2 / KTD-3: the detached post-receive Pinata replication task must enqueue +/// only the push's `(ref, old, new)` tuples and RE-DERIVE its object set inside the +/// worker once a pin slot frees — it must NOT move the full per-push object list into +/// the closure and hold it across the `pin_semaphore` acquire. Retaining the list makes +/// every parked task (under a slow Pinata backend) hold an MB-scale OID list, so +/// outstanding memory grows O(pushes x object-list). Coalescing/shedding the task is +/// forbidden (its per-ref effects are non-idempotent), so the fix bounds the retained +/// data, not the task count. +/// +/// Two load-bearing checks, both against the PRODUCTION half of `api/repos.rs` (the +/// `mod tests` half names the same identifiers in its own harness and would make the +/// scan vacuous): (a) the closure-local `object_list_pinata` binding — the retain form — +/// must be gone; reintroducing `let object_list_pinata = object_list;` turns this red. +/// (b) the re-derivation (`pinata_object_list_for_refs`) must appear AFTER the Pinata +/// pin permit is acquired, so the object list is materialized only inside the pin-bounded +/// section and a parked task holds O(ref tuples). +#[test] +fn f2_pinata_enqueues_refs_not_retained_object_lists() { + let repos = src("api/repos.rs"); + // Split at the TEST MODULE, not at the first `#[cfg(test)]`. api/repos.rs carries + // test-only items (the drain fault seam, the task entry point) ABOVE the production + // code these gates scan for, so splitting on the attribute truncated the production + // half above every line being checked and the gate went vacuously blind. + let production = repos + .split("\nmod tests {") + .next() + .expect("split always yields a first chunk"); + + // (a) the retained-list form must be gone from production. + assert!( + !production.contains("object_list_pinata"), + "F2/KTD-3 regression: the Pinata task retains a full per-push object list \ + (`object_list_pinata`) across the pin-permit acquire. Enqueue only the ref \ + tuples and re-derive the object set inside the worker (pinata_object_list_for_refs)." + ); + + // (b) the re-derivation runs AFTER the pin permit is acquired. Anchor on the + // Pinata pin-admission clone so the window is the Pinata task, not the sibling + // IPFS/encrypt spawn (which shares `pin_semaphore` but never re-derives). + let anchor = production + .find("let pin_sem_pinata") + .expect("F2 gate stale: the Pinata pin-admission clone (pin_sem_pinata) moved"); + let tail = &production[anchor..]; + let acquire = tail + .find(".acquire_owned()") + .expect("F2 gate: the Pinata task no longer acquires a pin permit (acquire_owned)"); + let rederive = tail.find("pinata_object_list_for_refs(").expect( + "F2 gate missing: the Pinata task must re-derive its object set via \ + pinata_object_list_for_refs; it can no longer move a pre-resolved list into the closure", + ); + assert!( + acquire < rederive, + "F2 gate bypassed: the Pinata object-set re-derivation must run AFTER the pin \ + permit is acquired, so a parked task never holds the MB-scale object list" + ); + + // The re-derivation is driven by the push's ref tuples (the enqueued unit), not a + // retained object list — tie "enqueue ref_updates" to the call explicitly. + assert!( + tail[rederive..].starts_with("pinata_object_list_for_refs(") + && tail[rederive..] + .get(..400) + .map(|w| w.contains("ref_updates_clone")) + .unwrap_or(false), + "F2 gate: pinata_object_list_for_refs must re-derive from the push's ref tuples \ + (ref_updates_clone), the small unit the parked task retains" + ); +} + +/// #174 U2 / F3 (second same-repo writer serialized until a disconnected first push's +/// git group is reaped): `git_receive_pack` must take the per-repo in-process write +/// lease and CARRY it on the write-path `AdmissionGuard` (via `.with_lease`), so the +/// lease rides `KillGroupOnDrop`'s detached reaper and frees only after the group is +/// reaped — supplementing the pg advisory lock, which `RepoWriteGuard::Drop` releases at +/// the disconnect instant. Tying the lease to `RepoWriteGuard` instead (or dropping the +/// `.with_lease` carry) reopens the race; the behavioral RED/GREEN test +/// (`f3_second_push_serialized_until_disconnected_group_reaped`) is the real bar, this +/// is the completeness tripwire. Scanned against the PRODUCTION half of `api/repos.rs` +/// (the `mod tests` half names these identifiers in its own harness). +#[test] +fn f3_second_writer_leased_until_reap() { + let repos = src("api/repos.rs"); + let smart_http = src("git/smart_http.rs"); + // Split at the TEST MODULE, not at the first `#[cfg(test)]`. api/repos.rs carries + // test-only items (the drain fault seam, the task entry point) ABOVE the production + // code these gates scan for, so splitting on the attribute truncated the production + // half above every line being checked and the gate went vacuously blind. + let repos_production = repos + .split("\nmod tests {") + .next() + .expect("split always yields a first chunk"); + + // The lease is acquired, then carried by the AdmissionGuard (.with_lease), before + // receive_pack runs the write. Severing any of the three turns this red. + let lease_acquire = repos_production.find("repo_write_leases").expect( + "F3 gate missing: git_receive_pack no longer takes the per-repo write lease \ + (state.repo_write_leases)", + ); + let with_lease = repos_production.find(".with_lease(").expect( + "F3 gate missing: the write-path AdmissionGuard no longer carries the lease \ + (.with_lease). The lease must ride the disconnect reaper via the AdmissionGuard, \ + NOT RepoWriteGuard (which drops at the disconnect instant, reopening F3).", + ); + let receive = repos_production + .find("smart_http::receive_pack(") + .expect("F3 gate stale: git_receive_pack no longer calls smart_http::receive_pack"); + assert!( + lease_acquire < with_lease && with_lease < receive, + "F3 gate bypassed: the write lease must be acquired, then carried by the \ + AdmissionGuard (.with_lease), BEFORE receive_pack runs the write" + ); + + // The AdmissionGuard must actually hold the lease (Option) via a + // with_lease setter, so it travels into the detached reaper. Removing the field or + // method turns this red. + assert!( + smart_http.contains("_lease: Option") + && smart_http.contains("pub fn with_lease("), + "F3 gate missing: AdmissionGuard must hold an Option set via \ + with_lease, so the lease rides the guard into KillGroupOnDrop's reaper" + ); +} + +/// #174 U1 — every blocking walk in the `/ipfs` scan carries the request's walk +/// admission. +/// +/// The admission is an `Arc` cloned into each `spawn_blocking` +/// closure, so the global + per-source permits release only when the last holder +/// drops: a client disconnect leaves the abandoned closure holding the slot, and a +/// panicking closure leaves the handler holding it. +/// +/// Only the walk site runs under `state.git_bin`, so only it can be pinned by the +/// fake-git harness and mutation-verified dynamically (see +/// `get_by_cid_walk_permit_held_through_blocking_walk`). The probe and the content +/// read deliberately shell to the real `git`, which is why this structural check +/// exists: it binds all three sites, and any blocking site added to this loop +/// later, without reversing that deliberate independence. +/// +/// MUTATION (RED): delete any one `Arc::clone(ctx.admission)` binding, or drop the +/// clone from inside its closure, and the count falls below three. +#[test] +fn inv22_ipfs_walk_admission_reaches_every_blocking_site() { + let ipfs = src("api/ipfs.rs"); + + // The shared owner must exist and be built once per request. + assert!( + ipfs.contains("struct WalkAdmission") && ipfs.contains("Arc::new(WalkAdmission {"), + "U1 gate missing: the /ipfs walk admission must be a shared WalkAdmission, \ + not a handler-local permit pair" + ); + + // Every blocking site in the scan takes its own clone... + // `Arc::clone(ctx.admission)` in the gate, `Arc::clone(&admission)` if a site is + // ever added back in the handler body itself; count both spellings. + let clones = ipfs.matches("Arc::clone(ctx.admission)").count() + + ipfs.matches("Arc::clone(&admission)").count(); + assert!( + clones >= 3, + "U1 gate bypassed: expected an admission clone for each of the three \ + /ipfs spawn_blocking sites (probe, walk, read); found {clones}. A blocking \ + walk that does not hold the admission lets a disconnect or a panic free the \ + slot while its git child is still running." + ); + + // ...and each clone is actually moved INTO the blocking closure, not merely + // created in the async frame (which would hold nothing across the join). + let held = ipfs.matches("let _admission = ").count(); + assert!( + held >= 3, + "U1 gate bypassed: each admission clone must be bound inside its \ + spawn_blocking closure so the blocking work owns it; found {held} of 3." + ); + + // The count of blocking sites is itself the thing being covered: if a fourth + // appears, it needs an admission clone too and this gate must be revisited. + let sites = ipfs.matches("spawn_blocking(move ||").count(); + assert_eq!( + sites, 3, + "the /ipfs scan grew or lost a spawn_blocking site ({sites} found, expected 3); \ + give any new blocking walk its own Arc::clone(&admission) and update this gate" + ); +} + +/// #174 U5: the post-receive replication tail is spawned at the DURABILITY BOUNDARY, +/// which is the moment receive-pack returns success, not the end of the handler and +/// not after `guard.release()`. +/// +/// The tail owes this push its pins, recovery copy, and announcements. Everything +/// below the spawn stays in the cancellable request future, so anything the tail is +/// spawned after is a window where a client disconnect drops that work while the pack +/// is already durable on disk. `guard.release()` is such a window: on success it +/// awaits the Tigris upload and then the advisory unlock. +/// +/// The lower bound matters just as much as the upper one: `release` runs on failure +/// too, so an ungated spawn would fire for a push git rejected, pinning and announcing +/// a half-applied repo. Above `release` the `?` on `receive_result` can no longer be +/// what gates it, so the success check is explicit and this gate binds it: the spawn +/// must sit inside `if push_succeeded`, and `release` must consume the same flag so +/// the two cannot drift apart. +/// +/// This is an ordering check rather than a cancellation-race test on purpose: it is +/// the companion to `receive_pack_tail_survives_a_disconnect_during_release`, which +/// drives the actual disconnect through a parked `release`. Same instrument the F3 +/// gate above uses. +/// +/// MUTATION (RED): move the `tokio::spawn(post_receive_replication_tail` call below +/// `guard.release(` and the ordering assertion fails; take it out of the +/// `if push_succeeded` block and the failed-push assertion fails. +#[test] +fn inv22_replication_tail_spawns_at_the_durability_boundary() { + let repos = src("api/repos.rs"); + // Production half only — the tests below name these identifiers too. + // Split at the TEST MODULE, not at the first `#[cfg(test)]`. api/repos.rs carries + // test-only items (the drain fault seam, the task entry point) ABOVE the production + // code these gates scan for, so splitting on the attribute truncated the production + // half above every line being checked and the gate went vacuously blind. + let production = repos + .split("\nmod tests {") + .next() + .expect("split always yields a first chunk"); + + let success_flag = production + .find("let push_succeeded = receive_result.is_ok();") + .expect( + "U5 gate missing: the tail's success gate must be bound from receive_result.is_ok()", + ); + let gate_open = production + .find("if push_succeeded {") + .expect("U5 gate missing: the tail spawn must be gated on the push having succeeded"); + let spawn = production + .find("tokio::spawn(post_receive_replication_tail(") + .expect("U5 gate missing: the replication tail must be spawned by git_receive_pack"); + let release = production + .find(".release(push_succeeded)") + .expect("U5 gate stale: release must consume the same success flag as the tail gate"); + let touch = production + .find("state.db.touch_repo(") + .expect("U5 gate stale: git_receive_pack no longer calls touch_repo"); + let webhook = production + .find("webhooks::fire_event(") + .expect("U5 gate stale: git_receive_pack no longer fires push webhooks"); + + assert!( + success_flag < gate_open && gate_open < spawn, + "U5 gate bypassed: the tail must be spawned inside `if push_succeeded`, or a \ + rejected push spawns a tail that pins and announces a half-applied repo" + ); + // Still inside that block: no `}` may close it between the gate and the spawn. + assert!( + !production[gate_open + "if push_succeeded {".len()..spawn].contains('}'), + "U5 gate bypassed: the tail spawn left the `if push_succeeded` block, so a \ + rejected push now spawns a tail" + ); + assert!( + spawn < release && spawn < touch && spawn < webhook, + "U5 gate bypassed: the tail must be spawned BEFORE guard.release, touch_repo \ + and the webhook fan-out, so a disconnect in any of those windows cannot drop \ + this push's pins, recovery copy, and announcements" + ); +} diff --git a/crates/gl/src/ipfs_cmd.rs b/crates/gl/src/ipfs_cmd.rs index d4cb64fc..fa7a3f3f 100644 --- a/crates/gl/src/ipfs_cmd.rs +++ b/crates/gl/src/ipfs_cmd.rs @@ -33,13 +33,16 @@ pub enum IpfsCmd { cid: String, #[arg(long, default_value = "https://node.gitlawb.com", env = "GITLAWB_NODE")] node: String, + /// Identity directory (default: ~/.gitlawb) + #[arg(long)] + dir: Option, }, } pub async fn run(args: IpfsArgs) -> Result<()> { match args.cmd { IpfsCmd::List { node, dir } => cmd_list(node, dir).await, - IpfsCmd::Get { cid, node } => cmd_get(cid, node).await, + IpfsCmd::Get { cid, node, dir } => cmd_get(cid, node, dir).await, } } @@ -86,11 +89,32 @@ async fn cmd_list(node: String, dir: Option) -> Result<()> { Ok(()) } -async fn cmd_get(cid: String, node: String) -> Result<()> { - let client = NodeClient::new(&node, None); - let path = format!("/ipfs/{cid}"); +async fn cmd_get(cid: String, node: String, dir: Option) -> Result<()> { + // #173 (F5): the resolver now serves path-scoped objects to authorized readers, + // so sign with an available identity like `gl ipfs list` — otherwise an owner or + // listed reader gets the opaque anonymous 404 for content they can read. + // `get_authed` signs when a keypair is present and falls back to unsigned. + // + // An explicit `--dir` is a request to use THAT identity: propagate a + // missing/corrupt-keystore error (like `list`) instead of silently sending an + // anonymous request the authorized reader would see as the node's opaque 404 + // (#173 review). Only the default (no `--dir`) keeps the best-effort unsigned + // fallback, so `get` stays usable for genuinely public content. + let keypair = match dir.as_deref() { + Some(dir) => Some(crate::identity::load_keypair_from_dir(Some(dir))?), + None => crate::identity::load_keypair_from_dir(None).ok(), + }; + let client = NodeClient::new(&node, keypair); + // #173 review (F1): the node now accepts equivalent multibase spellings, + // including base64 CIDs (prefix 'm'), whose alphabet contains '/', '+', '='. + // Interpolating the CID raw would make the client request (and sign) + // `/ipfs//`, which neither matches the single-segment Axum + // route nor points at the intended target. Percent-encode the CID as exactly + // one path segment so the signed and sent target agree and the server's + // `Path` extractor decodes it back to the original CID. + let path = format!("/ipfs/{}", encode_cid_segment(&cid)); let resp = client - .get(&path) + .get_authed(&path) .await .with_context(|| format!("failed to fetch CID {cid} from {node}"))?; @@ -119,6 +143,16 @@ async fn cmd_get(cid: String, node: String) -> Result<()> { Ok(()) } +/// Percent-encode a CID so it occupies exactly one path segment of `/ipfs/`. +/// `urlencoding::encode` escapes every byte outside the RFC 3986 unreserved set +/// (ALPHA / DIGIT / `-._~`), so the base64-CID characters that would otherwise +/// break the single-segment route — `/`, `+`, `=` — are all escaped, and the +/// server's `Path` extractor decodes the result back to the original CID (#173 +/// review, F1). +fn encode_cid_segment(cid: &str) -> String { + urlencoding::encode(cid).into_owned() +} + #[cfg(test)] mod tests { use super::*; @@ -235,4 +269,162 @@ mod tests { m.assert_async().await; } + + /// #173 (F5): `gl ipfs get` must SIGN with an available identity, like + /// `gl ipfs list`, so an owner/reader can retrieve a path-scoped object the node + /// now resolves by CID. RED before the fix: cmd_get ignores the identity dir and + /// sends an unsigned request, so the signature-matching mock is never hit + /// (cmd_get errors on the unmatched 501, and m.assert fails). GREEN after: the + /// signed request carries the RFC 9421 headers and is served 200. + #[tokio::test] + async fn test_cmd_get_signs_when_identity_present() { + let mut server = mockito::Server::new_async().await; + let keystore = seed_keystore(); + + let m = server + .mock("GET", "/ipfs/bafkreitestcid") + .match_header("signature", mockito::Matcher::Any) + .match_header("signature-input", mockito::Matcher::Any) + .with_status(200) + .with_header("content-type", "application/octet-stream") + .with_header("x-git-hash", "abc123") + .with_body("object bytes") + .create_async() + .await; + + cmd_get( + "bafkreitestcid".to_string(), + server.url(), + Some(keystore.path().to_path_buf()), + ) + .await + .expect("signed get of a resolvable object should succeed"); + + m.assert_async().await; + } + + /// #173 (F5) must-not: a genuine anonymous denial must surface as an error, not + /// be masked as success. With no identity dir the request is unsigned; a 404 + /// from the node must produce an Err mentioning the status. + #[tokio::test] + async fn test_cmd_get_anonymous_denial_is_error() { + let mut server = mockito::Server::new_async().await; + + let m = server + .mock("GET", "/ipfs/bafkreidenied") + .with_status(404) + .with_header("content-type", "text/plain") + .with_body("no git object found") + .create_async() + .await; + + let err = cmd_get("bafkreidenied".to_string(), server.url(), None) + .await + .expect_err("a 404 denial must be an error, not masked success"); + assert!( + err.to_string().contains("404"), + "error should mention the status, got: {err}" + ); + + m.assert_async().await; + } + + /// #173 (INV-8) must-not: the node's new 503 "search incomplete" (the legacy CID + /// scan hit its bound and could not prove absence) must surface as an actionable + /// Err naming the status, NOT be rendered as an empty/"not found" success — a + /// retryable outcome the caller has to see. Mirrors the 404 denial case for the + /// bounded-search response the resolver now emits. + #[tokio::test] + async fn test_cmd_get_search_incomplete_503_is_error() { + let mut server = mockito::Server::new_async().await; + + let m = server + .mock("GET", "/ipfs/bafkreiincomplete") + .with_status(503) + .with_header("content-type", "application/json") + .with_body(r#"{"error":"search_incomplete","message":"CID search incomplete — retry"}"#) + .create_async() + .await; + + let err = cmd_get("bafkreiincomplete".to_string(), server.url(), None) + .await + .expect_err("a 503 incomplete-search must be an error, not masked as not-found"); + assert!( + err.to_string().contains("503"), + "error should mention the status, got: {err}" + ); + + m.assert_async().await; + } + + /// #173 review (F1): a base64 CID (multibase prefix 'm') can contain '/', '+', + /// and '='. The client must percent-encode it into ONE path segment before + /// building and signing `/ipfs/`; otherwise the '/' splits the target so + /// it misses the single-segment Axum route and the signature covers the wrong + /// path. Assert the encoded segment carries no raw '/', '+', or '=', and that + /// it decodes back to the original CID (the server's `Path` extractor performs + /// that same decode). RED with the old raw `format!("/ipfs/{cid}")`: the + /// segment still contains '/'. + #[test] + fn test_encode_cid_segment_escapes_base64_alphabet() { + let cid = "mFoo/Bar+baz=="; + let encoded = encode_cid_segment(cid); + + assert!( + !encoded.contains('/'), + "encoded CID must be a single path segment (no raw '/'), got: {encoded}" + ); + assert!( + !encoded.contains('+'), + "encoded CID must escape '+', got: {encoded}" + ); + assert!( + !encoded.contains('='), + "encoded CID must escape '=', got: {encoded}" + ); + + let decoded = urlencoding::decode(&encoded).expect("encoded CID must decode"); + assert_eq!( + decoded, cid, + "encoding must round-trip back to the original CID" + ); + } + + /// #173 review: `gl ipfs get --dir ` must PROPAGATE a missing/corrupt + /// identity-load error like `gl ipfs list`, not silently fall back to an anonymous + /// request — otherwise an authorized reader pointing `--dir` at a broken keystore + /// gets the node's opaque 404 instead of the actionable key-load error. The + /// unsigned fallback is preserved only when NO `--dir` is given (covered by + /// `test_cmd_get_anonymous_denial_is_error`). RED before the fix (`.ok()` swallows + /// the error, an anonymous request is sent, and the `.expect(0)` mock is hit), + /// GREEN after. + #[tokio::test] + async fn test_cmd_get_explicit_dir_no_identity_errors_without_request() { + let mut server = mockito::Server::new_async().await; + // Empty keystore dir passed explicitly via --dir: no identity.pem present. + let empty = tempfile::TempDir::new().unwrap(); + + // The endpoint must never be hit when an explicit --dir fails to load. + let m = server + .mock("GET", "/ipfs/bafkreitestcid") + .expect(0) + .create_async() + .await; + + let err = cmd_get( + "bafkreitestcid".to_string(), + server.url(), + Some(empty.path().to_path_buf()), + ) + .await + .expect_err("an explicit --dir that fails to load must be an error"); + assert!( + err.to_string().contains("gl identity new") + || err.to_string().contains("no identity found") + || err.to_string().contains("failed to load keypair"), + "error should name the key-load failure, got: {err}" + ); + + m.assert_async().await; + } }