From 471713644806cbcc3f875722c64a919a42d12230 Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Fri, 18 Sep 2026 13:54:01 -0400 Subject: [PATCH 01/40] feat(telemetry): record the retry prompt as gen_ai.input.messages Under the Level 3 content gate, a retry iteration that composes a validation-feedback prompt (feedback_mode: append) now records it on the iteration's agent span as gen_ai.input.messages: one user message with one text part. The first iteration and retries without feedback send the runtime's default prompt and record nothing. The recorded copy passes through the collector's redaction, and its findings join the iteration's; Result no longer returns early when an iteration has findings or dropped bytes but no parts. The encoded input is charged against maxEncodedContentBytes so the two content attributes of one span stay within the proven size together. Signed-off-by: Dharit Shah --- internal/cli/content_collector.go | 52 +++++++++++++++++- internal/cli/content_collector_test.go | 64 ++++++++++++++++++++++ internal/cli/run.go | 1 + internal/cli/scan_output_telemetry_test.go | 2 +- internal/cli/telemetry_run_test.go | 14 +++++ internal/telemetry/telemetry.go | 15 ++--- 6 files changed, 138 insertions(+), 10 deletions(-) diff --git a/internal/cli/content_collector.go b/internal/cli/content_collector.go index fdd1a2b86b..5d53fb58fa 100644 --- a/internal/cli/content_collector.go +++ b/internal/cli/content_collector.go @@ -38,7 +38,11 @@ import ( const maxContentBytes = 256 * 1024 // maxEncodedContentBytes bounds gen_ai.output.messages as exported — the -// JSON string the backend receives, syntax and escaping included. It sits +// JSON string the backend receives, syntax and escaping included. On a +// retry that records gen_ai.input.messages, attachInput charges that +// attribute's encoded size against it first, so the output gets what the +// input left and the two together stay under it; the input itself is cut +// upstream (maxFeedbackBytes), not here. It sits // just under the one size the pilot backend is proven to accept (see // maxContentBytes), so no record is ever larger than the proof. The raw // budget runs first; on live streams a budget-binding record encodes 9 to @@ -125,6 +129,50 @@ func attachContent(span trace.Span, res contentResult) { } } +// attachInput records the prompt the runner composed for this iteration +// as gen_ai.input.messages: one user message with one text part. Only a +// retry under feedback_mode: append composes a prompt; otherwise prompt +// is empty and nothing is recorded, as with a nil collector (gate off). +// +// The recorded copy goes through the same redaction as output content. +// redactFeedback already scanned the feedback, before it was sanitized, +// cut and framed; when neither sanitizing nor the pipeline's folding +// changes the text, the pattern scan here repeats that one. Sanitizing can +// join a token that scan saw split by a zero-width character, and folding +// can spell out one it saw in fullwidth; then this scan is the first to +// see it whole. +// It masks the recorded copy only — the agent is sent the prompt as +// composed — and it does not repeat redactFeedback's literal pass over +// runner-env values. The mask that first scan leaves for a +// connection-string password of ten or more bytes ("abcd...") matches its +// own pattern again, so such a prompt counts a finding here; a shorter +// password is masked "***", which does not. +// The pipeline also folds compatibility characters the agent received +// unfolded. Findings join the iteration's and surface through Result. +// +// The prompt is cut upstream (maxFeedbackBytes), not here. Folding can +// grow the copy about elevenfold, so its encoded size is charged to +// maxEncoded: the two content attributes of one span together stay +// within the size maxEncodedContentBytes is proven for. +func (c *contentCollector) attachInput(span trace.Span, prompt string) { + if c == nil { + return + } + text := c.redact(prompt, &c.findings) + if text == "" { + return + } + raw, err := json.Marshal([]struct { + Role string `json:"role"` + Parts []contentPart `json:"parts"` + }{{"user", []contentPart{{Type: "text", Content: text}}}}) + if err != nil { + return // strings marshal unconditionally + } + c.maxEncoded -= len(raw) + span.SetAttributes(stringAttr("gen_ai.input.messages", string(raw))) +} + // contentPart is one part of the assembled assistant output message, // shaped for the GenAI output-messages JSON schema: TextPart // ({type:"text",content}), the schema's GenericPart extension point @@ -442,7 +490,7 @@ func bulkField(p *contentPart) *string { // failed. Redaction runs before the size budget: truncating first could // split a secret so the redactor no longer recognizes it. func (c *contentCollector) Result(finishReason string) contentResult { - if c == nil || len(c.parts) == 0 { + if c == nil { return contentResult{} } diff --git a/internal/cli/content_collector_test.go b/internal/cli/content_collector_test.go index 92a29236e0..8ad531165e 100644 --- a/internal/cli/content_collector_test.go +++ b/internal/cli/content_collector_test.go @@ -10,6 +10,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/sdk/trace/tracetest" agentruntime "github.com/fullsend-ai/fullsend/internal/runtime" ) @@ -1094,3 +1096,65 @@ func TestTailToRuneBoundary(t *testing.T) { assert.Equal(t, "fgh", tailToRuneBoundary("abcdéfgh", 4), "cut landing mid-rune walks forward, never splitting the rune") } + +// recordedInput runs attachInput on a recorded agent span and returns the +// attributes the span ended with. +func recordedInput(t *testing.T, c *contentCollector, prompt string) map[attribute.Key]attribute.Value { + t.Helper() + _, rec, span := toolSpanFixture(t) + c.attachInput(span, prompt) + span.End() + ended := rec.Ended() + require.Len(t, ended, 1) + return toolSpanAttrs(tracetest.SpanStubFromReadOnlySpan(ended[0])) +} + +func TestAttachInput_RecordsTheRetryPromptAsOneUserMessage(t *testing.T) { + prompt, _ := buildFeedbackPrompt("lint: main.go:3 unused variable ") + attrs := recordedInput(t, newContentCollector(maxContentBytes), prompt) + + var msgs []map[string]any + require.NoError(t, json.Unmarshal([]byte(attrs["gen_ai.input.messages"].AsString()), &msgs)) + require.Len(t, msgs, 1) + assert.Equal(t, map[string]any{ + "role": "user", + "parts": []any{map[string]any{"type": "text", "content": prompt}}, + }, msgs[0], "an input message carries role and parts; finish_reason belongs to output messages") + assert.Len(t, attrs, 1) +} + +func TestAttachInput_NoPromptOrNoCollectorAddsNothing(t *testing.T) { + assert.Empty(t, recordedInput(t, newContentCollector(maxContentBytes), ""), + "no composed prompt (the first iteration, or feedback_mode off)") + assert.Empty(t, recordedInput(t, nil, "would-be prompt"), "gate off") +} + +func TestAttachInput_RedactsAndCountsFindings(t *testing.T) { + secret := "ghp_" + strings.Repeat("k", 36) + c := newContentCollector(maxContentBytes) + in := recordedInput(t, c, "token "+secret+" leaked")["gen_ai.input.messages"].AsString() + + assert.NotContains(t, in, secret) + assert.Contains(t, in, "leaked") + // A retry can fail before the agent emits anything; the input's + // findings still have to reach the span. + assert.NotEmpty(t, c.Result("error").Findings) +} + +func TestAttachInput_WorstCaseFeedbackSharesTheEncodedCeiling(t *testing.T) { + // U+FDFA has the largest NFKC expansion in Unicode (3 bytes to 33) and + // redaction folds it, so the recorded copy of a 10 KiB feedback is an + // order of magnitude larger than the prompt the agent was sent. + prompt, _ := buildFeedbackPrompt(strings.Repeat("\uFDFA", maxFeedbackBytes)) + c := newContentCollector(maxContentBytes) + in := recordedInput(t, c, prompt)["gen_ai.input.messages"].AsString() + require.Greater(t, len(in), 10*maxFeedbackBytes, "fixture must expand for this test to prove anything") + + c.Handle(agentruntime.TextEvent{Text: strings.Repeat("<", maxContentBytes)}) // encodes sixfold + out := c.Result("stop").OutputMessages + require.NotEmpty(t, out) + assert.LessOrEqual(t, len(in)+len(out), maxEncodedContentBytes, + "both attributes ride one span; together they stay within the proven size") + assert.Greater(t, len(in)+len(out), maxEncodedContentBytes-100, + "the output keeps everything the input left") +} diff --git a/internal/cli/run.go b/internal/cli/run.go index b52e6f6dd4..2b8e117118 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -2319,6 +2319,7 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep // tool-span tracker is per iteration for the same reason and is not // gated: execute_tool spans are metadata. collector := newContentCollectorIfEnabled() + collector.attachInput(agentSpan, agentPrompt) toolSpans := newToolSpanTracker(tracer, agentCtx) var metrics agentruntime.RunMetrics hooksSettings := "" diff --git a/internal/cli/scan_output_telemetry_test.go b/internal/cli/scan_output_telemetry_test.go index eff57ddc90..296e1ecf2d 100644 --- a/internal/cli/scan_output_telemetry_test.go +++ b/internal/cli/scan_output_telemetry_test.go @@ -17,7 +17,7 @@ import ( // redaction scan does NOT rewrite the telemetry JSONL file. It is still held // open for append during the scan, so an in-place rewrite would truncate it // under the open handle; and any Level 3 conversation content it carries was -// already redacted at assembly (contentCollector.Result) before reaching a +// already redacted at assembly (contentCollector) before reaching a // span, so the file needs no post-hoc sweep. A normal output file must still // be sanitized. func TestScanOutputFiles_SkipsTelemetryArtifacts(t *testing.T) { diff --git a/internal/cli/telemetry_run_test.go b/internal/cli/telemetry_run_test.go index 87925a11da..9c53b531d6 100644 --- a/internal/cli/telemetry_run_test.go +++ b/internal/cli/telemetry_run_test.go @@ -1396,7 +1396,10 @@ func TestContentCapture_EndToEndFileSink(t *testing.T) { big := strings.Repeat("all work and no play makes claude a dull agent. ", 400) // ~19KB > 8192 c.Handle(agentruntime.TextEvent{Text: big}) + prompt, _ := buildFeedbackPrompt(strings.Repeat("the validator said no. ", 500)) // cut at 10 KiB > 8192 + _, span := tracer.Start(context.Background(), "agent") + c.attachInput(span, prompt) res := c.Result("stop") attachContent(span, res) span.End() @@ -1406,6 +1409,14 @@ func TestContentCapture_EndToEndFileSink(t *testing.T) { require.NoError(t, err) require.NotEmpty(t, raw) content := string(raw) + require.Greater(t, len(prompt), telemetry.MaxSpanAttrValueLen) + promptJSON, err := json.Marshal(prompt) + require.NoError(t, err) + escapedInput, err := json.Marshal(`[{"role":"user","parts":[{"type":"text","content":` + string(promptJSON) + `}]}]`) + require.NoError(t, err) + require.Contains(t, content, "gen_ai.input.messages") + assert.Contains(t, content, string(escapedInput[1:len(escapedInput)-1]), + "the input record must appear byte-intact in the file sink") require.Contains(t, content, "gen_ai.output.messages") assert.Contains(t, content, "dull agent", "content must reach the file sink") @@ -1440,12 +1451,15 @@ func TestContentCapture_GateOffProducesNoContent(t *testing.T) { c.Handle(agentruntime.TextEvent{Text: "would-be content"}) // nil-safe no-op _, span := tracer.Start(context.Background(), "agent") + c.attachInput(span, "would-be prompt") // nil-safe no-op attachContent(span, c.Result("stop")) span.End() cleanup(context.Background()) raw, err := os.ReadFile(filepath.Join(dir, telemetry.TelemetryFile)) require.NoError(t, err) + assert.NotContains(t, string(raw), "gen_ai.input.messages") + assert.NotContains(t, string(raw), "would-be prompt") assert.NotContains(t, string(raw), "gen_ai.output.messages") assert.NotContains(t, string(raw), "would-be content") assert.NotContains(t, string(raw), "fullsend.content.") diff --git a/internal/telemetry/telemetry.go b/internal/telemetry/telemetry.go index 4185e3725c..5734db6f42 100644 --- a/internal/telemetry/telemetry.go +++ b/internal/telemetry/telemetry.go @@ -215,12 +215,13 @@ func operatorAttrValueLimit() (int, bool) { // warnContentCaptureAttrLimit warns on stderr when the Level 3 content // gate is on but an operator's finite attribute value-length limit is // configured. The operator limit wins over the gate's cap lift -// (spanLimits), so the SDK will cut any gen_ai.output.messages value over -// the limit mid-JSON — in both sinks, with no fullsend.content.truncated -// marker (that marker reflects only collector-side cuts) — silently -// breaking the documented consumer contract. Telemetry never fails a run -// (ADR 0050), so the collision is surfaced, not fatal. An explicit -1 -// (unlimited) cannot cut and is not a conflict. +// (spanLimits), so the SDK will cut any gen_ai.input.messages or +// gen_ai.output.messages value over the limit mid-JSON — in both sinks, +// with no fullsend.content.truncated marker (that marker reflects only +// collector-side cuts) — silently breaking the documented consumer +// contract. Telemetry never fails a run (ADR 0050), so the collision is +// surfaced, not fatal. An explicit -1 (unlimited) cannot cut and is not a +// conflict. func warnContentCaptureAttrLimit() { if !ContentCaptureEnabled() { return @@ -228,7 +229,7 @@ func warnContentCaptureAttrLimit() { if limit, ok := operatorAttrValueLimit(); ok && limit >= 0 { fmt.Fprintf(os.Stderr, "fullsend: content capture is enabled but the operator attribute value length limit (%d) is set; "+ - "gen_ai.output.messages values over the limit will be cut mid-JSON (unparseable, and "+ + "gen_ai.input.messages and gen_ai.output.messages values over the limit will be cut mid-JSON (unparseable, and "+ "fullsend.content.truncated will not flag the cut) — raise the limit or unset it to keep content parseable\n", limit) } From 73d586b74d73291fb998e5db1228893d6c5c539b Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Fri, 18 Sep 2026 15:10:18 -0400 Subject: [PATCH 02/40] feat(telemetry): record tool arguments on tool_call parts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Under the Level 3 content gate a tool_call part now carries the call's arguments. ToolUseEvent gains Arguments — the input as the stream carried it, from both Claude Code parser paths; pi, codex and OpenCode leave it empty (#7414) and the renderer does not print it. The collector decodes the value, redacts each string and object key on its own, and encodes it again: the redactor's patterns are written for plain text, and over serialised JSON they miss an assignment that opens a string or follows an escaped newline, a value behind escaped quotes, and JSON nested in a string, while Unicode folding can close a string early. Each string member is scanned once more, already redacted, beside its key, so the member-name pattern still sees the pair; only that pattern's finding is kept from the second scan. Arguments are dropped whole, charged to the dropped bytes, and the part marked fullsend.truncated (it keeps its id, name and summary) when their redacted encoding exceeds maxToolArgumentsBytes (8 KiB; a cut object is not JSON), when the text is not one JSON value, or when two keys of one object redact to the same string. A call without a name carries none. On the three measured live review streams 0-3 calls per iteration exceed the bound, each an Agent dispatch prompt. The encoded-ceiling property test generates and counts arguments. Regression pins, green on arrival: execute_tool spans carry nothing of a call's arguments, and with the gate off arguments and results stay out of run-telemetry.jsonl while the tool span is still written. Signed-off-by: Dharit Shah --- internal/cli/content_collector.go | 204 +++++++++++++++--- internal/cli/content_collector_test.go | 255 ++++++++++++++++++++++- internal/cli/telemetry_run_test.go | 10 +- internal/cli/tool_spans_test.go | 16 ++ internal/runtime/claude_progress.go | 14 +- internal/runtime/claude_progress_test.go | 50 +++++ internal/runtime/event.go | 13 +- internal/runtime/renderer_test.go | 11 + 8 files changed, 535 insertions(+), 38 deletions(-) diff --git a/internal/cli/content_collector.go b/internal/cli/content_collector.go index 5d53fb58fa..14842b97ed 100644 --- a/internal/cli/content_collector.go +++ b/internal/cli/content_collector.go @@ -2,7 +2,10 @@ package cli import ( "encoding/json" + "maps" + "slices" "sort" + "strings" "unicode/utf8" "go.opentelemetry.io/otel/attribute" @@ -45,8 +48,8 @@ const maxContentBytes = 256 * 1024 // upstream (maxFeedbackBytes), not here. It sits // just under the one size the pilot backend is proven to accept (see // maxContentBytes), so no record is ever larger than the proof. The raw -// budget runs first; on live streams a budget-binding record encodes 9 to -// 11% larger than its raw bytes, and escape-dense content ('<', control +// budget runs first; on live streams a budget-binding record encodes 8 to +// 10% larger than its raw bytes, and escape-dense content ('<', control // bytes, invalid UTF-8) up to six times larger, so Result trims the // oldest content again until the encoding fits. const maxEncodedContentBytes = 255_000 @@ -77,6 +80,20 @@ const maxToolIDBytes = 256 // results; no consumer requirement has confirmed either direction yet. const maxToolResultBytes = 8 * 1024 +// maxToolArgumentsBytes bounds one tool call's arguments, measured on +// their redacted encoding. Arguments over it are dropped whole, not cut: +// a cut object is not JSON, and the part keeps its id, name and summary. +// Like maxToolResultBytes it exists because the total above is small; +// what blocks raising it is the same unproven backend ceiling named on +// maxContentBytes. +// Measured on the three live review-agent streams behind that constant +// (2026-09-18, sub-agent calls included): 117-255 calls per iteration +// carry 41-100KB of arguments in all (p50 about 110 bytes, p90 under 300), +// and 0-3 calls per iteration exceed 8KiB, each an Agent dispatch prompt +// (largest 24KB). No stream from an agent that writes files was +// measured; a Write call carries the file body as an argument. +const maxToolArgumentsBytes = 8 * 1024 + // newContentCollectorIfEnabled returns a live collector when the Level 3 // gate is on and nil otherwise — nil is the off state and is inert at // every call site, so the gate needs no second check. @@ -91,7 +108,8 @@ func newContentCollectorIfEnabled() *contentCollector { // renderer, the tool-span tracker and the Level 3 collector, in that // order. The tracker stamps a span's start and end when it handles the // event, so it runs ahead of the collector, whose redaction pass scales -// with the size of a tool result. It is always non-nil: tool spans are metadata and are emitted +// with the size of a tool result or a call's arguments. It is always +// non-nil: tool spans are metadata and are emitted // with the content gate off, and supplying any OnEvent replaces the // runtime's default renderer — losing it silences CI output — so the // renderer runs first whatever else is off. A nil collector (gate off) @@ -177,23 +195,26 @@ func (c *contentCollector) attachInput(span trace.Span, prompt string) { // shaped for the GenAI output-messages JSON schema: TextPart // ({type:"text",content}), the schema's GenericPart extension point // ({type:"reasoning",content}), ToolCallRequestPart -// ({type:"tool_call",id,name}+summary), and ToolCallResponsePart +// ({type:"tool_call",id,name,arguments}+summary), and ToolCallResponsePart // ({type:"tool_call_response",id,response}). A tool summary is not the -// tool's arguments, so no arguments field is ever fabricated from it. +// tool's arguments, so no arguments field is ever fabricated from it; +// arguments come only from an event that carries them (toolArguments). type contentPart struct { - Type string `json:"type"` - Content string `json:"content,omitempty"` - ID string `json:"id,omitempty"` - Name string `json:"name,omitempty"` - Summary string `json:"summary,omitempty"` - Response string `json:"response,omitempty"` + Type string `json:"type"` + Content string `json:"content,omitempty"` + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Summary string `json:"summary,omitempty"` + Response string `json:"response,omitempty"` + Arguments json.RawMessage `json:"arguments,omitempty"` // IsError mirrors the wire's is_error on failed tool calls; it is // content-bearing (an errored empty result is signal, not absence) // and accounts a fixed footprint. Truncated marks a part whose bulk // field was cut, so a consumer never reads a fragment as a whole - // result; it is set by the collector's own cuts and by parser-side - // loss (Partial, Oversized), and stays outside the accounting except - // for an oversized stand-in's fixed footprint (see oversized). + // result, and a tool_call whose arguments were dropped; it is set by + // the collector's own cuts and by parser-side loss (Partial, + // Oversized), and stays outside the accounting except for an oversized + // stand-in's fixed footprint (see oversized). IsError bool `json:"is_error,omitempty"` Truncated bool `json:"fullsend.truncated,omitempty"` // bulkScanned records that the bulk field was already redacted at @@ -238,7 +259,7 @@ func (p contentPart) MarshalJSON() ([]byte, error) { // content: a failed call with empty output must survive, so it counts // its serialized footprint. func contentBytes(p contentPart) int { - n := len(p.Content) + len(p.Name) + len(p.Summary) + len(p.Response) + n := len(p.Content) + len(p.Name) + len(p.Summary) + len(p.Response) + len(p.Arguments) if p.IsError { n += isErrorFootprint } @@ -301,10 +322,16 @@ type contentResult struct { // when the iteration produced no content. OutputMessages string // DroppedBytes counts raw part bytes removed by the size budget - // (content, tool name, summary, tool response, and id bytes alike). + // (content, tool name, summary, tool response, and id bytes alike) and + // the bytes of dropped tool arguments: as re-encoded JSON (on a key + // collision, without the member the later key in sorted order + // replaced), or as + // redacted text when they were not one JSON value. DroppedBytes int - // Truncated reports whether the budget cut or dropped anything, or a - // kept tool result was a parser-side fragment (its part is marked). + // Truncated reports whether the budget cut or dropped anything, a kept + // tool call's arguments were dropped (toolArguments; its part is + // marked), or a kept tool result was a parser-side fragment (its part + // is marked). Truncated bool // Findings are the security findings raised during redaction. Findings []security.Finding @@ -324,23 +351,30 @@ type contentCollector struct { // maxEncoded bounds the marshaled attribute; see maxEncodedContentBytes. maxEncoded int pipeline *security.Pipeline - parts []contentPart - total int - // evicted counts bytes of old parts discarded during accumulation. + // secrets is the pipeline's pattern stage alone, for secretNamed. + secrets *security.SecretRedactor + parts []contentPart + total int + // evicted counts bytes discarded before Result's budget runs: old + // parts and a lone oversized part's head (evictOverflow), the head a + // tool result loses to maxToolResultBytes, and dropped tool arguments + // (toolArguments). // Eviction keeps memory bounded on long sessions by approximating the // Result budget on sizes as accumulated — pre-redaction — so it can // drop content the post-redaction suffix budget would have kept. // Discarded content is always redacted first: its findings land in // findings below, and no cut ever runs on raw bytes. evicted int - // findings raised while redacting content during eviction; merged - // into the contentResult at Result so eviction-time redactions are - // counted exactly like assembly-time ones. + // findings raised by redaction that runs before Result: on evicted or + // capped content, on tool arguments — kept or dropped; Result does + // not scan them again — and on the input message (attachInput). + // Merged into the contentResult at Result so they count exactly like + // assembly-time ones. findings []security.Finding } func newContentCollector(maxBytes int) *contentCollector { - return &contentCollector{maxBytes: maxBytes, maxEncoded: maxEncodedContentBytes, pipeline: security.OutputPipeline()} + return &contentCollector{maxBytes: maxBytes, maxEncoded: maxEncodedContentBytes, pipeline: security.OutputPipeline(), secrets: security.NewSecretRedactor()} } // Handle consumes one normalized event. Contiguous text and reasoning @@ -357,7 +391,14 @@ func (c *contentCollector) Handle(evt agentruntime.AgentEvent) { case agentruntime.ThinkingEvent: c.appendText("reasoning", e.Text) case agentruntime.ToolUseEvent: - c.appendPart(contentPart{Type: "tool_call", ID: boundedID(e.ID), Name: e.Name, Summary: e.Summary}) + p := contentPart{Type: "tool_call", ID: boundedID(e.ID), Name: e.Name, Summary: e.Summary} + if e.Name != "" { + // The schema requires a name on a tool_call part. Arguments + // alone must not keep a nameless call, nor charge for one that + // appendPart then refuses. + p.Arguments, p.Truncated = c.toolArguments(e.Arguments) + } + c.appendPart(p) case agentruntime.ToolResultEvent: p := contentPart{Type: "tool_call_response", ID: boundedID(e.ID), Response: e.Result, IsError: e.IsError, oversized: e.Oversized} // A parser-side partial flatten (non-text blocks skipped) is a @@ -497,7 +538,7 @@ func (c *contentCollector) Result(finishReason string) contentResult { res := contentResult{ DroppedBytes: c.evicted, Truncated: c.evicted > 0, - // Findings raised while redacting evicted content count exactly + // Findings raised before Result (see findings) count exactly // like assembly-time ones — a consumer must see every redaction, // including ones inside content the budget dropped. Findings: append([]security.Finding(nil), c.findings...), @@ -516,6 +557,11 @@ func (c *contentCollector) Result(finishReason string) contentResult { p.Response = c.redact(p.Response, &res.Findings) } p.Name = c.redact(p.Name, &res.Findings) + if p.Name == "" { + // As in Handle, for a name that redacts to nothing. Arguments + // Handle had already dropped stay charged and marked. + p.Arguments = nil + } p.Summary = c.redact(p.Summary, &res.Findings) p.ID = c.redactID(p.ID, &res.Findings) if contentBytes(p) == 0 { @@ -670,6 +716,110 @@ func (c *contentCollector) redact(text string, findings *[]security.Finding) str return text } +// toolArguments returns a tool call's arguments redacted and re-encoded, +// and whether they were dropped instead. There are none to return when +// the event carries none or JSON null. +// +// The redactor's patterns are written for plain text: run over serialised +// JSON they miss an assignment that opens a string or follows an escaped +// newline, a value behind escaped quotes, and JSON nested in a string, and +// Unicode folding can turn a fullwidth quotation mark into one that closes +// the string. So the value is decoded, each string and object key is +// redacted on its own (redactValue), and the result is encoded again — +// key order, spacing and escapes are the encoder's, not the wire's. +// +// Text that is not one JSON value (see ToolUseEvent.Arguments) cannot be +// redacted that way. It is scanned as text so its findings count, then +// dropped and charged like any other discarded content. Arguments whose +// redacted encoding exceeds maxToolArgumentsBytes, or in which two keys +// of one object redact to the same string, are dropped the same way and +// charged as encoded. On a collision that is the encoding after the later +// key (in sorted order) replaced the earlier member, so the replaced +// member is not counted. +func (c *contentCollector) toolArguments(args string) (json.RawMessage, bool) { + if args == "" { + return nil, false + } + if !json.Valid([]byte(args)) { + c.evicted += len(c.redact(args, &c.findings)) + return nil, true + } + dec := json.NewDecoder(strings.NewReader(args)) + dec.UseNumber() // float64 would rewrite integers past 2^53 + var v any + if err := dec.Decode(&v); err != nil || v == nil { + return nil, false + } + collided := false + out, err := json.Marshal(c.redactValue(v, &collided)) + if err != nil { + return nil, false // decoded JSON values marshal unconditionally + } + if collided || len(out) > maxToolArgumentsBytes { + c.evicted += len(out) + return nil, true + } + return out, false +} + +// redactValue redacts every string and object key under v. A pattern +// keyed on a member name ("password": "...") cannot see the pair that +// way, so each string member is scanned once more, already redacted, +// beside its redacted key (secretNamed) and masked whole on a match. +// Keys are walked in sorted order, so neither the findings nor a dropped +// value's charge follow map order. *collided reports two keys of one +// object that redact to the same string: keeping either member would +// show a call the agent did not make. +func (c *contentCollector) redactValue(v any, collided *bool) any { + switch t := v.(type) { + case string: + return c.redact(t, &c.findings) + case []any: + for i := range t { + t[i] = c.redactValue(t[i], collided) + } + case map[string]any: + out := make(map[string]any, len(t)) + for _, k := range slices.Sorted(maps.Keys(t)) { + rk := c.redact(k, &c.findings) + e := c.redactValue(t[k], collided) + if s, ok := e.(string); ok && c.secretNamed(rk, s) { + e = "***" + } + if _, dup := out[rk]; dup { + *collided = true + } + out[rk] = e + } + return out + } + return v +} + +// secretNamed reports whether the redactor's member-name pattern +// (json_field) matches the pair, and records that finding. Any other +// finding of this second scan is discarded: both strings were scanned +// already, and a connection-string mask of the "abcd..." form (a password +// of ten or more bytes) matches its own pattern again. The scanned text +// is never exported, and it goes to the pattern stage alone: both strings +// are normalized already, and the normalizer is not idempotent over +// escape sequences — run again over the pair it can strip one the first +// pass left open, and the value or the key's keyword with it. A double +// quote would end the pattern's quoted run early, so ',' stands in for it: like the quote it is outside every +// pattern's token class, so it joins no two runs into a token that a +// prefix pattern would mask ahead of the member-name pattern. +func (c *contentCollector) secretNamed(key, value string) bool { + unquote := strings.NewReplacer(`"`, ",") + pair := `"` + unquote.Replace(key) + `":"` + unquote.Replace(value) + `"` + for _, f := range c.secrets.Scan(pair).Findings { + if f.Name == "json_field" { + c.findings = append(c.findings, f) + return true + } + } + return false +} + // redactID scans a part id like every other stream-derived string; on // any finding the id is dropped entirely — a substituted id could // falsely collide with another call's. diff --git a/internal/cli/content_collector_test.go b/internal/cli/content_collector_test.go index 8ad531165e..c03344f383 100644 --- a/internal/cli/content_collector_test.go +++ b/internal/cli/content_collector_test.go @@ -97,6 +97,252 @@ func TestContentCollector_ToolUseBecomesToolCallPart(t *testing.T) { "a summary is not the tool's arguments; do not fabricate them") } +func TestContentCollector_ToolCallPartCarriesArguments(t *testing.T) { + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{ID: "toolu_01", Name: "Bash", Summary: "ls -la", + Arguments: `{"timeout": 9007199254740993, "command":"ls -la"}`}) + + res := c.Result("stop") + assert.Contains(t, res.OutputMessages, `"arguments":{"command":"ls -la","timeout":9007199254740993}`, + "arguments are an object, re-encoded; integers keep their digits") + part := partAt(t, decodeOutputMessages(t, res.OutputMessages), 0) + assert.Equal(t, "ls -la", part["summary"], "the summary stays beside the arguments") + assert.Zero(t, res.DroppedBytes) + assert.False(t, res.Truncated) +} + +func TestContentCollector_RedactsSecretsInArguments(t *testing.T) { + const secret = "s3cr3tvalue99xyz" + pat := "ghp_" + strings.Repeat("m", 36) + for name, args := range map[string]string{ + "leading assignment": `{"command":"DEPLOY_TOKEN=` + secret + ` make"}`, + "assignment after newline": `{"command":"cd x\nAPI_KEY=` + secret + ` run"}`, + "assignment in quotes": `{"command":"export GH_TOKEN=\"` + secret + `\""}`, + "json inside a string": `{"content":"{\"password\": \"` + secret + `\"}"}`, + "secret-named member": `{"password":"` + secret + `"}`, + "nested member": `{"edits":[{"headers":{"api_key":"` + secret + `"}}]}`, + "token prefix": `{"command":"curl -u x:` + pat + ` h"}`, + "token as a key": `{"` + pat + `":"v"}`, + // A secret-named member whose value raises a finding of its own + // (Unicode or another secret) is still scanned beside its key. + "secret-named, zero width": `{"password":"s3cr3tva\u200Blue99xyz"}`, + "secret-named, folded": `{"password":"` + secret + `\uFF01"}`, + "secret-named, two secrets": `{"password":"` + secret + ` ` + pat + `"}`, + "secret-named, early quote": `{"password":"ab\"` + secret + `"}`, + "quote in a secret-named key": `{"api\"key":"` + secret + `"}`, + // Joined by the probe's stand-in for the quote, the two runs must + // not become one token a prefix pattern masks ahead of the + // member-name pattern. + "quote inside a token run": `{"api_key":"sk-abc\"` + secret + `"}`, + // The normalizer is not idempotent over escape sequences: run a + // second time over the pair, it strips one the first pass left + // open and takes the value, or the key's keyword, with it. + "escape sequence closed by the pair, value": `{"password":"\u001b]\u001b]x\u0007` + secret + `\u0007"}`, + "escape sequence closed by the pair, key": `{"password\u001b]":"ab\u0007` + secret + `"}`, + } { + t.Run(name, func(t *testing.T) { + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Arguments: args}) + + res := c.Result("stop") + assert.NotContains(t, res.OutputMessages, secret) + assert.NotContains(t, res.OutputMessages, pat) + assert.NotEmpty(t, res.Findings) + part := partAt(t, decodeOutputMessages(t, res.OutputMessages), 0) + assert.IsType(t, map[string]any{}, part["arguments"], "redaction keeps the arguments an object") + }) + } +} + +func TestContentCollector_AKeyFindingLeavesTheValueAndCountsOnce(t *testing.T) { + for name, tc := range map[string]struct{ args, want string }{ + "token": {`{"ghp_` + strings.Repeat("r", 36) + `":"hello"}`, `{"ghp_...":"hello"}`}, + "fullwidth": {`{"\uFF50ath":"/tmp/x"}`, `{"path":"/tmp/x"}`}, + } { + t.Run(name, func(t *testing.T) { + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{Name: "Read", Arguments: tc.args}) + + res := c.Result("stop") + assert.Contains(t, res.OutputMessages, `"arguments":`+tc.want) + assert.Len(t, res.Findings, 1) + }) + } +} + +func TestContentCollector_KeysThatRedactAlikeDropTheArguments(t *testing.T) { + // Folding makes the two keys one; keeping either member would show a + // call the agent did not make, and which one would follow map order. + const args = `{"command":"rm -rf /","\uFF43ommand":"ls"}` + var first contentResult + for i := 0; i < 100; i++ { + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Summary: "rm -rf /", Arguments: args}) + res := c.Result("stop") + if i == 0 { + first = res + } + require.Equal(t, first.OutputMessages, res.OutputMessages) + require.Equal(t, first.DroppedBytes, res.DroppedBytes) + } + part := partAt(t, decodeOutputMessages(t, first.OutputMessages), 0) + assert.NotContains(t, part, "arguments") + assert.Equal(t, true, part["fullsend.truncated"]) + assert.True(t, first.Truncated) + assert.Positive(t, first.DroppedBytes) +} + +func TestContentCollector_NamelessCallCarriesNoArgumentsAndNoCharge(t *testing.T) { + // The schema requires a name on a tool_call part. A nameless call is + // refused; its arguments must not survive alone, nor charge a part + // that was never kept. + for name, args := range map[string]string{ + "kept": `{"a":1}`, + "over bound": `{"a":"` + strings.Repeat("x", maxToolArgumentsBytes) + `"}`, + "not json": `{"a":"unterminated`, + } { + t.Run(name, func(t *testing.T) { + c := newContentCollector(maxContentBytes) + c.Handle(agentruntime.ToolUseEvent{ID: "toolu_01", Arguments: args}) + assert.Equal(t, contentResult{}, c.Result("stop")) + }) + } +} + +func TestContentCollector_ANameRedactedAwayTakesItsArgumentsAlong(t *testing.T) { + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{ID: "toolu_01", Name: "\u200B", Arguments: `{"a":1}`}) + + res := c.Result("stop") + assert.Empty(t, res.OutputMessages, "a tool_call part needs a name") + assert.Len(t, res.Findings, 1) +} + +func TestContentCollector_DroppedBytesCountArgumentsOnEveryDropPath(t *testing.T) { + const args = `{"command":"ls -la"}` + for name, tc := range map[string]struct { + text string + evicted bool + }{ + "evicted during Handle": {strings.Repeat("x", 25), true}, + "dropped by the raw budget": {"final", false}, + } { + t.Run(name, func(t *testing.T) { + c := newContentCollector(25) + c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Arguments: args}) + c.Handle(agentruntime.TextEvent{Text: tc.text}) + require.Equal(t, tc.evicted, len(c.parts) == 1, "which path drops the call") + + res := c.Result("stop") + assert.True(t, res.Truncated) + assert.Equal(t, len("Bash")+len(args), res.DroppedBytes) + assert.NotContains(t, res.OutputMessages, "arguments") + }) + } +} + +func TestContentCollector_ADroppedResultIsReportedWithNoPartsLeft(t *testing.T) { + // An over-cap result that redacts to nothing leaves no part; what was + // cut and found still has to reach the span. + c := newContentCollector(maxContentBytes) + c.Handle(agentruntime.ToolResultEvent{ID: "toolu_01", Result: strings.Repeat("\u200B", maxToolResultBytes)}) + + res := c.Result("stop") + assert.Empty(t, res.OutputMessages) + assert.NotEmpty(t, res.Findings) +} + +func TestContentCollector_ArgumentsAreScannedOnce(t *testing.T) { + // A masked connection string still matches its own pattern, so a second + // pass over redacted arguments would count the same secret twice. + c := newContentCollector(64) + c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Arguments: `{"command":"psql postgres://u:hunter2hunter2@db/x"}`}) + c.Handle(agentruntime.TextEvent{Text: strings.Repeat("x", 200)}) // evicts the call + + assert.Len(t, c.Result("stop").Findings, 1) +} + +func TestContentCollector_FullwidthQuotesCannotAddAMember(t *testing.T) { + // NFKC folds a fullwidth quotation mark to '"'. Folded inside the + // serialised text it would close the string and add a member. + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{Name: "Write", Arguments: `{"content":"a\uFF02,\uFF02injected\uFF02:\uFF02b"}`}) + + part := partAt(t, decodeOutputMessages(t, c.Result("stop").OutputMessages), 0) + assert.Equal(t, map[string]any{"content": `a","injected":"b`}, part["arguments"]) +} + +func TestContentCollector_IncompleteArgumentsAreDroppedAndMarked(t *testing.T) { + // Input assembled from stream deltas stops growing at the parser's + // cap, so it can be a fragment of a JSON value. A fragment cannot be + // redacted string by string, and as text this assignment hides behind + // the quote that opens it. + fragment := `{"content":"TOKEN=s3cr3tvalue99xyz ghp_` + strings.Repeat("n", 36) + ` and then the inp` + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{Name: "Write", Summary: "/x", Arguments: fragment}) + + res := c.Result("stop") + part := partAt(t, decodeOutputMessages(t, res.OutputMessages), 0) + assert.NotContains(t, part, "arguments") + assert.Equal(t, "/x", part["summary"], "the call itself is kept") + assert.Equal(t, true, part["fullsend.truncated"]) + assert.True(t, res.Truncated) + assert.Len(t, res.Findings, 1, "dropped content is scanned first") + assert.Equal(t, len(fragment)-len("ghp_"+strings.Repeat("n", 36))+len("ghp_..."), res.DroppedBytes, + "charged as redacted, like every other discarded byte") + assert.NotContains(t, res.OutputMessages, "s3cr3tvalue99xyz") +} + +func TestContentCollector_OverBoundArgumentsAreDroppedWholeAndMarked(t *testing.T) { + body := strings.Repeat("x", maxToolArgumentsBytes) + c := newContentCollector(maxContentBytes) + c.Handle(agentruntime.ToolUseEvent{ID: "toolu_01", Name: "Write", Summary: "/x", Arguments: `{"content": "` + body + `"}`}) + + res := c.Result("stop") + part := partAt(t, decodeOutputMessages(t, res.OutputMessages), 0) + assert.NotContains(t, part, "arguments", "a cut object is not JSON, so nothing partial is kept") + assert.Equal(t, "/x", part["summary"]) + assert.Equal(t, "toolu_01", part["id"]) + assert.Equal(t, true, part["fullsend.truncated"]) + assert.True(t, res.Truncated) + assert.Equal(t, len(`{"content":"`+body+`"}`), res.DroppedBytes, "charged as re-encoded") +} + +func TestContentCollector_ArgumentsAtTheBoundAreKept(t *testing.T) { + args := `{"content":"` + strings.Repeat("x", maxToolArgumentsBytes-len(`{"content":""}`)) + `"}` + require.Len(t, args, maxToolArgumentsBytes) + c := newContentCollector(maxContentBytes) + c.Handle(agentruntime.ToolUseEvent{Name: "Write", Arguments: args}) + + res := c.Result("stop") + assert.Contains(t, res.OutputMessages, args) + assert.False(t, res.Truncated) +} + +func TestContentCollector_ArgumentsBoundAppliesAfterRedaction(t *testing.T) { + // Over the bound only while unredacted: the token masks to seven + // bytes, and the secret inside over-bound arguments still counts. + token := "ghp_" + strings.Repeat("q", 2*maxToolArgumentsBytes) + c := newContentCollector(maxContentBytes) + c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Arguments: `{"command":"echo ` + token + `"}`}) + c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Arguments: `{"command":"echo ` + token + ` ` + strings.Repeat("x", maxToolArgumentsBytes) + `"}`}) + + res := c.Result("stop") + msgs := decodeOutputMessages(t, res.OutputMessages) + assert.Equal(t, map[string]any{"command": "echo ghp_..."}, partAt(t, msgs, 0)["arguments"]) + assert.NotContains(t, partAt(t, msgs, 1), "arguments") + assert.Len(t, res.Findings, 2) + assert.NotContains(t, res.OutputMessages, "qqqq") +} + +func TestContentCollector_NullArgumentsAreOmitted(t *testing.T) { + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Arguments: `null`}) + + part := partAt(t, decodeOutputMessages(t, c.Result("stop").OutputMessages), 0) + assert.NotContains(t, part, "arguments") +} + func TestContentCollector_ToolCallPartCarriesID(t *testing.T) { c := newContentCollector(4096) c.Handle(agentruntime.ToolUseEvent{ID: "toolu_09qrs", Name: "Read", Summary: "/src/main.go"}) @@ -1020,6 +1266,11 @@ func TestContentCollector_EncodedCeilingHoldsForEscapeDenseRecords(t *testing.T) n += len(v) } } + if args, ok := m["arguments"]; ok { + enc, err := json.Marshal(args) // arguments are charged as encoded + require.NoError(t, err) + n += len(enc) + } } return n } @@ -1035,7 +1286,9 @@ func TestContentCollector_EncodedCeilingHoldsForEscapeDenseRecords(t *testing.T) case 0: events = append(events, agentruntime.TextEvent{Text: b.String() + "."}) case 1: - events = append(events, agentruntime.ToolUseEvent{ID: id, Name: "Bash", Summary: b.String()}) + args, err := json.Marshal(map[string]string{"command": b.String()}) + require.NoError(t, err) + events = append(events, agentruntime.ToolUseEvent{ID: id, Name: "Bash", Summary: b.String(), Arguments: string(args)}) default: events = append(events, agentruntime.ToolResultEvent{ID: id, Result: b.String() + "."}) } diff --git a/internal/cli/telemetry_run_test.go b/internal/cli/telemetry_run_test.go index 9c53b531d6..5784cad7b6 100644 --- a/internal/cli/telemetry_run_test.go +++ b/internal/cli/telemetry_run_test.go @@ -1394,6 +1394,7 @@ func TestContentCapture_EndToEndFileSink(t *testing.T) { c := newContentCollectorIfEnabled() require.NotNil(t, c) big := strings.Repeat("all work and no play makes claude a dull agent. ", 400) // ~19KB > 8192 + c.Handle(agentruntime.ToolUseEvent{ID: "toolu_01", Name: "Bash", Arguments: `{"command":"make lint"}`}) c.Handle(agentruntime.TextEvent{Text: big}) prompt, _ := buildFeedbackPrompt(strings.Repeat("the validator said no. ", 500)) // cut at 10 KiB > 8192 @@ -1420,6 +1421,7 @@ func TestContentCapture_EndToEndFileSink(t *testing.T) { require.Contains(t, content, "gen_ai.output.messages") assert.Contains(t, content, "dull agent", "content must reach the file sink") + assert.Contains(t, res.OutputMessages, `"arguments":{"command":"make lint"}`) // The whole redacted content must survive — no SDK truncation at 8192. var decoded []map[string]any require.NoError(t, json.Unmarshal([]byte(res.OutputMessages), &decoded), @@ -1450,14 +1452,20 @@ func TestContentCapture_GateOffProducesNoContent(t *testing.T) { require.Nil(t, c, "gate off must mean no collector") c.Handle(agentruntime.TextEvent{Text: "would-be content"}) // nil-safe no-op - _, span := tracer.Start(context.Background(), "agent") + ctx, span := tracer.Start(context.Background(), "agent") c.attachInput(span, "would-be prompt") // nil-safe no-op + handle := iterationEventHandler(func(agentruntime.AgentEvent) {}, c, newToolSpanTracker(tracer, ctx)) + handle(agentruntime.ToolUseEvent{ID: "toolu_01", Name: "Bash", Arguments: `{"command":"would-be arguments"}`}) + handle(agentruntime.ToolResultEvent{ID: "toolu_01", Result: "would-be result"}) attachContent(span, c.Result("stop")) span.End() cleanup(context.Background()) raw, err := os.ReadFile(filepath.Join(dir, telemetry.TelemetryFile)) require.NoError(t, err) + assert.Contains(t, string(raw), "execute_tool Bash", "the tool span is metadata") + assert.NotContains(t, string(raw), "would-be arguments") + assert.NotContains(t, string(raw), "would-be result") assert.NotContains(t, string(raw), "gen_ai.input.messages") assert.NotContains(t, string(raw), "would-be prompt") assert.NotContains(t, string(raw), "gen_ai.output.messages") diff --git a/internal/cli/tool_spans_test.go b/internal/cli/tool_spans_test.go index 3db2880ed4..7e28816ff2 100644 --- a/internal/cli/tool_spans_test.go +++ b/internal/cli/tool_spans_test.go @@ -82,6 +82,22 @@ func TestToolSpanTracker_PairEmitsExecuteToolChild(t *testing.T) { assert.Empty(t, tr.open, "an answered call must not stay open (Finish would re-end it, and the map would grow per call)") } +// Arguments are content and ride the gated message record; tool spans are +// emitted with the gate off, so nothing of a call's arguments may reach one. +func TestToolSpanTracker_ArgumentsStayOffTheSpan(t *testing.T) { + tr, rec, _ := toolSpanFixture(t) + + tr.Handle(agentruntime.ToolUseEvent{ID: "toolu_01", Name: "Bash", Arguments: `{"command":"cat argument-text"}`}) + tr.Handle(agentruntime.ToolResultEvent{ID: "toolu_01", Result: "ok"}) + + spans := endedToolSpans(rec) + require.Len(t, spans, 1) + assert.Len(t, spans[0].Attributes, 3, "operation name, tool name and call id") + for _, kv := range spans[0].Attributes { + assert.NotContains(t, kv.Value.Emit(), "argument-text", string(kv.Key)) + } +} + func TestToolSpanTracker_ErrorResultSetsErrorTypeAndStatus(t *testing.T) { tr, rec, _ := toolSpanFixture(t) diff --git a/internal/runtime/claude_progress.go b/internal/runtime/claude_progress.go index 00f851a55e..3eae069d64 100644 --- a/internal/runtime/claude_progress.go +++ b/internal/runtime/claude_progress.go @@ -293,9 +293,10 @@ func parseClaudeStream(r io.Reader, onEvent func(AgentEvent)) error { case "content_block_stop": if currentToolName != "" { onEvent(ToolUseEvent{ - ID: currentToolID, - Name: currentToolName, - Summary: extractSafeContext(currentToolName, json.RawMessage(toolInputJSON.String())), + ID: currentToolID, + Name: currentToolName, + Summary: extractSafeContext(currentToolName, json.RawMessage(toolInputJSON.String())), + Arguments: toolInputJSON.String(), }) currentToolName = "" currentToolID = "" @@ -424,9 +425,10 @@ func parseClaudeStream(r io.Reader, onEvent func(AgentEvent)) error { } case "tool_use": onEvent(ToolUseEvent{ - ID: item.ID, - Name: item.Name, - Summary: extractSafeContext(item.Name, item.Input), + ID: item.ID, + Name: item.Name, + Summary: extractSafeContext(item.Name, item.Input), + Arguments: string(item.Input), }) } } diff --git a/internal/runtime/claude_progress_test.go b/internal/runtime/claude_progress_test.go index 5f0d05d90a..a2abedd0b7 100644 --- a/internal/runtime/claude_progress_test.go +++ b/internal/runtime/claude_progress_test.go @@ -825,6 +825,56 @@ func TestParseClaudeStreamAssistantFallbackToolUseCarriesID(t *testing.T) { } } +func collectToolUses(t *testing.T, input string) []ToolUseEvent { + t.Helper() + var tools []ToolUseEvent + for _, e := range collectEvents(t, input) { + if te, ok := e.(ToolUseEvent); ok { + tools = append(tools, te) + } + } + return tools +} + +func TestParseClaudeStreamToolUseCarriesArguments(t *testing.T) { + lines := []string{ + `{"type":"stream_event","event":{"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_01","name":"Read"}}}`, + `{"type":"stream_event","event":{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"file_path\""}}}`, + `{"type":"stream_event","event":{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":":\"/src/main.go\"}"}}}`, + `{"type":"stream_event","event":{"type":"content_block_stop","index":0}}`, + } + tools := collectToolUses(t, strings.Join(lines, "\n")) + if len(tools) != 1 { + t.Fatalf("expected 1 tool event, got %d", len(tools)) + } + if want := `{"file_path":"/src/main.go"}`; tools[0].Arguments != want { + t.Errorf("expected the joined input deltas %q, got %q", want, tools[0].Arguments) + } +} + +func TestParseClaudeStreamAssistantToolUseCarriesArguments(t *testing.T) { + // Key order, escapes and spacing are the wire's: the parser passes the + // input through without decoding it. + const input = `{"timeout": 5, "command":"grep -n \"a\u003cb\" x\nls"}` + tools := collectToolUses(t, `{"type":"assistant","message":{"content":[{"type":"tool_use","id":"toolu_02","name":"Bash","input":`+input+`}]}}`) + if len(tools) != 1 { + t.Fatalf("expected 1 tool event, got %d", len(tools)) + } + if tools[0].Arguments != input { + t.Errorf("expected the input as written %q, got %q", input, tools[0].Arguments) + } +} + +func TestParseClaudeStreamToolUseWithoutInputHasNoArguments(t *testing.T) { + tools := collectToolUses(t, `{"type":"assistant","message":{"content":[{"type":"tool_use","id":"toolu_03","name":"Bash"}]}}`) + if len(tools) != 1 { + t.Fatalf("expected 1 tool event, got %d", len(tools)) + } + if tools[0].Arguments != "" { + t.Errorf("expected no arguments, got %q", tools[0].Arguments) + } +} + func collectToolResults(t *testing.T, input string) []ToolResultEvent { t.Helper() var results []ToolResultEvent diff --git a/internal/runtime/event.go b/internal/runtime/event.go index 0b2faf0235..65cc32bc06 100644 --- a/internal/runtime/event.go +++ b/internal/runtime/event.go @@ -49,10 +49,17 @@ func (TextEvent) agentEvent() {} // Name is the raw tool name from the runtime stream. // Summary is a one-line context string from extractSafeContext; it is // empty for tools not recognized by that function. +// Arguments is the call's input as the stream carried it: JSON text, +// unredacted, for the Level 3 content collector only — the renderer does +// not print it. It is not checked here: input assembled from stream_event +// deltas stops growing at maxToolInputSize and can be incomplete JSON. +// Empty when the wire carried no input, and from the pi and codex +// parsers (#7414) and the OpenCode parser. type ToolUseEvent struct { - ID string - Name string - Summary string + ID string + Name string + Summary string + Arguments string } func (ToolUseEvent) agentEvent() {} diff --git a/internal/runtime/renderer_test.go b/internal/runtime/renderer_test.go index e83d23732f..f7e046abd6 100644 --- a/internal/runtime/renderer_test.go +++ b/internal/runtime/renderer_test.go @@ -68,6 +68,17 @@ func TestRendererToolResultEventIgnored(t *testing.T) { } } +func TestRendererToolUseEventNeverPrintsArguments(t *testing.T) { + var buf bytes.Buffer + r := newTestRenderer(&buf) + + r.Handle(ToolUseEvent{Name: "Bash", Summary: "make", Arguments: `{"command":"make SECRET=hunter2"}`}) + + if strings.Contains(buf.String(), "hunter2") { + t.Errorf("arguments must not reach the console, got %q", buf.String()) + } +} + func TestRendererToolUseEventCI(t *testing.T) { t.Setenv("GITHUB_ACTIONS", "true") From 19f3c5c106eeef6fd7b921bd23fbf000ad18d83b Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Fri, 18 Sep 2026 15:10:22 -0400 Subject: [PATCH 03/40] docs(tracing): describe input capture and tool arguments Level 3 now records tool-call arguments and, on a retry that carries validation feedback, the prompt the runner composed. The reference, the dev guide, the runtime matrix and the two user guides say what is recorded, what is still not (the rest of the model's input), how arguments are redacted, bounded and counted, and that the input message is cut upstream and not described by the truncation markers. The size figures are re-measured with arguments sharing the total: on the three captured review runs the shipped bounds evict 31-63% of tool results (28-56% before), a 1 MiB total still evicts none, and encoding adds 8-10% at these bounds. Signed-off-by: Dharit Shah --- docs/guides/dev/tracing.md | 60 ++++++++++++-- .../infrastructure/distributed-tracing.md | 78 +++++++++++++------ .../infrastructure/eval-measurements.md | 2 +- docs/guides/user/how-to-emit-traces.md | 10 ++- docs/guides/user/tracing-with-mlflow.md | 4 +- docs/runtimes.md | 2 +- 6 files changed, 122 insertions(+), 34 deletions(-) diff --git a/docs/guides/dev/tracing.md b/docs/guides/dev/tracing.md index 77d2c353d7..2f8a1b2ae3 100644 --- a/docs/guides/dev/tracing.md +++ b/docs/guides/dev/tracing.md @@ -204,7 +204,8 @@ renderer path. The collector (`internal/cli/content_collector.go`) coalesces contiguous text/reasoning deltas, maps tool use to `tool_call` parts and tool results to `tool_call_response` parts (only the Claude parser emits -`ToolResultEvent` and call ids today — pi and codex emit neither, +`ToolResultEvent`, call ids and `ToolUseEvent.Arguments` today — pi and +codex emit none of them, [#7414](https://github.com/fullsend-ai/fullsend/issues/7414); the schema's required result field is `response`), redacts every part through `security.OutputPipeline()` at assembly (redaction runs @@ -212,14 +213,18 @@ before the size budget — truncating first could split a secret past recognition), enforces a 256 KiB ordered-suffix budget (the ending survives — the final answer is what consumers judge) plus an 8 KiB per-tool-result bound (tail-kept, redacted before the cut, the part marked -`fullsend.truncated`), with exact dropped-byte -accounting across content, tool names, summaries, responses, and part -ids, then holds the marshaled string to `maxEncodedContentBytes` +`fullsend.truncated`) and an 8 KiB per-call arguments bound (over it the +arguments are dropped whole and the part marked, since a cut object is not +JSON), with exact dropped-byte +accounting across content, tool names, summaries, responses, part +ids, and dropped arguments (counted as re-encoded JSON, or as redacted +text when they were not a JSON value; on a key collision the member the +later key (in sorted order) replaced is not counted), then holds the marshaled string to `maxEncodedContentBytes` (255,000 — just under the one size the pilot backend is proven to accept) by trimming the oldest content again, measured on the encoding itself and still charged in raw bytes (a separate, earlier boundary — the parser's 1 MiB stream-line cap — skips oversized lines; an oversized `tool_result` line -still yields an empty part marked `fullsend.truncated`). None of the three +still yields an empty part marked `fullsend.truncated`). None of the four content bounds is a measured backend limit; the constants' comments and [Size limits](../infrastructure/distributed-tracing.md#content-capture-level-3) name what blocks raising them. The collector emits @@ -229,11 +234,56 @@ content and its marker attributes on the span before either `finalizeAgentSpan` path can end it, so failed iterations keep their content. +Arguments that parse as JSON are not redacted as one serialized text. The +redactor's patterns are written for plain text, and over JSON they miss an +assignment that opens a string or follows an escaped newline, a value +behind escaped quotes, and JSON nested in a string; Unicode folding can +also turn a fullwidth quotation mark into one that closes the string. +`toolArguments` decodes the value, redacts each string and object key on +its own, and encodes the result again. Each string member is also scanned +once more, already redacted, beside its key, and masked whole when the +member-name pattern (`json_field`) matches — a pattern keyed on a member +name has no other way to see the pair; that scan's other findings are +discarded, since both strings were already scanned. It runs the pattern +stage alone: the normalizer is not idempotent over escape sequences, and a +second pass over the pair could strip the value or the key's keyword. It reaches string +members only: a secret-named member holding an array, object or number is +redacted string by string, as it would be in text. Arguments that are not +one JSON value cannot be walked that way: they are scanned as text — with +the misses above — so their findings count, then dropped and charged. So +are arguments in which two keys of one object redact to the same string, +where keeping either member would misreport the call. A call the stream +reports without a name carries no arguments. This happens once, when the +event is handled; eviction and `Result` do not rescan it. + +The input message does not come from the stream. When `runAgent` composes +a retry prompt (`buildFeedbackPrompt`, under `feedback_mode: append`), +`attachInput` redacts it with the collector's pipeline — `redactFeedback` +ran before the feedback was sanitized and framed, and sanitizing can join +a token that scan saw split, so the recorded copy is pattern-scanned +again; the prompt the agent is sent is not changed, and the mask the first +scan leaves for a connection-string password of ten or more bytes matches +its own pattern and counts a finding — +and sets `gen_ai.input.messages` on the span +before the runtime starts, so each finalize path carries it. Its findings +join the iteration's, and its encoded size is charged against +`maxEncodedContentBytes`: the two content attributes of one span together +stay within the proven size. + **Consumer contract** (for eval scorers and other readers of `run-telemetry.jsonl`): parse the `gen_ai.output.messages` attribute as JSON; check `fullsend.content.truncated` / `fullsend.content.dropped_bytes` before treating content as complete; masked secrets appear as the redactor's mask tokens and are counted in `fullsend.content.redactions`. +A `tool_call` part's `arguments`, when present, is a JSON value (an object +for the tools seen so far); a `tool_call` part marked `fullsend.truncated` +had arguments that were dropped. `gen_ai.input.messages` is absent unless +the iteration is a retry that carried validation feedback — absence is +the normal case, not a gap. Masks `redactFeedback` left in the feedback +(`abcd...`, `***`, `[REDACTED:]`) are in `gen_ai.input.messages` +too, but that function discards its findings, so they are not counted in +`fullsend.content.redactions`; the connection-string mask described above +is the known exception. The attribute names and shapes above are the consumption contract — see the [Tracing reference](../infrastructure/distributed-tracing.md#content-capture-level-3). diff --git a/docs/guides/infrastructure/distributed-tracing.md b/docs/guides/infrastructure/distributed-tracing.md index ec85f41a45..a0f8116c62 100644 --- a/docs/guides/infrastructure/distributed-tracing.md +++ b/docs/guides/infrastructure/distributed-tracing.md @@ -12,7 +12,7 @@ For implementation details, see the |-------|-----------------|----------------------| | 1 | `run-telemetry.jsonl` file in the run output directory | None | | 2 | OTLP/HTTP export to a remote backend (metadata only) | `OTEL_EXPORTER_OTLP_*ENDPOINT` | -| 3 | Conversation content (assistant text, reasoning, tool calls, and — on Claude runs — tool results) on `agent` spans | `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true` | +| 3 | Conversation content (assistant text, reasoning, tool calls and — on Claude runs — their arguments and results; on a retry that carries validation feedback, the prompt the runner composed) on `agent` spans | `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true` | All levels produce metadata (timing, token counts, tool names, errors), including up to one `execute_tool` span per id-bearing tool call under each @@ -74,7 +74,10 @@ unset OTEL_EXPORTER_OTLP_TRACES_ENDPOINT Fullsend assembles Level 3 content from the normalized event stream the console renders, redacts it through the security output pipeline, and -attaches it to the per-iteration `agent` span. The agent runtime's own +attaches it to the per-iteration `agent` span. The one input it records, +a retry's validation-feedback prompt, is composed by the runner — fixed +framing around the previous iteration's validation output — rather than +read from the stream, and goes through the same redaction. The agent runtime's own content-logging variables (`OTEL_LOG_USER_PROMPTS`, `OTEL_LOG_ASSISTANT_RESPONSES`, etc.) are never set. @@ -87,31 +90,57 @@ The variable name and accepted values follow the Fullsend records content on span attributes only, so `event_only` stays off. An unrecognized value disables capture; telemetry never fails a run. -**Captured:** assistant text, reasoning, tool calls (name plus short -summary), and tool results — including any sub-agent activity, +**Captured:** assistant text, reasoning, tool calls (name, short summary +and `arguments`), and tool results — including any sub-agent activity, unattributed — as the `gen_ai.output.messages` span attribute: a JSON string following the [GenAI output-messages schema](https://github.com/open-telemetry/semantic-conventions/blob/v1.37.0/docs/gen-ai/gen-ai-output-messages.json) -with a `finish_reason` of `stop` or `error`. Tool results, and the `id` that correlates each with its call, are captured +with a `finish_reason` of `stop` or `error`. Tool arguments, tool results, and the `id` that correlates a result with its call, are captured only when the runtime's stream provides them: Claude runs do; the pi and -codex parsers emit neither yet +codex parsers emit none of the three yet ([#7414](https://github.com/fullsend-ai/fullsend/issues/7414)). - -**Not captured:** model input (`gen_ai.input.messages`) and -pre/post-script content. First-iteration runs have no meaningful -runner-side input; retry iterations carry the injected validation -feedback, a natural input-capture follow-up. +`arguments` is the call's input decoded, redacted string by string, and +encoded again, so key order, spacing and escapes are not the stream's. + +On a retry iteration under `validation_loop.feedback_mode: append`, the +prompt the runner composed — its fixed framing around the previous +iteration's validation output — is recorded as `gen_ai.input.messages`: one +`user` message with one text part, following the +[GenAI input-messages schema](https://github.com/open-telemetry/semantic-conventions/blob/v1.37.0/docs/gen-ai/gen-ai-input-messages.json). +The attribute is absent on the first iteration and on a retry without +feedback, which send the runtime's fixed default prompt. + +**Not captured:** the rest of the model's input, and pre/post-script +content. The fixed default prompt is a constant that carries no task and +is left out by choice; the system prompt and per-request context the +runtime builds from the agent definition are not visible to fullsend, +which sees the runtime's stream, not its API requests. **Redaction and size:** every part passes through security redaction (Unicode normalization, then secret masking) before reaching the span. -Content is bounded at 256 KiB per iteration — each tool result at 8 KiB — +Content is bounded at 256 KiB per iteration — each tool result at 8 KiB, +each call's arguments at 8 KiB — kept as an ordered suffix; overflow drops the oldest content first. Those -bounds count raw bytes; the exported JSON string is bounded as well, at -255,000 bytes, because encoding adds 9–11% at these bounds on real runs +bounds count raw bytes, except that arguments count as the +re-encoded JSON described above, against their own bound and against the +total. That encoding writes `<`, `>` and `&` as six bytes each, so +arguments dense in those characters reach their bound at about a sixth of +that size on the stream. The exported JSON string is bounded as well, at +255,000 bytes, because encoding adds 8–10% at these bounds on real runs and up to six times on escape-dense content — a record over that is trimmed again, oldest first. Truncation is marked via `fullsend.content.truncated` on the span and -`fullsend.truncated` on each cut part. A tool result whose stream line +`fullsend.truncated` on each cut part. A tool result over its bound keeps +its tail; arguments over theirs, not a complete JSON value, or with two +keys of one object that redact to the same string, are dropped +whole — a cut object is not JSON — and the marked `tool_call` part keeps +its `id`, name and summary. The input message is not cut at this stage: +the validation output inside it is cut at 10 KiB when the prompt is +composed (the prompt then says `[truncated]`), and the two markers above +do not describe it. The recorded copy is the redacted one, so it can +differ from the prompt the agent received: compatibility characters are +folded, which can make it about eleven times longer, and its encoded size +counts toward the 255,000 bytes. A tool result whose stream line exceeds the parser's 1 MiB bound is kept as an empty, marked `tool_call_response` part — the call was answered, its content is lost and its `is_error` unknown — provided the line shows the call id within its @@ -135,10 +164,14 @@ its own, and the file sink has none. The bounds stay because guessing wrong is costly — a backend that refuses an oversized request refuses the whole batch, up to 512 spans with their Level 1 metadata, and the exporter does not retry a refusal. On three captured review runs (117–255 tool -results per iteration, sub-agents included) these bounds evict 28–56% of -tool results. A 1 MiB total with the same per-result bound evicts none -(records of 412–938 KB); raising only the per-result bound to 32 KiB -evicts 47–91%; keeping every result whole takes 1.1–2.2 MB. So the total +calls per iteration and as many results, sub-agents included; 41–100 KB +of arguments, with the arguments of 0–3 calls over their bound) these +bounds evict 31–63% of tool results — 28–56% before arguments shared the +total. A 1 MiB total with the same per-part bounds evicts none +(records of 456–1,011 KB); raising only the per-result bound to 32 KiB +evicts 47–91%; keeping every result whole takes 1.1–2.3 MB. No run of an +agent that writes files was measured; a `Write` call carries the file +body as an argument. So the total is the bound to raise first, once the size is proven on the target backend: send attributes of increasing size and read each back whole ([#7415](https://github.com/fullsend-ai/fullsend/issues/7415)). @@ -249,10 +282,11 @@ The `agent` span's provider identity reflects only the parent run's serving endp | `fullsend.prescript.skipped` | `run` | Whether the pre-script signaled a skip | | `fullsend.prescript.skip_reason` | `run` | Human-readable skip reason from the pre-script | | `fullsend.transcript_error` | `agent` | Present (`true`) when the agent exited 0 but its transcript reported an error — the span's status is Error while `exit_code` keeps the raw process exit | +| `gen_ai.input.messages` | `agent` | Level 3 only, on a retry iteration that carries validation feedback: the prompt the runner composed, as a JSON string (see Content capture) | | `gen_ai.output.messages` | `agent` | Level 3 only: the iteration's conversation content as a JSON string (see Content capture) | -| `fullsend.content.truncated` | `agent` | Level 3 only: present (`true`) when the size budget cut or dropped content, or a kept tool result is a parser-side fragment or an oversized line's empty stand-in (`fullsend.truncated` on the part; no byte count) | -| `fullsend.content.dropped_bytes` | `agent` | Level 3 only: exact part bytes removed by the size budget — content, ids, and the fixed footprint of an errored-empty or oversized stand-in part — in raw bytes whichever bound made the cut; the bytes of a skipped oversized line were never decoded and are not counted | -| `fullsend.content.redactions` | `agent` | Level 3 only: number of security findings raised while redacting content at assembly (including findings from parts the size budget later dropped) | +| `fullsend.content.truncated` | `agent` | Level 3 only: present (`true`) when the size budget cut or dropped content, a kept tool call lost its arguments, or a kept tool result is a parser-side fragment or an oversized line's empty stand-in (`fullsend.truncated` on the part; no byte count for the last two) | +| `fullsend.content.dropped_bytes` | `agent` | Level 3 only: exact part bytes removed by the size budget — content, ids, dropped arguments (counted as re-encoded JSON — on a key collision, without the member the later key (in sorted key order, not stream order) replaced — or as redacted text when they were not a JSON value), and the fixed footprint of an errored-empty or oversized stand-in part — otherwise in raw bytes, whichever bound made the cut; the bytes of a skipped oversized line were never decoded and are not counted | +| `fullsend.content.redactions` | `agent` | Level 3 only: number of security findings raised while redacting content at assembly, the input message included (and findings from parts the size budget later dropped) | | `fullsend.tool.unmatched` | `execute_tool` | Present (`true`) when a result arrived for a call the stream never reported; the span has near-zero duration | | `fullsend.tool.result_oversized` | `execute_tool` | Present (`true`) when the result's stream line exceeded the parser's 1 MiB bound: the call was answered but nothing of the result was decoded, so the span has no status and no `error.type` | | `fullsend.tool_spans.dropped` | `agent` | Present when the iteration hit the 1,024-span cap and at least one id-bearing call was refused a span: the number of `tool_use` events with a usable id that arrived past the cap, each counted once whatever its result later does; a result with no open span past the cap is not counted | diff --git a/docs/guides/infrastructure/eval-measurements.md b/docs/guides/infrastructure/eval-measurements.md index bfb8090d38..d5f9f21526 100644 --- a/docs/guides/infrastructure/eval-measurements.md +++ b/docs/guides/infrastructure/eval-measurements.md @@ -183,7 +183,7 @@ newer `agents@v0` manifest this binary does not implement yet) also writes | Topic | Relationship to measurements | |---|---| -| Level 3 content capture ([ADR 0050](../../ADRs/0050-distributed-tracing-instrumentation.md); activation draft closed without merge in [#5947](https://github.com/fullsend-ai/fullsend/pull/5947)) | First ship scores Level 1/2 metadata fitness. **Planned:** content-aware scorers on Level 3 prompt/completion bodies once L3 is implemented — that is the real quality signal. Measure CLI is host-side after the sandbox exits. Tool-result content and `execute_tool` spans are Claude-only until the pi and codex parsers emit call ids and results ([#7414](https://github.com/fullsend-ai/fullsend/issues/7414)); a scorer that reads tool I/O should key on `fullsend.runtime` and skip, not fail, other runtimes. | +| Level 3 content capture ([ADR 0050](../../ADRs/0050-distributed-tracing-instrumentation.md); activation draft closed without merge in [#5947](https://github.com/fullsend-ai/fullsend/pull/5947)) | First ship scores Level 1/2 metadata fitness. **Planned:** content-aware scorers on Level 3 prompt/completion bodies once L3 is implemented — that is the real quality signal. Measure CLI is host-side after the sandbox exits. Tool arguments, tool-result content and `execute_tool` spans are Claude-only until the pi and codex parsers emit call ids, arguments and results ([#7414](https://github.com/fullsend-ai/fullsend/issues/7414)); a scorer that reads tool I/O should key on `fullsend.runtime` and skip, not fail, other runtimes. | | [#5944](https://github.com/fullsend-ai/fullsend/pull/5944) Span status from run outcome *(merged)* | Unblocks outcome scorers keyed on Status, not raw exit alone. | | Semantic observability / observer / lessons (draft closed without merge in [#2423](https://github.com/fullsend-ai/fullsend/pull/2423)) | Observer + lessons → fixtures remains a sibling idea; measurements are the online score path. | | [#5524](https://github.com/fullsend-ai/fullsend/pull/5524) Harness snapshot / forge join keys *(open)* | Complementary join/identity proposal beside telemetry; measurements are derived scores, not primary run facts. | diff --git a/docs/guides/user/how-to-emit-traces.md b/docs/guides/user/how-to-emit-traces.md index 77e9785133..dd7d2906c8 100644 --- a/docs/guides/user/how-to-emit-traces.md +++ b/docs/guides/user/how-to-emit-traces.md @@ -120,11 +120,13 @@ to a backend like MLflow, Jaeger, Grafana Tempo, etc. ## Capture conversation content Set `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT` to add the agent's -text, reasoning, tool calls, and tool results — when the runtime's stream -provides them; Claude runs do — to each `agent` span, in the +text, reasoning, tool calls, and their arguments and results — when the +runtime's stream provides them; Claude runs do — to each `agent` span, in the local file and -at the endpoint. Content is redacted for secrets and bounded per iteration, -but may still contain proprietary code or PII — make sure your backend's +at the endpoint. A retry that carries validation feedback also records the +prompt the runner composed. Content is redacted for secrets and bounded per iteration, +but may still contain proprietary code or PII — tool arguments include the +commands an agent ran and the file contents it wrote — so make sure your backend's access controls fit before enabling it. ```bash diff --git a/docs/guides/user/tracing-with-mlflow.md b/docs/guides/user/tracing-with-mlflow.md index 1a49aac22a..9b2401071d 100644 --- a/docs/guides/user/tracing-with-mlflow.md +++ b/docs/guides/user/tracing-with-mlflow.md @@ -84,7 +84,9 @@ attribute on `agent` spans (also in `run-telemetry.jsonl`). With content capture enabled (see [How To Emit Traces](how-to-emit-traces.md#capture-conversation-content)), -the conversation lives on each `agent` span as `gen_ai.output.messages`: +the conversation lives on each `agent` span as `gen_ai.output.messages` +(and, on a retry that carried validation feedback, the runner's prompt as +`gen_ai.input.messages`): open the trace and select the span to read it. The trace list's Request/Response preview columns derive from the root span only (capped at 1000 characters), so they stay empty for fullsend traces — content is diff --git a/docs/runtimes.md b/docs/runtimes.md index 5a2b9114a0..794160693d 100644 --- a/docs/runtimes.md +++ b/docs/runtimes.md @@ -62,7 +62,7 @@ sequenceDiagram | Tools | Native Claude permission syntax | `--tools` (strict) + a first-token Bash allowlist | Shell + `apply_patch` only; `tools:` is recorded, not enforced (the allowlist hook is opt-in) | | Security controls | Full matrix | Full matrix; stricter on failed-call sanitizing | Full matrix; post-tool hooks detect and block but cannot rewrite output | | Cost in `metrics.json` | Reported | Reported | Not reported — codex sends none | -| Content capture (Level 3) | Text, reasoning, tool calls and tool results (correlating ids) | Text, reasoning, tool calls (no correlating ids) — pi's parser emits neither ids nor tool results yet ([#7414](https://github.com/fullsend-ai/fullsend/issues/7414)) | Text, reasoning, tool calls (no correlating ids) — codex's parser emits neither ids nor tool results ([#7414](https://github.com/fullsend-ai/fullsend/issues/7414)) | +| Content capture (Level 3) | Text, reasoning, tool calls with their arguments, and tool results (correlating ids); the retry prompt under `feedback_mode: append` | Text, reasoning, tool calls (no correlating ids) — pi's parser emits no ids, arguments or tool results yet ([#7414](https://github.com/fullsend-ai/fullsend/issues/7414)); the retry prompt, as on Claude | Text, reasoning, tool calls (no correlating ids) — codex's parser emits no ids, arguments or tool results ([#7414](https://github.com/fullsend-ai/fullsend/issues/7414)); the retry prompt, as on Claude | | Tool spans (`execute_tool`) | One per id-bearing tool call (server-side tools get none), up to 1,024 per iteration, a child of the iteration's `agent` span, timed at receipt | None — the parser emits no call ids ([#7414](https://github.com/fullsend-ai/fullsend/issues/7414)) | None — the parser emits no call ids ([#7414](https://github.com/fullsend-ai/fullsend/issues/7414)) | All three run unattended in the same sandbox, behind the same egress allowlist. Stay on `claude` From 4fb9cd7909e8a216b9ac6f6e26948276e8c2070b Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Fri, 18 Sep 2026 15:10:24 -0400 Subject: [PATCH 04/40] docs(adr): annotate ADR 0050 and ADR 0108 for input capture and tool arguments Dated annotations only; no Decision text changes. ADR 0050 records that the retry prompt and tool arguments now ride the Level 3 record and what of the model's input is still not recorded. ADR 0108 records that its "full arguments next" is in place on the message record up to a per-call bound, while execute_tool spans stay metadata only. Signed-off-by: Dharit Shah --- .../0050-distributed-tracing-instrumentation.md | 11 +++++++++++ docs/ADRs/0108-tool-call-span-topology.md | 13 +++++++++++++ 2 files changed, 24 insertions(+) diff --git a/docs/ADRs/0050-distributed-tracing-instrumentation.md b/docs/ADRs/0050-distributed-tracing-instrumentation.md index 0f36a9c824..f05d74fc65 100644 --- a/docs/ADRs/0050-distributed-tracing-instrumentation.md +++ b/docs/ADRs/0050-distributed-tracing-instrumentation.md @@ -178,3 +178,14 @@ codex emit no call ids — becomes an `execute_tool` child of its iteration's `agent` span, up to 1,024 per iteration, metadata only; the message record on the `agent` span stays the content carrier. Sub-agent nesting (deferred item 1 above) remains deferred. + +**2026-09-18 — Level 3 input message and tool arguments ([ADR 0108](0108-tool-call-span-topology.md)):** +under the content gate, a retry iteration that carries validation feedback +records the prompt the runner composed as `gen_ai.input.messages`, and +`tool_call` parts on the message record carry the call's arguments — Claude +Code today ([#7414](https://github.com/fullsend-ai/fullsend/issues/7414)). +The rest of the model's input is not recorded. The fixed default prompt +that all other iterations send is a constant that carries no task and is left +out by choice; the requests the runtime builds from the agent definition and +the conversation are not visible to fullsend, which reads the runtime's +stream, not its API requests. The gate and the carrier are unchanged. diff --git a/docs/ADRs/0108-tool-call-span-topology.md b/docs/ADRs/0108-tool-call-span-topology.md index 8fe88dfdda..8a13b7efdb 100644 --- a/docs/ADRs/0108-tool-call-span-topology.md +++ b/docs/ADRs/0108-tool-call-span-topology.md @@ -121,3 +121,16 @@ convention bump. call ids and results ([#7414](https://github.com/fullsend-ai/fullsend/issues/7414)). - This settles the span-granularity question in #294; retention and access remain open there. + +## Annotations + +**2026-09-18 — Tool arguments on the message record:** the Decision's "full +arguments next" is in place up to a per-call bound: `tool_call` parts on the +`agent` span's `gen_ai.output.messages` record carry the call's arguments — +decoded, redacted and re-encoded, not the wire text — where the runtime's +stream provides them, Claude Code today. Arguments over 8 KiB re-encoded, not +a complete JSON value, or with two keys of one object that redact to the same +string, are dropped whole and the part marked `fullsend.truncated`; kept +arguments count toward the record's total bound +([reference](../guides/infrastructure/distributed-tracing.md#content-capture-level-3)). +`execute_tool` spans still carry metadata only. From 99f139915f7361e0f7430821b5a3c37328caddd0 Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Mon, 21 Sep 2026 14:47:26 -0400 Subject: [PATCH 05/40] fix(telemetry): charge and mark arguments lost to a redacted-away name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Result cleared the accepted arguments of a call whose name redacted to nothing without counting them. They are now added to the dropped bytes, and the span and the part — kept only when it has a summary — are marked truncated, as for every other way a call loses its arguments. Signed-off-by: Dharit Shah --- docs/guides/dev/tracing.md | 5 ++++- internal/cli/content_collector.go | 11 +++++++---- internal/cli/content_collector_test.go | 17 ++++++++++++++++- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/docs/guides/dev/tracing.md b/docs/guides/dev/tracing.md index 2f8a1b2ae3..eea97cfafb 100644 --- a/docs/guides/dev/tracing.md +++ b/docs/guides/dev/tracing.md @@ -254,7 +254,10 @@ the misses above — so their findings count, then dropped and charged. So are arguments in which two keys of one object redact to the same string, where keeping either member would misreport the call. A call the stream reports without a name carries no arguments. This happens once, when the -event is handled; eviction and `Result` do not rescan it. +event is handled; eviction and `Result` do not rescan it. A name that +redacts to nothing at `Result` takes the arguments with it: they are +charged to the dropped bytes, and the part — kept only when it has a +summary — is marked. The input message does not come from the stream. When `runAgent` composes a retry prompt (`buildFeedbackPrompt`, under `feedback_mode: append`), diff --git a/internal/cli/content_collector.go b/internal/cli/content_collector.go index 14842b97ed..7ae84116ea 100644 --- a/internal/cli/content_collector.go +++ b/internal/cli/content_collector.go @@ -557,10 +557,13 @@ func (c *contentCollector) Result(finishReason string) contentResult { p.Response = c.redact(p.Response, &res.Findings) } p.Name = c.redact(p.Name, &res.Findings) - if p.Name == "" { - // As in Handle, for a name that redacts to nothing. Arguments - // Handle had already dropped stay charged and marked. - p.Arguments = nil + if p.Name == "" && p.Arguments != nil { + // As in Handle, for a name that redacts to nothing — but these + // arguments were accepted, so losing them is charged and + // marked. Ones Handle had already dropped stay as they were. + res.DroppedBytes += len(p.Arguments) + res.Truncated = true + p.Arguments, p.Truncated = nil, true } p.Summary = c.redact(p.Summary, &res.Findings) p.ID = c.redactID(p.ID, &res.Findings) diff --git a/internal/cli/content_collector_test.go b/internal/cli/content_collector_test.go index c03344f383..e865dc4c30 100644 --- a/internal/cli/content_collector_test.go +++ b/internal/cli/content_collector_test.go @@ -210,12 +210,27 @@ func TestContentCollector_NamelessCallCarriesNoArgumentsAndNoCharge(t *testing.T } func TestContentCollector_ANameRedactedAwayTakesItsArgumentsAlong(t *testing.T) { + const args = `{"a":1}` c := newContentCollector(4096) - c.Handle(agentruntime.ToolUseEvent{ID: "toolu_01", Name: "\u200B", Arguments: `{"a":1}`}) + c.Handle(agentruntime.ToolUseEvent{ID: "toolu_01", Name: "\u200B", Arguments: args}) res := c.Result("stop") assert.Empty(t, res.OutputMessages, "a tool_call part needs a name") assert.Len(t, res.Findings, 1) + assert.True(t, res.Truncated) + assert.Equal(t, len(args), res.DroppedBytes) +} + +func TestContentCollector_ANameRedactedAwayMarksTheCallItsSummaryKeeps(t *testing.T) { + const args = `{"command":"ls"}` + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{Name: "\u200B", Summary: "ls", Arguments: args}) + + res := c.Result("stop") + assert.Contains(t, res.OutputMessages, `"summary":"ls","fullsend.truncated":true`) + assert.NotContains(t, res.OutputMessages, "arguments") + assert.True(t, res.Truncated) + assert.Equal(t, len(args), res.DroppedBytes) } func TestContentCollector_DroppedBytesCountArgumentsOnEveryDropPath(t *testing.T) { From 1c124cebf35ec97a6d49c867268cbc1b497d16e1 Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Mon, 21 Sep 2026 14:47:30 -0400 Subject: [PATCH 06/40] fix(runtime): hold accumulated tool input to maxToolInputSize The cap was checked before each delta was appended, so the delta that crossed it went in whole and the input could reach twice the cap. Now that the accumulated input travels on ToolUseEvent.Arguments, cut the crossing delta at the cap. Signed-off-by: Dharit Shah --- internal/runtime/claude_progress.go | 7 +++++-- internal/runtime/claude_progress_test.go | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/internal/runtime/claude_progress.go b/internal/runtime/claude_progress.go index 3eae069d64..6795573651 100644 --- a/internal/runtime/claude_progress.go +++ b/internal/runtime/claude_progress.go @@ -285,8 +285,11 @@ func parseClaudeStream(r io.Reader, onEvent func(AgentEvent)) error { case "thinking_delta": onEvent(ThinkingEvent{Text: d.Thinking}) case "input_json_delta": - if toolInputJSON.Len() < maxToolInputSize { - toolInputJSON.WriteString(d.PartialJSON) + // Cut the delta that crosses the cap: the input is + // incomplete past it either way, and Arguments + // carries whatever accumulated. + if room := maxToolInputSize - toolInputJSON.Len(); room > 0 { + toolInputJSON.WriteString(d.PartialJSON[:min(room, len(d.PartialJSON))]) } } diff --git a/internal/runtime/claude_progress_test.go b/internal/runtime/claude_progress_test.go index a2abedd0b7..998a3547f7 100644 --- a/internal/runtime/claude_progress_test.go +++ b/internal/runtime/claude_progress_test.go @@ -852,6 +852,23 @@ func TestParseClaudeStreamToolUseCarriesArguments(t *testing.T) { } } +func TestParseClaudeStreamArgumentsStopAtTheInputCap(t *testing.T) { + // Each delta fits a stream line; the third crosses maxToolInputSize. + delta := `{"type":"stream_event","event":{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"` + strings.Repeat("x", 400*1024) + `"}}}` + lines := []string{ + `{"type":"stream_event","event":{"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_01","name":"Write"}}}`, + delta, delta, delta, + `{"type":"stream_event","event":{"type":"content_block_stop","index":0}}`, + } + tools := collectToolUses(t, strings.Join(lines, "\n")) + if len(tools) != 1 { + t.Fatalf("expected 1 tool event, got %d", len(tools)) + } + if got := len(tools[0].Arguments); got != maxToolInputSize { + t.Errorf("expected the arguments cut at the %d-byte cap, got %d bytes", maxToolInputSize, got) + } +} + func TestParseClaudeStreamAssistantToolUseCarriesArguments(t *testing.T) { // Key order, escapes and spacing are the wire's: the parser passes the // input through without decoding it. From c697b84b24df4c6d7284ef4689965d0994876ff2 Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Mon, 21 Sep 2026 14:53:54 -0400 Subject: [PATCH 07/40] fix(telemetry): replace runner env credential values in Level 3 content MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The collector redacted by pattern only, so a credential with no recognizable shape — the case the literal pass in redactFeedback exists for — reached the record unchanged. Move that pass into replaceEnvSecrets, walked longest value first, and run it in the collector's redact ahead of the output pipeline: text, reasoning, tool results, arguments, names, summaries and the input message. An id that holds a value is dropped, like an id with any other finding. Each replacement counts in fullsend.content.redactions. Signed-off-by: Dharit Shah --- docs/guides/dev/tracing.md | 3 +- .../infrastructure/distributed-tracing.md | 5 ++- internal/cli/content_collector.go | 29 +++++++++++------ internal/cli/content_collector_test.go | 21 +++++++++++++ internal/cli/run.go | 31 ++++++++++++++----- internal/cli/run_test.go | 13 ++++++++ internal/cli/telemetry_run_test.go | 8 ++--- 7 files changed, 87 insertions(+), 23 deletions(-) diff --git a/docs/guides/dev/tracing.md b/docs/guides/dev/tracing.md index eea97cfafb..029cdc64bb 100644 --- a/docs/guides/dev/tracing.md +++ b/docs/guides/dev/tracing.md @@ -208,7 +208,8 @@ results to `tool_call_response` parts (only the Claude parser emits codex emit none of them, [#7414](https://github.com/fullsend-ai/fullsend/issues/7414); the schema's required result field is `response`), redacts every -part through `security.OutputPipeline()` at assembly (redaction runs +part — `replaceEnvSecrets` for the values of sensitive `runner_env` keys, +then `security.OutputPipeline()` — at assembly (redaction runs before the size budget — truncating first could split a secret past recognition), enforces a 256 KiB ordered-suffix budget (the ending survives — the final answer is what consumers judge) plus an 8 KiB per-tool-result diff --git a/docs/guides/infrastructure/distributed-tracing.md b/docs/guides/infrastructure/distributed-tracing.md index a0f8116c62..8976e49ea8 100644 --- a/docs/guides/infrastructure/distributed-tracing.md +++ b/docs/guides/infrastructure/distributed-tracing.md @@ -77,7 +77,10 @@ console renders, redacts it through the security output pipeline, and attaches it to the per-iteration `agent` span. The one input it records, a retry's validation-feedback prompt, is composed by the runner — fixed framing around the previous iteration's validation output — rather than -read from the stream, and goes through the same redaction. The agent runtime's own +read from the stream, and goes through the same redaction. Before the +pipeline runs, the value of every sensitive `runner_env` key (8 bytes or +longer) is replaced with `[REDACTED:]`, as it is in validation +feedback; each replacement counts as a redaction. The agent runtime's own content-logging variables (`OTEL_LOG_USER_PROMPTS`, `OTEL_LOG_ASSISTANT_RESPONSES`, etc.) are never set. diff --git a/internal/cli/content_collector.go b/internal/cli/content_collector.go index 7ae84116ea..d0b775dc2e 100644 --- a/internal/cli/content_collector.go +++ b/internal/cli/content_collector.go @@ -97,9 +97,15 @@ const maxToolArgumentsBytes = 8 * 1024 // newContentCollectorIfEnabled returns a live collector when the Level 3 // gate is on and nil otherwise — nil is the off state and is inert at // every call site, so the gate needs no second check. -func newContentCollectorIfEnabled() *contentCollector { +// +// runnerEnv is the harness runner environment. The sandbox is not meant to +// hold its credentials, but the record leaves the runner, so their values +// get the same literal pass redactFeedback gives script output. +func newContentCollectorIfEnabled(runnerEnv map[string]string) *contentCollector { if telemetry.ContentCaptureEnabled() { - return newContentCollector(maxContentBytes) + c := newContentCollector(maxContentBytes) + c.runnerEnv = runnerEnv + return c } return nil } @@ -353,8 +359,10 @@ type contentCollector struct { pipeline *security.Pipeline // secrets is the pipeline's pattern stage alone, for secretNamed. secrets *security.SecretRedactor - parts []contentPart - total int + // runnerEnv feeds redact's literal pass; see newContentCollectorIfEnabled. + runnerEnv map[string]string + parts []contentPart + total int // evicted counts bytes discarded before Result's budget runs: old // parts and a lone oversized part's head (evictOverflow), the head a // tool result loses to maxToolResultBytes, and dropped tool arguments @@ -708,6 +716,10 @@ func (c *contentCollector) redact(text string, findings *[]security.Finding) str if text == "" { return text } + text, keys := replaceEnvSecrets(text, c.runnerEnv) + for _, key := range keys { + *findings = append(*findings, security.Finding{Scanner: "runner_env", Name: key, Severity: "critical", Position: -1}) + } scanned := c.pipeline.Scan(text) *findings = append(*findings, scanned.Findings...) if scanned.Sanitized != "" { @@ -827,12 +839,9 @@ func (c *contentCollector) secretNamed(key, value string) bool { // any finding the id is dropped entirely — a substituted id could // falsely collide with another call's. func (c *contentCollector) redactID(id string, findings *[]security.Finding) string { - if id == "" { - return id - } - scanned := c.pipeline.Scan(id) - *findings = append(*findings, scanned.Findings...) - if len(scanned.Findings) > 0 { + before := len(*findings) + c.redact(id, findings) + if len(*findings) > before { return "" } return id diff --git a/internal/cli/content_collector_test.go b/internal/cli/content_collector_test.go index e865dc4c30..8eff9d735b 100644 --- a/internal/cli/content_collector_test.go +++ b/internal/cli/content_collector_test.go @@ -14,6 +14,7 @@ import ( "go.opentelemetry.io/otel/sdk/trace/tracetest" agentruntime "github.com/fullsend-ai/fullsend/internal/runtime" + "github.com/fullsend-ai/fullsend/internal/telemetry" ) // decodeOutputMessages unmarshals a collector's OutputMessages JSON and @@ -209,6 +210,26 @@ func TestContentCollector_NamelessCallCarriesNoArgumentsAndNoCharge(t *testing.T } } +func TestContentCollector_ReplacesRunnerEnvSecretValues(t *testing.T) { + // No prefix and no shape a pattern knows: only the literal pass sees it. + const secret = "runner-only-opaque-value" + t.Setenv(telemetry.ContentCaptureEnvVar, "true") + c := newContentCollectorIfEnabled(map[string]string{ + "PUSH_TOKEN": secret, + "BRANCH": "feature-branch-name", // not a sensitive key + "API_KEY": "short", // under minRedactableSecretLen + }) + c.Handle(agentruntime.TextEvent{Text: "pushing " + secret + " to feature-branch-name, short"}) + c.Handle(agentruntime.ToolUseEvent{ID: "toolu_01", Name: "Bash", Arguments: `{"command":"git push https://x:` + secret + `@host/repo"}`}) + c.Handle(agentruntime.ToolResultEvent{ID: "toolu_" + secret, Result: "remote: " + secret}) + + res := c.Result("stop") + assert.NotContains(t, res.OutputMessages, secret) + assert.Equal(t, 3, strings.Count(res.OutputMessages, "[REDACTED:PUSH_TOKEN]"), "an id is dropped, not rewritten") + assert.Contains(t, res.OutputMessages, "to feature-branch-name, short") + assert.Len(t, res.Findings, 4, "one finding per replaced key per scanned string") +} + func TestContentCollector_ANameRedactedAwayTakesItsArgumentsAlong(t *testing.T) { const args = `{"a":1}` c := newContentCollector(4096) diff --git a/internal/cli/run.go b/internal/cli/run.go index 2b8e117118..2c6feb9903 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -2,6 +2,7 @@ package cli import ( "bytes" + "cmp" "context" "crypto/sha256" "encoding/hex" @@ -9,6 +10,7 @@ import ( "errors" "fmt" "io" + "maps" "math" "net/http" "os" @@ -16,6 +18,7 @@ import ( "path/filepath" "regexp" "runtime" + "slices" "sort" "strconv" "strings" @@ -2318,7 +2321,7 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep // later spans. Nil when the Level 3 gate is off; nil is inert. The // tool-span tracker is per iteration for the same reason and is not // gated: execute_tool spans are metadata. - collector := newContentCollectorIfEnabled() + collector := newContentCollectorIfEnabled(h.RunnerEnv) collector.attachInput(agentSpan, agentPrompt) toolSpans := newToolSpanTracker(tracer, agentCtx) var metrics agentruntime.RunMetrics @@ -3427,12 +3430,7 @@ const minRedactableSecretLen = 8 // never passed through our env (a key baked into a fixture, a hook printing // its own). func redactFeedback(feedback string, runnerEnv map[string]string) string { - for key, value := range runnerEnv { - if len(value) < minRedactableSecretLen || !sensitiveEnvKey(key) { - continue - } - feedback = strings.ReplaceAll(feedback, value, "[REDACTED:"+key+"]") - } + feedback, _ = replaceEnvSecrets(feedback, runnerEnv) // ScanResult.Sanitized is empty when the scanner changed nothing, so the // original text is the fallback — not an empty prompt. if res := security.NewSecretRedactor().Scan(feedback); res.Sanitized != "" { @@ -3441,6 +3439,25 @@ func redactFeedback(feedback string, runnerEnv map[string]string) string { return feedback } +// replaceEnvSecrets is the literal pass: every value of a sensitive runner +// env key that occurs in text becomes [REDACTED:]. It returns the keys +// it replaced. Longer values go first, then key order: a value that +// contains another is replaced whole, and the text is the same on every run. +func replaceEnvSecrets(text string, runnerEnv map[string]string) (string, []string) { + var replaced []string + for _, key := range slices.SortedFunc(maps.Keys(runnerEnv), func(a, b string) int { + return cmp.Or(cmp.Compare(len(runnerEnv[b]), len(runnerEnv[a])), cmp.Compare(a, b)) + }) { + value := runnerEnv[key] + if len(value) < minRedactableSecretLen || !sensitiveEnvKey(key) || !strings.Contains(text, value) { + continue + } + text = strings.ReplaceAll(text, value, "[REDACTED:"+key+"]") + replaced = append(replaced, key) + } + return text, replaced +} + // writeValidationFeedback writes the validation failure output to a file in // the iteration directory for the audit trail, and returns the redacted // feedback string for prompt injection. The file is written on every diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index a66ac0e459..99434687bf 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -2878,6 +2878,19 @@ func TestTruncateUTF8_ShortInputUnchanged(t *testing.T) { assert.NotContains(t, truncateUTF8("short", maxFeedbackBytes), "[truncated]") } +func TestReplaceEnvSecrets_LongerValuesGoFirst(t *testing.T) { + // One value inside another: replacing the short one first would leave + // the rest of the long one in the text, and map order would pick. + short := strings.Repeat("v", minRedactableSecretLen) + long := short + "-and-more" + env := map[string]string{"A_TOKEN": short, "B_TOKEN": long} + for range 50 { + got, keys := replaceEnvSecrets("x "+long+" y "+short, env) + require.Equal(t, "x [REDACTED:B_TOKEN] y [REDACTED:A_TOKEN]", got) + require.Equal(t, []string{"B_TOKEN", "A_TOKEN"}, keys) + } +} + func TestRedactFeedback_RedactsRunnerCredentialLiterals(t *testing.T) { // The validation script runs with the full runner env, which for the code // and fix harnesses includes PUSH_TOKEN \u2014 a credential that must never diff --git a/internal/cli/telemetry_run_test.go b/internal/cli/telemetry_run_test.go index 5784cad7b6..f3c3082f76 100644 --- a/internal/cli/telemetry_run_test.go +++ b/internal/cli/telemetry_run_test.go @@ -1368,10 +1368,10 @@ func TestIterationEventHandler_TeesToRendererAndCollector(t *testing.T) { func TestNewContentCollectorIfEnabled_FollowsGate(t *testing.T) { t.Setenv(telemetry.ContentCaptureEnvVar, "") - assert.Nil(t, newContentCollectorIfEnabled(), "gate off => nil collector") + assert.Nil(t, newContentCollectorIfEnabled(nil), "gate off => nil collector") t.Setenv(telemetry.ContentCaptureEnvVar, "true") - assert.NotNil(t, newContentCollectorIfEnabled(), "gate on => live collector") + assert.NotNil(t, newContentCollectorIfEnabled(nil), "gate on => live collector") } // TestContentCapture_EndToEndFileSink drives the REAL telemetry.Setup — @@ -1391,7 +1391,7 @@ func TestContentCapture_EndToEndFileSink(t *testing.T) { dir := t.TempDir() tracer, cleanup := telemetry.Setup(dir, "test") - c := newContentCollectorIfEnabled() + c := newContentCollectorIfEnabled(nil) require.NotNil(t, c) big := strings.Repeat("all work and no play makes claude a dull agent. ", 400) // ~19KB > 8192 c.Handle(agentruntime.ToolUseEvent{ID: "toolu_01", Name: "Bash", Arguments: `{"command":"make lint"}`}) @@ -1448,7 +1448,7 @@ func TestContentCapture_GateOffProducesNoContent(t *testing.T) { dir := t.TempDir() tracer, cleanup := telemetry.Setup(dir, "test") - c := newContentCollectorIfEnabled() + c := newContentCollectorIfEnabled(nil) require.Nil(t, c, "gate off must mean no collector") c.Handle(agentruntime.TextEvent{Text: "would-be content"}) // nil-safe no-op From a1c1851a125bbe142e317cbfd7b0b7cca3b30ac1 Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Mon, 21 Sep 2026 17:04:38 -0400 Subject: [PATCH 08/40] fix(telemetry): repeat the runner env literal pass after the pipeline 3d19a5fe ran the literal pass ahead of the output pipeline only. The pipeline's normalizer joins a value the stream split with an invisible character or spelled in compatibility forms, so that value reached the record, the input message included. Run the pass on the pipeline's result as well. The pass ahead of it stays: it sees the value as the env has it, and keeps a pattern from masking part of it. Not covered, and pinned by a test for the assignment case: a value written that way which a pattern also recognises is masked by that pattern. A call loses its summary when redaction found a secret in its arguments: the parser cut the summary out of them before anything scanned it. A number leaf in tool arguments is scanned as its digits. Signed-off-by: Dharit Shah --- internal/cli/content_collector.go | 68 ++++++++--- internal/cli/content_collector_test.go | 159 ++++++++++++++++++++++++- internal/cli/run.go | 5 +- internal/cli/tool_spans.go | 3 +- 4 files changed, 209 insertions(+), 26 deletions(-) diff --git a/internal/cli/content_collector.go b/internal/cli/content_collector.go index d0b775dc2e..5486568ffa 100644 --- a/internal/cli/content_collector.go +++ b/internal/cli/content_collector.go @@ -82,7 +82,9 @@ const maxToolResultBytes = 8 * 1024 // maxToolArgumentsBytes bounds one tool call's arguments, measured on // their redacted encoding. Arguments over it are dropped whole, not cut: -// a cut object is not JSON, and the part keeps its id, name and summary. +// a cut object is not JSON, and the part keeps its id, name and summary +// (not the summary when redaction found a secret in the arguments; see +// Handle). // Like maxToolResultBytes it exists because the total above is small; // what blocks raising it is the same unproven backend ceiling named on // maxContentBytes. @@ -166,8 +168,9 @@ func attachContent(span trace.Span, res contentResult) { // can spell out one it saw in fullwidth; then this scan is the first to // see it whole. // It masks the recorded copy only — the agent is sent the prompt as -// composed — and it does not repeat redactFeedback's literal pass over -// runner-env values. The mask that first scan leaves for a +// composed. redact repeats redactFeedback's literal pass over runner env +// values, once more after the fold, which can spell out a value that pass +// saw in compatibility forms. The mask that first scan leaves for a // connection-string password of ten or more bytes ("abcd...") matches its // own pattern again, so such a prompt counts a finding here; a shorter // password is masked "***", which does not. @@ -334,10 +337,10 @@ type contentResult struct { // replaced), or as // redacted text when they were not one JSON value. DroppedBytes int - // Truncated reports whether the budget cut or dropped anything, a kept - // tool call's arguments were dropped (toolArguments; its part is - // marked), or a kept tool result was a parser-side fragment (its part - // is marked). + // Truncated reports whether the budget cut or dropped anything, a tool + // call's arguments were dropped (toolArguments, or Result when the name + // redacted away; a kept part is marked), or a kept tool result was a + // parser-side fragment (its part is marked). Truncated bool // Findings are the security findings raised during redaction. Findings []security.Finding @@ -404,7 +407,15 @@ func (c *contentCollector) Handle(evt agentruntime.AgentEvent) { // The schema requires a name on a tool_call part. Arguments // alone must not keep a nameless call, nor charge for one that // appendPart then refuses. + scanned := len(c.findings) p.Arguments, p.Truncated = c.toolArguments(e.Arguments) + if slices.ContainsFunc(c.findings[scanned:], func(f security.Finding) bool { return f.Scanner != "unicode_normalizer" }) { + // The parser cut the summary out of these arguments + // before anything scanned it, so a secret found in them + // can be in the summary as a beginning that neither the + // literal pass nor a pattern matches. + p.Summary = "" + } } c.appendPart(p) case agentruntime.ToolResultEvent: @@ -707,8 +718,8 @@ func encodedTail(s string, allow int) string { return tailToRuneBoundary(s, len(s)-start) } -// redact runs text through the output pipeline, returning the sanitized -// form and accumulating findings. ScanResult.Sanitized is empty when +// redact runs text through the runner env literal pass and the output +// pipeline, returning the sanitized form and accumulating findings. ScanResult.Sanitized is empty when // nothing changed — but also when sanitization removed everything (an // all-invisible-bytes input), so an empty Sanitized WITH findings means // fully redacted, not unchanged. @@ -716,14 +727,20 @@ func (c *contentCollector) redact(text string, findings *[]security.Finding) str if text == "" { return text } - text, keys := replaceEnvSecrets(text, c.runnerEnv) - for _, key := range keys { - *findings = append(*findings, security.Finding{Scanner: "runner_env", Name: key, Severity: "critical", Position: -1}) - } + // The literal pass runs on both sides of the pipeline. Ahead of it, on + // the text as the stream wrote it, so a pattern does not mask part of + // a value and keep its first bytes. After it, because the normalizer + // joins a value the stream split with an invisible character or + // spelled in compatibility forms. Not covered: a value written that way + // which a pattern also recognises — in an assignment, an auth header, a + // secret-named field or a connection string, or by its own prefix — is + // masked by that pattern, which shows what its mask shows; and a value + // the normalizer itself rewrites is matched only as the env has it. + text = c.replaceEnv(text, findings) scanned := c.pipeline.Scan(text) *findings = append(*findings, scanned.Findings...) if scanned.Sanitized != "" { - return scanned.Sanitized + return c.replaceEnv(scanned.Sanitized, findings) } if len(scanned.Findings) > 0 { return "" @@ -739,8 +756,8 @@ func (c *contentCollector) redact(text string, findings *[]security.Finding) str // JSON they miss an assignment that opens a string or follows an escaped // newline, a value behind escaped quotes, and JSON nested in a string, and // Unicode folding can turn a fullwidth quotation mark into one that closes -// the string. So the value is decoded, each string and object key is -// redacted on its own (redactValue), and the result is encoded again — +// the string. So the value is decoded, each string, number and object key +// is redacted on its own (redactValue), and the result is encoded again — // key order, spacing and escapes are the encoder's, not the wire's. // // Text that is not one JSON value (see ToolUseEvent.Arguments) cannot be @@ -777,7 +794,8 @@ func (c *contentCollector) toolArguments(args string) (json.RawMessage, bool) { return out, false } -// redactValue redacts every string and object key under v. A pattern +// redactValue redacts every string, number and object key under v; a +// number that redacts becomes the redacted string. A pattern // keyed on a member name ("password": "...") cannot see the pair that // way, so each string member is scanned once more, already redacted, // beside its redacted key (secretNamed) and masked whole on a match. @@ -789,6 +807,12 @@ func (c *contentCollector) redactValue(v any, collided *bool) any { switch t := v.(type) { case string: return c.redact(t, &c.findings) + case json.Number: + // Digits can be a credential too; a number that redacts is + // recorded as the redacted string. + if s := c.redact(t.String(), &c.findings); s != t.String() { + return s + } case []any: for i := range t { t[i] = c.redactValue(t[i], collided) @@ -835,6 +859,16 @@ func (c *contentCollector) secretNamed(key, value string) bool { return false } +// replaceEnv is redact's literal pass (replaceEnvSecrets over runnerEnv), +// with one finding for each key it replaced. +func (c *contentCollector) replaceEnv(text string, findings *[]security.Finding) string { + text, keys := replaceEnvSecrets(text, c.runnerEnv) + for _, key := range keys { + *findings = append(*findings, security.Finding{Scanner: "runner_env", Name: key, Severity: "critical", Position: -1}) + } + return text +} + // redactID scans a part id like every other stream-derived string; on // any finding the id is dropped entirely — a substituted id could // falsely collide with another call's. diff --git a/internal/cli/content_collector_test.go b/internal/cli/content_collector_test.go index 8eff9d735b..b250c5d75f 100644 --- a/internal/cli/content_collector_test.go +++ b/internal/cli/content_collector_test.go @@ -210,7 +210,18 @@ func TestContentCollector_NamelessCallCarriesNoArgumentsAndNoCharge(t *testing.T } } -func TestContentCollector_ReplacesRunnerEnvSecretValues(t *testing.T) { +// runnerEnvFindings counts the findings the runner env literal pass raised. +func runnerEnvFindings(res contentResult) int { + n := 0 + for _, f := range res.Findings { + if f.Scanner == "runner_env" { + n++ + } + } + return n +} + +func TestContentCollector_ReplacesRunnerEnvValues(t *testing.T) { // No prefix and no shape a pattern knows: only the literal pass sees it. const secret = "runner-only-opaque-value" t.Setenv(telemetry.ContentCaptureEnvVar, "true") @@ -225,9 +236,144 @@ func TestContentCollector_ReplacesRunnerEnvSecretValues(t *testing.T) { res := c.Result("stop") assert.NotContains(t, res.OutputMessages, secret) - assert.Equal(t, 3, strings.Count(res.OutputMessages, "[REDACTED:PUSH_TOKEN]"), "an id is dropped, not rewritten") - assert.Contains(t, res.OutputMessages, "to feature-branch-name, short") - assert.Len(t, res.Findings, 4, "one finding per replaced key per scanned string") + assert.Contains(t, res.OutputMessages, `"content":"pushing [REDACTED:PUSH_TOKEN] to feature-branch-name, short"`) + assert.Contains(t, res.OutputMessages, `"arguments":{"command":"git push https://x:[REDACTED:PUSH_TOKEN]@host/repo"}`) + assert.Contains(t, res.OutputMessages, `{"type":"tool_call_response","response":"remote: [REDACTED:PUSH_TOKEN]"}`, "an id that holds a value is dropped") + assert.Equal(t, 4, runnerEnvFindings(res), "one per replaced key per pass per scanned string") + assert.Len(t, res.Findings, 4) +} + +func TestContentCollector_ReplacesARunnerEnvValueBeforeAPatternMasksIt(t *testing.T) { + // The assignment pattern would mask this value and keep its first + // four bytes; the literal pass has to reach it first. + const secret = "zzqx-opaque-runner-value" + t.Setenv(telemetry.ContentCaptureEnvVar, "true") + c := newContentCollectorIfEnabled(map[string]string{"PUSH_TOKEN": secret}) + c.Handle(agentruntime.TextEvent{Text: "export PUSH_TOKEN=" + secret}) + + res := c.Result("stop") + require.NotEmpty(t, res.OutputMessages) + assert.NotContains(t, res.OutputMessages, secret[:4]) +} + +func TestContentCollector_ARunnerEnvValueSplitInsideAnAssignmentKeepsFourBytes(t *testing.T) { + // The documented limit: the pass ahead of the pipeline cannot see a + // value an invisible character splits, the normalizer joins it, and the + // assignment pattern masks it its own way before the second pass runs. + const secret = "zzqx-opaque-runner-value" + t.Setenv(telemetry.ContentCaptureEnvVar, "true") + c := newContentCollectorIfEnabled(map[string]string{"PUSH_TOKEN": secret}) + c.Handle(agentruntime.TextEvent{Text: "export PUSH_TOKEN=" + secret[:11] + "\u200B" + secret[11:]}) + + res := c.Result("stop") + assert.Contains(t, res.OutputMessages, `"content":"export PUSH_TOKEN=zzqx..."`) +} + +// fullwidth writes ASCII in its compatibility forms, which NFKC folds back. +func fullwidth(s string) string { + return strings.Map(func(r rune) rune { + if r > ' ' && r <= '~' { + return r + 0xFEE0 + } + return r + }, s) +} + +func TestContentCollector_ReplacesRunnerEnvValuesTheNormalizerSpellsOut(t *testing.T) { + // A value in fullwidth forms, or split by a zero-width space, is the + // value only once the pipeline's normalizer ran. + const secret = "runner-only-opaque-value" + t.Setenv(telemetry.ContentCaptureEnvVar, "true") + env := map[string]string{"PUSH_TOKEN": secret} + + c := newContentCollectorIfEnabled(env) + c.Handle(agentruntime.TextEvent{Text: "pushing " + fullwidth(secret)}) + c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Arguments: `{"command":"echo ` + secret[:11] + `\u200B` + secret[11:] + `"}`}) + res := c.Result("stop") + assert.NotContains(t, res.OutputMessages, secret) + assert.Equal(t, 2, strings.Count(res.OutputMessages, "[REDACTED:PUSH_TOKEN]")) + assert.Equal(t, 2, runnerEnvFindings(res), "a value found after the pipeline counts like any other") + + input := recordedInput(t, newContentCollectorIfEnabled(env), "validator said: "+fullwidth(secret))["gen_ai.input.messages"].AsString() + assert.Contains(t, input, "validator said: [REDACTED:PUSH_TOKEN]") +} + +func TestContentCollector_ReplacesARunnerEnvValueTheNormalizerWouldRewrite(t *testing.T) { + // ² folds to 2 and the ligature to "fi", and a combining mark after the + // value composes with its last letter: the pass ahead of the pipeline + // sees the value as the env has it. + secret := "opaque²-runner-" + "\uFB01" + "nal-value" + t.Setenv(telemetry.ContentCaptureEnvVar, "true") + c := newContentCollectorIfEnabled(map[string]string{"DEPLOY_PASSWORD": secret}) + c.Handle(agentruntime.TextEvent{Text: "deploying with " + secret + "\u0301 now"}) + + res := c.Result("stop") + assert.Contains(t, res.OutputMessages, "deploying with [REDACTED:DEPLOY_PASSWORD]") + assert.NotContains(t, res.OutputMessages, "runner-final-value") + assert.Equal(t, 1, runnerEnvFindings(res)) +} + +func TestContentCollector_TextThatOnlyBeginsARunnerEnvValueStays(t *testing.T) { + // Only a whole value is replaced: ordinary words that begin one stay, + // and so does a call id. + token := "github_pat_" + strings.Repeat("A", 30) + t.Setenv(telemetry.ContentCaptureEnvVar, "true") + c := newContentCollectorIfEnabled(map[string]string{"DEPLOY_TOKEN": token}) + c.Handle(agentruntime.TextEvent{Text: "opened a pull request on github"}) + c.Handle(agentruntime.ToolUseEvent{ID: "call_on_github", Name: "search_github", Summary: "querying github"}) + + res := c.Result("stop") + assert.Contains(t, res.OutputMessages, `"content":"opened a pull request on github"`) + assert.Contains(t, res.OutputMessages, `{"type":"tool_call","id":"call_on_github","name":"search_github","summary":"querying github"}`) + assert.Empty(t, res.Findings) +} + +func TestContentCollector_DropsTheSummaryWhenTheArgumentsHeldARunnerEnvValue(t *testing.T) { + // The parser cuts the summary out of the arguments before anything + // scans it, so the value can be there as a beginning no literal matches. + const secret = "runner-only-opaque-value" + t.Setenv(telemetry.ContentCaptureEnvVar, "true") + c := newContentCollectorIfEnabled(map[string]string{"PUSH_TOKEN": secret}) + c.Handle(agentruntime.ToolUseEvent{ + Name: "Bash", + Summary: "git push https://x:" + secret[:20] + "…", + Arguments: `{"command":"git push https://x:` + secret + `@host/repo"}`, + }) + c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Summary: "ls", Arguments: `{"command":"ls"}`}) + + res := c.Result("stop") + assert.NotContains(t, res.OutputMessages, secret[:5]) + assert.Contains(t, res.OutputMessages, `{"type":"tool_call","name":"Bash","arguments":{"command":"git push https://x:[REDACTED:PUSH_TOKEN]@host/repo"}}`) + assert.Contains(t, res.OutputMessages, `"summary":"ls"`, "a call whose arguments held no value keeps its summary") +} + +func TestContentCollector_DropsTheSummaryWhenAPatternFoundTheSecret(t *testing.T) { + // A zero-width space hides the token from the parser's scan of the + // summary; the collector's pattern finds it in the arguments once the + // normalizer joined it, and the summary holds its cut beginning. + token := "ghp_" + strings.Repeat("k", 36) + split := token[:20] + "\u200B" + token[20:] + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{ + Name: "Bash", + Summary: "curl -H x:" + token[:20] + "…", + Arguments: `{"command":"curl -H x:` + split + ` https://host"}`, + }) + c.Handle(agentruntime.ToolUseEvent{Name: "Write", Summary: "/docs/a.md", Arguments: `{"file_path":"/docs/a.md","content":"to be continued…"}`}) + + res := c.Result("stop") + assert.NotContains(t, res.OutputMessages, token[:20]) + assert.Contains(t, res.OutputMessages, `"summary":"/docs/a.md"`, "a normalizer finding alone does not cost the summary") +} + +func TestContentCollector_ReplacesARunnerEnvValueSentAsANumber(t *testing.T) { + digits := strings.Repeat("7", minRedactableSecretLen) + t.Setenv(telemetry.ContentCaptureEnvVar, "true") + c := newContentCollectorIfEnabled(map[string]string{"DEPLOY_PASSWORD": digits}) + c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Arguments: `{"pin":` + digits + `,"n":42}`}) + + res := c.Result("stop") + assert.Contains(t, res.OutputMessages, `"arguments":{"n":42,"pin":"[REDACTED:DEPLOY_PASSWORD]"}`) } func TestContentCollector_ANameRedactedAwayTakesItsArgumentsAlong(t *testing.T) { @@ -236,7 +382,7 @@ func TestContentCollector_ANameRedactedAwayTakesItsArgumentsAlong(t *testing.T) c.Handle(agentruntime.ToolUseEvent{ID: "toolu_01", Name: "\u200B", Arguments: args}) res := c.Result("stop") - assert.Empty(t, res.OutputMessages, "a tool_call part needs a name") + assert.Empty(t, res.OutputMessages, "nothing is left of the part") assert.Len(t, res.Findings, 1) assert.True(t, res.Truncated) assert.Equal(t, len(args), res.DroppedBytes) @@ -320,7 +466,8 @@ func TestContentCollector_IncompleteArgumentsAreDroppedAndMarked(t *testing.T) { res := c.Result("stop") part := partAt(t, decodeOutputMessages(t, res.OutputMessages), 0) assert.NotContains(t, part, "arguments") - assert.Equal(t, "/x", part["summary"], "the call itself is kept") + assert.Equal(t, "Write", part["name"], "the call itself is kept") + assert.NotContains(t, part, "summary", "a secret in the arguments costs the summary") assert.Equal(t, true, part["fullsend.truncated"]) assert.True(t, res.Truncated) assert.Len(t, res.Findings, 1, "dropped content is scanned first") diff --git a/internal/cli/run.go b/internal/cli/run.go index 2c6feb9903..2c9cab9962 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -3439,8 +3439,9 @@ func redactFeedback(feedback string, runnerEnv map[string]string) string { return feedback } -// replaceEnvSecrets is the literal pass: every value of a sensitive runner -// env key that occurs in text becomes [REDACTED:]. It returns the keys +// replaceEnvSecrets is the literal pass: the value of each sensitive runner +// env key, when it holds minRedactableSecretLen bytes or more and occurs in +// text, becomes [REDACTED:]. It returns the keys // it replaced. Longer values go first, then key order: a value that // contains another is replaced whole, and the text is the same on every run. func replaceEnvSecrets(text string, runnerEnv map[string]string) (string, []string) { diff --git a/internal/cli/tool_spans.go b/internal/cli/tool_spans.go index 45dd891e0c..770512293c 100644 --- a/internal/cli/tool_spans.go +++ b/internal/cli/tool_spans.go @@ -74,7 +74,8 @@ const maxToolSpanNameBytes = 128 // reports it without an id (stream_event lines) or not at all (assistant // lines, the live path). The // name and the call id go through the same output pipeline as span content -// (Unicode normalization, then secret redaction): the name is bounded, the id +// (Unicode normalization, then secret redaction; not the collector's runner +// env literal pass): the name is bounded, the id // is dropped on any finding (safeID) — both land on a Level 1 span in the // telemetry file the output scan exempts on the strength of that treatment. // Delivery is synchronous on one goroutine, so no lock. From 7944734cea164ef4dbb69e8465803114c73ab604 Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Mon, 21 Sep 2026 17:04:43 -0400 Subject: [PATCH 09/40] docs(tracing): describe the runner env literal pass and shorten the ADR annotations The reference says where the pass runs, how it counts, what it does not cover, when a call loses its summary, and the limit for runtimes that report no arguments; it adds the fourth way a call loses its arguments. The dev guide says a marked tool_call part lost its arguments whole, never in part, and that tool span names and ids do not get the literal pass. The contributing guide names replaceEnvSecrets. The two 2026-09-18 ADR annotations repeated what the reference holds; they are a short note and a link now. Signed-off-by: Dharit Shah --- ...050-distributed-tracing-instrumentation.md | 8 ++-- docs/ADRs/0108-tool-call-span-topology.md | 11 ++--- docs/contributing/go-code.md | 4 +- docs/guides/dev/tracing.md | 22 ++++++---- .../infrastructure/distributed-tracing.md | 42 ++++++++++++++----- 5 files changed, 55 insertions(+), 32 deletions(-) diff --git a/docs/ADRs/0050-distributed-tracing-instrumentation.md b/docs/ADRs/0050-distributed-tracing-instrumentation.md index f05d74fc65..77eb580f11 100644 --- a/docs/ADRs/0050-distributed-tracing-instrumentation.md +++ b/docs/ADRs/0050-distributed-tracing-instrumentation.md @@ -184,8 +184,6 @@ under the content gate, a retry iteration that carries validation feedback records the prompt the runner composed as `gen_ai.input.messages`, and `tool_call` parts on the message record carry the call's arguments — Claude Code today ([#7414](https://github.com/fullsend-ai/fullsend/issues/7414)). -The rest of the model's input is not recorded. The fixed default prompt -that all other iterations send is a constant that carries no task and is left -out by choice; the requests the runtime builds from the agent definition and -the conversation are not visible to fullsend, which reads the runtime's -stream, not its API requests. The gate and the carrier are unchanged. +The rest of the model's input is not recorded +([reference](../guides/infrastructure/distributed-tracing.md#content-capture-level-3)). +The gate and the carrier are unchanged. diff --git a/docs/ADRs/0108-tool-call-span-topology.md b/docs/ADRs/0108-tool-call-span-topology.md index 8a13b7efdb..d12f59975c 100644 --- a/docs/ADRs/0108-tool-call-span-topology.md +++ b/docs/ADRs/0108-tool-call-span-topology.md @@ -126,11 +126,8 @@ convention bump. **2026-09-18 — Tool arguments on the message record:** the Decision's "full arguments next" is in place up to a per-call bound: `tool_call` parts on the -`agent` span's `gen_ai.output.messages` record carry the call's arguments — -decoded, redacted and re-encoded, not the wire text — where the runtime's -stream provides them, Claude Code today. Arguments over 8 KiB re-encoded, not -a complete JSON value, or with two keys of one object that redact to the same -string, are dropped whole and the part marked `fullsend.truncated`; kept -arguments count toward the record's total bound -([reference](../guides/infrastructure/distributed-tracing.md#content-capture-level-3)). +`agent` span's `gen_ai.output.messages` record carry the call's redacted +arguments where the runtime's stream provides them, Claude Code today. The +bound and the drop rules are in the +[reference](../guides/infrastructure/distributed-tracing.md#content-capture-level-3). `execute_tool` spans still carry metadata only. diff --git a/docs/contributing/go-code.md b/docs/contributing/go-code.md index df23c1162f..90aa1678d7 100644 --- a/docs/contributing/go-code.md +++ b/docs/contributing/go-code.md @@ -471,7 +471,9 @@ function in `internal/cli/run.go` is the canonical implementation. `_SECRET`, `_PASSWORD`, `_KEY`, `_CREDENTIALS`) with `[REDACTED:]`. Skip values shorter than `minRedactableSecretLen` (currently 8) — short values like `"main"` - or `"true"` cause false-positive mangling. + or `"true"` cause false-positive mangling. `replaceEnvSecrets` is that + pass: it takes longer values first, so a value that contains another + is replaced whole. 2. **Apply `security.SecretRedactor` as a second-pass fallback.** The `RunnerEnv` scan only catches credentials the harness declared. diff --git a/docs/guides/dev/tracing.md b/docs/guides/dev/tracing.md index 029cdc64bb..4757c60415 100644 --- a/docs/guides/dev/tracing.md +++ b/docs/guides/dev/tracing.md @@ -169,8 +169,8 @@ an id — pi and codex emit none — produce no span, so the child count can be below `fullsend.tool_calls`, which counts every reported call, id or not; a `server_tool_use` block on an `assistant` line produces no event at all (its result never arrives as a `tool_result`), so it appears in neither count. The name passes through `security.OutputPipeline()` -— Unicode normalization, then secret redaction, the same pipeline as span -content — and is bounded to 256 bytes before it becomes the attribute; the +— Unicode normalization, then secret redaction, the pipeline span content +gets, without the collector's runner env literal pass — and is bounded to 256 bytes before it becomes the attribute; the span name keeps at most 128 bytes of it. The call id is scanned through the same pipeline and dropped from the span on any finding — never substituted, since a masked id could collide with another call's — while the raw bounded @@ -208,8 +208,8 @@ results to `tool_call_response` parts (only the Claude parser emits codex emit none of them, [#7414](https://github.com/fullsend-ai/fullsend/issues/7414); the schema's required result field is `response`), redacts every -part — `replaceEnvSecrets` for the values of sensitive `runner_env` keys, -then `security.OutputPipeline()` — at assembly (redaction runs +part — `security.OutputPipeline()`, with `replaceEnvSecrets` for the values +of sensitive runner env keys on both sides of it — at assembly (redaction runs before the size budget — truncating first could split a secret past recognition), enforces a 256 KiB ordered-suffix budget (the ending survives — the final answer is what consumers judge) plus an 8 KiB per-tool-result @@ -241,15 +241,17 @@ assignment that opens a string or follows an escaped newline, a value behind escaped quotes, and JSON nested in a string; Unicode folding can also turn a fullwidth quotation mark into one that closes the string. `toolArguments` decodes the value, redacts each string and object key on -its own, and encodes the result again. Each string member is also scanned +its own (a number as its digits; one that redacts becomes the redacted +string), and encodes the result again. Each string member is also scanned once more, already redacted, beside its key, and masked whole when the member-name pattern (`json_field`) matches — a pattern keyed on a member name has no other way to see the pair; that scan's other findings are discarded, since both strings were already scanned. It runs the pattern stage alone: the normalizer is not idempotent over escape sequences, and a second pass over the pair could strip the value or the key's keyword. It reaches string -members only: a secret-named member holding an array, object or number is -redacted string by string, as it would be in text. Arguments that are not +members only — a number that redacted is one by then: a +secret-named member holding an array, an object or any other number is +redacted leaf by leaf, as it would be in text. Arguments that are not one JSON value cannot be walked that way: they are scanned as text — with the misses above — so their findings count, then dropped and charged. So are arguments in which two keys of one object redact to the same string, @@ -278,10 +280,12 @@ stay within the proven size. `run-telemetry.jsonl`): parse the `gen_ai.output.messages` attribute as JSON; check `fullsend.content.truncated` / `fullsend.content.dropped_bytes` before treating content as complete; masked secrets appear as the -redactor's mask tokens and are counted in `fullsend.content.redactions`. +redactor's mask tokens, or as `[REDACTED:]` for a runner env value, +and are counted in `fullsend.content.redactions`. A `tool_call` part's `arguments`, when present, is a JSON value (an object for the tools seen so far); a `tool_call` part marked `fullsend.truncated` -had arguments that were dropped. `gen_ai.input.messages` is absent unless +had arguments that were dropped whole — on that part type the marker +never means a partial value, as it does on a tool result. `gen_ai.input.messages` is absent unless the iteration is a retry that carried validation feedback — absence is the normal case, not a gap. Masks `redactFeedback` left in the feedback (`abcd...`, `***`, `[REDACTED:]`) are in `gen_ai.input.messages` diff --git a/docs/guides/infrastructure/distributed-tracing.md b/docs/guides/infrastructure/distributed-tracing.md index 8976e49ea8..7f9b1f04be 100644 --- a/docs/guides/infrastructure/distributed-tracing.md +++ b/docs/guides/infrastructure/distributed-tracing.md @@ -77,10 +77,26 @@ console renders, redacts it through the security output pipeline, and attaches it to the per-iteration `agent` span. The one input it records, a retry's validation-feedback prompt, is composed by the runner — fixed framing around the previous iteration's validation output — rather than -read from the stream, and goes through the same redaction. Before the -pipeline runs, the value of every sensitive `runner_env` key (8 bytes or -longer) is replaced with `[REDACTED:]`, as it is in validation -feedback; each replacement counts as a redaction. The agent runtime's own +read from the stream, and goes through the same redaction. On both sides +of the pipeline — on the text as written, and again on what the pipeline +returns, since its normalization can join a value the stream split or +spelled in compatibility characters — the value of each sensitive runner +environment key (`env.runner`, or the deprecated `runner_env`; 8 bytes or +longer) is replaced with `[REDACTED:]`, the marker validation +feedback uses. A key counts one redaction for each of the two passes that +replaced it in a string, however often its value occurs there; in an +assignment, an authorization header or a secret-named field the patterns +then mask the marker as well and count again. Not covered: a value split +or spelled that way which a pattern also recognises — in one of those +contexts, in a connection string, or by its own prefix — is masked by +that pattern, and shows what the pattern's mask shows. A tool call loses +its `summary` when redaction found a secret in its arguments: the parser +cut the summary out of them before anything scanned it, so the secret +could be there as a beginning. Where a runtime reports a summary and no +arguments (pi, codex, OpenCode), its parser has already run the patterns +over the summary and, for pi and codex, cut it: a value that straddles a +cut stays in part, and one a pattern recognised stays as that pattern's +mask. The agent runtime's own content-logging variables (`OTEL_LOG_USER_PROMPTS`, `OTEL_LOG_ASSISTANT_RESPONSES`, etc.) are never set. @@ -102,7 +118,8 @@ with a `finish_reason` of `stop` or `error`. Tool arguments, tool results, and t only when the runtime's stream provides them: Claude runs do; the pi and codex parsers emit none of the three yet ([#7414](https://github.com/fullsend-ai/fullsend/issues/7414)). -`arguments` is the call's input decoded, redacted string by string, and +`arguments` is the call's input decoded, redacted string by string (a +number that redacts becomes the redacted string), and encoded again, so key order, spacing and escapes are not the stream's. On a retry iteration under `validation_loop.feedback_mode: append`, the @@ -120,7 +137,8 @@ runtime builds from the agent definition are not visible to fullsend, which sees the runtime's stream, not its API requests. **Redaction and size:** every part passes through security redaction -(Unicode normalization, then secret masking) before reaching the span. +(runner environment values, Unicode normalization, secret masking, then +runner environment values again) before reaching the span. Content is bounded at 256 KiB per iteration — each tool result at 8 KiB, each call's arguments at 8 KiB — kept as an ordered suffix; overflow drops the oldest content first. Those @@ -137,7 +155,10 @@ Truncation is marked via `fullsend.content.truncated` on the span and its tail; arguments over theirs, not a complete JSON value, or with two keys of one object that redact to the same string, are dropped whole — a cut object is not JSON — and the marked `tool_call` part keeps -its `id`, name and summary. The input message is not cut at this stage: +its `id`, name and summary — not the summary when redaction found a +secret in the arguments. Arguments also go, charged and marked, when +the call's name redacts to nothing; that part survives only if it has a +summary. The input message is not cut at this stage: the validation output inside it is cut at 10 KiB when the prompt is composed (the prompt then says `[truncated]`), and the two markers above do not describe it. The recorded copy is the redacted one, so it can @@ -218,8 +239,9 @@ result whose call was never reported (its stream line was skipped) is a near-zero-duration span marked `fullsend.tool.unmatched`. Runtimes whose parsers emit no call ids (pi, codex) produce no `execute_tool` spans, and neither do server-side tools, whose result never arrives as a `tool_result`. Tool -names and call ids pass through the same sanitizer as span content (Unicode -normalization, then secret redaction): a name is redacted in place and +names and call ids pass through the output pipeline span content gets (Unicode +normalization, then secret redaction; the collector's runner environment +pass is not applied to them): a name is redacted in place and bounded to 256 bytes for the attribute and 128 for the span name; an id with any finding is dropped from the span. At most 1,024 `execute_tool` spans are recorded per iteration; calls past that are counted in @@ -287,7 +309,7 @@ The `agent` span's provider identity reflects only the parent run's serving endp | `fullsend.transcript_error` | `agent` | Present (`true`) when the agent exited 0 but its transcript reported an error — the span's status is Error while `exit_code` keeps the raw process exit | | `gen_ai.input.messages` | `agent` | Level 3 only, on a retry iteration that carries validation feedback: the prompt the runner composed, as a JSON string (see Content capture) | | `gen_ai.output.messages` | `agent` | Level 3 only: the iteration's conversation content as a JSON string (see Content capture) | -| `fullsend.content.truncated` | `agent` | Level 3 only: present (`true`) when the size budget cut or dropped content, a kept tool call lost its arguments, or a kept tool result is a parser-side fragment or an oversized line's empty stand-in (`fullsend.truncated` on the part; no byte count for the last two) | +| `fullsend.content.truncated` | `agent` | Level 3 only: present (`true`) when the size budget cut or dropped content, a tool call lost its arguments, or a kept tool result is a parser-side fragment or an oversized line's empty stand-in (`fullsend.truncated` on the part; no byte count for the last two) | | `fullsend.content.dropped_bytes` | `agent` | Level 3 only: exact part bytes removed by the size budget — content, ids, dropped arguments (counted as re-encoded JSON — on a key collision, without the member the later key (in sorted key order, not stream order) replaced — or as redacted text when they were not a JSON value), and the fixed footprint of an errored-empty or oversized stand-in part — otherwise in raw bytes, whichever bound made the cut; the bytes of a skipped oversized line were never decoded and are not counted | | `fullsend.content.redactions` | `agent` | Level 3 only: number of security findings raised while redacting content at assembly, the input message included (and findings from parts the size budget later dropped) | | `fullsend.tool.unmatched` | `execute_tool` | Present (`true`) when a result arrived for a call the stream never reported; the span has near-zero duration | From 67d08abb8c91394ad31e4f9398f6228372ed0a1e Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Wed, 23 Sep 2026 09:49:22 -0400 Subject: [PATCH 10/40] docs(tracing): say fullsend.truncated reads by part type On a tool result the marker means part of the response is kept; on a tool call it means none of the arguments is. The consumer contract in the dev guide already says so; the reference now says it where the marker is introduced. Signed-off-by: Dharit Shah --- docs/guides/infrastructure/distributed-tracing.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/guides/infrastructure/distributed-tracing.md b/docs/guides/infrastructure/distributed-tracing.md index 7f9b1f04be..c38f7ecc02 100644 --- a/docs/guides/infrastructure/distributed-tracing.md +++ b/docs/guides/infrastructure/distributed-tracing.md @@ -158,7 +158,9 @@ whole — a cut object is not JSON — and the marked `tool_call` part keeps its `id`, name and summary — not the summary when redaction found a secret in the arguments. Arguments also go, charged and marked, when the call's name redacts to nothing; that part survives only if it has a -summary. The input message is not cut at this stage: +summary. So `fullsend.truncated` means different things by part type: on a +tool result, part of the response is kept; on a tool call, none of the +arguments is. Read it together with the part's `type`. The input message is not cut at this stage: the validation output inside it is cut at 10 KiB when the prompt is composed (the prompt then says `[truncated]`), and the two markers above do not describe it. The recorded copy is the redacted one, so it can From 3ecc9b50b76502cf4539fe4a6a4fef4f43938fea Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Tue, 6 Oct 2026 14:36:12 -0400 Subject: [PATCH 11/40] fix(telemetry): judge values nested under a secret-named member The member-name check reached only a string directly under its key, so {"api_keys":["..."]}, {"credentials":{"value":"..."}} and a numeric password kept an opaque credential. Carry the nearest secret-named key down the walk: a string or number in an array or nested object under it is judged as the member's own value would be, the pattern's eight-character floor included. Booleans and nulls stay. A direct string member is still checked against its own key as well. The collector's literal pass also covers the provider-only keys the runner keeps out of RunnerEnv (GH_WORKFLOW_TOKEN, #6649), through replaceProviderOnlySecrets, which redactFeedback now calls too: the process value is replaced whatever RunnerEnv holds. Signed-off-by: Dharit Shah --- docs/contributing/go-code.md | 5 +- docs/guides/dev/tracing.md | 18 +++-- .../infrastructure/distributed-tracing.md | 9 ++- internal/cli/content_collector.go | 70 +++++++++++++---- internal/cli/content_collector_test.go | 78 +++++++++++++++++++ internal/cli/run.go | 25 ++++-- 6 files changed, 167 insertions(+), 38 deletions(-) diff --git a/docs/contributing/go-code.md b/docs/contributing/go-code.md index eff0feffa9..efe4911138 100644 --- a/docs/contributing/go-code.md +++ b/docs/contributing/go-code.md @@ -484,8 +484,9 @@ function in `internal/cli/run.go` is the canonical implementation. can't reach them, which means the `RunnerEnv` scan in invariant 1 never sees them. Iterate `providerOnlyKeys`, read each value with `os.Getenv`, and replace it the same way (skipping values shorter - than `minRedactableSecretLen`). A future credential class kept out - of `RunnerEnv` for the same reason needs the same treatment here. + than `minRedactableSecretLen`). `replaceProviderOnlySecrets` is that + pass. A future credential class kept out of `RunnerEnv` for the same + reason needs the same treatment here. 3. **Apply `security.SecretRedactor` as a fallback pass.** The `RunnerEnv` and `providerOnlyKeys` scans only catch credentials diff --git a/docs/guides/dev/tracing.md b/docs/guides/dev/tracing.md index 6560a52868..00b56eea73 100644 --- a/docs/guides/dev/tracing.md +++ b/docs/guides/dev/tracing.md @@ -209,7 +209,8 @@ codex emit none of them, [#7414](https://github.com/fullsend-ai/fullsend/issues/7414); the schema's required result field is `response`), redacts every part — `security.OutputPipeline()`, with `replaceEnvSecrets` for the values -of sensitive runner env keys on both sides of it — at assembly (redaction runs +of sensitive runner env keys, and `replaceProviderOnlySecrets` for +`providerOnlyKeys`, on both sides of it — at assembly (redaction runs before the size budget — truncating first could split a secret past recognition), enforces a 256 KiB ordered-suffix budget (the ending survives — the final answer is what consumers judge) plus an 8 KiB per-tool-result @@ -242,16 +243,17 @@ behind escaped quotes, and JSON nested in a string; Unicode folding can also turn a fullwidth quotation mark into one that closes the string. `toolArguments` decodes the value, redacts each string and object key on its own (a number as its digits; one that redacts becomes the redacted -string), and encodes the result again. Each string member is also scanned -once more, already redacted, beside its key, and masked whole when the -member-name pattern (`json_field`) matches — a pattern keyed on a member +string), and encodes the result again. Each string and number is also +scanned once more, already redacted, beside the nearest enclosing key the +member-name pattern (`json_field`) names, and masked whole when that pattern +matches the pair — a pattern keyed on a member name has no other way to see the pair; that scan's other findings are discarded, since both strings were already scanned. It runs the pattern stage alone: the normalizer is not idempotent over escape sequences, and a -second pass over the pair could strip the value or the key's keyword. It reaches string -members only — a number that redacted is one by then: a -secret-named member holding an array, an object or any other number is -redacted leaf by leaf, as it would be in text. Arguments that are not +second pass over the pair could strip the value or the key's keyword. A value in an +array or a nested object under a secret-named member is judged as that +member's own value would be — the pattern's eight-character floor included, +so a short count stays; booleans and nulls are left as they are. Arguments that are not one JSON value cannot be walked that way: they are scanned as text — with the misses above — so their findings count, then dropped and charged. So are arguments in which two keys of one object redact to the same string, diff --git a/docs/guides/infrastructure/distributed-tracing.md b/docs/guides/infrastructure/distributed-tracing.md index 82b329bc8d..f4883b7c89 100644 --- a/docs/guides/infrastructure/distributed-tracing.md +++ b/docs/guides/infrastructure/distributed-tracing.md @@ -81,10 +81,11 @@ read from the stream, and goes through the same redaction. On both sides of the pipeline — on the text as written, and again on what the pipeline returns, since its normalization can join a value the stream split or spelled in compatibility characters — the value of each sensitive runner -environment key (`env.runner`, or the deprecated `runner_env`; 8 bytes or -longer) is replaced with `[REDACTED:]`, the marker validation -feedback uses. A key counts one redaction for each of the two passes that -replaced it in a string, however often its value occurs there; in an +environment key (`env.runner`, or the deprecated `runner_env`), and of each +provider-only key the runner keeps out of it (`GH_WORKFLOW_TOKEN`), 8 bytes or +longer, is replaced with `[REDACTED:]`, the marker validation +feedback uses. Each value replaced in a string counts one redaction per +pass, however often it occurs there; in an assignment, an authorization header or a secret-named field the patterns then mask the marker as well and count again. Not covered: a value split or spelled that way which a pattern also recognises — in one of those diff --git a/internal/cli/content_collector.go b/internal/cli/content_collector.go index 5486568ffa..8577497aef 100644 --- a/internal/cli/content_collector.go +++ b/internal/cli/content_collector.go @@ -101,8 +101,9 @@ const maxToolArgumentsBytes = 8 * 1024 // every call site, so the gate needs no second check. // // runnerEnv is the harness runner environment. The sandbox is not meant to -// hold its credentials, but the record leaves the runner, so their values -// get the same literal pass redactFeedback gives script output. +// hold its credentials, but the record leaves the runner, so their values, +// and the provider-only keys', get the same literal pass redactFeedback +// gives script output. func newContentCollectorIfEnabled(runnerEnv map[string]string) *contentCollector { if telemetry.ContentCaptureEnabled() { c := newContentCollector(maxContentBytes) @@ -783,7 +784,7 @@ func (c *contentCollector) toolArguments(args string) (json.RawMessage, bool) { return nil, false } collided := false - out, err := json.Marshal(c.redactValue(v, &collided)) + out, err := json.Marshal(c.redactValue(v, "", &collided)) if err != nil { return nil, false // decoded JSON values marshal unconditionally } @@ -797,37 +798,55 @@ func (c *contentCollector) toolArguments(args string) (json.RawMessage, bool) { // redactValue redacts every string, number and object key under v; a // number that redacts becomes the redacted string. A pattern // keyed on a member name ("password": "...") cannot see the pair that -// way, so each string member is scanned once more, already redacted, -// beside its redacted key (secretNamed) and masked whole on a match. +// way, so a string or number under a key that pattern names — the nearest +// such enclosing key, under — is scanned once more, already redacted, +// beside it and masked whole on a match (secretNamed); a direct string +// member is also scanned beside its own key. A value in +// an array or a nested object under a secret-named member is judged as +// that member's own value would be. // Keys are walked in sorted order, so neither the findings nor a dropped // value's charge follow map order. *collided reports two keys of one // object that redact to the same string: keeping either member would // show a call the agent did not make. -func (c *contentCollector) redactValue(v any, collided *bool) any { +func (c *contentCollector) redactValue(v any, under string, collided *bool) any { switch t := v.(type) { case string: - return c.redact(t, &c.findings) + s := c.redact(t, &c.findings) + if under != "" && c.secretNamed(under, s) { + return "***" + } + return s case json.Number: // Digits can be a credential too; a number that redacts is // recorded as the redacted string. - if s := c.redact(t.String(), &c.findings); s != t.String() { + s := c.redact(t.String(), &c.findings) + if under != "" && c.secretNamed(under, s) { + return "***" + } + if s != t.String() { return s } case []any: for i := range t { - t[i] = c.redactValue(t[i], collided) + t[i] = c.redactValue(t[i], under, collided) } case map[string]any: out := make(map[string]any, len(t)) for _, k := range slices.Sorted(maps.Keys(t)) { rk := c.redact(k, &c.findings) - e := c.redactValue(t[k], collided) - if s, ok := e.(string); ok && c.secretNamed(rk, s) { - e = "***" + member := under + if c.namesASecret(rk) { + member = rk } if _, dup := out[rk]; dup { *collided = true } + e := c.redactValue(t[k], member, collided) + if s, ok := e.(string); ok && member != rk && c.secretNamed(rk, s) { + // A single quote in the key can let the pattern match + // the pair even when the key names no secret. + e = "***" + } out[rk] = e } return out @@ -835,6 +854,15 @@ func (c *contentCollector) redactValue(v any, collided *bool) any { return v } +// namesASecret reports whether the member-name pattern matches key with +// any value long enough: a value placed under such a key is then judged +// against it. It records no finding; secretNamed does, for a value that +// matches. +func (c *contentCollector) namesASecret(key string) bool { + return slices.ContainsFunc(c.secrets.Scan(memberPair(key, strings.Repeat("x", 8))).Findings, + func(f security.Finding) bool { return f.Name == "json_field" }) +} + // secretNamed reports whether the redactor's member-name pattern // (json_field) matches the pair, and records that finding. Any other // finding of this second scan is discarded: both strings were scanned @@ -848,9 +876,7 @@ func (c *contentCollector) redactValue(v any, collided *bool) any { // pattern's token class, so it joins no two runs into a token that a // prefix pattern would mask ahead of the member-name pattern. func (c *contentCollector) secretNamed(key, value string) bool { - unquote := strings.NewReplacer(`"`, ",") - pair := `"` + unquote.Replace(key) + `":"` + unquote.Replace(value) + `"` - for _, f := range c.secrets.Scan(pair).Findings { + for _, f := range c.secrets.Scan(memberPair(key, value)).Findings { if f.Name == "json_field" { c.findings = append(c.findings, f) return true @@ -859,10 +885,20 @@ func (c *contentCollector) secretNamed(key, value string) bool { return false } -// replaceEnv is redact's literal pass (replaceEnvSecrets over runnerEnv), -// with one finding for each key it replaced. +// memberPair writes key and value as the JSON member secretNamed scans, +// ',' standing in for a double quote. +func memberPair(key, value string) string { + unquote := strings.NewReplacer(`"`, ",") + return `"` + unquote.Replace(key) + `":"` + unquote.Replace(value) + `"` +} + +// replaceEnv is redact's literal pass — replaceEnvSecrets over runnerEnv, +// then replaceProviderOnlySecrets — with one finding for each key it +// replaced. func (c *contentCollector) replaceEnv(text string, findings *[]security.Finding) string { text, keys := replaceEnvSecrets(text, c.runnerEnv) + text, more := replaceProviderOnlySecrets(text) + keys = append(keys, more...) for _, key := range keys { *findings = append(*findings, security.Finding{Scanner: "runner_env", Name: key, Severity: "critical", Position: -1}) } diff --git a/internal/cli/content_collector_test.go b/internal/cli/content_collector_test.go index b250c5d75f..64f2cfddef 100644 --- a/internal/cli/content_collector_test.go +++ b/internal/cli/content_collector_test.go @@ -155,6 +155,40 @@ func TestContentCollector_RedactsSecretsInArguments(t *testing.T) { } } +func TestContentCollector_SecretNamedMembersReachTheirNestedValues(t *testing.T) { + // A value at any depth under a secret-named member is judged as if it + // were that member's own string value. + const opaque = "opaque-credential-123" + for name, tc := range map[string]struct{ args, want string }{ + "array": {`{"api_keys":["` + opaque + `"]}`, `{"api_keys":["***"]}`}, + "object": {`{"credentials":{"value":"` + opaque + `"}}`, `{"credentials":{"value":"***"}}`}, + "array of object": {`{"tokens":[{"v":"` + opaque + `"}]}`, `{"tokens":[{"v":"***"}]}`}, + "number": {`{"password":31415926535}`, `{"password":"***"}`}, + "nested number": {`{"secrets":{"pin":31415926535}}`, `{"secrets":{"pin":"***"}}`}, + // The member-name pattern wants eight bytes or more, as it does of + // a string member: short values and a token count stay. + "short values stay": {`{"max_tokens":4096,"api_keys":["ab",true,null]}`, `{"api_keys":["ab",true,null],"max_tokens":4096}`}, + // No secret-named member above: nothing to judge against. + "plain nesting stays": {`{"edits":[{"value":"` + opaque + `"}]}`, `{"edits":[{"value":"` + opaque + `"}]}`}, + // A quote in the key lets the pattern's single-quote form match + // across the pair: the member's own key is still checked. + "quote in a plain key": {`{"it's":"password': 'hunter2hunter2'"}`, `{"it's":"***"}`}, + } { + t.Run(name, func(t *testing.T) { + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Arguments: tc.args}) + + res := c.Result("stop") + assert.Contains(t, res.OutputMessages, `"arguments":`+tc.want) + if tc.want == tc.args || strings.Contains(name, "stay") { + assert.Empty(t, res.Findings) + } else { + assert.Len(t, res.Findings, 1, "one json_field finding") + } + }) + } +} + func TestContentCollector_AKeyFindingLeavesTheValueAndCountsOnce(t *testing.T) { for name, tc := range map[string]struct{ args, want string }{ "token": {`{"ghp_` + strings.Repeat("r", 36) + `":"hello"}`, `{"ghp_...":"hello"}`}, @@ -366,6 +400,50 @@ func TestContentCollector_DropsTheSummaryWhenAPatternFoundTheSecret(t *testing.T assert.Contains(t, res.OutputMessages, `"summary":"/docs/a.md"`, "a normalizer finding alone does not cost the summary") } +func TestContentCollector_ReplacesProviderOnlyValuesFromTheProcessEnv(t *testing.T) { + // GH_WORKFLOW_TOKEN is kept out of RunnerEnv (#6649); the collector + // reads it where redactFeedback does. + const secret = "workflow-only-opaque-value" + t.Setenv(telemetry.ContentCaptureEnvVar, "true") + t.Setenv(workflowTokenEnv, secret) + c := newContentCollectorIfEnabled(nil) + c.Handle(agentruntime.TextEvent{Text: "token " + secret}) + c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Arguments: `{"command":"echo ` + secret + `"}`}) + + res := c.Result("stop") + assert.NotContains(t, res.OutputMessages, secret) + assert.Equal(t, 2, strings.Count(res.OutputMessages, "[REDACTED:"+workflowTokenEnv+"]")) + assert.Equal(t, 2, runnerEnvFindings(res), "counted like a runner env value") +} + +func TestReplaceProviderOnlySecrets(t *testing.T) { + t.Setenv(workflowTokenEnv, "abcdefgh") + got, keys := replaceProviderOnlySecrets("x abcdefgh y") + assert.Equal(t, "x [REDACTED:"+workflowTokenEnv+"] y", got) + assert.Equal(t, []string{workflowTokenEnv}, keys) + + got, keys = replaceProviderOnlySecrets("nothing here") + assert.Equal(t, "nothing here", got) + assert.Empty(t, keys, "a value absent from the text is not reported") + + t.Setenv(workflowTokenEnv, "abcdefg") // under minRedactableSecretLen + got, keys = replaceProviderOnlySecrets("x abcdefg y") + assert.Equal(t, "x abcdefg y", got) + assert.Empty(t, keys) +} + +func TestContentCollector_ReplacesBothValuesWhenRunnerEnvReusesAProviderOnlyName(t *testing.T) { + // Nothing refuses GH_WORKFLOW_TOKEN as a runner env key; the process + // value is still the one redactFeedback replaces. + t.Setenv(telemetry.ContentCaptureEnvVar, "true") + t.Setenv(workflowTokenEnv, "from-the-process-env") + c := newContentCollectorIfEnabled(map[string]string{workflowTokenEnv: "from-the-runner-env"}) + c.Handle(agentruntime.TextEvent{Text: "a from-the-process-env b from-the-runner-env"}) + + res := c.Result("stop") + assert.Contains(t, res.OutputMessages, `"a [REDACTED:`+workflowTokenEnv+`] b [REDACTED:`+workflowTokenEnv+`]"`) +} + func TestContentCollector_ReplacesARunnerEnvValueSentAsANumber(t *testing.T) { digits := strings.Repeat("7", minRedactableSecretLen) t.Setenv(telemetry.ContentCaptureEnvVar, "true") diff --git a/internal/cli/run.go b/internal/cli/run.go index cba2d0265a..6ba5240931 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -3633,13 +3633,7 @@ func redactFeedback(feedback string, runnerEnv map[string]string) string { // Provider-only keys live in the process environment, not RunnerEnv. // Redact their literals the same way so they cannot reach the agent // prompt or the uploaded run directory (#6649). - for key := range providerOnlyKeys { - value := os.Getenv(key) - if len(value) < minRedactableSecretLen { - continue - } - feedback = strings.ReplaceAll(feedback, value, "[REDACTED:"+key+"]") - } + feedback, _ = replaceProviderOnlySecrets(feedback) // ScanResult.Sanitized is empty when the scanner changed nothing, so the // original text is the fallback — not an empty prompt. if res := security.NewSecretRedactor().Scan(feedback); res.Sanitized != "" { @@ -3668,6 +3662,23 @@ func replaceEnvSecrets(text string, runnerEnv map[string]string) (string, []stri return text, replaced } +// replaceProviderOnlySecrets is the literal pass over providerOnlyKeys, +// whose values live in the process environment, not RunnerEnv (#6649): +// each value of minRedactableSecretLen bytes or more that occurs in text +// becomes [REDACTED:]. It returns the keys it replaced, in key order. +func replaceProviderOnlySecrets(text string) (string, []string) { + var replaced []string + for _, key := range slices.Sorted(maps.Keys(providerOnlyKeys)) { + value := os.Getenv(key) + if len(value) < minRedactableSecretLen || !strings.Contains(text, value) { + continue + } + text = strings.ReplaceAll(text, value, "[REDACTED:"+key+"]") + replaced = append(replaced, key) + } + return text, replaced +} + // writeValidationFeedback writes the validation failure output to a file in // the iteration directory for the audit trail, and returns the redacted // feedback string for prompt injection. The file is written on every From 1fd2b7d49d3b2d3439de9dada26477882767b85b Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Tue, 6 Oct 2026 15:34:14 -0400 Subject: [PATCH 12/40] fix(telemetry): one literal pass for Level 3 strings and execute_tool names and ids replaceEnvSecrets now takes the runner env values and the provider-only values from the process environment in one longest-first pass, so a value that contains another is replaced whole whichever source holds it; redactFeedback calls it once. replaceProviderOnlySecrets is gone. execute_tool span names and ids get the same redaction as span content: redactText, the literal pass on both sides of the output pipeline, now shared by the collector and the tool span tracker. Signed-off-by: Dharit Shah --- docs/contributing/go-code.md | 6 +- docs/guides/dev/tracing.md | 18 +++--- .../infrastructure/distributed-tracing.md | 6 +- internal/cli/content_collector.go | 47 ++++++++------- internal/cli/content_collector_test.go | 28 ++++----- internal/cli/run.go | 58 +++++++++---------- internal/cli/run_test.go | 37 ++++++++++++ internal/cli/tool_spans.go | 25 ++++---- internal/cli/tool_spans_test.go | 23 ++++++++ 9 files changed, 153 insertions(+), 95 deletions(-) diff --git a/docs/contributing/go-code.md b/docs/contributing/go-code.md index efe4911138..c492a0b2aa 100644 --- a/docs/contributing/go-code.md +++ b/docs/contributing/go-code.md @@ -484,9 +484,9 @@ function in `internal/cli/run.go` is the canonical implementation. can't reach them, which means the `RunnerEnv` scan in invariant 1 never sees them. Iterate `providerOnlyKeys`, read each value with `os.Getenv`, and replace it the same way (skipping values shorter - than `minRedactableSecretLen`). `replaceProviderOnlySecrets` is that - pass. A future credential class kept out of `RunnerEnv` for the same - reason needs the same treatment here. + than `minRedactableSecretLen`). `replaceEnvSecrets` does this in the + same longest-first pass as invariant 1. A future credential class kept + out of `RunnerEnv` for the same reason needs the same treatment here. 3. **Apply `security.SecretRedactor` as a fallback pass.** The `RunnerEnv` and `providerOnlyKeys` scans only catch credentials diff --git a/docs/guides/dev/tracing.md b/docs/guides/dev/tracing.md index 00b56eea73..1e9018e6cd 100644 --- a/docs/guides/dev/tracing.md +++ b/docs/guides/dev/tracing.md @@ -168,11 +168,11 @@ near-zero-duration span marked `fullsend.tool.unmatched=true`. Events without an id — pi and codex emit none — produce no span, so the child count can be below `fullsend.tool_calls`, which counts every reported call, id or not; a `server_tool_use` block on an `assistant` line produces no event at all (its -result never arrives as a `tool_result`), so it appears in neither count. The name passes through `security.OutputPipeline()` -— Unicode normalization, then secret redaction, the pipeline span content -gets, without the collector's runner env literal pass — and is bounded to 256 bytes before it becomes the attribute; the -span name keeps at most 128 bytes of it. The call id is scanned through the -same pipeline and dropped from the span on any finding — never substituted, +result never arrives as a `tool_result`), so it appears in neither count. The name passes through `redactText` +— the runner env literal pass on both sides of `security.OutputPipeline()`, +the redaction span content gets — and is bounded to 256 bytes before it becomes the attribute; the +span name keeps at most 128 bytes of it. The call id goes through the +same `redactText` and dropped from the span on any finding — never substituted, since a masked id could collide with another call's — while the raw bounded id still keys the open-call map, so correlation is unaffected. The tracker records at most `maxToolSpansPerIteration` (1,024) spans per @@ -209,8 +209,8 @@ codex emit none of them, [#7414](https://github.com/fullsend-ai/fullsend/issues/7414); the schema's required result field is `response`), redacts every part — `security.OutputPipeline()`, with `replaceEnvSecrets` for the values -of sensitive runner env keys, and `replaceProviderOnlySecrets` for -`providerOnlyKeys`, on both sides of it — at assembly (redaction runs +of sensitive runner env keys and of `providerOnlyKeys`, in one pass, on both +sides of it (`redactText`) — at assembly (redaction runs before the size budget — truncating first could split a secret past recognition), enforces a 256 KiB ordered-suffix budget (the ending survives — the final answer is what consumers judge) plus an 8 KiB per-tool-result @@ -282,8 +282,8 @@ stay within the proven size. `run-telemetry.jsonl`): parse the `gen_ai.output.messages` attribute as JSON; check `fullsend.content.truncated` / `fullsend.content.dropped_bytes` before treating content as complete; masked secrets appear as the -redactor's mask tokens, or as `[REDACTED:]` for a runner env value, -and are counted in `fullsend.content.redactions`. +redactor's mask tokens, or as `[REDACTED:]` for a runner env or +provider-only value, and are counted in `fullsend.content.redactions`. A `tool_call` part's `arguments`, when present, is a JSON value (an object for the tools seen so far); a `tool_call` part marked `fullsend.truncated` had arguments that were dropped whole — on that part type the marker diff --git a/docs/guides/infrastructure/distributed-tracing.md b/docs/guides/infrastructure/distributed-tracing.md index f4883b7c89..6ebb298700 100644 --- a/docs/guides/infrastructure/distributed-tracing.md +++ b/docs/guides/infrastructure/distributed-tracing.md @@ -242,9 +242,9 @@ result whose call was never reported (its stream line was skipped) is a near-zero-duration span marked `fullsend.tool.unmatched`. Runtimes whose parsers emit no call ids (pi, codex) produce no `execute_tool` spans, and neither do server-side tools, whose result never arrives as a `tool_result`. Tool -names and call ids pass through the output pipeline span content gets (Unicode -normalization, then secret redaction; the collector's runner environment -pass is not applied to them): a name is redacted in place and +names and call ids get the redaction span content gets (runner environment +and provider-only values, Unicode normalization, secret redaction, then +those values again): a name is redacted in place and bounded to 256 bytes for the attribute and 128 for the span name; an id with any finding is dropped from the span. At most 1,024 `execute_tool` spans are recorded per iteration; calls past that are counted in diff --git a/internal/cli/content_collector.go b/internal/cli/content_collector.go index 8577497aef..28e53ef151 100644 --- a/internal/cli/content_collector.go +++ b/internal/cli/content_collector.go @@ -725,23 +725,29 @@ func encodedTail(s string, allow int) string { // all-invisible-bytes input), so an empty Sanitized WITH findings means // fully redacted, not unchanged. func (c *contentCollector) redact(text string, findings *[]security.Finding) string { + return redactText(c.pipeline, c.runnerEnv, text, findings) +} + +// redactText is the redaction every Level 3 string and every execute_tool +// name and id gets: the runner env literal pass (replaceEnvSecrets) on both +// sides of pipeline. Ahead of it, on the text as the stream wrote it, so a +// pattern does not mask part of a value and keep its first bytes. After +// it, because the normalizer joins a value the stream split with an +// invisible character or spelled in compatibility forms. Not covered: a +// value written that way which a pattern also recognises — in an +// assignment, an auth header, a secret-named field or a connection string, +// or by its own prefix — is masked by that pattern, which shows what its +// mask shows; and a value the normalizer itself rewrites is matched only as +// the env has it. An empty result with findings means fully redacted. +func redactText(pipeline *security.Pipeline, runnerEnv map[string]string, text string, findings *[]security.Finding) string { if text == "" { return text } - // The literal pass runs on both sides of the pipeline. Ahead of it, on - // the text as the stream wrote it, so a pattern does not mask part of - // a value and keep its first bytes. After it, because the normalizer - // joins a value the stream split with an invisible character or - // spelled in compatibility forms. Not covered: a value written that way - // which a pattern also recognises — in an assignment, an auth header, a - // secret-named field or a connection string, or by its own prefix — is - // masked by that pattern, which shows what its mask shows; and a value - // the normalizer itself rewrites is matched only as the env has it. - text = c.replaceEnv(text, findings) - scanned := c.pipeline.Scan(text) + text = replaceEnv(runnerEnv, text, findings) + scanned := pipeline.Scan(text) *findings = append(*findings, scanned.Findings...) if scanned.Sanitized != "" { - return c.replaceEnv(scanned.Sanitized, findings) + return replaceEnv(runnerEnv, scanned.Sanitized, findings) } if len(scanned.Findings) > 0 { return "" @@ -838,15 +844,15 @@ func (c *contentCollector) redactValue(v any, under string, collided *bool) any if c.namesASecret(rk) { member = rk } - if _, dup := out[rk]; dup { - *collided = true - } e := c.redactValue(t[k], member, collided) if s, ok := e.(string); ok && member != rk && c.secretNamed(rk, s) { // A single quote in the key can let the pattern match // the pair even when the key names no secret. e = "***" } + if _, dup := out[rk]; dup { + *collided = true + } out[rk] = e } return out @@ -892,13 +898,10 @@ func memberPair(key, value string) string { return `"` + unquote.Replace(key) + `":"` + unquote.Replace(value) + `"` } -// replaceEnv is redact's literal pass — replaceEnvSecrets over runnerEnv, -// then replaceProviderOnlySecrets — with one finding for each key it -// replaced. -func (c *contentCollector) replaceEnv(text string, findings *[]security.Finding) string { - text, keys := replaceEnvSecrets(text, c.runnerEnv) - text, more := replaceProviderOnlySecrets(text) - keys = append(keys, more...) +// replaceEnv is redactText's literal pass, with one finding for each key +// it replaced. +func replaceEnv(runnerEnv map[string]string, text string, findings *[]security.Finding) string { + text, keys := replaceEnvSecrets(text, runnerEnv) for _, key := range keys { *findings = append(*findings, security.Finding{Scanner: "runner_env", Name: key, Severity: "critical", Position: -1}) } diff --git a/internal/cli/content_collector_test.go b/internal/cli/content_collector_test.go index 64f2cfddef..d3cbee1b67 100644 --- a/internal/cli/content_collector_test.go +++ b/internal/cli/content_collector_test.go @@ -416,20 +416,20 @@ func TestContentCollector_ReplacesProviderOnlyValuesFromTheProcessEnv(t *testing assert.Equal(t, 2, runnerEnvFindings(res), "counted like a runner env value") } -func TestReplaceProviderOnlySecrets(t *testing.T) { - t.Setenv(workflowTokenEnv, "abcdefgh") - got, keys := replaceProviderOnlySecrets("x abcdefgh y") - assert.Equal(t, "x [REDACTED:"+workflowTokenEnv+"] y", got) - assert.Equal(t, []string{workflowTokenEnv}, keys) - - got, keys = replaceProviderOnlySecrets("nothing here") - assert.Equal(t, "nothing here", got) - assert.Empty(t, keys, "a value absent from the text is not reported") - - t.Setenv(workflowTokenEnv, "abcdefg") // under minRedactableSecretLen - got, keys = replaceProviderOnlySecrets("x abcdefg y") - assert.Equal(t, "x abcdefg y", got) - assert.Empty(t, keys) +func TestContentCollector_SecretNamedChecksSeeNormalizedText(t *testing.T) { + // The key and the value are judged as the normalizer writes them: a + // fullwidth key names a secret once folded, and four ligatures are + // eight characters once folded. + for name, tc := range map[string]struct{ args, want string }{ + "fullwidth key over an array": {`{"\uFF50\uFF41\uFF53\uFF53\uFF57\uFF4F\uFF52\uFF44":["opaque-credential-123"]}`, `{"password":["***"]}`}, + "value folded past the floor": {`{"password":"\uFB01\uFB01\uFB01\uFB01"}`, `{"password":"***"}`}, + } { + t.Run(name, func(t *testing.T) { + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Arguments: tc.args}) + assert.Contains(t, c.Result("stop").OutputMessages, `"arguments":`+tc.want) + }) + } } func TestContentCollector_ReplacesBothValuesWhenRunnerEnvReusesAProviderOnlyName(t *testing.T) { diff --git a/internal/cli/run.go b/internal/cli/run.go index 6ba5240931..a8347f5bce 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -10,7 +10,6 @@ import ( "errors" "fmt" "io" - "maps" "math" "net/http" "os" @@ -2423,6 +2422,7 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep collector := newContentCollectorIfEnabled(h.RunnerEnv) collector.attachInput(agentSpan, agentPrompt) toolSpans := newToolSpanTracker(tracer, agentCtx) + toolSpans.runnerEnv = h.RunnerEnv var metrics agentruntime.RunMetrics hooksSettings := "" if h.SecurityEnabled() { @@ -3629,11 +3629,10 @@ const minRedactableSecretLen = 8 // never passed through our env (a key baked into a fixture, a hook printing // its own). func redactFeedback(feedback string, runnerEnv map[string]string) string { + // Provider-only keys live in the process environment, not RunnerEnv; + // replaceEnvSecrets redacts their literals in the same pass so they + // cannot reach the agent prompt or the uploaded run directory (#6649). feedback, _ = replaceEnvSecrets(feedback, runnerEnv) - // Provider-only keys live in the process environment, not RunnerEnv. - // Redact their literals the same way so they cannot reach the agent - // prompt or the uploaded run directory (#6649). - feedback, _ = replaceProviderOnlySecrets(feedback) // ScanResult.Sanitized is empty when the scanner changed nothing, so the // original text is the fallback — not an empty prompt. if res := security.NewSecretRedactor().Scan(feedback); res.Sanitized != "" { @@ -3643,38 +3642,35 @@ func redactFeedback(feedback string, runnerEnv map[string]string) string { } // replaceEnvSecrets is the literal pass: the value of each sensitive runner -// env key, when it holds minRedactableSecretLen bytes or more and occurs in -// text, becomes [REDACTED:]. It returns the keys -// it replaced. Longer values go first, then key order: a value that -// contains another is replaced whole, and the text is the same on every run. +// env key, and of each providerOnlyKeys entry in the process environment +// (kept out of RunnerEnv, #6649), when it holds minRedactableSecretLen bytes +// or more and occurs in text, becomes [REDACTED:]. It returns the keys +// it replaced. Both sources go in one pass, longer values first, then key +// order: a value that contains another is replaced whole, whichever source +// holds it, and the text is the same on every run. func replaceEnvSecrets(text string, runnerEnv map[string]string) (string, []string) { - var replaced []string - for _, key := range slices.SortedFunc(maps.Keys(runnerEnv), func(a, b string) int { - return cmp.Or(cmp.Compare(len(runnerEnv[b]), len(runnerEnv[a])), cmp.Compare(a, b)) - }) { - value := runnerEnv[key] - if len(value) < minRedactableSecretLen || !sensitiveEnvKey(key) || !strings.Contains(text, value) { - continue + type literal struct{ key, value string } + var literals []literal + for key, value := range runnerEnv { + if len(value) >= minRedactableSecretLen && sensitiveEnvKey(key) { + literals = append(literals, literal{key, value}) } - text = strings.ReplaceAll(text, value, "[REDACTED:"+key+"]") - replaced = append(replaced, key) } - return text, replaced -} - -// replaceProviderOnlySecrets is the literal pass over providerOnlyKeys, -// whose values live in the process environment, not RunnerEnv (#6649): -// each value of minRedactableSecretLen bytes or more that occurs in text -// becomes [REDACTED:]. It returns the keys it replaced, in key order. -func replaceProviderOnlySecrets(text string) (string, []string) { + for key := range providerOnlyKeys { + if value := os.Getenv(key); len(value) >= minRedactableSecretLen { + literals = append(literals, literal{key, value}) + } + } + slices.SortFunc(literals, func(a, b literal) int { + return cmp.Or(cmp.Compare(len(b.value), len(a.value)), cmp.Compare(a.key, b.key), cmp.Compare(a.value, b.value)) + }) var replaced []string - for _, key := range slices.Sorted(maps.Keys(providerOnlyKeys)) { - value := os.Getenv(key) - if len(value) < minRedactableSecretLen || !strings.Contains(text, value) { + for _, l := range literals { + if !strings.Contains(text, l.value) { continue } - text = strings.ReplaceAll(text, value, "[REDACTED:"+key+"]") - replaced = append(replaced, key) + text = strings.ReplaceAll(text, l.value, "[REDACTED:"+l.key+"]") + replaced = append(replaced, l.key) } return text, replaced } diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index bf0f6fc7fd..423cd2c4f1 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -3241,6 +3241,43 @@ func TestWriteValidationFeedback_RedactsBeforeWritingFile(t *testing.T) { assert.Contains(t, string(data), "[REDACTED:PUSH_TOKEN]") } +func TestReplaceEnvSecrets_OneLongestFirstPassOverBothSources(t *testing.T) { + short := strings.Repeat("v", minRedactableSecretLen) + long := short + "-and-more" + for name, tc := range map[string]struct{ runner, process string }{ + "runner value inside the provider value": {short, long}, + "provider value inside the runner value": {long, short}, + } { + t.Run(name, func(t *testing.T) { + t.Setenv(workflowTokenEnv, tc.process) + env := map[string]string{"PUSH_TOKEN": tc.runner} + got, _ := replaceEnvSecrets("a "+long+" b "+short, env) + longKey, shortKey := "PUSH_TOKEN", workflowTokenEnv + if tc.process == long { + longKey, shortKey = workflowTokenEnv, "PUSH_TOKEN" + } + assert.Equal(t, "a [REDACTED:"+longKey+"] b [REDACTED:"+shortKey+"]", got) + assert.Equal(t, got, redactFeedback("a "+long+" b "+short, env), "redactFeedback runs the same pass") + }) + } +} + +func TestReplaceEnvSecrets_ProviderOnlyValues(t *testing.T) { + t.Setenv(workflowTokenEnv, "abcdefgh") + got, keys := replaceEnvSecrets("x abcdefgh y", nil) + assert.Equal(t, "x [REDACTED:"+workflowTokenEnv+"] y", got) + assert.Equal(t, []string{workflowTokenEnv}, keys) + + got, keys = replaceEnvSecrets("nothing here", nil) + assert.Equal(t, "nothing here", got) + assert.Empty(t, keys, "a value absent from the text is not reported") + + t.Setenv(workflowTokenEnv, "abcdefg") // under minRedactableSecretLen + got, keys = replaceEnvSecrets("x abcdefg y", nil) + assert.Equal(t, "x abcdefg y", got) + assert.Empty(t, keys) +} + func TestRedactFeedback_RedactsWorkflowTokenFromProcessEnv(t *testing.T) { const token = "ghs_workflow_redact_me_xx" t.Setenv(workflowTokenEnv, token) diff --git a/internal/cli/tool_spans.go b/internal/cli/tool_spans.go index 770512293c..cce27bead0 100644 --- a/internal/cli/tool_spans.go +++ b/internal/cli/tool_spans.go @@ -73,9 +73,9 @@ const maxToolSpanNameBytes = 128 // a server-side tool: its result never arrives as a tool_result, so the parser // reports it without an id (stream_event lines) or not at all (assistant // lines, the live path). The -// name and the call id go through the same output pipeline as span content -// (Unicode normalization, then secret redaction; not the collector's runner -// env literal pass): the name is bounded, the id +// name and the call id get the same redaction as span content (redactText: +// the runner env literal pass on both sides of the output pipeline): the +// name is bounded, the id // is dropped on any finding (safeID) — both land on a Level 1 span in the // telemetry file the output scan exempts on the strength of that treatment. // Delivery is synchronous on one goroutine, so no lock. @@ -83,9 +83,11 @@ type toolSpanTracker struct { tracer trace.Tracer ctx context.Context pipeline *security.Pipeline - open map[string]trace.Span - created int - dropped int + // runnerEnv feeds the literal pass on names and ids (redactText). + runnerEnv map[string]string + open map[string]trace.Span + created int + dropped int } func newToolSpanTracker(tracer trace.Tracer, agentCtx context.Context) *toolSpanTracker { @@ -175,12 +177,8 @@ func (t *toolSpanTracker) allow() bool { // result with findings means the whole name was sanitized away (the same // reading contentCollector.redact applies), so nothing is shown. func (t *toolSpanTracker) safeName(name string) string { - scanned := t.pipeline.Scan(name) - if scanned.Sanitized != "" { - name = scanned.Sanitized - } else if len(scanned.Findings) > 0 { - return "" - } + var findings []security.Finding + name = redactText(t.pipeline, t.runnerEnv, name, &findings) return strings.ToValidUTF8(truncateStatusMsgTo(name, maxToolNameBytes), "") } @@ -189,7 +187,8 @@ func (t *toolSpanTracker) safeName(name string) string { // id could falsely collide with another call's. The raw id still keys // use/result correlation, so a dropped attribute never breaks a pair. func (t *toolSpanTracker) safeID(id string) string { - if len(t.pipeline.Scan(id).Findings) > 0 { + var findings []security.Finding + if redactText(t.pipeline, t.runnerEnv, id, &findings); len(findings) > 0 { return "" } return id diff --git a/internal/cli/tool_spans_test.go b/internal/cli/tool_spans_test.go index 7e28816ff2..1cb992e85d 100644 --- a/internal/cli/tool_spans_test.go +++ b/internal/cli/tool_spans_test.go @@ -82,6 +82,29 @@ func TestToolSpanTracker_PairEmitsExecuteToolChild(t *testing.T) { assert.Empty(t, tr.open, "an answered call must not stay open (Finish would re-end it, and the map would grow per call)") } +func TestToolSpanTracker_ReplacesRunnerEnvValuesInNamesAndDropsIDs(t *testing.T) { + const runnerValue, workflowValue = "runner-only-opaque-value", "workflow-only-opaque-value" + t.Setenv(workflowTokenEnv, workflowValue) + tr, rec, _ := toolSpanFixture(t) + tr.runnerEnv = map[string]string{"PUSH_TOKEN": runnerValue} + + for _, value := range []string{runnerValue, workflowValue} { + tr.Handle(agentruntime.ToolUseEvent{ID: "toolu_" + value, Name: "mcp__x__" + value}) + tr.Handle(agentruntime.ToolResultEvent{ID: "toolu_" + value, Result: "ok"}) + } + + spans := endedToolSpans(rec) + require.Len(t, spans, 2) + assert.Equal(t, "execute_tool mcp__x__[REDACTED:PUSH_TOKEN]", spans[0].Name) + assert.Equal(t, "execute_tool mcp__x__[REDACTED:"+workflowTokenEnv+"]", spans[1].Name) + for _, span := range spans { + for _, kv := range span.Attributes { + assert.NotContains(t, kv.Value.Emit(), runnerValue, string(kv.Key)) + assert.NotContains(t, kv.Value.Emit(), workflowValue, string(kv.Key)) + } + } +} + // Arguments are content and ride the gated message record; tool spans are // emitted with the gate off, so nothing of a call's arguments may reach one. func TestToolSpanTracker_ArgumentsStayOffTheSpan(t *testing.T) { From e7c1e44e00ddf48212d9fa3d1eecc1733be390e5 Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Tue, 6 Oct 2026 15:50:47 -0400 Subject: [PATCH 13/40] fix(telemetry): judge keys under a secret-named member A credential used as a key under a secret-named member kept its place: {"credentials":{"":"user"}} was recorded unchanged. Judge such keys as the member's values are judged and mask a match as ***. Two keys masked alike take the existing collision path, so an ordinary object such as {"auth":{"username":...,"password":...}} now drops its call's arguments, charged and marked; the reference says so. Signed-off-by: Dharit Shah --- docs/guides/dev/tracing.md | 7 +++-- .../infrastructure/distributed-tracing.md | 6 ++-- internal/cli/content_collector.go | 8 +++++- internal/cli/content_collector_test.go | 28 +++++++++++++++++++ 4 files changed, 44 insertions(+), 5 deletions(-) diff --git a/docs/guides/dev/tracing.md b/docs/guides/dev/tracing.md index 1e9018e6cd..0d8262d261 100644 --- a/docs/guides/dev/tracing.md +++ b/docs/guides/dev/tracing.md @@ -220,7 +220,7 @@ arguments are dropped whole and the part marked, since a cut object is not JSON), with exact dropped-byte accounting across content, tool names, summaries, responses, part ids, and dropped arguments (counted as re-encoded JSON, or as redacted -text when they were not a JSON value; on a key collision the member the +text when they were not a JSON value; on a key collision each member a later key (in sorted order) replaced is not counted), then holds the marshaled string to `maxEncodedContentBytes` (255,000 — just under the one size the pilot backend is proven to accept) by trimming the oldest content again, measured on the encoding itself and @@ -253,7 +253,10 @@ stage alone: the normalizer is not idempotent over escape sequences, and a second pass over the pair could strip the value or the key's keyword. A value in an array or a nested object under a secret-named member is judged as that member's own value would be — the pattern's eight-character floor included, -so a short count stays; booleans and nulls are left as they are. Arguments that are not +so a short count stays; booleans and nulls are left as they are. Keys under +such a member are judged the same way, since a credential can be the key: a +key of eight characters or more is masked `***`, field names included, and +two masked alike collide (below). Arguments that are not one JSON value cannot be walked that way: they are scanned as text — with the misses above — so their findings count, then dropped and charged. So are arguments in which two keys of one object redact to the same string, diff --git a/docs/guides/infrastructure/distributed-tracing.md b/docs/guides/infrastructure/distributed-tracing.md index 6ebb298700..fe6f3f6aff 100644 --- a/docs/guides/infrastructure/distributed-tracing.md +++ b/docs/guides/infrastructure/distributed-tracing.md @@ -154,7 +154,9 @@ oldest first. Truncation is marked via `fullsend.content.truncated` on the span and `fullsend.truncated` on each cut part. A tool result over its bound keeps its tail; arguments over theirs, not a complete JSON value, or with two -keys of one object that redact to the same string, are dropped +keys of one object that redact to the same string (two keys of eight +characters or more under a secret-named member, such as `username` and +`password` under `auth`, are both masked and so collide), are dropped whole — a cut object is not JSON — and the marked `tool_call` part keeps its `id`, name and summary — not the summary when redaction found a secret in the arguments. Arguments also go, charged and marked, when @@ -313,7 +315,7 @@ The `agent` span's provider identity reflects only the parent run's serving endp | `gen_ai.input.messages` | `agent` | Level 3 only, on a retry iteration that carries validation feedback: the prompt the runner composed, as a JSON string (see Content capture) | | `gen_ai.output.messages` | `agent` | Level 3 only: the iteration's conversation content as a JSON string (see Content capture) | | `fullsend.content.truncated` | `agent` | Level 3 only: present (`true`) when the size budget cut or dropped content, a tool call lost its arguments, or a kept tool result is a parser-side fragment or an oversized line's empty stand-in (`fullsend.truncated` on the part; no byte count for the last two) | -| `fullsend.content.dropped_bytes` | `agent` | Level 3 only: exact part bytes removed by the size budget — content, ids, dropped arguments (counted as re-encoded JSON — on a key collision, without the member the later key (in sorted key order, not stream order) replaced — or as redacted text when they were not a JSON value), and the fixed footprint of an errored-empty or oversized stand-in part — otherwise in raw bytes, whichever bound made the cut; the bytes of a skipped oversized line were never decoded and are not counted | +| `fullsend.content.dropped_bytes` | `agent` | Level 3 only: exact part bytes removed by the size budget — content, ids, dropped arguments (counted as re-encoded JSON — on a key collision, without each member a later key (in sorted key order, not stream order) replaced — or as redacted text when they were not a JSON value), and the fixed footprint of an errored-empty or oversized stand-in part — otherwise in raw bytes, whichever bound made the cut; the bytes of a skipped oversized line were never decoded and are not counted | | `fullsend.content.redactions` | `agent` | Level 3 only: number of security findings raised while redacting content at assembly, the input message included (and findings from parts the size budget later dropped) | | `fullsend.tool.unmatched` | `execute_tool` | Present (`true`) when a result arrived for a call the stream never reported; the span has near-zero duration | | `fullsend.tool.result_oversized` | `execute_tool` | Present (`true`) when the result's stream line exceeded the parser's 1 MiB bound: the call was answered but nothing of the result was decoded, so the span has no status and no `error.type` | diff --git a/internal/cli/content_collector.go b/internal/cli/content_collector.go index 28e53ef151..1691335c52 100644 --- a/internal/cli/content_collector.go +++ b/internal/cli/content_collector.go @@ -809,7 +809,7 @@ func (c *contentCollector) toolArguments(args string) (json.RawMessage, bool) { // beside it and masked whole on a match (secretNamed); a direct string // member is also scanned beside its own key. A value in // an array or a nested object under a secret-named member is judged as -// that member's own value would be. +// that member's own value would be, and so is a key there. // Keys are walked in sorted order, so neither the findings nor a dropped // value's charge follow map order. *collided reports two keys of one // object that redact to the same string: keeping either member would @@ -850,6 +850,12 @@ func (c *contentCollector) redactValue(v any, under string, collided *bool) any // the pair even when the key names no secret. e = "***" } + if under != "" && c.secretNamed(under, rk) { + // A key under a secret-named member is judged like its + // values: a credential can be the key. Two masked alike + // take the collision path below. + rk = "***" + } if _, dup := out[rk]; dup { *collided = true } diff --git a/internal/cli/content_collector_test.go b/internal/cli/content_collector_test.go index d3cbee1b67..9c7128b2a6 100644 --- a/internal/cli/content_collector_test.go +++ b/internal/cli/content_collector_test.go @@ -173,6 +173,11 @@ func TestContentCollector_SecretNamedMembersReachTheirNestedValues(t *testing.T) // A quote in the key lets the pattern's single-quote form match // across the pair: the member's own key is still checked. "quote in a plain key": {`{"it's":"password': 'hunter2hunter2'"}`, `{"it's":"***"}`}, + // Keys are judged too: a credential can be the key. + "key under a secret-named member": {`{"credentials":{"` + opaque + `":"user"}}`, `{"credentials":{"***":"user"}}`}, + "key under a secret-named array": {`{"api_keys":[{"` + opaque + `":"prod"}]}`, `{"api_keys":[{"***":"prod"}]}`}, + "short keys stay": {`{"auth":{"user":"x"}}`, `{"auth":{"user":"x"}}`}, + "keys with no secret-named ancestor": {`{"edits":{"` + opaque + `":"x"}}`, `{"edits":{"` + opaque + `":"x"}}`}, } { t.Run(name, func(t *testing.T) { c := newContentCollector(4096) @@ -416,6 +421,29 @@ func TestContentCollector_ReplacesProviderOnlyValuesFromTheProcessEnv(t *testing assert.Equal(t, 2, runnerEnvFindings(res), "counted like a runner env value") } +func TestContentCollector_KeysMaskedAlikeDropTheArguments(t *testing.T) { + // Masked keys go the way of keys that redact alike: keeping either + // member would show a call the agent did not make. Field names of eight + // characters or more under a secret-named member are masked too, so an + // ordinary credentials object can cost the arguments. + for name, args := range map[string]string{ + "two opaque keys": `{"tokens":{"opaque-credential-123":"a","opaque-credential-456":"b"}}`, + "username and password": `{"auth":{"username":"alice","password":"hunter2hunter2"}}`, + } { + t.Run(name, func(t *testing.T) { + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Summary: "s", Arguments: args}) + + res := c.Result("stop") + assert.NotContains(t, res.OutputMessages, "arguments") + assert.NotContains(t, res.OutputMessages, "opaque-credential") + assert.NotContains(t, res.OutputMessages, "hunter2") + assert.True(t, res.Truncated) + assert.Positive(t, res.DroppedBytes) + }) + } +} + func TestContentCollector_SecretNamedChecksSeeNormalizedText(t *testing.T) { // The key and the value are judged as the normalizer writes them: a // fullwidth key names a secret once folded, and four ligatures are From 558f06098cdaedb84a9e3b20504398ff1b1da242 Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Tue, 6 Oct 2026 16:44:12 -0400 Subject: [PATCH 14/40] fix(cli): mask a string holding a document with a secret-named member A string argument holding a JSON document was scanned only as text, so the member-name pattern, which wants the value right after the key, missed a secret-named member nested deeper: {"content":"{\"credentials\":{\"value\":\"...\"}}"} kept the value. The string is still scanned whole, since a secret can span a document's strings. The document it holds - as scanned, or as written when folding or a mask broke it - is then walked as the arguments are; when that walk would mask a value or key under a secret-named member, the string is masked whole, with one finding. The walk's output and other findings are discarded: nothing is re-encoded, and a mask can match again. Documents held more than four deep are not walked. Signed-off-by: Dharit Shah --- docs/guides/dev/tracing.md | 16 ++- .../infrastructure/distributed-tracing.md | 5 + internal/cli/content_collector.go | 72 +++++++++++- internal/cli/content_collector_test.go | 110 ++++++++++++++++++ 4 files changed, 198 insertions(+), 5 deletions(-) diff --git a/docs/guides/dev/tracing.md b/docs/guides/dev/tracing.md index 0d8262d261..50c7b6ecaa 100644 --- a/docs/guides/dev/tracing.md +++ b/docs/guides/dev/tracing.md @@ -243,7 +243,21 @@ behind escaped quotes, and JSON nested in a string; Unicode folding can also turn a fullwidth quotation mark into one that closes the string. `toolArguments` decodes the value, redacts each string and object key on its own (a number as its digits; one that redacts becomes the redacted -string), and encodes the result again. Each string and number is also +string), and encodes the result again. A string value that holds one +JSON object or array — as scanned, or as written when the scan left none, +since folding or a mask can break one — is walked the same way as well +(`heldSecret`): the string is scanned whole, since a secret can span a +document's strings, but a secret-named member nested in it is out of the +pattern's sight. The walk's output and its other findings are discarded, +since a mask can match again; when it would mask a value or key under a +secret-named member, the string is masked `***` whole, with that one +finding. Not walked: documents held more than `maxHeldDepth` (4) deep — +each level is scanned once more, so a document the scan breaks costs up to +five scans of its text — a document in an object key, a member a later +duplicate key replaces, a document whose structure folding or a mask +changes while the text holding it still parses (a document held in it +then breaks too), and structure in any other text (several documents, +YAML, a document inside code). Each string and number is also scanned once more, already redacted, beside the nearest enclosing key the member-name pattern (`json_field`) names, and masked whole when that pattern matches the pair — a pattern keyed on a member diff --git a/docs/guides/infrastructure/distributed-tracing.md b/docs/guides/infrastructure/distributed-tracing.md index c7d03e766b..597ac45575 100644 --- a/docs/guides/infrastructure/distributed-tracing.md +++ b/docs/guides/infrastructure/distributed-tracing.md @@ -122,6 +122,11 @@ codex parsers emit none of the three yet `arguments` is the call's input decoded, redacted string by string (a number that redacts becomes the redacted string), and encoded again, so key order, spacing and escapes are not the stream's. +A string value in it that holds one JSON object or array — a JSON file +passed to `Write`, say — is masked `***` whole when that document nests a +value or key under a secret-named member that redaction would otherwise +leave; a string directly under such a member is masked in place, as +anywhere else. On a retry iteration under `validation_loop.feedback_mode: append`, the prompt the runner composed — its fixed framing around the previous diff --git a/internal/cli/content_collector.go b/internal/cli/content_collector.go index 1691335c52..91d14be42e 100644 --- a/internal/cli/content_collector.go +++ b/internal/cli/content_collector.go @@ -96,6 +96,11 @@ const maxToolResultBytes = 8 * 1024 // measured; a Write call carries the file body as an argument. const maxToolArgumentsBytes = 8 * 1024 +// maxHeldDepth bounds how deep heldSecret walks documents held in +// strings: each level is scanned once more, and with \u escapes a +// document can nest about the square root of its length deep. +const maxHeldDepth = 4 + // newContentCollectorIfEnabled returns a live collector when the Level 3 // gate is on and nil otherwise — nil is the off state and is inert at // every call site, so the gate needs no second check. @@ -383,6 +388,9 @@ type contentCollector struct { // Merged into the contentResult at Result so they count exactly like // assembly-time ones. findings []security.Finding + // held counts the documents held in strings that heldSecret is + // walking; see maxHeldDepth. + held int } func newContentCollector(maxBytes int) *contentCollector { @@ -783,10 +791,8 @@ func (c *contentCollector) toolArguments(args string) (json.RawMessage, bool) { c.evicted += len(c.redact(args, &c.findings)) return nil, true } - dec := json.NewDecoder(strings.NewReader(args)) - dec.UseNumber() // float64 would rewrite integers past 2^53 - var v any - if err := dec.Decode(&v); err != nil || v == nil { + v, err := decodeJSON(args) + if err != nil || v == nil { return nil, false } collided := false @@ -801,6 +807,60 @@ func (c *contentCollector) toolArguments(args string) (json.RawMessage, bool) { return out, false } +// decodeJSON decodes one JSON value; float64 would rewrite integers past +// 2^53, so numbers keep their digits. +func decodeJSON(s string) (any, error) { + dec := json.NewDecoder(strings.NewReader(s)) + dec.UseNumber() + var v any + err := dec.Decode(&v) + return v, err +} + +// jsonDocument decodes s when it is one JSON object or array, after a +// byte order mark: one with a BOM must parse as written, should folding +// break it once the normalizer has stripped the BOM. +func jsonDocument(s string) (any, bool) { + s = strings.TrimSpace(strings.TrimPrefix(s, "\uFEFF")) + if s == "" || (s[0] != '{' && s[0] != '[') || !json.Valid([]byte(s)) { + return nil, false + } + v, err := decodeJSON(s) + return v, err == nil +} + +// heldSecret walks the JSON object or array a string holds — as scanned, +// or as written when the scan left none (folding or a mask can break +// one) — as the arguments are walked, and returns the member-name +// finding of a value or key there under a secret-named member: the +// pattern wants the value right after the key, so the text scan misses +// one nested deeper. The walk's output and its other findings are +// discarded — the string was scanned already, and a mask can match +// again — so a string holding such a member is masked whole. +func (c *contentCollector) heldSecret(scanned, written string) (security.Finding, bool) { + if c.held == maxHeldDepth { + return security.Finding{}, false + } + doc, ok := jsonDocument(scanned) + if !ok { + doc, ok = jsonDocument(written) + } + if !ok { + return security.Finding{}, false + } + mark := len(c.findings) + c.held++ + c.redactValue(doc, "", new(bool)) + c.held-- + var f security.Finding + i := slices.IndexFunc(c.findings[mark:], func(f security.Finding) bool { return f.Name == "json_field" }) + if i >= 0 { + f = c.findings[mark+i] + } + c.findings = c.findings[:mark] + return f, i >= 0 +} + // redactValue redacts every string, number and object key under v; a // number that redacts becomes the redacted string. A pattern // keyed on a member name ("password": "...") cannot see the pair that @@ -818,6 +878,10 @@ func (c *contentCollector) redactValue(v any, under string, collided *bool) any switch t := v.(type) { case string: s := c.redact(t, &c.findings) + if f, ok := c.heldSecret(s, t); ok { + c.findings = append(c.findings, f) + return "***" + } if under != "" && c.secretNamed(under, s) { return "***" } diff --git a/internal/cli/content_collector_test.go b/internal/cli/content_collector_test.go index 9c7128b2a6..de29f1ad8a 100644 --- a/internal/cli/content_collector_test.go +++ b/internal/cli/content_collector_test.go @@ -194,6 +194,116 @@ func TestContentCollector_SecretNamedMembersReachTheirNestedValues(t *testing.T) } } +func TestContentCollector_JSONInAStringIsMaskedWholeForASecretNamedMember(t *testing.T) { + // The member-name pattern wants the value right after the key, so a + // secret-named member in a document held in a string is seen only by + // walking that document; the string is then masked whole. + const opaque = "opaque-credential-123" + cred := `"credentials":{"value":"` + opaque + `"}` + quote := func(s string) string { b, _ := json.Marshal(s); return string(b) } + for name, tc := range map[string]struct { + content, want string + findings int + }{ + "object under credentials": {`{` + cred + `}`, "***", 1}, + "array under password": {`{"password":["` + opaque + `"]}`, "***", 1}, + "document that is an array": {`[{"token":{"v":"` + opaque + `"}}]`, "***", 1}, + "key under credentials": {`{"credentials":{"` + opaque + `":"user"}}`, "***", 1}, + "document in a document": {`{"inner":` + quote(`{"token":["`+opaque+`"]}`) + `}`, "***", 1}, + // Folding can make the document, or break it: the scan's text is + // walked, or else the text as written. + "folded into a document": {"\uFF5B" + cred + "}", "***", 2}, + "folding breaks a document": {"{\"note\":\"\uFF02\"," + cred + "}", "***", 2}, + "a BOM and folding that breaks a document": {"\uFEFF{\"note\":\"\uFF02\"," + cred + "}", "***", 3}, + // So can a mask: this one ends on the escape of a quote. + "a mask breaks a document": {`{"token":"abcdefgh\\\"x",` + cred + `}`, "***", 2}, + // Otherwise the string is kept as the scan left it. + "plain document": {`{ "b": "` + opaque + `", "a": 1 }`, `{ "b": "` + opaque + `", "a": 1 }`, 0}, + "a pair stays masked in place": {`{"password":"hunter2hunter2","n":1}`, `{"password":"hunt...","n":1}`, 1}, + } { + t.Run(name, func(t *testing.T) { + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{Name: "Write", Summary: "kept", Arguments: `{"content":` + quote(tc.content) + `}`}) + + res := c.Result("stop") + assert.Contains(t, res.OutputMessages, `"arguments":{"content":`+quote(tc.want)+`}`) + assert.Len(t, res.Findings, tc.findings) + assert.Equal(t, tc.findings > 0 && tc.want != tc.content, !strings.Contains(res.OutputMessages, `"summary"`), "a secret drops the summary") + }) + } +} + +func TestContentCollector_JSONInAStringIsAlsoScannedWhole(t *testing.T) { + // A secret can span the document's strings: a notebook keeps each line + // of a cell in its own string, and a runner env value can hold quotes. + begin, end := "-----BEGIN RSA PRIVATE "+"KEY-----", "-----END RSA PRIVATE "+"KEY-----" + body := strings.Repeat("Q", 40) + notebook, _ := json.Marshal(map[string]any{"cells": []any{map[string]any{"source": []string{begin + "\n", body + "\n", end + "\n"}}}}) + for name, tc := range map[string]struct { + env map[string]string + content, gone string + }{ + "key over a cell's lines": {nil, string(notebook), body}, + "runner env value with structure": {map[string]string{"DB_PASSWORD": `hunter22","port`}, `{"pw":"hunter22","port":5432}`, "hunter22"}, + } { + t.Run(name, func(t *testing.T) { + t.Setenv(telemetry.ContentCaptureEnvVar, "true") + c := newContentCollectorIfEnabled(tc.env) + args, _ := json.Marshal(map[string]string{"content": tc.content}) + c.Handle(agentruntime.ToolUseEvent{Name: "Write", Arguments: string(args)}) + + res := c.Result("stop") + assert.NotContains(t, res.OutputMessages, tc.gone) + assert.Len(t, res.Findings, 1) + }) + } +} + +func TestContentCollector_JSONInAStringIsWalkedWithoutCountingAgain(t *testing.T) { + // A masked connection string matches its own pattern again; the walk's + // findings are not the string's. + args, _ := json.Marshal(map[string]string{"content": `{"db":"postgres://u:hunter2hunter2@db/x"}`}) + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{Name: "Write", Arguments: string(args)}) + + res := c.Result("stop") + want, _ := json.Marshal(map[string]string{"content": `{"db":"postgres://u:hunt...@db/x"}`}) + assert.Contains(t, res.OutputMessages, `"arguments":`+string(want)) + assert.Len(t, res.Findings, 1) +} + +func TestContentCollector_JSONHeldDeeperThanTheBoundIsScannedAsText(t *testing.T) { + // Each level is scanned once more: past maxHeldDepth documents are + // not walked, and a member nested in one goes unseen. + held := func(depth int) string { + doc := `{"credentials":{"value":"opaque-credential-123"}}` + for range depth - 1 { + b, _ := json.Marshal(map[string]string{"d": doc}) + doc = string(b) + } + b, _ := json.Marshal(map[string]string{"content": doc}) + return string(b) + } + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{Name: "Write", Arguments: held(maxHeldDepth)}) + assert.Contains(t, c.Result("stop").OutputMessages, `"arguments":{"content":"***"}`) + + c = newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{Name: "Write", Arguments: held(maxHeldDepth + 1)}) + assert.Contains(t, c.Result("stop").OutputMessages, "opaque-credential-123") +} + +func TestContentCollector_JSONInASecretNamedStringIsMaskedWholeOnce(t *testing.T) { + // The member-name pattern takes any string of eight bytes or more under + // the member, a document's encoding too: one finding, not one per value. + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{Name: "Write", Arguments: `{"credentials":"{\"value\":\"opaque-credential-123\"}"}`}) + + res := c.Result("stop") + assert.Contains(t, res.OutputMessages, `"arguments":{"credentials":"***"}`) + assert.Len(t, res.Findings, 1) +} + func TestContentCollector_AKeyFindingLeavesTheValueAndCountsOnce(t *testing.T) { for name, tc := range map[string]struct{ args, want string }{ "token": {`{"ghp_` + strings.Repeat("r", 36) + `":"hello"}`, `{"ghp_...":"hello"}`}, From 60fa4dbd68255e648bef693c317e3841e37fc886 Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Tue, 6 Oct 2026 20:43:04 -0400 Subject: [PATCH 15/40] fix(cli): judge a held document as exported, and fail closed heldSecret kept only member-name findings of its walk, found no secret at the depth bound, decoded past a repeated name, and never judged an object key. It now judges, decoded, the document the exported string holds: a walk finding other than the normalizer's masks the string whole (an assignment that opens a string, a token or runner env value written with an escape), as does the normalizer's removal of an ST-terminated escape sequence or of tag characters. A document that cannot be judged is masked whole too: one held past maxHeldDepth, one naming a member twice (decoding keeps the last; RFC 7493 forbids it), and text the scan changed that is shaped like a document but no longer parses. Keys go through the same check; a key masked whole still names a secret as scanned. Each whole mask records one held_document finding. Signed-off-by: Dharit Shah --- docs/guides/dev/tracing.md | 46 ++++-- .../infrastructure/distributed-tracing.md | 19 ++- internal/cli/content_collector.go | 142 +++++++++++------ internal/cli/content_collector_test.go | 146 +++++++++++++----- 4 files changed, 245 insertions(+), 108 deletions(-) diff --git a/docs/guides/dev/tracing.md b/docs/guides/dev/tracing.md index 50c7b6ecaa..f13fb083ee 100644 --- a/docs/guides/dev/tracing.md +++ b/docs/guides/dev/tracing.md @@ -243,21 +243,37 @@ behind escaped quotes, and JSON nested in a string; Unicode folding can also turn a fullwidth quotation mark into one that closes the string. `toolArguments` decodes the value, redacts each string and object key on its own (a number as its digits; one that redacts becomes the redacted -string), and encodes the result again. A string value that holds one -JSON object or array — as scanned, or as written when the scan left none, -since folding or a mask can break one — is walked the same way as well -(`heldSecret`): the string is scanned whole, since a secret can span a -document's strings, but a secret-named member nested in it is out of the -pattern's sight. The walk's output and its other findings are discarded, -since a mask can match again; when it would mask a value or key under a -secret-named member, the string is masked `***` whole, with that one -finding. Not walked: documents held more than `maxHeldDepth` (4) deep — -each level is scanned once more, so a document the scan breaks costs up to -five scans of its text — a document in an object key, a member a later -duplicate key replaces, a document whose structure folding or a mask -changes while the text holding it still parses (a document held in it -then breaks too), and structure in any other text (several documents, -YAML, a document inside code). Each string and number is also +string), and encodes the result again. A string value or key that holds +one JSON object or array once scanned — what is exported — is judged +decoded as well (`heldSecret`). The scan reads the string whole, since a +secret can span a document's strings, but as text: it misses a +secret-named member nested deeper than the value right after the key, and +a secret only decoding spells out (an assignment that opens a string, a +token or runner environment value written with an escape). So the +document is walked the same way, its output and findings discarded; when +the walk raises a finding — any but the normalizer's, save its removal of +an ST-terminated escape sequence (OSC and its kin) or of tag characters, +which can carry text no pattern sees; a secret the scan masked in place +no longer shows — the string is +masked `***` whole, with one `held_document` finding. So is a document +that cannot be judged: held more than `maxHeldDepth` (4) deep, each level +being scanned once more; naming a member twice, since decoding keeps only +the last (RFC 7493 forbids it); or broken by the scan — text the scan +changed, folding or masking it, that begins and ends like a document but +no longer parses. That masks whole text with no secret left in it too — +a document with a repeated name, one a pattern misreads once decoded (a +notebook line `key = …`), text shaped like a document whose secret the +scan masked (a Python dict) — and a mask the walk +matches again (a connection string's password of ten bytes or more, the +runner environment marker in an assignment, a mask of eight bytes or more +under a secret-named member) masks its string whole and counts its secret +twice. Not judged: a document whose structure folding or a mask changes +while the text holding it still parses (one held in it then breaks too); +text that never parsed — JSON with comments or trailing commas, several +documents, a raw control character — unless the scan changed it and it +is shaped like a document, when it is masked whole as above; and +structure in any other text (YAML, a document inside code). Each string +and number is also scanned once more, already redacted, beside the nearest enclosing key the member-name pattern (`json_field`) names, and masked whole when that pattern matches the pair — a pattern keyed on a member diff --git a/docs/guides/infrastructure/distributed-tracing.md b/docs/guides/infrastructure/distributed-tracing.md index 597ac45575..eea341669e 100644 --- a/docs/guides/infrastructure/distributed-tracing.md +++ b/docs/guides/infrastructure/distributed-tracing.md @@ -91,9 +91,9 @@ then mask the marker as well and count again. Not covered: a value split or spelled that way which a pattern also recognises — in one of those contexts, in a connection string, or by its own prefix — is masked by that pattern, and shows what the pattern's mask shows. A tool call loses -its `summary` when redaction found a secret in its arguments: the parser -cut the summary out of them before anything scanned it, so the secret -could be there as a beginning. Where a runtime reports a summary and no +its `summary` when redaction found a secret in its arguments, or masked a +document there it could not judge: the parser cut the summary out of them +before anything scanned it, so the secret could be there as a beginning. Where a runtime reports a summary and no arguments (pi, codex, OpenCode), its parser has already run the patterns over the summary and, for pi and codex, cut it: a value that straddles a cut stays in part, and one a pattern recognised stays as that pattern's @@ -122,10 +122,13 @@ codex parsers emit none of the three yet `arguments` is the call's input decoded, redacted string by string (a number that redacts becomes the redacted string), and encoded again, so key order, spacing and escapes are not the stream's. -A string value in it that holds one JSON object or array — a JSON file -passed to `Write`, say — is masked `***` whole when that document nests a -value or key under a secret-named member that redaction would otherwise -leave; a string directly under such a member is masked in place, as +A string value or key in it that holds one JSON object or array — a JSON +file passed to `Write`, say — is masked `***` whole when that document, +decoded, still shows a secret: a value or key nested under a secret-named +member, or one only decoding spells out. So is one that cannot be judged: +held more than four deep, naming a member twice, or broken by redaction +— the [developer guide](../dev/tracing.md) lists what is not judged. +A string directly under a secret-named member is masked in place, as anywhere else. On a retry iteration under `validation_loop.feedback_mode: append`, the @@ -164,7 +167,7 @@ characters or more under a secret-named member, such as `username` and `password` under `auth`, are both masked and so collide), are dropped whole — a cut object is not JSON — and the marked `tool_call` part keeps its `id`, name and summary — not the summary when redaction found a -secret in the arguments. Arguments also go, charged and marked, when +secret in the arguments, or masked a document there it could not judge. Arguments also go, charged and marked, when the call's name redacts to nothing; that part survives only if it has a summary. So `fullsend.truncated` means different things by part type: on a tool result, part of the response is kept; on a tool call, none of the diff --git a/internal/cli/content_collector.go b/internal/cli/content_collector.go index 91d14be42e..5a0178ede9 100644 --- a/internal/cli/content_collector.go +++ b/internal/cli/content_collector.go @@ -2,6 +2,7 @@ package cli import ( "encoding/json" + "io" "maps" "slices" "sort" @@ -83,8 +84,8 @@ const maxToolResultBytes = 8 * 1024 // maxToolArgumentsBytes bounds one tool call's arguments, measured on // their redacted encoding. Arguments over it are dropped whole, not cut: // a cut object is not JSON, and the part keeps its id, name and summary -// (not the summary when redaction found a secret in the arguments; see -// Handle). +// (not the summary when redaction found a secret in the arguments, or +// masked a document there it could not judge; see Handle). // Like maxToolResultBytes it exists because the total above is small; // what blocks raising it is the same unproven backend ceiling named on // maxContentBytes. @@ -98,7 +99,8 @@ const maxToolArgumentsBytes = 8 * 1024 // maxHeldDepth bounds how deep heldSecret walks documents held in // strings: each level is scanned once more, and with \u escapes a -// document can nest about the square root of its length deep. +// document can nest about the square root of its length deep. A string +// holding one deeper is masked whole. const maxHeldDepth = 4 // newContentCollectorIfEnabled returns a live collector when the Level 3 @@ -420,9 +422,10 @@ func (c *contentCollector) Handle(evt agentruntime.AgentEvent) { p.Arguments, p.Truncated = c.toolArguments(e.Arguments) if slices.ContainsFunc(c.findings[scanned:], func(f security.Finding) bool { return f.Scanner != "unicode_normalizer" }) { // The parser cut the summary out of these arguments - // before anything scanned it, so a secret found in them - // can be in the summary as a beginning that neither the - // literal pass nor a pattern matches. + // before anything scanned it, so a secret found in them, + // or in a document masked unjudged, can be in the summary + // as a beginning that neither the literal pass nor a + // pattern matches. p.Summary = "" } } @@ -817,48 +820,99 @@ func decodeJSON(s string) (any, error) { return v, err } -// jsonDocument decodes s when it is one JSON object or array, after a -// byte order mark: one with a BOM must parse as written, should folding -// break it once the normalizer has stripped the BOM. -func jsonDocument(s string) (any, bool) { +// jsonDocument returns s trimmed when it is one JSON object or array. +func jsonDocument(s string) (string, bool) { + s = strings.TrimSpace(s) + return s, s != "" && (s[0] == '{' || s[0] == '[') && json.Valid([]byte(s)) +} + +// documentLike reports text that begins and ends as a JSON object or +// array does, after a byte order mark. +func documentLike(s string) bool { s = strings.TrimSpace(strings.TrimPrefix(s, "\uFEFF")) - if s == "" || (s[0] != '{' && s[0] != '[') || !json.Valid([]byte(s)) { - return nil, false + return len(s) >= 2 && (s[0] == '{' && s[len(s)-1] == '}' || s[0] == '[' && s[len(s)-1] == ']') +} + +// duplicateName reports an object in the JSON document s that names a +// member twice, the names compared decoded, as RFC 7493 does. +func duplicateName(s string) bool { + dec := json.NewDecoder(strings.NewReader(s)) + dec.UseNumber() // a number past float64 is valid JSON too + var names []map[string]bool // one set for each open object + for { + tok, err := dec.Token() + if err != nil { + return err != io.EOF // s is valid; fail closed all the same + } + switch tok { + case json.Delim('{'): + names = append(names, map[string]bool{}) + case json.Delim('}'): + names = names[:len(names)-1] + default: + // A string a colon follows names a member of the innermost + // object open. + name, ok := tok.(string) + if !ok || !strings.HasPrefix(strings.TrimLeft(s[dec.InputOffset():], " \t\r\n"), ":") { + continue + } + if names[len(names)-1][name] { + return true + } + names[len(names)-1][name] = true + } } - v, err := decodeJSON(s) - return v, err == nil } -// heldSecret walks the JSON object or array a string holds — as scanned, -// or as written when the scan left none (folding or a mask can break -// one) — as the arguments are walked, and returns the member-name -// finding of a value or key there under a secret-named member: the -// pattern wants the value right after the key, so the text scan misses -// one nested deeper. The walk's output and its other findings are -// discarded — the string was scanned already, and a mask can match -// again — so a string holding such a member is masked whole. -func (c *contentCollector) heldSecret(scanned, written string) (security.Finding, bool) { - if c.held == maxHeldDepth { - return security.Finding{}, false +// maskHeld masks a string or an object key of the arguments whole, with +// one held_document finding, when heldSecret finds against the document +// its scanned text s holds; t is the text as written. +func (c *contentCollector) maskHeld(s, t string) string { + if reason, ok := c.heldSecret(s, t); ok { + c.findings = append(c.findings, security.Finding{Scanner: "held_document", Name: reason, Severity: "high", Detail: "string holding a JSON document masked whole", Position: -1}) + return "***" } + return s +} + +// heldSecret judges, decoded, the JSON object or array a string holds +// once scanned — what is exported. The scan reads the document's text, +// so it misses a secret-named member nested deeper than the value right +// after the key, and a secret that only decoding spells out: an +// assignment that opens a string, a token or runner env value written +// with an escape. The document is walked as the arguments are, its +// output and findings discarded; a revealing finding makes the reason +// "secret". A document that cannot be judged makes it "uninspected": +// one held deeper than maxHeldDepth, one naming a member twice — +// decoding keeps the last — and one the scan broke: text the scan +// changed, folding or masking it, that begins and ends like a document +// but no longer parses. A level above records that mask as "secret": it +// is a finding of that level's walk. +func (c *contentCollector) heldSecret(scanned, written string) (string, bool) { doc, ok := jsonDocument(scanned) if !ok { - doc, ok = jsonDocument(written) + return "uninspected", scanned != written && (documentLike(scanned) || documentLike(written)) } - if !ok { - return security.Finding{}, false + if c.held == maxHeldDepth || duplicateName(doc) { + return "uninspected", true } + v, _ := decodeJSON(doc) // a valid document decodes mark := len(c.findings) c.held++ - c.redactValue(doc, "", new(bool)) + c.redactValue(v, "", new(bool)) c.held-- - var f security.Finding - i := slices.IndexFunc(c.findings[mark:], func(f security.Finding) bool { return f.Name == "json_field" }) - if i >= 0 { - f = c.findings[mark+i] - } + found := slices.ContainsFunc(c.findings[mark:], revealing) c.findings = c.findings[:mark] - return f, i >= 0 + return "secret", found +} + +// revealing reports a finding of heldSecret's walk that shows a secret in +// the decoded document: any but the normalizer's, and its removal of an +// ST-terminated escape sequence (OSC and its kin) or of tag characters, +// which can carry text the patterns never see. A secret the scan masked in place is gone from the +// document by then, but for a mask that matches again. +func revealing(f security.Finding) bool { + return f.Scanner != "unicode_normalizer" || f.Name == "osc_escape" || f.Name == "tag_char" } // redactValue redacts every string, number and object key under v; a @@ -877,11 +931,7 @@ func (c *contentCollector) heldSecret(scanned, written string) (security.Finding func (c *contentCollector) redactValue(v any, under string, collided *bool) any { switch t := v.(type) { case string: - s := c.redact(t, &c.findings) - if f, ok := c.heldSecret(s, t); ok { - c.findings = append(c.findings, f) - return "***" - } + s := c.maskHeld(c.redact(t, &c.findings), t) if under != "" && c.secretNamed(under, s) { return "***" } @@ -903,18 +953,20 @@ func (c *contentCollector) redactValue(v any, under string, collided *bool) any case map[string]any: out := make(map[string]any, len(t)) for _, k := range slices.Sorted(maps.Keys(t)) { - rk := c.redact(k, &c.findings) + // The key as scanned names the secret, masked whole or not. + sk := c.redact(k, &c.findings) + rk := c.maskHeld(sk, k) member := under - if c.namesASecret(rk) { - member = rk + if c.namesASecret(sk) { + member = sk } e := c.redactValue(t[k], member, collided) - if s, ok := e.(string); ok && member != rk && c.secretNamed(rk, s) { + if s, ok := e.(string); ok && member != sk && c.secretNamed(sk, s) { // A single quote in the key can let the pattern match // the pair even when the key names no secret. e = "***" } - if under != "" && c.secretNamed(under, rk) { + if under != "" && c.secretNamed(under, sk) { // A key under a secret-named member is judged like its // values: a credential can be the key. Two masked alike // take the collision path below. diff --git a/internal/cli/content_collector_test.go b/internal/cli/content_collector_test.go index de29f1ad8a..7e2b1e5a72 100644 --- a/internal/cli/content_collector_test.go +++ b/internal/cli/content_collector_test.go @@ -194,12 +194,13 @@ func TestContentCollector_SecretNamedMembersReachTheirNestedValues(t *testing.T) } } -func TestContentCollector_JSONInAStringIsMaskedWholeForASecretNamedMember(t *testing.T) { - // The member-name pattern wants the value right after the key, so a - // secret-named member in a document held in a string is seen only by - // walking that document; the string is then masked whole. +func TestContentCollector_JSONInAStringIsMaskedWholeWhenItsDocumentHoldsASecret(t *testing.T) { + // The exported string is judged as decoded too: a secret-named member + // nested in the document it holds, or a secret only decoding spells + // out, masks the string whole. const opaque = "opaque-credential-123" cred := `"credentials":{"value":"` + opaque + `"}` + token := "ghp_" + strings.Repeat("r", 36) quote := func(s string) string { b, _ := json.Marshal(s); return string(b) } for name, tc := range map[string]struct { content, want string @@ -210,16 +211,40 @@ func TestContentCollector_JSONInAStringIsMaskedWholeForASecretNamedMember(t *tes "document that is an array": {`[{"token":{"v":"` + opaque + `"}}]`, "***", 1}, "key under credentials": {`{"credentials":{"` + opaque + `":"user"}}`, "***", 1}, "document in a document": {`{"inner":` + quote(`{"token":["`+opaque+`"]}`) + `}`, "***", 1}, - // Folding can make the document, or break it: the scan's text is - // walked, or else the text as written. - "folded into a document": {"\uFF5B" + cred + "}", "***", 2}, - "folding breaks a document": {"{\"note\":\"\uFF02\"," + cred + "}", "***", 2}, - "a BOM and folding that breaks a document": {"\uFEFF{\"note\":\"\uFF02\"," + cred + "}", "***", 3}, - // So can a mask: this one ends on the escape of a quote. - "a mask breaks a document": {`{"token":"abcdefgh\\\"x",` + cred + `}`, "***", 2}, + // The scan sees the document's text; these only its strings. + "an assignment that opens a string": {`{"command":"API_KEY=` + opaque + `"}`, "***", 1}, + "an assignment after an escaped break": {`{"c":"cd x\nAPI_KEY=` + opaque + ` run"}`, "***", 1}, + "a token spelled with an escape": {`{"t":"ghp_\u0072` + token[5:] + `"}`, "***", 1}, + "tag characters written with escapes": {`{"t":"\uDB40\uDC67"}`, "***", 1}, + "an escape sequence around a token": {`{"n":"see \u001b]ghp_\u0072` + token[5:] + `\u0007"}`, "***", 1}, + // Folding can make the document; a document the scan breaks, + // folding or masking it, cannot be judged. + "folded into a document": {"\uFF5B" + cred + "}", "***", 2}, + "folding breaks a document": {"{\"note\":\"\uFF02\"," + cred + "}", "***", 2}, + "a BOM and folding that breaks a document": {"\uFEFF{\"note\":\"\uFF02\"," + cred + "}", "***", 3}, + "a fold that makes and breaks a document": {"\uFF5B\"note\":\"\uFF02\"," + cred + "}", "***", 2}, + "an escape before a document folding breaks": {"\x1b[0m{\"note\":\"\uFF02\"," + cred + "}", "***", 3}, + "a mask breaks a document": {`{"token":"abcdefgh\\\"x",` + cred + `}`, "***", 2}, + // Decoding keeps the last of two members of one name; the string + // keeps both, so a repeated name cannot be judged either. + "a repeated name": {`{` + cred + `,"credentials":{}}`, "***", 1}, + "a name repeated with an escape": {`{` + cred + `,"\u0063redentials":{}}`, "***", 1}, + "a name repeated in an array item": {`[{"token":{"v":"` + opaque + `"},"token":{}}]`, "***", 1}, + "a repeated name and no secret": {`{"a":1,"a":2}`, "***", 1}, + // A mask the walk matches again, and a pattern's false positive in + // a decoded string, mask it whole as well. + "a mask that matches again": {`{"db":"postgres://u:hunter2hunter2@db/x"}`, "***", 2}, + "a notebook assignment": {`{"cells":[{"source":["key = jax.random.PRNGKey(0)\n"]}]}`, "***", 1}, // Otherwise the string is kept as the scan left it. - "plain document": {`{ "b": "` + opaque + `", "a": 1 }`, `{ "b": "` + opaque + `", "a": 1 }`, 0}, - "a pair stays masked in place": {`{"password":"hunter2hunter2","n":1}`, `{"password":"hunt...","n":1}`, 1}, + "plain document": {`{ "b": "` + opaque + `", "a": 1 }`, `{ "b": "` + opaque + `", "a": 1 }`, 0}, + "a pair stays masked in place": {`{"password":"hunter2hunter2","n":1}`, `{"password":"hunt...","n":1}`, 1}, + "a letter folded once decoded": {`{"v":"\uFF21"}`, `{"v":"\uFF21"}`, 0}, + "a control character the scan keeps": {"{\"a\":\"x\x01y\"}", "{\"a\":\"x\x01y\"}", 0}, + "a number past float64": {`{"maximum":1e999}`, `{"maximum":1e999}`, 0}, + "text like a document the scan changed": {`{'password': 'hunter2hunter2'}`, "***", 2}, + "a command that opens a brace": {`{ export API_KEY=` + opaque + `; } 2>/dev/null`, `{ export API_KEY=opaq...; } 2>/dev/null`, 1}, + "a value naming a sibling": {`{"name":"name","from":"to","to":"x"}`, `{"name":"name","from":"to","to":"x"}`, 0}, + "names repeated in other objects": {`[{"a":1},{"a":2},{"x":{"a":1},"y":{"a":2}},{"a":{"x":1},"x":2}]`, `[{"a":1},{"a":2},{"x":{"a":1},"y":{"a":2}},{"a":{"x":1},"x":2}]`, 0}, } { t.Run(name, func(t *testing.T) { c := newContentCollector(4096) @@ -242,9 +267,15 @@ func TestContentCollector_JSONInAStringIsAlsoScannedWhole(t *testing.T) { for name, tc := range map[string]struct { env map[string]string content, gone string + findings int }{ - "key over a cell's lines": {nil, string(notebook), body}, - "runner env value with structure": {map[string]string{"DB_PASSWORD": `hunter22","port`}, `{"pw":"hunter22","port":5432}`, "hunter22"}, + "key over a cell's lines": {nil, string(notebook), body, 1}, + // Its marker breaks the document, which cannot then be judged. + "runner env value with structure": {map[string]string{"DB_PASSWORD": `hunter22","port`}, `{"pw":"hunter22","port":5432}`, "hunter22", 2}, + "runner env value that ends the document": {map[string]string{"DB_PASSWORD": `hunter22"}`}, `{"pw":"hunter22"}`, "hunter22", 2}, + "and one behind a BOM": {map[string]string{"DB_PASSWORD": `hunter22"}`}, "\uFEFF{\"credentials\":{\"value\":\"opaque-credential-123\"},\"pw\":\"hunter22\"}", "opaque-credential-123", 3}, + // Decoded, the literal pass sees it. + "runner env value spelled with an escape": {map[string]string{"DB_PASSWORD": "hunter2hunter2"}, `{"pw":"\u0068unter2hunter2"}`, "unter2hunter2", 1}, } { t.Run(name, func(t *testing.T) { t.Setenv(telemetry.ContentCaptureEnvVar, "true") @@ -254,29 +285,15 @@ func TestContentCollector_JSONInAStringIsAlsoScannedWhole(t *testing.T) { res := c.Result("stop") assert.NotContains(t, res.OutputMessages, tc.gone) - assert.Len(t, res.Findings, 1) + assert.Len(t, res.Findings, tc.findings) }) } } -func TestContentCollector_JSONInAStringIsWalkedWithoutCountingAgain(t *testing.T) { - // A masked connection string matches its own pattern again; the walk's - // findings are not the string's. - args, _ := json.Marshal(map[string]string{"content": `{"db":"postgres://u:hunter2hunter2@db/x"}`}) - c := newContentCollector(4096) - c.Handle(agentruntime.ToolUseEvent{Name: "Write", Arguments: string(args)}) - - res := c.Result("stop") - want, _ := json.Marshal(map[string]string{"content": `{"db":"postgres://u:hunt...@db/x"}`}) - assert.Contains(t, res.OutputMessages, `"arguments":`+string(want)) - assert.Len(t, res.Findings, 1) -} - -func TestContentCollector_JSONHeldDeeperThanTheBoundIsScannedAsText(t *testing.T) { - // Each level is scanned once more: past maxHeldDepth documents are - // not walked, and a member nested in one goes unseen. - held := func(depth int) string { - doc := `{"credentials":{"value":"opaque-credential-123"}}` +func TestContentCollector_JSONHeldPastTheBoundIsMaskedWhole(t *testing.T) { + // Each level is scanned once more, so documents are walked down to + // maxHeldDepth; one held deeper cannot be judged. + held := func(doc string, depth int) string { for range depth - 1 { b, _ := json.Marshal(map[string]string{"d": doc}) doc = string(b) @@ -284,13 +301,62 @@ func TestContentCollector_JSONHeldDeeperThanTheBoundIsScannedAsText(t *testing.T b, _ := json.Marshal(map[string]string{"content": doc}) return string(b) } - c := newContentCollector(4096) - c.Handle(agentruntime.ToolUseEvent{Name: "Write", Arguments: held(maxHeldDepth)}) - assert.Contains(t, c.Result("stop").OutputMessages, `"arguments":{"content":"***"}`) + run := func(args string) contentResult { + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{Name: "Write", Arguments: args}) + return c.Result("stop") + } + const secret, plain = `{"credentials":{"value":"opaque-credential-123"}}`, `{"v":"plain"}` - c = newContentCollector(4096) - c.Handle(agentruntime.ToolUseEvent{Name: "Write", Arguments: held(maxHeldDepth + 1)}) - assert.Contains(t, c.Result("stop").OutputMessages, "opaque-credential-123") + res := run(held(secret, maxHeldDepth)) + assert.Contains(t, res.OutputMessages, `"arguments":{"content":"***"}`) + assert.Len(t, res.Findings, 1) + + res = run(held(plain, maxHeldDepth)) + assert.Contains(t, res.OutputMessages, `"arguments":`+held(plain, maxHeldDepth)) + assert.Empty(t, res.Findings) + + for _, doc := range []string{plain, secret} { + res = run(held(doc, maxHeldDepth+1)) + assert.Contains(t, res.OutputMessages, `"arguments":{"content":"***"}`) + assert.Len(t, res.Findings, 1) + } +} + +func TestContentCollector_AKeyHoldingADocumentIsJudgedLikeAValue(t *testing.T) { + quote := func(s string) string { b, _ := json.Marshal(s); return string(b) } + doc := quote(`{"credentials":{"value":"opaque-credential-123"}}`) + for name, tc := range map[string]struct { + args, want string + findings int + }{ + "key": {`{` + doc + `:"x"}`, `{"***":"x"}`, 1}, + "key of a nested object": {`{"files":{` + doc + `:"x"}}`, `{"files":{"***":"x"}}`, 1}, + "key in a held document": {`{"content":` + quote(`{`+doc+`:"x"}`) + `}`, `{"content":"***"}`, 1}, + // Masked whole, the key still names a secret as scanned: its + // value is judged against it. + "a value under it": {`{` + doc + `:"opaque-credential-456"}`, `{"***":"***"}`, 2}, + "an object under it": {`{` + doc + `:{"u":"opaque-credential-456"}}`, `{"***":{"u":"***"}}`, 2}, + "a value under a repeated name": {`{` + quote(`{"auth":1,"auth":2}`) + `:"opaque-credential-456"}`, `{"***":"***"}`, 2}, + } { + t.Run(name, func(t *testing.T) { + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{Name: "Write", Arguments: tc.args}) + + res := c.Result("stop") + assert.Contains(t, res.OutputMessages, `"arguments":`+tc.want) + assert.NotContains(t, res.OutputMessages, "opaque-credential-") + assert.Len(t, res.Findings, tc.findings) + }) + } + + // Two such keys are masked alike: the arguments go, as for any collision. + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{Name: "Write", Arguments: `{` + doc + `:"x",` + quote(`{"token":["opaque-credential-456"]}`) + `:"y"}`}) + res := c.Result("stop") + part := partAt(t, decodeOutputMessages(t, res.OutputMessages), 0) + assert.NotContains(t, part, "arguments") + assert.Equal(t, true, part["fullsend.truncated"]) } func TestContentCollector_JSONInASecretNamedStringIsMaskedWholeOnce(t *testing.T) { From 4d3c7bace534e7ebf798bd1441b8b15c820f54a6 Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Wed, 7 Oct 2026 07:28:51 -0400 Subject: [PATCH 16/40] fix(cli): count stripped escape sequences as revealing; trim invisibles A credential can hide where the normalizer deletes it unseen: digits as a CSI parameter under a secret-named member of a held document, or a token inside an OSC sequence whose terminator the parser's summary cut lost. The removal of any escape sequence now counts as revealing, in the held-document walk and for the summary, as OSC and tag characters already did in the walk; Handle uses the same predicate. documentLike trims white space, control and format characters (a BOM, a zero-width space) together, so one before a document the scan broke no longer hides it. Signed-off-by: Dharit Shah --- docs/guides/dev/tracing.md | 11 ++++--- .../infrastructure/distributed-tracing.md | 10 +++--- internal/cli/content_collector.go | 32 +++++++++++-------- internal/cli/content_collector_test.go | 30 ++++++++++++++--- 4 files changed, 56 insertions(+), 27 deletions(-) diff --git a/docs/guides/dev/tracing.md b/docs/guides/dev/tracing.md index f13fb083ee..2869668423 100644 --- a/docs/guides/dev/tracing.md +++ b/docs/guides/dev/tracing.md @@ -252,9 +252,9 @@ a secret only decoding spells out (an assignment that opens a string, a token or runner environment value written with an escape). So the document is walked the same way, its output and findings discarded; when the walk raises a finding — any but the normalizer's, save its removal of -an ST-terminated escape sequence (OSC and its kin) or of tag characters, -which can carry text no pattern sees; a secret the scan masked in place -no longer shows — the string is +an escape sequence or of tag characters, which can carry text no pattern +sees (digits as a CSI parameter, a token inside an OSC sequence); a +secret the scan masked in place no longer shows — the string is masked `***` whole, with one `held_document` finding. So is a document that cannot be judged: held more than `maxHeldDepth` (4) deep, each level being scanned once more; naming a member twice, since decoding keeps only @@ -262,8 +262,9 @@ the last (RFC 7493 forbids it); or broken by the scan — text the scan changed, folding or masking it, that begins and ends like a document but no longer parses. That masks whole text with no secret left in it too — a document with a repeated name, one a pattern misreads once decoded (a -notebook line `key = …`), text shaped like a document whose secret the -scan masked (a Python dict) — and a mask the walk +notebook line `key = …`), one holding an escaped terminal colour code, +text shaped like a document whose secret the scan masked (a Python +dict) — and a mask the walk matches again (a connection string's password of ten bytes or more, the runner environment marker in an assignment, a mask of eight bytes or more under a secret-named member) masks its string whole and counts its secret diff --git a/docs/guides/infrastructure/distributed-tracing.md b/docs/guides/infrastructure/distributed-tracing.md index eea341669e..c05558aab2 100644 --- a/docs/guides/infrastructure/distributed-tracing.md +++ b/docs/guides/infrastructure/distributed-tracing.md @@ -91,9 +91,10 @@ then mask the marker as well and count again. Not covered: a value split or spelled that way which a pattern also recognises — in one of those contexts, in a connection string, or by its own prefix — is masked by that pattern, and shows what the pattern's mask shows. A tool call loses -its `summary` when redaction found a secret in its arguments, or masked a -document there it could not judge: the parser cut the summary out of them -before anything scanned it, so the secret could be there as a beginning. Where a runtime reports a summary and no +its `summary` when redaction found a secret in its arguments, stripped an +escape sequence or tag characters from them, or masked a document there +it could not judge: the parser cut the summary out of them before +anything scanned it, so the secret could be there as a beginning. Where a runtime reports a summary and no arguments (pi, codex, OpenCode), its parser has already run the patterns over the summary and, for pi and codex, cut it: a value that straddles a cut stays in part, and one a pattern recognised stays as that pattern's @@ -167,7 +168,8 @@ characters or more under a secret-named member, such as `username` and `password` under `auth`, are both masked and so collide), are dropped whole — a cut object is not JSON — and the marked `tool_call` part keeps its `id`, name and summary — not the summary when redaction found a -secret in the arguments, or masked a document there it could not judge. Arguments also go, charged and marked, when +secret in the arguments, stripped an escape sequence or tag characters +from them, or masked a document there it could not judge. Arguments also go, charged and marked, when the call's name redacts to nothing; that part survives only if it has a summary. So `fullsend.truncated` means different things by part type: on a tool result, part of the response is kept; on a tool call, none of the diff --git a/internal/cli/content_collector.go b/internal/cli/content_collector.go index 5a0178ede9..fd3ce36c3a 100644 --- a/internal/cli/content_collector.go +++ b/internal/cli/content_collector.go @@ -7,6 +7,7 @@ import ( "slices" "sort" "strings" + "unicode" "unicode/utf8" "go.opentelemetry.io/otel/attribute" @@ -84,8 +85,9 @@ const maxToolResultBytes = 8 * 1024 // maxToolArgumentsBytes bounds one tool call's arguments, measured on // their redacted encoding. Arguments over it are dropped whole, not cut: // a cut object is not JSON, and the part keeps its id, name and summary -// (not the summary when redaction found a secret in the arguments, or -// masked a document there it could not judge; see Handle). +// (not the summary when redaction found a secret in the arguments, +// stripped an escape sequence or tag characters from them, or masked a +// document there it could not judge; see Handle). // Like maxToolResultBytes it exists because the total above is small; // what blocks raising it is the same unproven backend ceiling named on // maxContentBytes. @@ -420,12 +422,14 @@ func (c *contentCollector) Handle(evt agentruntime.AgentEvent) { // appendPart then refuses. scanned := len(c.findings) p.Arguments, p.Truncated = c.toolArguments(e.Arguments) - if slices.ContainsFunc(c.findings[scanned:], func(f security.Finding) bool { return f.Scanner != "unicode_normalizer" }) { + if slices.ContainsFunc(c.findings[scanned:], revealing) { // The parser cut the summary out of these arguments // before anything scanned it, so a secret found in them, - // or in a document masked unjudged, can be in the summary - // as a beginning that neither the literal pass nor a - // pattern matches. + // stripped with an escape sequence or tag characters, or + // in a document masked unjudged, can be in the summary as + // a beginning that neither the literal pass nor a pattern + // matches, nor the normalizer strips once its terminator + // is cut. p.Summary = "" } } @@ -827,9 +831,11 @@ func jsonDocument(s string) (string, bool) { } // documentLike reports text that begins and ends as a JSON object or -// array does, after a byte order mark. +// array does, white space, control and format characters (a byte order +// mark, a zero-width space) aside: the normalizer strips the latter from +// the scanned text, not from the written one. func documentLike(s string) bool { - s = strings.TrimSpace(strings.TrimPrefix(s, "\uFEFF")) + s = strings.TrimFunc(s, func(r rune) bool { return unicode.IsSpace(r) || unicode.IsControl(r) || unicode.Is(unicode.Cf, r) }) return len(s) >= 2 && (s[0] == '{' && s[len(s)-1] == '}' || s[0] == '[' && s[len(s)-1] == ']') } @@ -906,13 +912,13 @@ func (c *contentCollector) heldSecret(scanned, written string) (string, bool) { return "secret", found } -// revealing reports a finding of heldSecret's walk that shows a secret in -// the decoded document: any but the normalizer's, and its removal of an -// ST-terminated escape sequence (OSC and its kin) or of tag characters, -// which can carry text the patterns never see. A secret the scan masked in place is gone from the +// revealing reports a finding that shows a secret was there: any but the +// normalizer's, and its removal of an escape sequence or of tag +// characters, which can carry text the patterns never see — digits as a +// CSI parameter, a token inside an OSC sequence. A secret the scan masked in place is gone from the // document by then, but for a mask that matches again. func revealing(f security.Finding) bool { - return f.Scanner != "unicode_normalizer" || f.Name == "osc_escape" || f.Name == "tag_char" + return f.Scanner != "unicode_normalizer" || f.Name == "ansi_escape" || f.Name == "osc_escape" || f.Name == "tag_char" } // redactValue redacts every string, number and object key under v; a diff --git a/internal/cli/content_collector_test.go b/internal/cli/content_collector_test.go index 7e2b1e5a72..93ccb44a71 100644 --- a/internal/cli/content_collector_test.go +++ b/internal/cli/content_collector_test.go @@ -212,11 +212,12 @@ func TestContentCollector_JSONInAStringIsMaskedWholeWhenItsDocumentHoldsASecret( "key under credentials": {`{"credentials":{"` + opaque + `":"user"}}`, "***", 1}, "document in a document": {`{"inner":` + quote(`{"token":["`+opaque+`"]}`) + `}`, "***", 1}, // The scan sees the document's text; these only its strings. - "an assignment that opens a string": {`{"command":"API_KEY=` + opaque + `"}`, "***", 1}, - "an assignment after an escaped break": {`{"c":"cd x\nAPI_KEY=` + opaque + ` run"}`, "***", 1}, - "a token spelled with an escape": {`{"t":"ghp_\u0072` + token[5:] + `"}`, "***", 1}, - "tag characters written with escapes": {`{"t":"\uDB40\uDC67"}`, "***", 1}, - "an escape sequence around a token": {`{"n":"see \u001b]ghp_\u0072` + token[5:] + `\u0007"}`, "***", 1}, + "an assignment that opens a string": {`{"command":"API_KEY=` + opaque + `"}`, "***", 1}, + "an assignment after an escaped break": {`{"c":"cd x\nAPI_KEY=` + opaque + ` run"}`, "***", 1}, + "a token spelled with an escape": {`{"t":"ghp_\u0072` + token[5:] + `"}`, "***", 1}, + "tag characters written with escapes": {`{"t":"\uDB40\uDC67"}`, "***", 1}, + "digits in an escape under a secret name": {`{"auth":{"pin":"\u001b[31415926535m"}}`, "***", 1}, + "an escape sequence around a token": {`{"n":"see \u001b]ghp_\u0072` + token[5:] + `\u0007"}`, "***", 1}, // Folding can make the document; a document the scan breaks, // folding or masking it, cannot be judged. "folded into a document": {"\uFF5B" + cred + "}", "***", 2}, @@ -274,6 +275,8 @@ func TestContentCollector_JSONInAStringIsAlsoScannedWhole(t *testing.T) { "runner env value with structure": {map[string]string{"DB_PASSWORD": `hunter22","port`}, `{"pw":"hunter22","port":5432}`, "hunter22", 2}, "runner env value that ends the document": {map[string]string{"DB_PASSWORD": `hunter22"}`}, `{"pw":"hunter22"}`, "hunter22", 2}, "and one behind a BOM": {map[string]string{"DB_PASSWORD": `hunter22"}`}, "\uFEFF{\"credentials\":{\"value\":\"opaque-credential-123\"},\"pw\":\"hunter22\"}", "opaque-credential-123", 3}, + "and one behind a space and a BOM": {map[string]string{"DB_PASSWORD": `hunter22"}`}, " \uFEFF{\"credentials\":{\"value\":\"opaque-credential-123\"},\"pw\":\"hunter22\"}", "opaque-credential-123", 3}, + "and one behind a zero-width space": {map[string]string{"DB_PASSWORD": `hunter22"}`}, "\u200B{\"credentials\":{\"value\":\"opaque-credential-123\"},\"pw\":\"hunter22\"}", "opaque-credential-123", 3}, // Decoded, the literal pass sees it. "runner env value spelled with an escape": {map[string]string{"DB_PASSWORD": "hunter2hunter2"}, `{"pw":"\u0068unter2hunter2"}`, "unter2hunter2", 1}, } { @@ -562,6 +565,23 @@ func TestContentCollector_DropsTheSummaryWhenTheArgumentsHeldARunnerEnvValue(t * assert.Contains(t, res.OutputMessages, `"summary":"ls"`, "a call whose arguments held no value keeps its summary") } +func TestContentCollector_DropsTheSummaryWhenAnEscapeSequenceHidAToken(t *testing.T) { + // The normalizer strips the whole sequence from the arguments, token + // and all; the parser's cut summary lost the terminator, so nothing + // strips the beginning of the token there. + token := "ghp_" + strings.Repeat("r", 36) + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{ + Name: "Bash", + Summary: "echo \x1b]" + token[:36], + Arguments: `{"command":"echo \u001b]` + token + `\u0007"}`, + }) + + res := c.Result("stop") + assert.NotContains(t, res.OutputMessages, token[:36]) + assert.NotContains(t, res.OutputMessages, `"summary"`) +} + func TestContentCollector_DropsTheSummaryWhenAPatternFoundTheSecret(t *testing.T) { // A zero-width space hides the token from the parser's scan of the // summary; the collector's pattern finds it in the arguments once the From 96612eaaf9ebde9c9a2b3d18668024d7836c9898 Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Wed, 7 Oct 2026 08:08:17 -0400 Subject: [PATCH 17/40] fix(cli): judge a held JSON string literal decoded too A string argument holding a JSON string literal can spell a runner env value or a token with escapes that only decoding joins: a literal written as backslash-u0061 followed by the rest of the value hid the value of DEPLOY_PASSWORD. jsonDocument accepts a string literal as well as an object or array, so heldSecret decodes it and masks the holding string whole when the decoded text reveals one; documentLike knows the shape, so a literal the scan broke is masked whole too. Signed-off-by: Dharit Shah --- docs/guides/dev/tracing.md | 4 ++-- .../infrastructure/distributed-tracing.md | 6 +++--- internal/cli/content_collector.go | 19 ++++++++++--------- internal/cli/content_collector_test.go | 8 ++++++-- 4 files changed, 21 insertions(+), 16 deletions(-) diff --git a/docs/guides/dev/tracing.md b/docs/guides/dev/tracing.md index 2869668423..bb5f56c751 100644 --- a/docs/guides/dev/tracing.md +++ b/docs/guides/dev/tracing.md @@ -244,8 +244,8 @@ also turn a fullwidth quotation mark into one that closes the string. `toolArguments` decodes the value, redacts each string and object key on its own (a number as its digits; one that redacts becomes the redacted string), and encodes the result again. A string value or key that holds -one JSON object or array once scanned — what is exported — is judged -decoded as well (`heldSecret`). The scan reads the string whole, since a +one JSON object, array or string literal once scanned — what is exported +— is judged decoded as well (`heldSecret`). The scan reads the string whole, since a secret can span a document's strings, but as text: it misses a secret-named member nested deeper than the value right after the key, and a secret only decoding spells out (an assignment that opens a string, a diff --git a/docs/guides/infrastructure/distributed-tracing.md b/docs/guides/infrastructure/distributed-tracing.md index c05558aab2..3fa7710d94 100644 --- a/docs/guides/infrastructure/distributed-tracing.md +++ b/docs/guides/infrastructure/distributed-tracing.md @@ -123,9 +123,9 @@ codex parsers emit none of the three yet `arguments` is the call's input decoded, redacted string by string (a number that redacts becomes the redacted string), and encoded again, so key order, spacing and escapes are not the stream's. -A string value or key in it that holds one JSON object or array — a JSON -file passed to `Write`, say — is masked `***` whole when that document, -decoded, still shows a secret: a value or key nested under a secret-named +A string value or key in it that holds one JSON object, array or string +literal — a JSON file passed to `Write`, say — is masked `***` whole when that +document, decoded, still shows a secret: a value or key nested under a secret-named member, or one only decoding spells out. So is one that cannot be judged: held more than four deep, naming a member twice, or broken by redaction — the [developer guide](../dev/tracing.md) lists what is not judged. diff --git a/internal/cli/content_collector.go b/internal/cli/content_collector.go index fd3ce36c3a..403ad6f358 100644 --- a/internal/cli/content_collector.go +++ b/internal/cli/content_collector.go @@ -824,19 +824,20 @@ func decodeJSON(s string) (any, error) { return v, err } -// jsonDocument returns s trimmed when it is one JSON object or array. +// jsonDocument returns s trimmed when it is one JSON object, array or +// string literal: a string literal can spell a secret with escapes too. func jsonDocument(s string) (string, bool) { s = strings.TrimSpace(s) - return s, s != "" && (s[0] == '{' || s[0] == '[') && json.Valid([]byte(s)) + return s, s != "" && strings.IndexByte(`{["`, s[0]) >= 0 && json.Valid([]byte(s)) } -// documentLike reports text that begins and ends as a JSON object or -// array does, white space, control and format characters (a byte order -// mark, a zero-width space) aside: the normalizer strips the latter from -// the scanned text, not from the written one. +// documentLike reports text that begins and ends as a JSON object, array +// or string literal does, white space, control and format characters (a +// byte order mark, a zero-width space) aside: the normalizer strips the +// latter from the scanned text, not from the written one. func documentLike(s string) bool { s = strings.TrimFunc(s, func(r rune) bool { return unicode.IsSpace(r) || unicode.IsControl(r) || unicode.Is(unicode.Cf, r) }) - return len(s) >= 2 && (s[0] == '{' && s[len(s)-1] == '}' || s[0] == '[' && s[len(s)-1] == ']') + return len(s) >= 2 && (s[0] == '{' && s[len(s)-1] == '}' || s[0] == '[' && s[len(s)-1] == ']' || s[0] == '"' && s[len(s)-1] == '"') } // duplicateName reports an object in the JSON document s that names a @@ -881,8 +882,8 @@ func (c *contentCollector) maskHeld(s, t string) string { return s } -// heldSecret judges, decoded, the JSON object or array a string holds -// once scanned — what is exported. The scan reads the document's text, +// heldSecret judges, decoded, the JSON object, array or string literal a +// string holds once scanned — what is exported. The scan reads the document's text, // so it misses a secret-named member nested deeper than the value right // after the key, and a secret that only decoding spells out: an // assignment that opens a string, a token or runner env value written diff --git a/internal/cli/content_collector_test.go b/internal/cli/content_collector_test.go index 93ccb44a71..a47a412040 100644 --- a/internal/cli/content_collector_test.go +++ b/internal/cli/content_collector_test.go @@ -244,6 +244,7 @@ func TestContentCollector_JSONInAStringIsMaskedWholeWhenItsDocumentHoldsASecret( "a number past float64": {`{"maximum":1e999}`, `{"maximum":1e999}`, 0}, "text like a document the scan changed": {`{'password': 'hunter2hunter2'}`, "***", 2}, "a command that opens a brace": {`{ export API_KEY=` + opaque + `; } 2>/dev/null`, `{ export API_KEY=opaq...; } 2>/dev/null`, 1}, + "a held string literal": {`"plain text"`, `"plain text"`, 0}, "a value naming a sibling": {`{"name":"name","from":"to","to":"x"}`, `{"name":"name","from":"to","to":"x"}`, 0}, "names repeated in other objects": {`[{"a":1},{"a":2},{"x":{"a":1},"y":{"a":2}},{"a":{"x":1},"x":2}]`, `[{"a":1},{"a":2},{"x":{"a":1},"y":{"a":2}},{"a":{"x":1},"x":2}]`, 0}, } { @@ -277,8 +278,11 @@ func TestContentCollector_JSONInAStringIsAlsoScannedWhole(t *testing.T) { "and one behind a BOM": {map[string]string{"DB_PASSWORD": `hunter22"}`}, "\uFEFF{\"credentials\":{\"value\":\"opaque-credential-123\"},\"pw\":\"hunter22\"}", "opaque-credential-123", 3}, "and one behind a space and a BOM": {map[string]string{"DB_PASSWORD": `hunter22"}`}, " \uFEFF{\"credentials\":{\"value\":\"opaque-credential-123\"},\"pw\":\"hunter22\"}", "opaque-credential-123", 3}, "and one behind a zero-width space": {map[string]string{"DB_PASSWORD": `hunter22"}`}, "\u200B{\"credentials\":{\"value\":\"opaque-credential-123\"},\"pw\":\"hunter22\"}", "opaque-credential-123", 3}, - // Decoded, the literal pass sees it. - "runner env value spelled with an escape": {map[string]string{"DB_PASSWORD": "hunter2hunter2"}, `{"pw":"\u0068unter2hunter2"}`, "unter2hunter2", 1}, + // Decoded, the literal pass sees it — in a document, or in a + // string a held JSON string literal spells with escapes. + "runner env value spelled with an escape": {map[string]string{"DB_PASSWORD": "hunter2hunter2"}, `{"pw":"\u0068unter2hunter2"}`, "unter2hunter2", 1}, + "runner env value in a held string literal": {map[string]string{"DEPLOY_PASSWORD": "abcdefghijklmno"}, `"\u0061bcdefghijklmno"`, "bcdefghijklmno", 1}, + "and in one the scan breaks": {map[string]string{"DEPLOY_PASSWORD": "abcdefghijklmno"}, "\"\\u0061bcdefghijklmno\uFF02\"", "bcdefghijklmno", 2}, } { t.Run(name, func(t *testing.T) { t.Setenv(telemetry.ContentCaptureEnvVar, "true") From d3195708bfb14775797a32f3c658cb01363a8595 Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Wed, 7 Oct 2026 08:51:08 -0400 Subject: [PATCH 18/40] fix(cli): mask whole document-shaped text that does not parse heldSecret failed closed on a document the scan broke, but kept text shaped like a document that never parsed - JSON with a comment or a trailing comma, an object literal - with whatever it nested under a secret-named member, since neither the text patterns nor a walk could see that. Text that begins and ends like a JSON object, array or string literal, quotes something, and does not parse is now masked whole, whether the scan broke it or not. A mask is bracketed too and quotes nothing, so [REDACTED PRIVATE KEY] and the runner env marker stay as they are. Signed-off-by: Dharit Shah --- docs/guides/dev/tracing.md | 24 +++++++++++------------- internal/cli/content_collector.go | 18 ++++++++++-------- internal/cli/content_collector_test.go | 18 ++++++++++++++---- 3 files changed, 35 insertions(+), 25 deletions(-) diff --git a/docs/guides/dev/tracing.md b/docs/guides/dev/tracing.md index bb5f56c751..5c33f62d4f 100644 --- a/docs/guides/dev/tracing.md +++ b/docs/guides/dev/tracing.md @@ -258,23 +258,21 @@ secret the scan masked in place no longer shows — the string is masked `***` whole, with one `held_document` finding. So is a document that cannot be judged: held more than `maxHeldDepth` (4) deep, each level being scanned once more; naming a member twice, since decoding keeps only -the last (RFC 7493 forbids it); or broken by the scan — text the scan -changed, folding or masking it, that begins and ends like a document but -no longer parses. That masks whole text with no secret left in it too — -a document with a repeated name, one a pattern misreads once decoded (a -notebook line `key = …`), one holding an escaped terminal colour code, -text shaped like a document whose secret the scan masked (a Python -dict) — and a mask the walk +the last (RFC 7493 forbids it); or shaped like a document — it begins +and ends like one, white space and invisible characters aside, and +quotes something — but not parsing, whether the scan broke it, folding +or masking it, or it never parsed. That masks whole text with no secret in it too — a document with +a repeated name, one a pattern misreads once decoded (a notebook line +`key = …`), one holding an escaped terminal colour code, JSON with +comments or trailing commas, an object literal or a Python dict with a +quoted value, an `Edit` fragment shaped `{…}` with one — and a mask the walk matches again (a connection string's password of ten bytes or more, the runner environment marker in an assignment, a mask of eight bytes or more under a secret-named member) masks its string whole and counts its secret twice. Not judged: a document whose structure folding or a mask changes -while the text holding it still parses (one held in it then breaks too); -text that never parsed — JSON with comments or trailing commas, several -documents, a raw control character — unless the scan changed it and it -is shaped like a document, when it is masked whole as above; and -structure in any other text (YAML, a document inside code). Each string -and number is also +while the text holding it still parses (one held in it then breaks too), +and structure in text not shaped like a document (several documents, +YAML, a document inside code). Each string and number is also scanned once more, already redacted, beside the nearest enclosing key the member-name pattern (`json_field`) names, and masked whole when that pattern matches the pair — a pattern keyed on a member diff --git a/internal/cli/content_collector.go b/internal/cli/content_collector.go index 403ad6f358..536d1ae8cc 100644 --- a/internal/cli/content_collector.go +++ b/internal/cli/content_collector.go @@ -832,12 +832,14 @@ func jsonDocument(s string) (string, bool) { } // documentLike reports text that begins and ends as a JSON object, array -// or string literal does, white space, control and format characters (a -// byte order mark, a zero-width space) aside: the normalizer strips the -// latter from the scanned text, not from the written one. +// or string literal does and holds a quoted token, white space, control +// and format characters (a byte order mark, a zero-width space) aside: +// the normalizer strips the latter from the scanned text, not from the +// written one. A mask is bracketed too — [REDACTED PRIVATE KEY], the +// runner env marker — but quotes nothing. func documentLike(s string) bool { s = strings.TrimFunc(s, func(r rune) bool { return unicode.IsSpace(r) || unicode.IsControl(r) || unicode.Is(unicode.Cf, r) }) - return len(s) >= 2 && (s[0] == '{' && s[len(s)-1] == '}' || s[0] == '[' && s[len(s)-1] == ']' || s[0] == '"' && s[len(s)-1] == '"') + return len(s) >= 2 && (s[0] == '{' && s[len(s)-1] == '}' || s[0] == '[' && s[len(s)-1] == ']' || s[0] == '"' && s[len(s)-1] == '"') && strings.ContainsAny(s, `"'`) } // duplicateName reports an object in the JSON document s that names a @@ -891,14 +893,14 @@ func (c *contentCollector) maskHeld(s, t string) string { // output and findings discarded; a revealing finding makes the reason // "secret". A document that cannot be judged makes it "uninspected": // one held deeper than maxHeldDepth, one naming a member twice — -// decoding keeps the last — and one the scan broke: text the scan -// changed, folding or masking it, that begins and ends like a document -// but no longer parses. A level above records that mask as "secret": it +// decoding keeps the last — and text shaped like a document that does +// not parse: one the scan broke, folding or masking it, or one that +// never parsed (a comment, a trailing comma, an object literal). A level above records that mask as "secret": it // is a finding of that level's walk. func (c *contentCollector) heldSecret(scanned, written string) (string, bool) { doc, ok := jsonDocument(scanned) if !ok { - return "uninspected", scanned != written && (documentLike(scanned) || documentLike(written)) + return "uninspected", documentLike(scanned) || documentLike(written) } if c.held == maxHeldDepth || duplicateName(doc) { return "uninspected", true diff --git a/internal/cli/content_collector_test.go b/internal/cli/content_collector_test.go index a47a412040..e54bdf116d 100644 --- a/internal/cli/content_collector_test.go +++ b/internal/cli/content_collector_test.go @@ -237,10 +237,20 @@ func TestContentCollector_JSONInAStringIsMaskedWholeWhenItsDocumentHoldsASecret( "a mask that matches again": {`{"db":"postgres://u:hunter2hunter2@db/x"}`, "***", 2}, "a notebook assignment": {`{"cells":[{"source":["key = jax.random.PRNGKey(0)\n"]}]}`, "***", 1}, // Otherwise the string is kept as the scan left it. - "plain document": {`{ "b": "` + opaque + `", "a": 1 }`, `{ "b": "` + opaque + `", "a": 1 }`, 0}, - "a pair stays masked in place": {`{"password":"hunter2hunter2","n":1}`, `{"password":"hunt...","n":1}`, 1}, - "a letter folded once decoded": {`{"v":"\uFF21"}`, `{"v":"\uFF21"}`, 0}, - "a control character the scan keeps": {"{\"a\":\"x\x01y\"}", "{\"a\":\"x\x01y\"}", 0}, + "plain document": {`{ "b": "` + opaque + `", "a": 1 }`, `{ "b": "` + opaque + `", "a": 1 }`, 0}, + "a pair stays masked in place": {`{"password":"hunter2hunter2","n":1}`, `{"password":"hunt...","n":1}`, 1}, + "a letter folded once decoded": {`{"v":"\uFF21"}`, `{"v":"\uFF21"}`, 0}, + // Document-shaped text that does not parse cannot be judged either: + // a raw control character, a trailing comma, a comment, an object + // literal. Masked whole, secret or none. + "a control character the scan keeps": {"{\"a\":\"x\x01y\"}", "***", 1}, + "a trailing comma": {`{` + cred + `,}`, "***", 1}, + "a trailing comma and no secret": {`{ "a": 1, }`, "***", 1}, + "a comment": {"{\n // dev \n " + cred + "\n}", "***", 1}, + "an object literal": {`{ apiKey: "hunter2hunter2" }`, "***", 1}, + // Shaped like one but quoting nothing: code, or a mask. + "an object literal quoting nothing": {`{ apiKey: process.env.KEY }`, `{ apiKey: process.env.KEY }`, 0}, + "a mask": {"-----BEGIN RSA PRIVATE " + "KEY-----\nQQQQQQQQ\n-----END RSA PRIVATE " + "KEY-----", "[REDACTED PRIVATE KEY]", 1}, "a number past float64": {`{"maximum":1e999}`, `{"maximum":1e999}`, 0}, "text like a document the scan changed": {`{'password': 'hunter2hunter2'}`, "***", 2}, "a command that opens a brace": {`{ export API_KEY=` + opaque + `; } 2>/dev/null`, `{ export API_KEY=opaq...; } 2>/dev/null`, 1}, From 69583ad8e887ce512b87dcd0b253214ead0c8cca Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Wed, 7 Oct 2026 08:51:42 -0400 Subject: [PATCH 19/40] docs(tracing): a document-shaped string that does not parse is masked whole Signed-off-by: Dharit Shah --- docs/guides/infrastructure/distributed-tracing.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/guides/infrastructure/distributed-tracing.md b/docs/guides/infrastructure/distributed-tracing.md index 3fa7710d94..4763cd9f56 100644 --- a/docs/guides/infrastructure/distributed-tracing.md +++ b/docs/guides/infrastructure/distributed-tracing.md @@ -127,8 +127,9 @@ A string value or key in it that holds one JSON object, array or string literal — a JSON file passed to `Write`, say — is masked `***` whole when that document, decoded, still shows a secret: a value or key nested under a secret-named member, or one only decoding spells out. So is one that cannot be judged: -held more than four deep, naming a member twice, or broken by redaction -— the [developer guide](../dev/tracing.md) lists what is not judged. +held more than four deep, naming a member twice, or shaped like a +document but not parsing (JSON with comments or trailing commas, an +object literal) — the [developer guide](../dev/tracing.md) has the list. A string directly under a secret-named member is masked in place, as anywhere else. From 6ba0c6390630ec0e78d0f12799799b7dcbe63e71 Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Thu, 8 Oct 2026 08:38:36 -0400 Subject: [PATCH 20/40] fix(cli): bound the arguments walk before it starts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A secret-named key was passed down as it is to every value under it, and each value was scanned beside the whole key: the cost was the key's length times the number of values, and maxToolArgumentsBytes applied only after the walk. A 32 KiB key over 2,000 numbers took 14 s in the event handler, on one call. The values under a secret-named member are now judged beside a stand-in name, decided once for the member — the pattern reads any name it matches alike, so each scan costs the value's length — and arguments over four times the bound as written are scanned as text and dropped before they are decoded, charged as redacted text. Both guides say so. A test pins each: the first as a ratio against a short key, so load cannot flake it. Signed-off-by: Dharit Shah --- docs/guides/dev/tracing.md | 6 +- .../infrastructure/distributed-tracing.md | 5 +- internal/cli/content_collector.go | 29 ++++++--- internal/cli/content_collector_test.go | 64 +++++++++++++++++++ 4 files changed, 92 insertions(+), 12 deletions(-) diff --git a/docs/guides/dev/tracing.md b/docs/guides/dev/tracing.md index 5c33f62d4f..68e2ce6f87 100644 --- a/docs/guides/dev/tracing.md +++ b/docs/guides/dev/tracing.md @@ -217,10 +217,12 @@ final answer is what consumers judge) plus an 8 KiB per-tool-result bound (tail-kept, redacted before the cut, the part marked `fullsend.truncated`) and an 8 KiB per-call arguments bound (over it the arguments are dropped whole and the part marked, since a cut object is not -JSON), with exact dropped-byte +JSON; over four times it as written they are dropped before they are +decoded, so no walk does work the bound does not limit), with exact dropped-byte accounting across content, tool names, summaries, responses, part ids, and dropped arguments (counted as re-encoded JSON, or as redacted -text when they were not a JSON value; on a key collision each member a +text when they were not a JSON value or were dropped before they were +decoded; on a key collision each member a later key (in sorted order) replaced is not counted), then holds the marshaled string to `maxEncodedContentBytes` (255,000 — just under the one size the pilot backend is proven to accept) by trimming the oldest content again, measured on the encoding itself and diff --git a/docs/guides/infrastructure/distributed-tracing.md b/docs/guides/infrastructure/distributed-tracing.md index 4763cd9f56..ee26c32835 100644 --- a/docs/guides/infrastructure/distributed-tracing.md +++ b/docs/guides/infrastructure/distributed-tracing.md @@ -151,11 +151,12 @@ which sees the runtime's stream, not its API requests. (runner environment values, Unicode normalization, secret masking, then runner environment values again) before reaching the span. Content is bounded at 256 KiB per iteration — each tool result at 8 KiB, -each call's arguments at 8 KiB — +each call's arguments at 8 KiB, and at four times that as written, before +they are decoded — kept as an ordered suffix; overflow drops the oldest content first. Those bounds count raw bytes, except that arguments count as the re-encoded JSON described above, against their own bound and against the -total. That encoding writes `<`, `>` and `&` as six bytes each, so +total (as redacted text when they were dropped before they were decoded). That encoding writes `<`, `>` and `&` as six bytes each, so arguments dense in those characters reach their bound at about a sixth of that size on the stream. The exported JSON string is bounded as well, at 255,000 bytes, because encoding adds 8–10% at these bounds on real runs diff --git a/internal/cli/content_collector.go b/internal/cli/content_collector.go index 536d1ae8cc..1e7452287f 100644 --- a/internal/cli/content_collector.go +++ b/internal/cli/content_collector.go @@ -99,6 +99,13 @@ const maxToolResultBytes = 8 * 1024 // measured; a Write call carries the file body as an argument. const maxToolArgumentsBytes = 8 * 1024 +// maxRawToolArgumentsBytes bounds the arguments as written, before they +// are decoded. Redaction can bring them under maxToolArgumentsBytes (a +// token masks to seven bytes), but not from far over it, and the walk +// must do no work that bound does not limit. Over it they are scanned +// as text, so their findings count, then dropped and charged. +const maxRawToolArgumentsBytes = 4 * maxToolArgumentsBytes + // maxHeldDepth bounds how deep heldSecret walks documents held in // strings: each level is scanned once more, and with \u escapes a // document can nest about the square root of its length deep. A string @@ -783,7 +790,8 @@ func redactText(pipeline *security.Pipeline, runnerEnv map[string]string, text s // key order, spacing and escapes are the encoder's, not the wire's. // // Text that is not one JSON value (see ToolUseEvent.Arguments) cannot be -// redacted that way. It is scanned as text so its findings count, then +// redacted that way, and text over maxRawToolArgumentsBytes is not +// decoded at all. Either is scanned as text so its findings count, then // dropped and charged like any other discarded content. Arguments whose // redacted encoding exceeds maxToolArgumentsBytes, or in which two keys // of one object redact to the same string, are dropped the same way and @@ -794,7 +802,7 @@ func (c *contentCollector) toolArguments(args string) (json.RawMessage, bool) { if args == "" { return nil, false } - if !json.Valid([]byte(args)) { + if len(args) > maxRawToolArgumentsBytes || !json.Valid([]byte(args)) { c.evicted += len(c.redact(args, &c.findings)) return nil, true } @@ -928,9 +936,10 @@ func revealing(f security.Finding) bool { // number that redacts becomes the redacted string. A pattern // keyed on a member name ("password": "...") cannot see the pair that // way, so a string or number under a key that pattern names — the nearest -// such enclosing key, under — is scanned once more, already redacted, -// beside it and masked whole on a match (secretNamed); a direct string -// member is also scanned beside its own key. A value in +// such enclosing key, under, which a stand-in name represents — is scanned +// once more, already redacted, beside it and masked whole on a match +// (secretNamed); a direct string member is also scanned beside its own +// key. A value in // an array or a nested object under a secret-named member is judged as // that member's own value would be, and so is a key there. // Keys are walked in sorted order, so neither the findings nor a dropped @@ -966,11 +975,15 @@ func (c *contentCollector) redactValue(v any, under string, collided *bool) any sk := c.redact(k, &c.findings) rk := c.maskHeld(sk, k) member := under - if c.namesASecret(sk) { - member = sk + named := c.namesASecret(sk) + if named { + // Decided once for the member: the pattern reads any + // name it matches alike, so a stand-in judges the values + // under it at a cost the key's length does not multiply. + member = "secret" } e := c.redactValue(t[k], member, collided) - if s, ok := e.(string); ok && member != sk && c.secretNamed(sk, s) { + if s, ok := e.(string); ok && !named && c.secretNamed(sk, s) { // A single quote in the key can let the pattern match // the pair even when the key names no secret. e = "***" diff --git a/internal/cli/content_collector_test.go b/internal/cli/content_collector_test.go index e54bdf116d..d17168ba5d 100644 --- a/internal/cli/content_collector_test.go +++ b/internal/cli/content_collector_test.go @@ -6,6 +6,7 @@ import ( "math/rand" "strings" "testing" + "time" "unicode/utf8" "github.com/stretchr/testify/assert" @@ -14,6 +15,7 @@ import ( "go.opentelemetry.io/otel/sdk/trace/tracetest" agentruntime "github.com/fullsend-ai/fullsend/internal/runtime" + "github.com/fullsend-ai/fullsend/internal/security" "github.com/fullsend-ai/fullsend/internal/telemetry" ) @@ -834,6 +836,68 @@ func TestContentCollector_ArgumentsBoundAppliesAfterRedaction(t *testing.T) { assert.NotContains(t, res.OutputMessages, "qqqq") } +func TestContentCollector_ArgumentsFarOverTheBoundAreDroppedUnjudged(t *testing.T) { + // Redaction can bring arguments under the bound, but not from far + // over it, and the walk must do no work the bound does not limit: a + // secret-named key over an array once cost the key's length for each + // element. Over four times the bound as written, the arguments are + // scanned as text, so a secret still counts, then dropped and charged. + for name, tc := range map[string]struct { + args string + findings int + summary bool + }{ + "a long secret-named key over an array": {`{"password` + strings.Repeat("a", 32<<10) + `":[` + strings.Repeat("0,", 1999) + `0]}`, 0, true}, + "a token in a value": {`{"command":"echo ghp_` + strings.Repeat("q", 36) + ` ` + strings.Repeat("x", maxRawToolArgumentsBytes) + `"}`, 1, false}, + "spacing the encoder would drop": {`{"content": "` + strings.Repeat("x", maxRawToolArgumentsBytes) + `"}`, 0, true}, + } { + t.Run(name, func(t *testing.T) { + c := newContentCollector(maxContentBytes) + c.Handle(agentruntime.ToolUseEvent{Name: "Write", Summary: "/x", Arguments: tc.args}) + + res := c.Result("stop") + part := partAt(t, decodeOutputMessages(t, res.OutputMessages), 0) + assert.NotContains(t, part, "arguments") + assert.Equal(t, tc.summary, part["summary"] != nil) + assert.Equal(t, true, part["fullsend.truncated"]) + assert.Len(t, res.Findings, tc.findings) + assert.NotContains(t, res.OutputMessages, "qqqq") + assert.Equal(t, len(c.redact(tc.args, new([]security.Finding))), res.DroppedBytes, "charged as redacted text, like arguments that are not JSON") + }) + } + t.Run("at four times the bound, redaction still brings them under it", func(t *testing.T) { + args := `{"command":"echo ghp_` + strings.Repeat("q", maxRawToolArgumentsBytes-len(`{"command":"echo ghp_"}`)) + `"}` + require.Len(t, args, maxRawToolArgumentsBytes) + c := newContentCollector(maxContentBytes) + c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Arguments: args}) + + part := partAt(t, decodeOutputMessages(t, c.Result("stop").OutputMessages), 0) + assert.Equal(t, map[string]any{"command": "echo ghp_..."}, part["arguments"]) + }) +} + +func TestContentCollector_ASecretNamedKeyIsJudgedOnceForItsValues(t *testing.T) { + // The values under a secret-named member are judged beside a stand-in + // name decided once for the member, so each scan costs the value's + // length, not the key's: a long key over a thousand values must cost + // about what a short one does. A ratio, not an absolute time, so load + // cannot flake it. + values := `[` + strings.Repeat(`"12345678",`, 999) + `"12345678"]` + handle := func(key string) (time.Duration, contentResult) { + c := newContentCollector(maxContentBytes) + entry := time.Now() + c.Handle(agentruntime.ToolUseEvent{Name: "Write", Arguments: `{"` + key + `":` + values + `}`}) + return time.Since(entry), c.Result("stop") + } + short, res := handle("password") + require.Len(t, res.Findings, 1000, "every value is judged beside the name") + assert.NotContains(t, res.OutputMessages, "12345678") + long, res := handle("password" + strings.Repeat("a", 16<<10)) + require.Len(t, res.Findings, 1000) + assert.NotContains(t, res.OutputMessages, "12345678") + assert.Less(t, long, 20*short, "the key's length must not multiply the cost of its values") +} + func TestContentCollector_NullArgumentsAreOmitted(t *testing.T) { c := newContentCollector(4096) c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Arguments: `null`}) From 938a07c5f8187945363025916e08b96833dbb864 Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Thu, 8 Oct 2026 09:25:36 -0400 Subject: [PATCH 21/40] fix(cli): judge tool arguments across their strings and folds Three review findings on the decoded walk, each fixed by masking more. A private key written over the lines of an array - a notebook cell's source - reached the record whole: each string was judged on its own, and none held the block. Each level, the arguments and each JSON document a string holds, is now also judged as a reader of its keys, strings and numbers in the order written sees them, joined by a line break, decoded and as the normalizer renders them, for a secret that can run across them: the private key block, and an exact runtime value. The arguments are then dropped and charged as encoded, the held string masked whole. Every other pattern's secret stops at white space or at a quote, so a match over the break has only run its context into the next string, which stays unjudged, as a token split in two is. Folding can move a value out of its secret-named member while the document it is in still parses - fullwidth quotation marks - and the decoded walk then judged it under the wrong name. A held document the normalizer changes at all, as written, is masked whole. A terminal colour code ends at the next letter, so stripping one can take a token's first letter and leave the rest, and a title code's payload is text the patterns never see: a string, a value or a key, the normalizer stripped an escape sequence or tag characters from is masked whole, and a member is named as written as well as as scanned. SecretRedactor.Matches reports secrets with their bounds, for a caller that judges text it does not export. A test through the real event handler, collector, tool span tracker and file sink pins the record that leaves the runner: no fragment of the three reaches it, the clean call's arguments and every call's name do, and the markers say what was dropped. Signed-off-by: Dharit Shah --- docs/guides/dev/tracing.md | 30 +++- .../infrastructure/distributed-tracing.md | 13 +- internal/cli/content_capture_record_test.go | 146 +++++++++++++++ internal/cli/content_collector.go | 167 +++++++++++++++--- internal/cli/content_collector_test.go | 125 ++++++++++++- internal/security/redactor.go | 37 ++++ internal/security/redactor_matches_test.go | 29 +++ 7 files changed, 516 insertions(+), 31 deletions(-) create mode 100644 internal/cli/content_capture_record_test.go create mode 100644 internal/security/redactor_matches_test.go diff --git a/docs/guides/dev/tracing.md b/docs/guides/dev/tracing.md index 68e2ce6f87..cdd33ee130 100644 --- a/docs/guides/dev/tracing.md +++ b/docs/guides/dev/tracing.md @@ -263,7 +263,10 @@ being scanned once more; naming a member twice, since decoding keeps only the last (RFC 7493 forbids it); or shaped like a document — it begins and ends like one, white space and invisible characters aside, and quotes something — but not parsing, whether the scan broke it, folding -or masking it, or it never parsed. That masks whole text with no secret in it too — a document with +or masking it, or it never parsed; or one the normalizer changes at all, +as written: folding can move a value out of its secret-named member while +the document still parses (fullwidth quotation marks), so structure the +normalizer makes, or moves, is not trusted. That masks whole text with no secret in it too — a document with a repeated name, one a pattern misreads once decoded (a notebook line `key = …`), one holding an escaped terminal colour code, JSON with comments or trailing commas, an object literal or a Python dict with a @@ -271,10 +274,27 @@ quoted value, an `Edit` fragment shaped `{…}` with one — and a mask the walk matches again (a connection string's password of ten bytes or more, the runner environment marker in an assignment, a mask of eight bytes or more under a secret-named member) masks its string whole and counts its secret -twice. Not judged: a document whose structure folding or a mask changes -while the text holding it still parses (one held in it then breaks too), -and structure in text not shaped like a document (several documents, -YAML, a document inside code). Each string and number is also +twice. A string the normalizer stripped an escape sequence or tag +characters from — a value or a key — is masked `***` whole: a colour code +ends at the next letter, so the stripping can take a token's first letter +and leave the rest, and a title code's payload is text no pattern sees. A +member is named as written as well as as scanned, so a key so masked +still names the values under it. Each level — the arguments, and each +held document — is also judged as a reader of its keys, strings and +numbers in the order written sees them (`spanning`): joined by a line +break, each as decoded and as the normalizer renders it, before any is +masked, for a secret that can hold a line break — the private key block +over the lines of an array or over a key and a value, an exact runtime +value over the lines of one. The arguments are then dropped and charged +as encoded; a held document is masked whole (its text was scanned whole +first, so a block that scan masked in place stays in place). Every other +pattern's secret stops at white space or at a quote, so a match over the +break has only run its context into the next string, which is left +unjudged. Not judged: structure in text not shaped like a document +(several documents, YAML, a document inside code); an assignment, a +header or a connection string whose value begins in the next string; a +token or an exact value split into pieces between strings; a member-name +pair in single quotes over two strings' apostrophes. Each string and number is also scanned once more, already redacted, beside the nearest enclosing key the member-name pattern (`json_field`) names, and masked whole when that pattern matches the pair — a pattern keyed on a member diff --git a/docs/guides/infrastructure/distributed-tracing.md b/docs/guides/infrastructure/distributed-tracing.md index ee26c32835..819d7a609a 100644 --- a/docs/guides/infrastructure/distributed-tracing.md +++ b/docs/guides/infrastructure/distributed-tracing.md @@ -129,9 +129,16 @@ document, decoded, still shows a secret: a value or key nested under a secret-na member, or one only decoding spells out. So is one that cannot be judged: held more than four deep, naming a member twice, or shaped like a document but not parsing (JSON with comments or trailing commas, an -object literal) — the [developer guide](../dev/tracing.md) has the list. -A string directly under a secret-named member is masked in place, as -anywhere else. +object literal), or one that Unicode normalization changes at all, since +folding can move a value between members — the +[developer guide](../dev/tracing.md) has the list. A secret that runs +across the strings of the arguments, or of a held document, in the order +written — a private key over the lines of an array — can be masked +nowhere but whole: the arguments are dropped, the held string masked +`***`. A string a terminal escape sequence was stripped from is masked +whole too, since the stripping can leave most of a token. A string +directly under a secret-named member is masked in place, as anywhere +else. On a retry iteration under `validation_loop.feedback_mode: append`, the prompt the runner composed — its fixed framing around the previous diff --git a/internal/cli/content_capture_record_test.go b/internal/cli/content_capture_record_test.go new file mode 100644 index 0000000000..fa75f8c3cd --- /dev/null +++ b/internal/cli/content_capture_record_test.go @@ -0,0 +1,146 @@ +package cli + +import ( + "bufio" + "context" + "encoding/json" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + agentruntime "github.com/fullsend-ai/fullsend/internal/runtime" + "github.com/fullsend-ai/fullsend/internal/telemetry" +) + +// recordAttribute returns the attribute named key, a string or a bool +// printed, from the first span in the run-telemetry file that carries it, +// searching the exporter's JSON however it nests attributes, and reports +// whether one was found. +func recordAttribute(t *testing.T, dir, key string) (string, bool) { + t.Helper() + f, err := os.Open(filepath.Join(dir, telemetry.TelemetryFile)) + require.NoError(t, err) + defer f.Close() + var find func(v any) (string, bool) + find = func(v any) (string, bool) { + switch t := v.(type) { + case map[string]any: + if t["key"] == key || t["Key"] == key { + for _, val := range []any{t["value"], t["Value"]} { + if m, ok := val.(map[string]any); ok { + for _, s := range []any{m["stringValue"], m["boolValue"], m["Value"]} { + switch s := s.(type) { + case string: + return s, true + case bool: + return strconv.FormatBool(s), true + } + } + } + } + } + for _, e := range t { + if s, ok := find(e); ok { + return s, true + } + } + case []any: + for _, e := range t { + if s, ok := find(e); ok { + return s, true + } + } + } + return "", false + } + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 1<<20), 1<<24) + for sc.Scan() { + var line any + if json.Unmarshal(sc.Bytes(), &line) != nil { + continue + } + if s, ok := find(line); ok { + return s, true + } + } + return "", false +} + +func TestContentCapture_SecretsAcrossArgumentsNeverReachTheRecord(t *testing.T) { + // The record that leaves the runner, run-telemetry.jsonl, written + // through the real event handler, collector, tool span tracker and + // file sink with Level 3 on: a private key over the lines of a + // notebook cell passed as arguments, a credential that folding moves + // between members of a held document, and a token a colour code hides + // the first letter of are not in it; the clean call's arguments and + // every call's name are; and the markers say what was dropped. + for _, k := range []string{"OTEL_SDK_DISABLED", "OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT", "OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT"} { + t.Setenv(k, "") + } + t.Setenv(telemetry.ContentCaptureEnvVar, "true") + + begin, end := "-----BEGIN RSA PRIVATE "+"KEY-----", "-----END RSA PRIVATE "+"KEY-----" + body := strings.Repeat("Q", 40) + const opaque = "opaque-credential-123" + token := "ghp_" + strings.Repeat("r", 36) + cells, _ := json.Marshal(map[string]any{"notebook_path": "notes.ipynb", "cells": []any{map[string]any{"source": []string{begin + "\n", body + "\n", end + "\n"}}}}) + folded := "{\"credentials\":{\"value\":\"＂},＂note＂:{＂value＂:＂" + opaque + "\"}}" + held, _ := json.Marshal(map[string]string{"file_path": "cfg.json", "content": folded}) + + dir := t.TempDir() + tracer, cleanup := telemetry.Setup(dir, "test") + agentCtx, agentSpan := tracer.Start(context.Background(), "agent") + c := newContentCollectorIfEnabled(nil) + require.NotNil(t, c, "Level 3 is on") + tr := newToolSpanTracker(tracer, agentCtx) + handler := iterationEventHandler(func(agentruntime.AgentEvent) {}, c, tr) + require.NotNil(t, handler) + + for _, call := range []agentruntime.ToolUseEvent{ + {ID: "toolu_01", Name: "NotebookEdit", Summary: "notes.ipynb", Arguments: string(cells)}, + {ID: "toolu_02", Name: "Write", Summary: "cfg.json", Arguments: string(held)}, + {ID: "toolu_03", Name: "Bash", Summary: "echo", Arguments: `{"command":"echo \u001b[` + token + `"}`}, + {ID: "toolu_04", Name: "Read", Summary: "README.md", Arguments: `{"file_path":"README.md"}`}, + } { + handler(call) + handler(agentruntime.ToolResultEvent{ID: call.ID, Result: "ok"}) + } + attachContent(agentSpan, c.Result("stop")) + tr.Finish() + agentSpan.End() + cleanup(context.Background()) + + raw, err := os.ReadFile(filepath.Join(dir, telemetry.TelemetryFile)) + require.NoError(t, err) + for _, secret := range []string{body, opaque, token[1:]} { + assert.NotContains(t, string(raw), secret, "no fragment of a secret reaches the file") + } + + messages, ok := recordAttribute(t, dir, "gen_ai.output.messages") + require.True(t, ok, "the agent span carries the content attribute") + parts := decodeOutputMessages(t, messages)[0]["parts"].([]any) + calls := map[string]map[string]any{} + for _, p := range parts { + if part := p.(map[string]any); part["type"] == "tool_call" { + calls[part["name"].(string)] = part + } + } + require.Len(t, calls, 4, "every call is in the record, arguments or not") + assert.NotContains(t, calls["NotebookEdit"], "arguments", "a private key over the cell's lines drops the arguments") + assert.Equal(t, true, calls["NotebookEdit"]["fullsend.truncated"]) + assert.NotContains(t, calls["NotebookEdit"], "summary", "a secret in the arguments costs the summary") + assert.Equal(t, map[string]any{"file_path": "cfg.json", "content": "***"}, calls["Write"]["arguments"], "the held document folding changes is masked whole") + assert.Equal(t, map[string]any{"command": "***"}, calls["Bash"]["arguments"], "the string the colour code was stripped from is masked whole") + assert.Equal(t, map[string]any{"file_path": "README.md"}, calls["Read"]["arguments"], "the clean call's arguments are recorded") + assert.Equal(t, "README.md", calls["Read"]["summary"]) + + truncated, ok := recordAttribute(t, dir, "fullsend.content.truncated") + assert.True(t, ok, "the dropped arguments are marked on the span") + assert.Equal(t, "true", truncated) +} diff --git a/internal/cli/content_collector.go b/internal/cli/content_collector.go index 1e7452287f..73e8aeb8d2 100644 --- a/internal/cli/content_collector.go +++ b/internal/cli/content_collector.go @@ -377,8 +377,11 @@ type contentCollector struct { // maxEncoded bounds the marshaled attribute; see maxEncodedContentBytes. maxEncoded int pipeline *security.Pipeline - // secrets is the pipeline's pattern stage alone, for secretNamed. - secrets *security.SecretRedactor + // secrets is the pipeline's pattern stage alone, for secretNamed and + // spanning; normalizer its first stage alone, for the detection copies + // heldSecret and spanning judge. + secrets *security.SecretRedactor + normalizer *security.UnicodeNormalizer // runnerEnv feeds redact's literal pass; see newContentCollectorIfEnabled. runnerEnv map[string]string parts []contentPart @@ -405,7 +408,7 @@ type contentCollector struct { } func newContentCollector(maxBytes int) *contentCollector { - return &contentCollector{maxBytes: maxBytes, maxEncoded: maxEncodedContentBytes, pipeline: security.OutputPipeline(), secrets: security.NewSecretRedactor()} + return &contentCollector{maxBytes: maxBytes, maxEncoded: maxEncodedContentBytes, pipeline: security.OutputPipeline(), secrets: security.NewSecretRedactor(), normalizer: security.NewUnicodeNormalizer()} } // Handle consumes one normalized event. Contiguous text and reasoning @@ -793,11 +796,13 @@ func redactText(pipeline *security.Pipeline, runnerEnv map[string]string, text s // redacted that way, and text over maxRawToolArgumentsBytes is not // decoded at all. Either is scanned as text so its findings count, then // dropped and charged like any other discarded content. Arguments whose -// redacted encoding exceeds maxToolArgumentsBytes, or in which two keys -// of one object redact to the same string, are dropped the same way and -// charged as encoded. On a collision that is the encoding after the later -// key (in sorted order) replaced the earlier member, so the replaced -// member is not counted. +// redacted encoding exceeds maxToolArgumentsBytes, in which two keys of +// one object redact to the same string, or across whose keys, strings +// and numbers a secret runs (spanning: a private key over the lines of +// an array), which no one of them can mask, are dropped the same way +// and charged as encoded. On a collision that is the encoding after the +// later key (in sorted order) replaced the earlier member, so the +// replaced member is not counted. func (c *contentCollector) toolArguments(args string) (json.RawMessage, bool) { if args == "" { return nil, false @@ -815,7 +820,7 @@ func (c *contentCollector) toolArguments(args string) (json.RawMessage, bool) { if err != nil { return nil, false // decoded JSON values marshal unconditionally } - if collided || len(out) > maxToolArgumentsBytes { + if collided || c.spanning(args) || len(out) > maxToolArgumentsBytes { c.evicted += len(out) return nil, true } @@ -885,6 +890,9 @@ func duplicateName(s string) bool { // one held_document finding, when heldSecret finds against the document // its scanned text s holds; t is the text as written. func (c *contentCollector) maskHeld(s, t string) string { + if s == "***" { + return s // masked whole already: nothing left to judge + } if reason, ok := c.heldSecret(s, t); ok { c.findings = append(c.findings, security.Finding{Scanner: "held_document", Name: reason, Severity: "high", Detail: "string holding a JSON document masked whole", Position: -1}) return "***" @@ -903,18 +911,31 @@ func (c *contentCollector) maskHeld(s, t string) string { // one held deeper than maxHeldDepth, one naming a member twice — // decoding keeps the last — and text shaped like a document that does // not parse: one the scan broke, folding or masking it, or one that -// never parsed (a comment, a trailing comma, an object literal). A level above records that mask as "secret": it -// is a finding of that level's walk. +// never parsed (a comment, a trailing comma, an object literal) — and +// one the normalizer changes at all, as written: folding can move a +// value out of its secret-named member while the document still parses, +// so structure the normalizer makes, or moves, is not trusted. A secret +// across the strings of the document as written (spanning: a private key +// over the lines of an array, an exact value over them) makes the reason +// "secret" as well — as written, since the scan can mask a block's last +// line as a member's value before the block's own pattern runs, and +// leave its first line in the key. A level above records that mask as +// "secret": it is a finding of that level's walk. func (c *contentCollector) heldSecret(scanned, written string) (string, bool) { doc, ok := jsonDocument(scanned) if !ok { return "uninspected", documentLike(scanned) || documentLike(written) } - if c.held == maxHeldDepth || duplicateName(doc) { + raw, _ := jsonDocument(written) + if c.held == maxHeldDepth || duplicateName(doc) || c.normalized(raw) != raw || !json.Valid([]byte(raw)) { return "uninspected", true } - v, _ := decodeJSON(doc) // a valid document decodes mark := len(c.findings) + if c.spanning(raw) { + c.findings = c.findings[:mark] + return "secret", true + } + v, _ := decodeJSON(doc) // a valid document decodes c.held++ c.redactValue(v, "", new(bool)) c.held-- @@ -924,12 +945,113 @@ func (c *contentCollector) heldSecret(scanned, written string) (string, bool) { } // revealing reports a finding that shows a secret was there: any but the -// normalizer's, and its removal of an escape sequence or of tag -// characters, which can carry text the patterns never see — digits as a -// CSI parameter, a token inside an OSC sequence. A secret the scan masked in place is gone from the -// document by then, but for a mask that matches again. +// normalizer's, and the normalizer's stripping of content. A secret the +// scan masked in place is gone from the document by then, but for a mask +// that matches again. func revealing(f security.Finding) bool { - return f.Scanner != "unicode_normalizer" || f.Name == "ansi_escape" || f.Name == "osc_escape" || f.Name == "tag_char" + return f.Scanner != "unicode_normalizer" || stripped(f) +} + +// stripped reports the normalizer's removal of an escape sequence or of +// tag characters, which can carry text the patterns never see — digits +// as a CSI parameter, a token inside an OSC sequence — or take the first +// letter of a token with it, since a CSI sequence ends at a letter. +func stripped(f security.Finding) bool { + return f.Scanner == "unicode_normalizer" && (f.Name == "ansi_escape" || f.Name == "osc_escape" || f.Name == "tag_char") +} + +// scanLeaf redacts one string of the arguments, a value or a key: through +// the pipeline, which masks in place, and masked whole when the +// normalizer stripped an escape sequence or tag characters from it +// (stripped), since what is left can be most of a token. +func (c *contentCollector) scanLeaf(s string) string { + mark := len(c.findings) + r := c.redact(s, &c.findings) + if slices.ContainsFunc(c.findings[mark:], stripped) { + return "***" + } + return r +} + +// normalized returns the normalizer's rendering of s: a detection copy, +// which nothing exports. +func (c *contentCollector) normalized(s string) string { + return c.normalizer.Scan(s).Sanitized +} + +// spanning reports, and records as one critical finding, a secret that +// runs across the keys, strings and numbers of raw, one valid JSON value, +// as a reader of them in the order written sees them: a private key over +// the lines of an array or over a key and a value, a runtime value over +// the lines of one. Each is judged on its own (redactValue); here they +// are joined by a line break, each as decoded and as the normalizer +// renders it — the one can hold a marker inside an escape sequence the +// normalizer strips, the other a marker folding spells — before any is +// masked, since a string masked whole can take a secret's first or last +// line with it. Only a secret that can hold a line break is judged +// across them: the private key block, and an exact runtime value. Every +// other pattern's secret stops at white space or at a quote, so a match +// of one over the break has only run its context into the next string +// (an assignment whose value is the next string, a pair in single quotes +// over two strings' apostrophes), which is left unjudged, as a token +// split between two strings is. +func (c *contentCollector) spanning(raw string) bool { + var decoded, normalized []string + for _, sp := range atomSpans(raw) { + a := raw[sp[0]:sp[1]] + if sp[0] > 0 && raw[sp[0]-1] == '"' { + _ = json.Unmarshal([]byte(`"`+a+`"`), &a) // a string's escapes, decoded + } + decoded = append(decoded, a) + normalized = append(normalized, c.normalized(a)) + } + for _, atoms := range [][]string{decoded, normalized} { + text := strings.Join(atoms, "\n") + starts := make([]int, len(atoms)) // where each atom begins in text + for n := 1; n < len(atoms); n++ { + starts[n] = starts[n-1] + len(atoms[n-1]) + 1 + } + for _, m := range c.secrets.Matches(text) { + n := sort.Search(len(starts), func(n int) bool { return starts[n] > m.Start }) - 1 + if m.Name != "private_key" && m.Name != "runtime_secret" || m.End <= starts[n]+len(atoms[n]) { + continue // within one key, string or number: its own + } + c.findings = append(c.findings, security.Finding{Scanner: "secret_redactor", Name: m.Name, Severity: "critical", Detail: "secret spanning keys, strings or numbers", Position: -1}) + return true + } + } + return false +} + +// atomSpans lists where raw, a valid JSON value, spells each string +// (between its quotes) and each number, true, false and null, in order. +func atomSpans(raw string) [][2]int { + const structure = "{}[],: \t\r\n" + var spans [][2]int + for i := 0; i < len(raw); { + switch { + case raw[i] == '"': + j := i + 1 + for raw[j] != '"' { + if raw[j] == '\\' { + j++ + } + j++ + } + spans = append(spans, [2]int{i + 1, j}) + i = j + 1 + case strings.IndexByte(structure, raw[i]) >= 0: + i++ + default: + j := i + for j < len(raw) && strings.IndexByte(structure+`"`, raw[j]) < 0 { + j++ + } + spans = append(spans, [2]int{i, j}) + i = j + } + } + return spans } // redactValue redacts every string, number and object key under v; a @@ -949,7 +1071,7 @@ func revealing(f security.Finding) bool { func (c *contentCollector) redactValue(v any, under string, collided *bool) any { switch t := v.(type) { case string: - s := c.maskHeld(c.redact(t, &c.findings), t) + s := c.maskHeld(c.scanLeaf(t), t) if under != "" && c.secretNamed(under, s) { return "***" } @@ -972,10 +1094,13 @@ func (c *contentCollector) redactValue(v any, under string, collided *bool) any out := make(map[string]any, len(t)) for _, k := range slices.Sorted(maps.Keys(t)) { // The key as scanned names the secret, masked whole or not. - sk := c.redact(k, &c.findings) + sk := c.scanLeaf(k) rk := c.maskHeld(sk, k) member := under - named := c.namesASecret(sk) + // Named as scanned, or as written: the normalizer can strip + // the name's first letter with a colour code, or mask the key + // whole, and the value under it is still that name's. + named := c.namesASecret(sk) || c.namesASecret(k) if named { // Decided once for the member: the pattern reads any // name it matches alike, so a stand-in judges the values diff --git a/internal/cli/content_collector_test.go b/internal/cli/content_collector_test.go index d17168ba5d..a362432006 100644 --- a/internal/cli/content_collector_test.go +++ b/internal/cli/content_collector_test.go @@ -226,8 +226,13 @@ func TestContentCollector_JSONInAStringIsMaskedWholeWhenItsDocumentHoldsASecret( "folding breaks a document": {"{\"note\":\"\uFF02\"," + cred + "}", "***", 2}, "a BOM and folding that breaks a document": {"\uFEFF{\"note\":\"\uFF02\"," + cred + "}", "***", 3}, "a fold that makes and breaks a document": {"\uFF5B\"note\":\"\uFF02\"," + cred + "}", "***", 2}, - "an escape before a document folding breaks": {"\x1b[0m{\"note\":\"\uFF02\"," + cred + "}", "***", 3}, + "an escape before a document folding breaks": {"\x1b[0m{\"note\":\"\uFF02\"," + cred + "}", "***", 2}, "a mask breaks a document": {`{"token":"abcdefgh\\\"x",` + cred + `}`, "***", 2}, + // Folding can move a value out of its secret-named member while the + // document still parses; a document the normalizer changes at all + // is not judged. + "folding that moves a value between members": {"{\"credentials\":{\"value\":\"\uFF02},\uFF02note\uFF02:{\uFF02value\uFF02:\uFF02" + opaque + "\"}}", "***", 2}, + "folding that moves nothing": {"{\"name\":\"Widget\u2122\"}", "***", 2}, // Decoding keeps the last of two members of one name; the string // keeps both, so a repeated name cannot be judged either. "a repeated name": {`{` + cred + `,"credentials":{}}`, "***", 1}, @@ -278,12 +283,17 @@ func TestContentCollector_JSONInAStringIsAlsoScannedWhole(t *testing.T) { begin, end := "-----BEGIN RSA PRIVATE "+"KEY-----", "-----END RSA PRIVATE "+"KEY-----" body := strings.Repeat("Q", 40) notebook, _ := json.Marshal(map[string]any{"cells": []any{map[string]any{"source": []string{begin + "\n", body + "\n", end + "\n"}}}}) + security.RegisterRuntimeSecret("rtline1\nrtline2x") for name, tc := range map[string]struct { env map[string]string content, gone string findings int }{ - "key over a cell's lines": {nil, string(notebook), body, 1}, + "key over a cell's lines": {nil, string(notebook), body, 2}, + // Its strings in order, as written: the scan masks the block's last + // line as a member's value before the block's pattern runs. + "key from a member name into its value": {nil, `{"` + begin + `\n` + body[:20] + `":"` + body[20:] + `\n` + end + `"}`, body[:20], 2}, + "a runtime value over two strings": {nil, `{"lines":["rtline1","rtline2x"]}`, "rtline2x", 1}, // Its marker breaks the document, which cannot then be judged. "runner env value with structure": {map[string]string{"DB_PASSWORD": `hunter22","port`}, `{"pw":"hunter22","port":5432}`, "hunter22", 2}, "runner env value that ends the document": {map[string]string{"DB_PASSWORD": `hunter22"}`}, `{"pw":"hunter22"}`, "hunter22", 2}, @@ -309,6 +319,117 @@ func TestContentCollector_JSONInAStringIsAlsoScannedWhole(t *testing.T) { } } +func TestContentCollector_SecretsSpanningTheArgumentsStrings(t *testing.T) { + // Each key, string and number is judged on its own, and the level as a + // reader of them in the order written sees them: a private key over + // the lines of an array or over a key and a value, a runtime value + // over the lines of one. A secret found that way can be masked + // nowhere but the level whole: the arguments are dropped and charged + // as encoded. (An assignment or a header whose value is the next + // string is not judged: its secret stops at white space.) + begin, end := "-----BEGIN RSA PRIVATE "+"KEY-----", "-----END RSA PRIVATE "+"KEY-----" + body := strings.Repeat("Q", 40) + security.RegisterRuntimeSecret("rtline1\nrtline2x") + for name, tc := range map[string]struct { + args, gone string + findings int + }{ + "a private key over the lines of a cell": {`{"cells":[{"source":["` + begin + `\n","` + body + `\n","` + end + `\n"]}]}`, body, 1}, + "a private key over members against key order": {`{"b":"` + begin + `\n` + body[:20] + `","a":"` + body[20:] + `\n` + end + `"}`, body[:20], 1}, + "a private key over a key and a value": {`{"` + begin + `\n` + body[:20] + `":"` + body[20:] + `\n` + end + `"}`, body[:20], 2}, + "a runtime value over two strings": {`{"lines":["rtline1","rtline2x"]}`, "rtline2x", 1}, + "a marker inside an escape the normalizer strips": {`{"lines":["\u001b]0;` + begin + `\u0007\n","` + body + `\n","` + end + `\n"]}`, body, 2}, + "a marker folded": {`{"lines":["\uFF0D\uFF0D\uFF0D\uFF0D\uFF0DBEGIN RSA PRIVATE ` + `KEY-----\n","` + body + `\n","` + end + `\n"]}`, body, 2}, + } { + t.Run(name, func(t *testing.T) { + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{Name: "Write", Summary: "s", Arguments: tc.args}) + + res := c.Result("stop") + part := partAt(t, decodeOutputMessages(t, res.OutputMessages), 0) + assert.NotContains(t, part, "arguments") + assert.NotContains(t, part, "summary") + assert.Equal(t, true, part["fullsend.truncated"]) + assert.NotContains(t, res.OutputMessages, tc.gone) + assert.Len(t, res.Findings, tc.findings) + }) + } + t.Run("a private key in one string is that string's own", func(t *testing.T) { + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{Name: "Write", Summary: "s", Arguments: `{"content":"` + begin + `\n` + body + `\n` + end + `","file_path":"k.pem"}`}) + + res := c.Result("stop") + assert.Contains(t, res.OutputMessages, `"arguments":{"content":"[REDACTED PRIVATE KEY]","file_path":"k.pem"}`) + assert.Len(t, res.Findings, 1) + }) + t.Run("dropped arguments are charged as encoded", func(t *testing.T) { + c := newContentCollector(4096) + args := `{"lines": ["` + begin + `", "` + body + `", "` + end + `"]}` + c.Handle(agentruntime.ToolUseEvent{Name: "Write", Arguments: args}) + assert.Equal(t, len(args)-3*len(" "), c.Result("stop").DroppedBytes) + }) +} + +func TestContentCollector_JoinedNeighboursMakeNoSecret(t *testing.T) { + // Judged in order, two strings with no secret between them must not + // make one: a token stops at the end of its string, and so does a + // connection string's user. + for name, args := range map[string]string{ + "a database URL beside an address": `{"DATABASE_URL":"postgres://localhost:5432/dev","SMTP_FROM":"noreply@acme.io"}`, + "an empty assignment beside a description": `{"command":"export NPM_TOKEN=","description":"Configure npm auth"}`, + "names that run into a token's prefix": `{"service":"task-scheduler","namespace":"production_workloads"}`, + "a command's flags": `{"command":["flask","--app","hello","run","--no-reload"]}`, + "apostrophes of two strings": `{"old_string":"print(key + ': ' + str(cfg[key]))","new_string":"print(key + ': ' + str(cfg[key]))\nlog.debug('dumped')"}`, + "a connection string with no host": `{"pattern":"postgres://user:pass@","path":"src","output_mode":"files_with_matches"}`, + "a database URL before a remote": `{"database":"postgres://localhost/app","remote":"git+ssh://git@github.com/org/app.git"}`, + "and before a registry, nested": `{"environment":{"REDIS_URL":"redis://redis","NPM_CONFIG_REGISTRY":"https://npm.pkg.github.com/@acme"}}`, + } { + t.Run(name, func(t *testing.T) { + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{Name: "Write", Summary: "s", Arguments: args}) + + res := c.Result("stop") + var want, got any + require.NoError(t, json.Unmarshal([]byte(args), &want)) + got = partAt(t, decodeOutputMessages(t, res.OutputMessages), 0)["arguments"] + assert.Equal(t, want, got) + assert.Empty(t, res.Findings) + }) + } +} + +func TestContentCollector_AStringTheNormalizerStrippedIsMaskedWhole(t *testing.T) { + // The normalizer strips a terminal escape sequence or tag characters + // before the patterns run, and the stripping can take the first + // letter of a token (a CSI sequence ends at the next letter) or a + // whole payload the patterns never see: such a string, a value or a + // key, is masked whole. + token := "ghp_" + strings.Repeat("r", 36) + for name, tc := range map[string]struct { + args, want, gone string + }{ + "a colour code that eats a token's first letter": {`{"n":"\u001b[` + token + `"}`, `{"n":"***"}`, token[1:]}, + "a token inside a title code": {`{"command":"echo \u001b]` + token + `\u0007"}`, `{"command":"***"}`, token[4:]}, + "tag characters": {`{"t":"x\uDB40\uDC67y"}`, `{"t":"***"}`, "xy"}, + "a key": {`{"\u001b[0mpassword":"hunter2hunter2"}`, `{"***":"***"}`, "hunter2"}, + } { + t.Run(name, func(t *testing.T) { + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Summary: "s", Arguments: tc.args}) + + res := c.Result("stop") + assert.Contains(t, res.OutputMessages, `"arguments":`+tc.want) + assert.NotContains(t, res.OutputMessages, tc.gone) + assert.NotContains(t, res.OutputMessages, `"summary"`) + }) + } + t.Run("two keys masked alike collide", func(t *testing.T) { + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Arguments: `{"\u001b[0ma":1,"\u001b[1mb":2}`}) + assert.NotContains(t, c.Result("stop").OutputMessages, `"arguments"`) + }) +} + func TestContentCollector_JSONHeldPastTheBoundIsMaskedWhole(t *testing.T) { // Each level is scanned once more, so documents are walked down to // maxHeldDepth; one held deeper cannot be judged. diff --git a/internal/security/redactor.go b/internal/security/redactor.go index 6c29e01c09..5d49a76653 100644 --- a/internal/security/redactor.go +++ b/internal/security/redactor.go @@ -183,6 +183,43 @@ func (s *SecretRedactor) Scan(text string) ScanResult { return result } +// Match locates one secret in a text — a runtime value, or a pattern's +// whole match, the pair for a member-name one. Matches reports them +// without masking, over the text as given, for a caller that judges +// text it does not export: a scan would mask one match ahead of the +// next, and a mask can match a pattern again. +type Match struct { + Start, End int + Name string +} + +// Matches reports every secret in text; see Match. +func (s *SecretRedactor) Matches(text string) []Match { + var out []Match + for _, secret := range runtimeSecretSnapshot() { + for i := 0; ; { + j := strings.Index(text[i:], secret) + if j < 0 { + break + } + out = append(out, Match{i + j, i + j + len(secret), "runtime_secret"}) + i += j + len(secret) + } + } + add := func(p secretPattern) { + for _, loc := range p.regex.FindAllStringIndex(text, -1) { + out = append(out, Match{loc[0], loc[1], p.name}) + } + } + for _, p := range s.prefixPatterns { + add(p) + } + for _, p := range s.structuralPatterns { + add(p) + } + return out +} + func mask(value string) string { if len(value) < 10 { return "***" diff --git a/internal/security/redactor_matches_test.go b/internal/security/redactor_matches_test.go new file mode 100644 index 0000000000..36ccae94c5 --- /dev/null +++ b/internal/security/redactor_matches_test.go @@ -0,0 +1,29 @@ +package security + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSecretRedactor_Matches(t *testing.T) { + // Matches locates secrets without masking, over the text as given: a + // runtime value, a prefix token and a structural pattern's whole match, + // each by its bounds, so a caller can judge text it does not export. + RegisterRuntimeSecret("runtime-opaque-value-42") + t.Cleanup(resetRuntimeSecrets) + token := "ghp_" + strings.Repeat("a", 36) + text := "x runtime-opaque-value-42 " + token + " API_KEY=s3cr3tvalue99 end" + + got := map[string]string{} + for _, m := range NewSecretRedactor().Matches(text) { + got[m.Name] = text[m.Start:m.End] + } + assert.Equal(t, map[string]string{ + "runtime_secret": "runtime-opaque-value-42", + "github_pat": token, + "env_assignment": " API_KEY=s3cr3tvalue99", + }, got) + assert.Empty(t, NewSecretRedactor().Matches("nothing here")) +} From b1eb40b2f1029c408c0f780d0771e6101f06b872 Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Thu, 8 Oct 2026 09:53:23 -0400 Subject: [PATCH 22/40] fix(cli): judge across the strings whatever else drops the level Three review findings on 938a07c5f, each fixed by judging more. A level that two keys folding alike already dropped was not judged across its strings, so a private key split between a Bash command and its description raised no finding, and the summary the parser cut from the command kept the key's first line. The level is now judged across its strings whatever else drops it. A key the normalizer stripped a colour code from is masked whole, and its name was then not to be had: not from the mask, not from a spelling in compatibility forms the pattern does not read, and not from the normalizer's rendering when the stripping ate the first letter of the name folding spelled. A key masked whole now names a secret whatever it was called, so the values under it are judged as a secret-named member's. The judgement across the strings knew the patterns and the runtime secrets but not the runner env values the literal pass replaces, so one holding a line break, written over two strings of an array, was in neither string and reached the record. envLiterals, shared with replaceEnvSecrets, now lists them for it. Signed-off-by: Dharit Shah --- docs/guides/dev/tracing.md | 10 ++-- internal/cli/content_collector.go | 39 +++++++++++--- internal/cli/content_collector_test.go | 73 ++++++++++++++++++++++++++ internal/cli/run.go | 38 +++++++++----- 4 files changed, 134 insertions(+), 26 deletions(-) diff --git a/docs/guides/dev/tracing.md b/docs/guides/dev/tracing.md index cdd33ee130..d498a2ca03 100644 --- a/docs/guides/dev/tracing.md +++ b/docs/guides/dev/tracing.md @@ -278,14 +278,16 @@ twice. A string the normalizer stripped an escape sequence or tag characters from — a value or a key — is masked `***` whole: a colour code ends at the next letter, so the stripping can take a token's first letter and leave the rest, and a title code's payload is text no pattern sees. A -member is named as written as well as as scanned, so a key so masked -still names the values under it. Each level — the arguments, and each +member is named as written as well as as scanned, and a key masked whole +names a secret whatever it was called — its name is not to be had from +its mask, and stripping can eat the first letter of one folding spells — +so the values under it are judged as a secret-named member's. Each level — the arguments, and each held document — is also judged as a reader of its keys, strings and numbers in the order written sees them (`spanning`): joined by a line break, each as decoded and as the normalizer renders it, before any is masked, for a secret that can hold a line break — the private key block -over the lines of an array or over a key and a value, an exact runtime -value over the lines of one. The arguments are then dropped and charged +over the lines of an array or over a key and a value, an exact value (a +runtime secret, a runner environment value) over the lines of one. The arguments are then dropped and charged as encoded; a held document is masked whole (its text was scanned whole first, so a block that scan masked in place stays in place). Every other pattern's secret stops at white space or at a quote, so a match over the diff --git a/internal/cli/content_collector.go b/internal/cli/content_collector.go index 73e8aeb8d2..2d168421e8 100644 --- a/internal/cli/content_collector.go +++ b/internal/cli/content_collector.go @@ -820,7 +820,10 @@ func (c *contentCollector) toolArguments(args string) (json.RawMessage, bool) { if err != nil { return nil, false // decoded JSON values marshal unconditionally } - if collided || c.spanning(args) || len(out) > maxToolArgumentsBytes { + // Judged across its strings whatever else drops it: the secret found + // there is a finding of the call, and costs it its summary. + spanning := c.spanning(args) + if collided || spanning || len(out) > maxToolArgumentsBytes { c.evicted += len(out) return nil, true } @@ -989,7 +992,8 @@ func (c *contentCollector) normalized(s string) string { // normalizer strips, the other a marker folding spells — before any is // masked, since a string masked whole can take a secret's first or last // line with it. Only a secret that can hold a line break is judged -// across them: the private key block, and an exact runtime value. Every +// across them: the private key block, and an exact value — a runtime +// secret, or a runner env value the literal pass replaces. Every // other pattern's secret stops at white space or at a quote, so a match // of one over the break has only run its context into the next string // (an assignment whose value is the next string, a pair in single quotes @@ -1011,14 +1015,30 @@ func (c *contentCollector) spanning(raw string) bool { for n := 1; n < len(atoms); n++ { starts[n] = starts[n-1] + len(atoms[n-1]) + 1 } + within := func(start, end int) bool { // one key, string or number holds it: its own + n := sort.Search(len(starts), func(n int) bool { return starts[n] > start }) - 1 + return end <= starts[n]+len(atoms[n]) + } for _, m := range c.secrets.Matches(text) { - n := sort.Search(len(starts), func(n int) bool { return starts[n] > m.Start }) - 1 - if m.Name != "private_key" && m.Name != "runtime_secret" || m.End <= starts[n]+len(atoms[n]) { - continue // within one key, string or number: its own + if m.Name != "private_key" && m.Name != "runtime_secret" || within(m.Start, m.End) { + continue } c.findings = append(c.findings, security.Finding{Scanner: "secret_redactor", Name: m.Name, Severity: "critical", Detail: "secret spanning keys, strings or numbers", Position: -1}) return true } + for _, l := range envLiterals(c.runnerEnv) { + for i := 0; ; { + j := strings.Index(text[i:], l.value) + if j < 0 { + break + } + if i += j; !within(i, i+len(l.value)) { + c.findings = append(c.findings, security.Finding{Scanner: "runner_env", Name: l.key, Severity: "critical", Detail: "secret spanning keys, strings or numbers", Position: -1}) + return true + } + i += len(l.value) + } + } } return false } @@ -1098,9 +1118,12 @@ func (c *contentCollector) redactValue(v any, under string, collided *bool) any rk := c.maskHeld(sk, k) member := under // Named as scanned, or as written: the normalizer can strip - // the name's first letter with a colour code, or mask the key - // whole, and the value under it is still that name's. - named := c.namesASecret(sk) || c.namesASecret(k) + // the name's first letter with a colour code, and the value + // under it is still that name's. A key masked whole names a + // secret whatever it was called: its name is not to be had + // from its mask, and stripping can eat the first letter of + // one that folding spells. + named := sk == "***" || c.namesASecret(sk) || c.namesASecret(k) if named { // Decided once for the member: the pattern reads any // name it matches alike, so a stand-in judges the values diff --git a/internal/cli/content_collector_test.go b/internal/cli/content_collector_test.go index a362432006..0815b414e7 100644 --- a/internal/cli/content_collector_test.go +++ b/internal/cli/content_collector_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "math/rand" + "slices" "strings" "testing" "time" @@ -430,6 +431,78 @@ func TestContentCollector_AStringTheNormalizerStrippedIsMaskedWhole(t *testing.T }) } +func TestContentCollector_ACollisionStillCountsASecretAcrossTheStrings(t *testing.T) { + // Two keys folding alike drop the arguments on their own; the level is + // still judged across its strings, since the secret found there is + // what costs the call its summary — the parser cut it from these + // arguments, and a private key's first line can be in it. + begin, end := "-----BEGIN RSA PRIVATE "+"KEY-----", "-----END RSA PRIVATE "+"KEY-----" + body := strings.Repeat("Q", 40) + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Summary: begin + "\n" + body, Arguments: `{"a":"x","\uFF41":"y","command":"` + begin + `\n` + body + `","description":"` + end + `"}`}) + + res := c.Result("stop") + part := partAt(t, decodeOutputMessages(t, res.OutputMessages), 0) + assert.NotContains(t, part, "arguments") + assert.NotContains(t, part, "summary", "the secret across the strings costs the summary") + assert.NotContains(t, res.OutputMessages, body) + assert.True(t, slices.ContainsFunc(res.Findings, func(f security.Finding) bool { return f.Name == "private_key" }), "the spanning secret is recorded") +} + +func TestContentCollector_AKeyMaskedWholeNamesASecret(t *testing.T) { + // A key the normalizer stripped an escape sequence from is masked + // whole, and its name is then not to be had from its mask, nor from + // its spelling when that is in compatibility forms the pattern does + // not read, nor from the normalizer's rendering when the stripping ate + // the name's first letter: the values under such a key are judged as + // a secret-named member's, whatever it was called. + const opaque = "opaque-credential-123" + password := "\uFF50\uFF41\uFF53\uFF53\uFF57\uFF4F\uFF52\uFF44" // fullwidth + for name, tc := range map[string]struct{ args, want string }{ + "a colour code before a fullwidth name": {`{"\u001b[0m` + password + `":"` + opaque + `"}`, `{"***":"***"}`}, + "an unfinished colour code the fold completes": {`{"\u001b[` + password + `":"` + opaque + `"}`, `{"***":"***"}`}, + "and the values nested under it": {`{"\u001b[0m` + password + `":{"v":["` + opaque + `"]}}`, `{"***":{"v":["***"]}}`}, + "a colour code alone": {`{"\u001b[0mnote":"` + opaque + `"}`, `{"***":"***"}`}, + } { + t.Run(name, func(t *testing.T) { + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{Name: "Write", Summary: "s", Arguments: tc.args}) + + res := c.Result("stop") + assert.Contains(t, res.OutputMessages, `"arguments":`+tc.want) + assert.NotContains(t, res.OutputMessages, opaque) + }) + } +} + +func TestContentCollector_ARunnerEnvValueOverTheLinesOfAnArray(t *testing.T) { + // A runner env value can hold a line break; written over two strings + // of an array, no string holds it whole and the literal pass sees no + // part of it. Judged across the strings it is found: the arguments + // are dropped, a held document masked whole. + env := map[string]string{"DEPLOY_SECRET": "opaque-line-one\nopaque-line-two"} + t.Setenv(telemetry.ContentCaptureEnvVar, "true") + t.Run("arguments", func(t *testing.T) { + c := newContentCollectorIfEnabled(env) + c.Handle(agentruntime.ToolUseEvent{Name: "Write", Summary: "s", Arguments: `{"lines":["opaque-line-one","opaque-line-two"]}`}) + + res := c.Result("stop") + part := partAt(t, decodeOutputMessages(t, res.OutputMessages), 0) + assert.NotContains(t, part, "arguments") + assert.NotContains(t, part, "summary") + assert.NotContains(t, res.OutputMessages, "opaque-line") + assert.True(t, slices.ContainsFunc(res.Findings, func(f security.Finding) bool { return f.Scanner == "runner_env" && f.Name == "DEPLOY_SECRET" })) + }) + t.Run("held", func(t *testing.T) { + c := newContentCollectorIfEnabled(env) + c.Handle(agentruntime.ToolUseEvent{Name: "Write", Arguments: `{"content":"{\"lines\":[\"opaque-line-one\",\"opaque-line-two\"]}"}`}) + + res := c.Result("stop") + assert.Contains(t, res.OutputMessages, `"arguments":{"content":"***"}`) + assert.NotContains(t, res.OutputMessages, "opaque-line") + }) +} + func TestContentCollector_JSONHeldPastTheBoundIsMaskedWhole(t *testing.T) { // Each level is scanned once more, so documents are walked down to // maxHeldDepth; one held deeper cannot be judged. diff --git a/internal/cli/run.go b/internal/cli/run.go index a8347f5bce..657253b423 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -3649,30 +3649,40 @@ func redactFeedback(feedback string, runnerEnv map[string]string) string { // order: a value that contains another is replaced whole, whichever source // holds it, and the text is the same on every run. func replaceEnvSecrets(text string, runnerEnv map[string]string) (string, []string) { - type literal struct{ key, value string } - var literals []literal + var replaced []string + for _, l := range envLiterals(runnerEnv) { + if !strings.Contains(text, l.value) { + continue + } + text = strings.ReplaceAll(text, l.value, "[REDACTED:"+l.key+"]") + replaced = append(replaced, l.key) + } + return text, replaced +} + +// envLiteral is one value the literal pass replaces, with the key it +// names in the mask. +type envLiteral struct{ key, value string } + +// envLiterals lists the values replaceEnvSecrets replaces — longer values +// first, then key order — for it and for a caller that must find one +// where no replacement can mask it (the collector's spanning). +func envLiterals(runnerEnv map[string]string) []envLiteral { + var literals []envLiteral for key, value := range runnerEnv { if len(value) >= minRedactableSecretLen && sensitiveEnvKey(key) { - literals = append(literals, literal{key, value}) + literals = append(literals, envLiteral{key, value}) } } for key := range providerOnlyKeys { if value := os.Getenv(key); len(value) >= minRedactableSecretLen { - literals = append(literals, literal{key, value}) + literals = append(literals, envLiteral{key, value}) } } - slices.SortFunc(literals, func(a, b literal) int { + slices.SortFunc(literals, func(a, b envLiteral) int { return cmp.Or(cmp.Compare(len(b.value), len(a.value)), cmp.Compare(a.key, b.key), cmp.Compare(a.value, b.value)) }) - var replaced []string - for _, l := range literals { - if !strings.Contains(text, l.value) { - continue - } - text = strings.ReplaceAll(text, l.value, "[REDACTED:"+l.key+"]") - replaced = append(replaced, l.key) - } - return text, replaced + return literals } // writeValidationFeedback writes the validation failure output to a file in From 2b07a78ccbf07bd4e333d9e0583ac8584527ef95 Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Thu, 8 Oct 2026 10:17:22 -0400 Subject: [PATCH 23/40] fix(cli): judge the strings joined by nothing as well A line of a notebook cell keeps its own line break, so an exact value holding one, written over two such lines, was whole in neither string nor in the strings joined by a line break, which put a second break between them. Each level is now judged over its strings joined by nothing as well, for the private key block and the exact values alone. Signed-off-by: Dharit Shah --- docs/guides/dev/tracing.md | 3 +- internal/cli/content_collector.go | 56 ++++++++++++++------------ internal/cli/content_collector_test.go | 43 ++++++++++++-------- 3 files changed, 59 insertions(+), 43 deletions(-) diff --git a/docs/guides/dev/tracing.md b/docs/guides/dev/tracing.md index d498a2ca03..6705f9cb5c 100644 --- a/docs/guides/dev/tracing.md +++ b/docs/guides/dev/tracing.md @@ -284,7 +284,8 @@ its mask, and stripping can eat the first letter of one folding spells — so the values under it are judged as a secret-named member's. Each level — the arguments, and each held document — is also judged as a reader of its keys, strings and numbers in the order written sees them (`spanning`): joined by a line -break, each as decoded and as the normalizer renders it, before any is +break, and by nothing (a line of a notebook cell keeps its own line +break), each as decoded and as the normalizer renders it, before any is masked, for a secret that can hold a line break — the private key block over the lines of an array or over a key and a value, an exact value (a runtime secret, a runner environment value) over the lines of one. The arguments are then dropped and charged diff --git a/internal/cli/content_collector.go b/internal/cli/content_collector.go index 2d168421e8..f22194f8fc 100644 --- a/internal/cli/content_collector.go +++ b/internal/cli/content_collector.go @@ -987,7 +987,8 @@ func (c *contentCollector) normalized(s string) string { // as a reader of them in the order written sees them: a private key over // the lines of an array or over a key and a value, a runtime value over // the lines of one. Each is judged on its own (redactValue); here they -// are joined by a line break, each as decoded and as the normalizer +// are joined by a line break, and by nothing — a line of a notebook +// cell keeps its own line break — each as decoded and as the normalizer // renders it — the one can hold a marker inside an escape sequence the // normalizer strips, the other a marker folding spells — before any is // masked, since a string masked whole can take a secret's first or last @@ -1009,34 +1010,39 @@ func (c *contentCollector) spanning(raw string) bool { decoded = append(decoded, a) normalized = append(normalized, c.normalized(a)) } + // Joined by a line break, and joined by nothing: a line of a + // notebook cell keeps its own line break, so an exact value over two + // such lines is whole only in the second text. for _, atoms := range [][]string{decoded, normalized} { - text := strings.Join(atoms, "\n") - starts := make([]int, len(atoms)) // where each atom begins in text - for n := 1; n < len(atoms); n++ { - starts[n] = starts[n-1] + len(atoms[n-1]) + 1 - } - within := func(start, end int) bool { // one key, string or number holds it: its own - n := sort.Search(len(starts), func(n int) bool { return starts[n] > start }) - 1 - return end <= starts[n]+len(atoms[n]) - } - for _, m := range c.secrets.Matches(text) { - if m.Name != "private_key" && m.Name != "runtime_secret" || within(m.Start, m.End) { - continue + for _, sep := range []string{"\n", ""} { + text := strings.Join(atoms, sep) + starts := make([]int, len(atoms)) // where each atom begins in text + for n := 1; n < len(atoms); n++ { + starts[n] = starts[n-1] + len(atoms[n-1]) + len(sep) } - c.findings = append(c.findings, security.Finding{Scanner: "secret_redactor", Name: m.Name, Severity: "critical", Detail: "secret spanning keys, strings or numbers", Position: -1}) - return true - } - for _, l := range envLiterals(c.runnerEnv) { - for i := 0; ; { - j := strings.Index(text[i:], l.value) - if j < 0 { - break + within := func(start, end int) bool { // one key, string or number holds it: its own + n := sort.Search(len(starts), func(n int) bool { return starts[n] > start }) - 1 + return end <= starts[n]+len(atoms[n]) + } + for _, m := range c.secrets.Matches(text) { + if m.Name != "private_key" && m.Name != "runtime_secret" || within(m.Start, m.End) { + continue } - if i += j; !within(i, i+len(l.value)) { - c.findings = append(c.findings, security.Finding{Scanner: "runner_env", Name: l.key, Severity: "critical", Detail: "secret spanning keys, strings or numbers", Position: -1}) - return true + c.findings = append(c.findings, security.Finding{Scanner: "secret_redactor", Name: m.Name, Severity: "critical", Detail: "secret spanning keys, strings or numbers", Position: -1}) + return true + } + for _, l := range envLiterals(c.runnerEnv) { + for i := 0; ; { + j := strings.Index(text[i:], l.value) + if j < 0 { + break + } + if i += j; !within(i, i+len(l.value)) { + c.findings = append(c.findings, security.Finding{Scanner: "runner_env", Name: l.key, Severity: "critical", Detail: "secret spanning keys, strings or numbers", Position: -1}) + return true + } + i += len(l.value) } - i += len(l.value) } } } diff --git a/internal/cli/content_collector_test.go b/internal/cli/content_collector_test.go index 0815b414e7..2af58ad85e 100644 --- a/internal/cli/content_collector_test.go +++ b/internal/cli/content_collector_test.go @@ -339,6 +339,7 @@ func TestContentCollector_SecretsSpanningTheArgumentsStrings(t *testing.T) { "a private key over members against key order": {`{"b":"` + begin + `\n` + body[:20] + `","a":"` + body[20:] + `\n` + end + `"}`, body[:20], 1}, "a private key over a key and a value": {`{"` + begin + `\n` + body[:20] + `":"` + body[20:] + `\n` + end + `"}`, body[:20], 2}, "a runtime value over two strings": {`{"lines":["rtline1","rtline2x"]}`, "rtline2x", 1}, + "and over two lines keeping their breaks": {`{"lines":["rtline1\n","rtline2x"]}`, "rtline2x", 1}, "a marker inside an escape the normalizer strips": {`{"lines":["\u001b]0;` + begin + `\u0007\n","` + body + `\n","` + end + `\n"]}`, body, 2}, "a marker folded": {`{"lines":["\uFF0D\uFF0D\uFF0D\uFF0D\uFF0DBEGIN RSA PRIVATE ` + `KEY-----\n","` + body + `\n","` + end + `\n"]}`, body, 2}, } { @@ -480,27 +481,35 @@ func TestContentCollector_ARunnerEnvValueOverTheLinesOfAnArray(t *testing.T) { // of an array, no string holds it whole and the literal pass sees no // part of it. Judged across the strings it is found: the arguments // are dropped, a held document masked whole. + // Each line of a notebook cell keeps its own line break: the strings + // are judged joined end to end as well. env := map[string]string{"DEPLOY_SECRET": "opaque-line-one\nopaque-line-two"} t.Setenv(telemetry.ContentCaptureEnvVar, "true") - t.Run("arguments", func(t *testing.T) { - c := newContentCollectorIfEnabled(env) - c.Handle(agentruntime.ToolUseEvent{Name: "Write", Summary: "s", Arguments: `{"lines":["opaque-line-one","opaque-line-two"]}`}) + for name, lines := range map[string]string{ + "lines": `["opaque-line-one","opaque-line-two"]`, + "lines keeping the break": `["opaque-line-one\n","opaque-line-two"]`, + } { + t.Run(name+", arguments", func(t *testing.T) { + c := newContentCollectorIfEnabled(env) + c.Handle(agentruntime.ToolUseEvent{Name: "Write", Summary: "s", Arguments: `{"lines":` + lines + `}`}) - res := c.Result("stop") - part := partAt(t, decodeOutputMessages(t, res.OutputMessages), 0) - assert.NotContains(t, part, "arguments") - assert.NotContains(t, part, "summary") - assert.NotContains(t, res.OutputMessages, "opaque-line") - assert.True(t, slices.ContainsFunc(res.Findings, func(f security.Finding) bool { return f.Scanner == "runner_env" && f.Name == "DEPLOY_SECRET" })) - }) - t.Run("held", func(t *testing.T) { - c := newContentCollectorIfEnabled(env) - c.Handle(agentruntime.ToolUseEvent{Name: "Write", Arguments: `{"content":"{\"lines\":[\"opaque-line-one\",\"opaque-line-two\"]}"}`}) + res := c.Result("stop") + part := partAt(t, decodeOutputMessages(t, res.OutputMessages), 0) + assert.NotContains(t, part, "arguments") + assert.NotContains(t, part, "summary") + assert.NotContains(t, res.OutputMessages, "opaque-line") + assert.True(t, slices.ContainsFunc(res.Findings, func(f security.Finding) bool { return f.Scanner == "runner_env" && f.Name == "DEPLOY_SECRET" })) + }) + t.Run(name+", held", func(t *testing.T) { + c := newContentCollectorIfEnabled(env) + content, _ := json.Marshal(`{"lines":` + lines + `}`) + c.Handle(agentruntime.ToolUseEvent{Name: "Write", Arguments: `{"content":` + string(content) + `}`}) - res := c.Result("stop") - assert.Contains(t, res.OutputMessages, `"arguments":{"content":"***"}`) - assert.NotContains(t, res.OutputMessages, "opaque-line") - }) + res := c.Result("stop") + assert.Contains(t, res.OutputMessages, `"arguments":{"content":"***"}`) + assert.NotContains(t, res.OutputMessages, "opaque-line") + }) + } } func TestContentCollector_JSONHeldPastTheBoundIsMaskedWhole(t *testing.T) { From f713132bc747ea8dd22bff0bace70e2b0a1b4062 Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Thu, 8 Oct 2026 10:36:04 -0400 Subject: [PATCH 24/40] fix(cli): seek an exact value in the strings with white space aside An exact value over the lines of an array was sought in the strings joined by a line break, then joined by nothing as well, since a line of a notebook cell keeps its own break; a Windows line ending or a trailing space would have been the next case, and each case a view of its own. It is now sought once, in the strings in order with white space set aside, which reads the value as a reader of the lines does, however a line ends and wherever the value was cut. The private key block keeps the line-break join its pattern reads over. RuntimeSecrets lists the registered values for it. Signed-off-by: Dharit Shah --- docs/guides/dev/tracing.md | 13 ++- internal/cli/content_collector.go | 129 ++++++++++++++------- internal/cli/content_collector_test.go | 13 ++- internal/security/redactor.go | 4 + internal/security/redactor_matches_test.go | 13 +++ 5 files changed, 120 insertions(+), 52 deletions(-) diff --git a/docs/guides/dev/tracing.md b/docs/guides/dev/tracing.md index 6705f9cb5c..76f7e45ecc 100644 --- a/docs/guides/dev/tracing.md +++ b/docs/guides/dev/tracing.md @@ -283,12 +283,13 @@ names a secret whatever it was called — its name is not to be had from its mask, and stripping can eat the first letter of one folding spells — so the values under it are judged as a secret-named member's. Each level — the arguments, and each held document — is also judged as a reader of its keys, strings and -numbers in the order written sees them (`spanning`): joined by a line -break, and by nothing (a line of a notebook cell keeps its own line -break), each as decoded and as the normalizer renders it, before any is -masked, for a secret that can hold a line break — the private key block -over the lines of an array or over a key and a value, an exact value (a -runtime secret, a runner environment value) over the lines of one. The arguments are then dropped and charged +numbers in the order written sees them (`spanning`), each as decoded and +as the normalizer renders it, before any is masked, for a secret that +can hold a line break: the private key block, sought in them joined by +a line break (over the lines of an array, over a key and a value), and +an exact value (a runtime secret, a runner environment value), sought in +them in order with white space set aside — however a line ends and +wherever the value was cut. The arguments are then dropped and charged as encoded; a held document is masked whole (its text was scanned whole first, so a block that scan masked in place stays in place). Every other pattern's secret stops at white space or at a quote, so a match over the diff --git a/internal/cli/content_collector.go b/internal/cli/content_collector.go index f22194f8fc..2bb1d7b5a4 100644 --- a/internal/cli/content_collector.go +++ b/internal/cli/content_collector.go @@ -985,21 +985,25 @@ func (c *contentCollector) normalized(s string) string { // spanning reports, and records as one critical finding, a secret that // runs across the keys, strings and numbers of raw, one valid JSON value, // as a reader of them in the order written sees them: a private key over -// the lines of an array or over a key and a value, a runtime value over +// the lines of an array or over a key and a value, an exact value over // the lines of one. Each is judged on its own (redactValue); here they -// are joined by a line break, and by nothing — a line of a notebook -// cell keeps its own line break — each as decoded and as the normalizer -// renders it — the one can hold a marker inside an escape sequence the -// normalizer strips, the other a marker folding spells — before any is -// masked, since a string masked whole can take a secret's first or last -// line with it. Only a secret that can hold a line break is judged -// across them: the private key block, and an exact value — a runtime -// secret, or a runner env value the literal pass replaces. Every +// are judged together, each as decoded and as the normalizer renders it +// — the one can hold a marker inside an escape sequence the normalizer +// strips, the other a marker folding spells — before any is masked, +// since a string masked whole can take a secret's first or last line +// with it. Only a secret that can hold a line break is judged across +// them. The private key block is sought in the strings joined by a line +// break, which its pattern reads over. An exact value — a runtime +// secret, a runner env value the literal pass replaces — is sought in +// the strings in order with white space set aside, so however a line +// ends (its own line break, a Windows one, a space) and wherever the +// value was cut, it is read as a reader of the lines reads it. Every // other pattern's secret stops at white space or at a quote, so a match -// of one over the break has only run its context into the next string -// (an assignment whose value is the next string, a pair in single quotes +// of one over a break has only run its context into the next string (an +// assignment whose value is the next string, a pair in single quotes // over two strings' apostrophes), which is left unjudged, as a token -// split between two strings is. +// split between two strings is. A match within one key, string or +// number is that one's own. func (c *contentCollector) spanning(raw string) bool { var decoded, normalized []string for _, sp := range atomSpans(raw) { @@ -1010,45 +1014,86 @@ func (c *contentCollector) spanning(raw string) bool { decoded = append(decoded, a) normalized = append(normalized, c.normalized(a)) } - // Joined by a line break, and joined by nothing: a line of a - // notebook cell keeps its own line break, so an exact value over two - // such lines is whole only in the second text. + found := func(scanner, name string) bool { + c.findings = append(c.findings, security.Finding{Scanner: scanner, Name: name, Severity: "critical", Detail: "secret spanning keys, strings or numbers", Position: -1}) + return true + } for _, atoms := range [][]string{decoded, normalized} { - for _, sep := range []string{"\n", ""} { - text := strings.Join(atoms, sep) - starts := make([]int, len(atoms)) // where each atom begins in text - for n := 1; n < len(atoms); n++ { - starts[n] = starts[n-1] + len(atoms[n-1]) + len(sep) - } - within := func(start, end int) bool { // one key, string or number holds it: its own - n := sort.Search(len(starts), func(n int) bool { return starts[n] > start }) - 1 - return end <= starts[n]+len(atoms[n]) + lines, bounds := joined(atoms, "\n", plain) + for _, m := range c.secrets.Matches(lines) { + if m.Name == "private_key" && !bounds.within(m.Start, m.End) { + return found("secret_redactor", m.Name) } - for _, m := range c.secrets.Matches(text) { - if m.Name != "private_key" && m.Name != "runtime_secret" || within(m.Start, m.End) { - continue - } - c.findings = append(c.findings, security.Finding{Scanner: "secret_redactor", Name: m.Name, Severity: "critical", Detail: "secret spanning keys, strings or numbers", Position: -1}) - return true + } + dense, bounds := joined(atoms, "", unspaced) + for _, v := range security.RuntimeSecrets() { + if bounds.holds(dense, unspaced(v)) { + return found("secret_redactor", "runtime_secret") } - for _, l := range envLiterals(c.runnerEnv) { - for i := 0; ; { - j := strings.Index(text[i:], l.value) - if j < 0 { - break - } - if i += j; !within(i, i+len(l.value)) { - c.findings = append(c.findings, security.Finding{Scanner: "runner_env", Name: l.key, Severity: "critical", Detail: "secret spanning keys, strings or numbers", Position: -1}) - return true - } - i += len(l.value) - } + } + for _, l := range envLiterals(c.runnerEnv) { + if bounds.holds(dense, unspaced(l.value)) { + return found("runner_env", l.key) } } } return false } +// atomBounds is where each atom begins and ends in a text joined of them. +type atomBounds [][2]int + +// joined writes atoms as render renders them, sep between, and where +// each is in the text. +func joined(atoms []string, sep string, render func(string) string) (string, atomBounds) { + var b strings.Builder + bounds := make(atomBounds, len(atoms)) + for n, a := range atoms { + if n > 0 { + b.WriteString(sep) + } + bounds[n][0] = b.Len() + b.WriteString(render(a)) + bounds[n][1] = b.Len() + } + return b.String(), bounds +} + +// within reports whether one atom holds text[start:end] whole. +func (bounds atomBounds) within(start, end int) bool { + n := sort.Search(len(bounds), func(n int) bool { return bounds[n][0] > start }) - 1 + return n >= 0 && end <= bounds[n][1] +} + +// holds reports whether text holds value other than within one atom. +func (bounds atomBounds) holds(text, value string) bool { + if value == "" { + return false + } + for i := 0; ; { + j := strings.Index(text[i:], value) + if j < 0 { + return false + } + if i += j; !bounds.within(i, i+len(value)) { + return true + } + i += len(value) + } +} + +// plain renders a string as it is; unspaced without its white space. +func plain(s string) string { return s } + +func unspaced(s string) string { + return strings.Map(func(r rune) rune { + if unicode.IsSpace(r) { + return -1 + } + return r + }, s) +} + // atomSpans lists where raw, a valid JSON value, spells each string // (between its quotes) and each number, true, false and null, in order. func atomSpans(raw string) [][2]int { diff --git a/internal/cli/content_collector_test.go b/internal/cli/content_collector_test.go index 2af58ad85e..4e2f2766f8 100644 --- a/internal/cli/content_collector_test.go +++ b/internal/cli/content_collector_test.go @@ -340,6 +340,7 @@ func TestContentCollector_SecretsSpanningTheArgumentsStrings(t *testing.T) { "a private key over a key and a value": {`{"` + begin + `\n` + body[:20] + `":"` + body[20:] + `\n` + end + `"}`, body[:20], 2}, "a runtime value over two strings": {`{"lines":["rtline1","rtline2x"]}`, "rtline2x", 1}, "and over two lines keeping their breaks": {`{"lines":["rtline1\n","rtline2x"]}`, "rtline2x", 1}, + "and over two lines keeping a Windows break": {`{"lines":["rtline1\r\n","rtline2x"]}`, "rtline2x", 1}, "a marker inside an escape the normalizer strips": {`{"lines":["\u001b]0;` + begin + `\u0007\n","` + body + `\n","` + end + `\n"]}`, body, 2}, "a marker folded": {`{"lines":["\uFF0D\uFF0D\uFF0D\uFF0D\uFF0DBEGIN RSA PRIVATE ` + `KEY-----\n","` + body + `\n","` + end + `\n"]}`, body, 2}, } { @@ -481,13 +482,17 @@ func TestContentCollector_ARunnerEnvValueOverTheLinesOfAnArray(t *testing.T) { // of an array, no string holds it whole and the literal pass sees no // part of it. Judged across the strings it is found: the arguments // are dropped, a held document masked whole. - // Each line of a notebook cell keeps its own line break: the strings - // are judged joined end to end as well. + // Each line of a notebook cell keeps its own line break, a Windows + // one or a trailing space: an exact value is sought in the strings in + // order, white space aside. env := map[string]string{"DEPLOY_SECRET": "opaque-line-one\nopaque-line-two"} t.Setenv(telemetry.ContentCaptureEnvVar, "true") for name, lines := range map[string]string{ - "lines": `["opaque-line-one","opaque-line-two"]`, - "lines keeping the break": `["opaque-line-one\n","opaque-line-two"]`, + "lines": `["opaque-line-one","opaque-line-two"]`, + "lines keeping the break": `["opaque-line-one\n","opaque-line-two"]`, + "lines keeping a Windows break": `["opaque-line-one\r\n","opaque-line-two"]`, + "a line ending in a space": `["opaque-line-one \n","opaque-line-two"]`, + "a value split mid-word": `["opaque-li","ne-one\nopaque-line-two"]`, } { t.Run(name+", arguments", func(t *testing.T) { c := newContentCollectorIfEnabled(env) diff --git a/internal/security/redactor.go b/internal/security/redactor.go index 5d49a76653..482e58a2c2 100644 --- a/internal/security/redactor.go +++ b/internal/security/redactor.go @@ -39,6 +39,10 @@ func RegisterRuntimeSecret(value string) bool { return true } +// RuntimeSecrets returns the registered values, for a caller that must +// find one where no scan can mask it. +func RuntimeSecrets() []string { return runtimeSecretSnapshot() } + func runtimeSecretSnapshot() []string { runtimeSecretsMu.RLock() defer runtimeSecretsMu.RUnlock() diff --git a/internal/security/redactor_matches_test.go b/internal/security/redactor_matches_test.go index 36ccae94c5..74a9307bb6 100644 --- a/internal/security/redactor_matches_test.go +++ b/internal/security/redactor_matches_test.go @@ -27,3 +27,16 @@ func TestSecretRedactor_Matches(t *testing.T) { }, got) assert.Empty(t, NewSecretRedactor().Matches("nothing here")) } + +func TestRuntimeSecrets(t *testing.T) { + // The registered values, for a caller that must find one where no scan + // can mask it; a copy, so the caller cannot change the registry. + t.Cleanup(resetRuntimeSecrets) + resetRuntimeSecrets() + assert.Empty(t, RuntimeSecrets()) + RegisterRuntimeSecret("runtime-opaque-value-42") + got := RuntimeSecrets() + assert.Equal(t, []string{"runtime-opaque-value-42"}, got) + got[0] = "changed" + assert.Equal(t, []string{"runtime-opaque-value-42"}, RuntimeSecrets()) +} From fa2f0b43f020b721c899d26e8d82efb03127773d Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Thu, 8 Oct 2026 11:02:58 -0400 Subject: [PATCH 25/40] fix(cli): a key masked whole as a document names a secret too A key masked whole names a secret whatever it was called, but the check read the key as scanned, before the document it holds could mask it: a key holding a document that names a secret in a spelling only decoding reads was masked, and the value under it kept. The check now reads the key as it is exported. Signed-off-by: Dharit Shah --- internal/cli/content_collector.go | 10 +++++----- internal/cli/content_collector_test.go | 4 ++++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/internal/cli/content_collector.go b/internal/cli/content_collector.go index 2bb1d7b5a4..cac42071fa 100644 --- a/internal/cli/content_collector.go +++ b/internal/cli/content_collector.go @@ -1170,11 +1170,11 @@ func (c *contentCollector) redactValue(v any, under string, collided *bool) any member := under // Named as scanned, or as written: the normalizer can strip // the name's first letter with a colour code, and the value - // under it is still that name's. A key masked whole names a - // secret whatever it was called: its name is not to be had - // from its mask, and stripping can eat the first letter of - // one that folding spells. - named := sk == "***" || c.namesASecret(sk) || c.namesASecret(k) + // under it is still that name's. A key masked whole — by the + // scan, or as a document it holds — names a secret whatever + // it was called: its name is not to be had from its mask, and + // a document can name one in a spelling only decoding reads. + named := rk == "***" || c.namesASecret(sk) || c.namesASecret(k) if named { // Decided once for the member: the pattern reads any // name it matches alike, so a stand-in judges the values diff --git a/internal/cli/content_collector_test.go b/internal/cli/content_collector_test.go index 4e2f2766f8..03b3e06ca6 100644 --- a/internal/cli/content_collector_test.go +++ b/internal/cli/content_collector_test.go @@ -465,6 +465,10 @@ func TestContentCollector_AKeyMaskedWholeNamesASecret(t *testing.T) { "an unfinished colour code the fold completes": {`{"\u001b[` + password + `":"` + opaque + `"}`, `{"***":"***"}`}, "and the values nested under it": {`{"\u001b[0m` + password + `":{"v":["` + opaque + `"]}}`, `{"***":{"v":["***"]}}`}, "a colour code alone": {`{"\u001b[0mnote":"` + opaque + `"}`, `{"***":"***"}`}, + // Masked whole as a document it holds, with a name only decoding + // spells: no keyword to read in the key itself. + "a key holding a document naming a secret": {`{"{\"\\u0070assword\":{\"v\":\"opaque-inner-value\"}}":"` + opaque + `"}`, `{"***":"***"}`}, + "and the values nested under that": {`{"{\"\\u0070assword\":{\"v\":\"opaque-inner-value\"}}":{"v":["` + opaque + `"]}}`, `{"***":{"v":["***"]}}`}, } { t.Run(name, func(t *testing.T) { c := newContentCollector(4096) From bb50b213bafb8e2dfd7bd1160bd8dd70d88537e4 Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Thu, 8 Oct 2026 12:21:07 -0400 Subject: [PATCH 26/40] refactor(cli): judge tool arguments per leaf and drop on two coarse rules The cross-string judgement masked as little as it could: it located each key, string and number in the text as written with a scanner of its own, judged their joins by a line break and by nothing against match offsets, and judged a held document both as scanned and as written. Each view had an edge, and each review round found the input past it. Each level is now judged in three coarse ways, every one of which only masks or drops more. Each key, string and number is redacted on its own, as before. A string holding a JSON document is judged as the arguments are: decoded from the string as written, walked under the same member, encoded again, so the structure a reader decodes is the structure that was judged, at the cost of the document's layout. One the normalizer changes at all, one held past the depth bound, one whose keys redact alike, text shaped like a document that does not parse, and one whose text holds an exact value with structure of its own are masked whole. Of two members of one name decoding keeps the last and the record keeps what was decoded, so the duplicate-name walk is gone. The atoms of the arguments, their strings and numbers decoded in the order written by the standard decoder's token walk, a held document's in its place, are judged joined by a line break for the private key block, as decoded and as the normalizer renders them, and in order with white space set aside for an exact runtime or runner env value, sought in the text as written as well when it holds structure of its own; either drops the arguments. A private key inside one string drops them too, rather than masking the block in place. The exported set shrinks or holds in every case the tests had. The fidelity changes: a held document is encoded again; a document with a compatibility character is masked whole; a private key or an exact value over the lines of a held document drops the arguments instead of masking the string. The shelved rewrite's leak rows replay as before: only the documented limits stay red (a member-name pair a string's own quote closes, an assignment whose value is the next number). Signed-off-by: Dharit Shah --- docs/guides/dev/tracing.md | 105 +++-- .../infrastructure/distributed-tracing.md | 28 +- internal/cli/content_collector.go | 388 ++++++++---------- internal/cli/content_collector_test.go | 147 +++---- 4 files changed, 313 insertions(+), 355 deletions(-) diff --git a/docs/guides/dev/tracing.md b/docs/guides/dev/tracing.md index 76f7e45ecc..e9e728c61e 100644 --- a/docs/guides/dev/tracing.md +++ b/docs/guides/dev/tracing.md @@ -246,59 +246,58 @@ also turn a fullwidth quotation mark into one that closes the string. `toolArguments` decodes the value, redacts each string and object key on its own (a number as its digits; one that redacts becomes the redacted string), and encodes the result again. A string value or key that holds -one JSON object, array or string literal once scanned — what is exported -— is judged decoded as well (`heldSecret`). The scan reads the string whole, since a -secret can span a document's strings, but as text: it misses a -secret-named member nested deeper than the value right after the key, and -a secret only decoding spells out (an assignment that opens a string, a -token or runner environment value written with an escape). So the -document is walked the same way, its output and findings discarded; when -the walk raises a finding — any but the normalizer's, save its removal of -an escape sequence or of tag characters, which can carry text no pattern -sees (digits as a CSI parameter, a token inside an OSC sequence); a -secret the scan masked in place no longer shows — the string is -masked `***` whole, with one `held_document` finding. So is a document -that cannot be judged: held more than `maxHeldDepth` (4) deep, each level -being scanned once more; naming a member twice, since decoding keeps only -the last (RFC 7493 forbids it); or shaped like a document — it begins -and ends like one, white space and invisible characters aside, and -quotes something — but not parsing, whether the scan broke it, folding -or masking it, or it never parsed; or one the normalizer changes at all, -as written: folding can move a value out of its secret-named member while -the document still parses (fullwidth quotation marks), so structure the -normalizer makes, or moves, is not trusted. That masks whole text with no secret in it too — a document with -a repeated name, one a pattern misreads once decoded (a notebook line -`key = …`), one holding an escaped terminal colour code, JSON with -comments or trailing commas, an object literal or a Python dict with a -quoted value, an `Edit` fragment shaped `{…}` with one — and a mask the walk -matches again (a connection string's password of ten bytes or more, the -runner environment marker in an assignment, a mask of eight bytes or more -under a secret-named member) masks its string whole and counts its secret -twice. A string the normalizer stripped an escape sequence or tag -characters from — a value or a key — is masked `***` whole: a colour code -ends at the next letter, so the stripping can take a token's first letter -and leave the rest, and a title code's payload is text no pattern sees. A -member is named as written as well as as scanned, and a key masked whole -names a secret whatever it was called — its name is not to be had from -its mask, and stripping can eat the first letter of one folding spells — -so the values under it are judged as a secret-named member's. Each level — the arguments, and each -held document — is also judged as a reader of its keys, strings and -numbers in the order written sees them (`spanning`), each as decoded and -as the normalizer renders it, before any is masked, for a secret that -can hold a line break: the private key block, sought in them joined by -a line break (over the lines of an array, over a key and a value), and -an exact value (a runtime secret, a runner environment value), sought in -them in order with white space set aside — however a line ends and -wherever the value was cut. The arguments are then dropped and charged -as encoded; a held document is masked whole (its text was scanned whole -first, so a block that scan masked in place stays in place). Every other -pattern's secret stops at white space or at a quote, so a match over the -break has only run its context into the next string, which is left -unjudged. Not judged: structure in text not shaped like a document -(several documents, YAML, a document inside code); an assignment, a -header or a connection string whose value begins in the next string; a -token or an exact value split into pieces between strings; a member-name -pair in single quotes over two strings' apostrophes. Each string and number is also +one JSON object, array or string literal is judged as the arguments are +(`redactHeld`): decoded from the string as written, walked the same way +under the same member, and encoded again, so the structure a reader +decodes from the exported string is the structure that was judged — a +fold or an escape is inside the encoder's quoting — at the cost of the +document's layout (the encoder's key order, spacing and escapes). A +document that cannot be judged is masked `***` whole, with one +`held_document` finding: one the normalizer changes at all, as written, +since folding can make a document of text that is none as written, break +one, or spell a member-name pair into a key (fullwidth braces and +quotation marks); one held more than `maxHeldDepth` (4) deep, each level +being decoded and encoded once more; one in which two keys redact alike; +text shaped like a document — it begins and ends like one, white space +and invisible characters aside, and quotes something — that does not +parse (a comment, a trailing comma, an object literal, a Python dict with +a quoted value); and one whose text holds an exact value (a runtime +secret, a runner environment value) with structure of its own, which can +run over the document's (`valueAcross` over the text alone). Of two +members of one name decoding keeps the last, and the record keeps what +was decoded. That masks whole text with no secret in it too — a document +with a compatibility character (™, a fullwidth letter), JSON with +comments or trailing commas, an `Edit` fragment shaped `{…}` with a +quoted value. A key holding a document is masked whole as well. A string +the normalizer stripped an escape sequence or tag characters from — a +value or a key — is masked `***` whole: a colour code ends at the next +letter, so the stripping can take a token's first letter and leave the +rest, and a title code's payload is text no pattern sees. A member is +named as written as well as as scanned, and a key masked whole names a +secret whatever it was called — its name is not to be had from its mask +— so the values under it are judged as a secret-named member's. The +arguments are also judged across their keys, strings and numbers +(`atoms`: decoded, in the order written, a held document's in its place, +so the lines of a notebook cell held in a `Write`'s content are read as +lines) for a secret that can hold a line break, which no one of them can +mask. The private key block is sought in them joined by a line break, as +decoded and as the normalizer renders them (`blockAcross`): over the +lines of an array, over a key and a value, and inside one string as +well, which drops the arguments rather than masking the block in place. +An exact value is sought in them in order with white space set aside — +however a line ends and wherever the value was cut — an occurrence +within one atom being the literal pass's to replace; one holding +structure of its own (a credentials document in an env value) is sought +in the text as written too (`valueAcross`). The arguments are then +dropped and charged as encoded. Every other pattern's secret stops at +white space or at a quote, so a match over the break has only run its +context into the next string, which is left unjudged. Not judged: +structure in text not shaped like a document (several documents, YAML, a +document inside code); an assignment, a header or a connection string +whose value begins in the next string or number; a token split in pieces +between strings, or an exact value whose pieces other strings separate +or a string's escape spells; a member-name pair in single quotes over +two strings' apostrophes, or one a string's own quote opens or closes. Each string and number is also scanned once more, already redacted, beside the nearest enclosing key the member-name pattern (`json_field`) names, and masked whole when that pattern matches the pair — a pattern keyed on a member diff --git a/docs/guides/infrastructure/distributed-tracing.md b/docs/guides/infrastructure/distributed-tracing.md index 819d7a609a..b4a672184c 100644 --- a/docs/guides/infrastructure/distributed-tracing.md +++ b/docs/guides/infrastructure/distributed-tracing.md @@ -123,19 +123,21 @@ codex parsers emit none of the three yet `arguments` is the call's input decoded, redacted string by string (a number that redacts becomes the redacted string), and encoded again, so key order, spacing and escapes are not the stream's. -A string value or key in it that holds one JSON object, array or string -literal — a JSON file passed to `Write`, say — is masked `***` whole when that -document, decoded, still shows a secret: a value or key nested under a secret-named -member, or one only decoding spells out. So is one that cannot be judged: -held more than four deep, naming a member twice, or shaped like a -document but not parsing (JSON with comments or trailing commas, an -object literal), or one that Unicode normalization changes at all, since -folding can move a value between members — the -[developer guide](../dev/tracing.md) has the list. A secret that runs -across the strings of the arguments, or of a held document, in the order -written — a private key over the lines of an array — can be masked -nowhere but whole: the arguments are dropped, the held string masked -`***`. A string a terminal escape sequence was stripped from is masked +A string value in it that holds one JSON object, array or string +literal — a JSON file passed to `Write`, say — is judged the same way, +decoded, and recorded encoded again, so its layout is the encoder's and a +secret nested in it is masked in place. It is masked `***` whole when it +cannot be judged: held more than four deep, shaped like a document but +not parsing (JSON with comments or trailing commas, an object literal), +holding a runner environment or runtime secret value across its +structure, or changed at all by Unicode normalization, since +folding can make or break a document or spell a secret-named pair into a +key — the [developer guide](../dev/tracing.md) has the list. A key that +holds a document is masked whole. A secret that runs across the strings +of the arguments, or of a held document, in the order written — a private +key over the lines of an array, a runner environment value over them — +can be masked nowhere but whole: the arguments are dropped. A private +key inside one string drops them as well. A string a terminal escape sequence was stripped from is masked whole too, since the stripping can leave most of a token. A string directly under a secret-named member is masked in place, as anywhere else. diff --git a/internal/cli/content_collector.go b/internal/cli/content_collector.go index cac42071fa..b0df97b683 100644 --- a/internal/cli/content_collector.go +++ b/internal/cli/content_collector.go @@ -2,7 +2,6 @@ package cli import ( "encoding/json" - "io" "maps" "slices" "sort" @@ -106,10 +105,10 @@ const maxToolArgumentsBytes = 8 * 1024 // as text, so their findings count, then dropped and charged. const maxRawToolArgumentsBytes = 4 * maxToolArgumentsBytes -// maxHeldDepth bounds how deep heldSecret walks documents held in -// strings: each level is scanned once more, and with \u escapes a -// document can nest about the square root of its length deep. A string -// holding one deeper is masked whole. +// maxHeldDepth bounds how deep redactHeld walks documents held in +// strings: each level is decoded and encoded once more, and with \u +// escapes a document can nest about the square root of its length deep. +// A string holding one deeper is masked whole. const maxHeldDepth = 4 // newContentCollectorIfEnabled returns a live collector when the Level 3 @@ -378,8 +377,8 @@ type contentCollector struct { maxEncoded int pipeline *security.Pipeline // secrets is the pipeline's pattern stage alone, for secretNamed and - // spanning; normalizer its first stage alone, for the detection copies - // heldSecret and spanning judge. + // blockAcross; normalizer its first stage alone, for the detection + // copies holdsDocument and blockAcross judge. secrets *security.SecretRedactor normalizer *security.UnicodeNormalizer // runnerEnv feeds redact's literal pass; see newContentCollectorIfEnabled. @@ -402,7 +401,7 @@ type contentCollector struct { // Merged into the contentResult at Result so they count exactly like // assembly-time ones. findings []security.Finding - // held counts the documents held in strings that heldSecret is + // held counts the documents held in strings that redactHeld is // walking; see maxHeldDepth. held int } @@ -798,11 +797,12 @@ func redactText(pipeline *security.Pipeline, runnerEnv map[string]string, text s // dropped and charged like any other discarded content. Arguments whose // redacted encoding exceeds maxToolArgumentsBytes, in which two keys of // one object redact to the same string, or across whose keys, strings -// and numbers a secret runs (spanning: a private key over the lines of -// an array), which no one of them can mask, are dropped the same way -// and charged as encoded. On a collision that is the encoding after the -// later key (in sorted order) replaced the earlier member, so the -// replaced member is not counted. +// and numbers a secret runs — a private key block over the lines of an +// array (blockAcross), an exact value over them (valueAcross) — which +// no one of them can mask, are dropped the same way and charged as +// encoded. On a collision that is the encoding after the later key (in +// sorted order) replaced the earlier member, so the replaced member is +// not counted. func (c *contentCollector) toolArguments(args string) (json.RawMessage, bool) { if args == "" { return nil, false @@ -820,10 +820,11 @@ func (c *contentCollector) toolArguments(args string) (json.RawMessage, bool) { if err != nil { return nil, false // decoded JSON values marshal unconditionally } - // Judged across its strings whatever else drops it: the secret found + // Judged across its atoms whatever else drops it: the secret found // there is a finding of the call, and costs it its summary. - spanning := c.spanning(args) - if collided || spanning || len(out) > maxToolArgumentsBytes { + as := atoms(args, 0) + across := c.blockAcross(as) || c.valueAcross(as, args) + if collided || across || len(out) > maxToolArgumentsBytes { c.evicted += len(out) return nil, true } @@ -858,103 +859,63 @@ func documentLike(s string) bool { return len(s) >= 2 && (s[0] == '{' && s[len(s)-1] == '}' || s[0] == '[' && s[len(s)-1] == ']' || s[0] == '"' && s[len(s)-1] == '"') && strings.ContainsAny(s, `"'`) } -// duplicateName reports an object in the JSON document s that names a -// member twice, the names compared decoded, as RFC 7493 does. -func duplicateName(s string) bool { - dec := json.NewDecoder(strings.NewReader(s)) - dec.UseNumber() // a number past float64 is valid JSON too - var names []map[string]bool // one set for each open object - for { - tok, err := dec.Token() - if err != nil { - return err != io.EOF // s is valid; fail closed all the same - } - switch tok { - case json.Delim('{'): - names = append(names, map[string]bool{}) - case json.Delim('}'): - names = names[:len(names)-1] - default: - // A string a colon follows names a member of the innermost - // object open. - name, ok := tok.(string) - if !ok || !strings.HasPrefix(strings.TrimLeft(s[dec.InputOffset():], " \t\r\n"), ":") { - continue - } - if names[len(names)-1][name] { - return true - } - names[len(names)-1][name] = true - } - } +// heldFinding is the finding for a string holding a JSON document that +// is masked whole, for reason. +func heldFinding(reason string) security.Finding { + return security.Finding{Scanner: "held_document", Name: reason, Severity: "high", Detail: "string holding a JSON document masked whole", Position: -1} } -// maskHeld masks a string or an object key of the arguments whole, with -// one held_document finding, when heldSecret finds against the document -// its scanned text s holds; t is the text as written. -func (c *contentCollector) maskHeld(s, t string) string { - if s == "***" { - return s // masked whole already: nothing left to judge - } - if reason, ok := c.heldSecret(s, t); ok { - c.findings = append(c.findings, security.Finding{Scanner: "held_document", Name: reason, Severity: "high", Detail: "string holding a JSON document masked whole", Position: -1}) +// redactHeld redacts the JSON object, array or string literal doc that a +// string of the arguments holds, as the arguments are redacted: decoded, +// walked (redactValue, under the same member), and encoded again. The +// structure a reader decodes from the string is then the structure that +// was judged — a fold or an escape is inside the encoder's quoting, and +// moves no value between members — at the cost of the document's layout. +// Of two members of one name decoding keeps the last, and the record +// keeps what was decoded. A document that cannot be judged is masked +// whole: one held deeper than maxHeldDepth, and one in which two keys +// redact alike. +func (c *contentCollector) redactHeld(doc, under string) string { + if c.held == maxHeldDepth { + c.findings = append(c.findings, heldFinding("uninspected")) return "***" } - return s -} - -// heldSecret judges, decoded, the JSON object, array or string literal a -// string holds once scanned — what is exported. The scan reads the document's text, -// so it misses a secret-named member nested deeper than the value right -// after the key, and a secret that only decoding spells out: an -// assignment that opens a string, a token or runner env value written -// with an escape. The document is walked as the arguments are, its -// output and findings discarded; a revealing finding makes the reason -// "secret". A document that cannot be judged makes it "uninspected": -// one held deeper than maxHeldDepth, one naming a member twice — -// decoding keeps the last — and text shaped like a document that does -// not parse: one the scan broke, folding or masking it, or one that -// never parsed (a comment, a trailing comma, an object literal) — and -// one the normalizer changes at all, as written: folding can move a -// value out of its secret-named member while the document still parses, -// so structure the normalizer makes, or moves, is not trusted. A secret -// across the strings of the document as written (spanning: a private key -// over the lines of an array, an exact value over them) makes the reason -// "secret" as well — as written, since the scan can mask a block's last -// line as a member's value before the block's own pattern runs, and -// leave its first line in the key. A level above records that mask as -// "secret": it is a finding of that level's walk. -func (c *contentCollector) heldSecret(scanned, written string) (string, bool) { - doc, ok := jsonDocument(scanned) - if !ok { - return "uninspected", documentLike(scanned) || documentLike(written) - } - raw, _ := jsonDocument(written) - if c.held == maxHeldDepth || duplicateName(doc) || c.normalized(raw) != raw || !json.Valid([]byte(raw)) { - return "uninspected", true - } - mark := len(c.findings) - if c.spanning(raw) { - c.findings = c.findings[:mark] - return "secret", true - } v, _ := decodeJSON(doc) // a valid document decodes + collided := false c.held++ - c.redactValue(v, "", new(bool)) + e := c.redactValue(v, under, &collided) c.held-- - found := slices.ContainsFunc(c.findings[mark:], revealing) - c.findings = c.findings[:mark] - return "secret", found + if collided { + c.findings = append(c.findings, heldFinding("collision")) + return "***" + } + b, _ := json.Marshal(e) // decoded JSON values marshal unconditionally + return string(b) } // revealing reports a finding that shows a secret was there: any but the -// normalizer's, and the normalizer's stripping of content. A secret the -// scan masked in place is gone from the document by then, but for a mask -// that matches again. +// normalizer's, and the normalizer's stripping of content. func revealing(f security.Finding) bool { return f.Scanner != "unicode_normalizer" || stripped(f) } +// holdsDocument returns the JSON document s holds and can be judged, +// or whether s is shaped like one that cannot (like): a document the +// normalizer changes at all — folding can spell a member-name pair +// into a key, or make a document of text that is none as written (a +// fullwidth brace or quotation mark) — and text shaped like a document +// (documentLike) as written or as rendered. +func (c *contentCollector) holdsDocument(s string) (doc string, ok, like bool) { + n := c.normalized(s) + if doc, ok = jsonDocument(s); ok { + return doc, n == s, n != s + } + if _, ok = jsonDocument(n); ok { + return "", false, true + } + return "", false, documentLike(s) || documentLike(n) +} + // stripped reports the normalizer's removal of an escape sequence or of // tag characters, which can carry text the patterns never see — digits // as a CSI parameter, a token inside an OSC sequence — or take the first @@ -982,109 +943,101 @@ func (c *contentCollector) normalized(s string) string { return c.normalizer.Scan(s).Sanitized } -// spanning reports, and records as one critical finding, a secret that -// runs across the keys, strings and numbers of raw, one valid JSON value, -// as a reader of them in the order written sees them: a private key over -// the lines of an array or over a key and a value, an exact value over -// the lines of one. Each is judged on its own (redactValue); here they -// are judged together, each as decoded and as the normalizer renders it -// — the one can hold a marker inside an escape sequence the normalizer -// strips, the other a marker folding spells — before any is masked, -// since a string masked whole can take a secret's first or last line -// with it. Only a secret that can hold a line break is judged across -// them. The private key block is sought in the strings joined by a line -// break, which its pattern reads over. An exact value — a runtime -// secret, a runner env value the literal pass replaces — is sought in -// the strings in order with white space set aside, so however a line -// ends (its own line break, a Windows one, a space) and wherever the -// value was cut, it is read as a reader of the lines reads it. Every -// other pattern's secret stops at white space or at a quote, so a match -// of one over a break has only run its context into the next string (an -// assignment whose value is the next string, a pair in single quotes -// over two strings' apostrophes), which is left unjudged, as a token -// split between two strings is. A match within one key, string or -// number is that one's own. -func (c *contentCollector) spanning(raw string) bool { - var decoded, normalized []string - for _, sp := range atomSpans(raw) { - a := raw[sp[0]:sp[1]] - if sp[0] > 0 && raw[sp[0]-1] == '"' { - _ = json.Unmarshal([]byte(`"`+a+`"`), &a) // a string's escapes, decoded - } - decoded = append(decoded, a) - normalized = append(normalized, c.normalized(a)) - } - found := func(scanner, name string) bool { - c.findings = append(c.findings, security.Finding{Scanner: scanner, Name: name, Severity: "critical", Detail: "secret spanning keys, strings or numbers", Position: -1}) - return true - } - for _, atoms := range [][]string{decoded, normalized} { - lines, bounds := joined(atoms, "\n", plain) - for _, m := range c.secrets.Matches(lines) { - if m.Name == "private_key" && !bounds.within(m.Start, m.End) { - return found("secret_redactor", m.Name) - } +// atoms lists the strings and numbers of raw, one valid JSON value, +// decoded, in the order written — as a reader of the arguments reads +// them, whatever order the encoder gives the keys. A string holding a +// document (to maxHeldDepth) gives the document's atoms in its place, so +// the lines of a notebook cell held in a Write's content are read as +// lines. +func atoms(raw string, depth int) []string { + dec := json.NewDecoder(strings.NewReader(raw)) + dec.UseNumber() + var out []string + for { + tok, err := dec.Token() + if err != nil { + return out // io.EOF: raw is valid } - dense, bounds := joined(atoms, "", unspaced) - for _, v := range security.RuntimeSecrets() { - if bounds.holds(dense, unspaced(v)) { - return found("secret_redactor", "runtime_secret") + switch t := tok.(type) { + case string: + if doc, ok := jsonDocument(t); ok && depth < maxHeldDepth { + out = append(out, atoms(doc, depth+1)...) + } else { + out = append(out, t) } + case json.Number: + out = append(out, t.String()) } - for _, l := range envLiterals(c.runnerEnv) { - if bounds.holds(dense, unspaced(l.value)) { - return found("runner_env", l.key) + } +} + +// blockAcross reports, and records as one critical finding, a private +// key block in the atoms of the arguments — over the lines of an array, +// over a key and a value, over the lines of a held document — which no +// one of them can mask: its pattern reads over a line break, so the +// atoms are judged joined by one, as decoded and as the normalizer +// renders them, since a marker can be folded. A block inside one atom +// drops the arguments as well: coarser than the mask in place the atom +// gets, and nothing left to judge. +func (c *contentCollector) blockAcross(atoms []string) bool { + lines := strings.Join(atoms, "\n") + for _, text := range []string{lines, c.normalized(lines)} { + for _, m := range c.secrets.Matches(text) { + if m.Name == "private_key" { + c.findings = append(c.findings, security.Finding{Scanner: "secret_redactor", Name: m.Name, Severity: "critical", Detail: "secret spanning keys, strings or numbers", Position: -1}) + return true } } } return false } -// atomBounds is where each atom begins and ends in a text joined of them. -type atomBounds [][2]int - -// joined writes atoms as render renders them, sep between, and where -// each is in the text. -func joined(atoms []string, sep string, render func(string) string) (string, atomBounds) { - var b strings.Builder - bounds := make(atomBounds, len(atoms)) - for n, a := range atoms { - if n > 0 { - b.WriteString(sep) +// valueAcross reports, and records as one critical finding, an exact +// value — a runtime secret, a runner env value the literal pass replaces +// — that runs across the atoms of the arguments, in the order written, +// with white space set aside: however a line ends (its own line break, +// a Windows one, a space) and wherever the value was cut, it is read as +// a reader of the lines reads it. An occurrence within one atom is the +// literal pass's to replace, and is set aside first. A value holding +// structure of its own — a quote, a comma, a colon, a bracket, as a +// credentials document in an env value does — is sought in the text as +// written as well, where it can run over the structure between the +// atoms. +func (c *contentCollector) valueAcross(atoms []string, raw string) bool { + for _, l := range exactValues(c.runnerEnv) { + v := unspaced(l.value) + if v == "" { + continue + } + var b strings.Builder + for _, a := range atoms { + b.WriteString(strings.ReplaceAll(unspaced(a), v, "")) + } + if strings.Contains(b.String(), v) || strings.ContainsAny(v, `",:{}[]`) && strings.Contains(unspaced(raw), v) { + c.findings = append(c.findings, security.Finding{Scanner: l.scanner, Name: l.key, Severity: "critical", Detail: "secret spanning keys, strings or numbers", Position: -1}) + return true } - bounds[n][0] = b.Len() - b.WriteString(render(a)) - bounds[n][1] = b.Len() } - return b.String(), bounds + return false } -// within reports whether one atom holds text[start:end] whole. -func (bounds atomBounds) within(start, end int) bool { - n := sort.Search(len(bounds), func(n int) bool { return bounds[n][0] > start }) - 1 - return n >= 0 && end <= bounds[n][1] -} +// exactValue is a value sought exactly, and how a finding names it. +type exactValue struct{ scanner, key, value string } -// holds reports whether text holds value other than within one atom. -func (bounds atomBounds) holds(text, value string) bool { - if value == "" { - return false +// exactValues lists the runtime secrets and the runner env values the +// literal pass replaces. +func exactValues(runnerEnv map[string]string) []exactValue { + var out []exactValue + for _, v := range security.RuntimeSecrets() { + out = append(out, exactValue{"secret_redactor", "runtime_secret", v}) } - for i := 0; ; { - j := strings.Index(text[i:], value) - if j < 0 { - return false - } - if i += j; !bounds.within(i, i+len(value)) { - return true - } - i += len(value) + for _, l := range envLiterals(runnerEnv) { + out = append(out, exactValue{"runner_env", l.key, l.value}) } + return out } -// plain renders a string as it is; unspaced without its white space. -func plain(s string) string { return s } - +// unspaced renders s without its white space. func unspaced(s string) string { return strings.Map(func(r rune) rune { if unicode.IsSpace(r) { @@ -1094,39 +1047,10 @@ func unspaced(s string) string { }, s) } -// atomSpans lists where raw, a valid JSON value, spells each string -// (between its quotes) and each number, true, false and null, in order. -func atomSpans(raw string) [][2]int { - const structure = "{}[],: \t\r\n" - var spans [][2]int - for i := 0; i < len(raw); { - switch { - case raw[i] == '"': - j := i + 1 - for raw[j] != '"' { - if raw[j] == '\\' { - j++ - } - j++ - } - spans = append(spans, [2]int{i + 1, j}) - i = j + 1 - case strings.IndexByte(structure, raw[i]) >= 0: - i++ - default: - j := i - for j < len(raw) && strings.IndexByte(structure+`"`, raw[j]) < 0 { - j++ - } - spans = append(spans, [2]int{i, j}) - i = j - } - } - return spans -} - // redactValue redacts every string, number and object key under v; a -// number that redacts becomes the redacted string. A pattern +// number that redacts becomes the redacted string, and a string holding +// a JSON document is judged as the arguments are and encoded again +// (redactHeld), or masked whole when it cannot be judged. A pattern // keyed on a member name ("password": "...") cannot see the pair that // way, so a string or number under a key that pattern names — the nearest // such enclosing key, under, which a stand-in name represents — is scanned @@ -1142,7 +1066,25 @@ func atomSpans(raw string) [][2]int { func (c *contentCollector) redactValue(v any, under string, collided *bool) any { switch t := v.(type) { case string: - s := c.maskHeld(c.scanLeaf(t), t) + doc, ok, like := c.holdsDocument(t) + if ok { + if c.valueAcross(nil, t) { + // An exact value holding structure of its own can run + // over the document's, which no leaf of the walk holds. + return "***" + } + return c.redactHeld(doc, under) + } + if like { + // A document that cannot be judged: one the normalizer + // changes, or text shaped like one that does not parse — a + // comment, a trailing comma, an object literal — holding + // whatever it nests under a secret-named member, which + // neither the patterns nor a walk can see. + c.findings = append(c.findings, heldFinding("uninspected")) + return "***" + } + s := c.scanLeaf(t) if under != "" && c.secretNamed(under, s) { return "***" } @@ -1166,14 +1108,20 @@ func (c *contentCollector) redactValue(v any, under string, collided *bool) any for _, k := range slices.Sorted(maps.Keys(t)) { // The key as scanned names the secret, masked whole or not. sk := c.scanLeaf(k) - rk := c.maskHeld(sk, k) + rk := sk + if _, ok, like := c.holdsDocument(k); ok || like { + // A key holding a document is masked whole: a reader + // decodes nothing from a key, and a document can name a + // secret in a spelling only decoding reads. + c.findings = append(c.findings, heldFinding("key")) + rk = "***" + } member := under // Named as scanned, or as written: the normalizer can strip // the name's first letter with a colour code, and the value // under it is still that name's. A key masked whole — by the // scan, or as a document it holds — names a secret whatever - // it was called: its name is not to be had from its mask, and - // a document can name one in a spelling only decoding reads. + // it was called: its name is not to be had from its mask. named := rk == "***" || c.namesASecret(sk) || c.namesASecret(k) if named { // Decided once for the member: the pattern reads any diff --git a/internal/cli/content_collector_test.go b/internal/cli/content_collector_test.go index 03b3e06ca6..966228e6e5 100644 --- a/internal/cli/content_collector_test.go +++ b/internal/cli/content_collector_test.go @@ -198,9 +198,10 @@ func TestContentCollector_SecretNamedMembersReachTheirNestedValues(t *testing.T) } func TestContentCollector_JSONInAStringIsMaskedWholeWhenItsDocumentHoldsASecret(t *testing.T) { - // The exported string is judged as decoded too: a secret-named member - // nested in the document it holds, or a secret only decoding spells - // out, masks the string whole. + // A string holding a document is judged as the arguments are: decoded, + // walked and encoded again, so a secret-named member nested in it, or + // a secret only decoding spells out, is masked in place; what cannot + // be judged is masked whole. const opaque = "opaque-credential-123" cred := `"credentials":{"value":"` + opaque + `"}` token := "ghp_" + strings.Repeat("r", 36) @@ -209,45 +210,45 @@ func TestContentCollector_JSONInAStringIsMaskedWholeWhenItsDocumentHoldsASecret( content, want string findings int }{ - "object under credentials": {`{` + cred + `}`, "***", 1}, - "array under password": {`{"password":["` + opaque + `"]}`, "***", 1}, - "document that is an array": {`[{"token":{"v":"` + opaque + `"}}]`, "***", 1}, - "key under credentials": {`{"credentials":{"` + opaque + `":"user"}}`, "***", 1}, - "document in a document": {`{"inner":` + quote(`{"token":["`+opaque+`"]}`) + `}`, "***", 1}, - // The scan sees the document's text; these only its strings. - "an assignment that opens a string": {`{"command":"API_KEY=` + opaque + `"}`, "***", 1}, - "an assignment after an escaped break": {`{"c":"cd x\nAPI_KEY=` + opaque + ` run"}`, "***", 1}, - "a token spelled with an escape": {`{"t":"ghp_\u0072` + token[5:] + `"}`, "***", 1}, - "tag characters written with escapes": {`{"t":"\uDB40\uDC67"}`, "***", 1}, - "digits in an escape under a secret name": {`{"auth":{"pin":"\u001b[31415926535m"}}`, "***", 1}, - "an escape sequence around a token": {`{"n":"see \u001b]ghp_\u0072` + token[5:] + `\u0007"}`, "***", 1}, - // Folding can make the document; a document the scan breaks, - // folding or masking it, cannot be judged. - "folded into a document": {"\uFF5B" + cred + "}", "***", 2}, - "folding breaks a document": {"{\"note\":\"\uFF02\"," + cred + "}", "***", 2}, - "a BOM and folding that breaks a document": {"\uFEFF{\"note\":\"\uFF02\"," + cred + "}", "***", 3}, - "a fold that makes and breaks a document": {"\uFF5B\"note\":\"\uFF02\"," + cred + "}", "***", 2}, - "an escape before a document folding breaks": {"\x1b[0m{\"note\":\"\uFF02\"," + cred + "}", "***", 2}, - "a mask breaks a document": {`{"token":"abcdefgh\\\"x",` + cred + `}`, "***", 2}, - // Folding can move a value out of its secret-named member while the - // document still parses; a document the normalizer changes at all - // is not judged. - "folding that moves a value between members": {"{\"credentials\":{\"value\":\"\uFF02},\uFF02note\uFF02:{\uFF02value\uFF02:\uFF02" + opaque + "\"}}", "***", 2}, - "folding that moves nothing": {"{\"name\":\"Widget\u2122\"}", "***", 2}, - // Decoding keeps the last of two members of one name; the string - // keeps both, so a repeated name cannot be judged either. - "a repeated name": {`{` + cred + `,"credentials":{}}`, "***", 1}, - "a name repeated with an escape": {`{` + cred + `,"\u0063redentials":{}}`, "***", 1}, - "a name repeated in an array item": {`[{"token":{"v":"` + opaque + `"},"token":{}}]`, "***", 1}, - "a repeated name and no secret": {`{"a":1,"a":2}`, "***", 1}, - // A mask the walk matches again, and a pattern's false positive in - // a decoded string, mask it whole as well. - "a mask that matches again": {`{"db":"postgres://u:hunter2hunter2@db/x"}`, "***", 2}, - "a notebook assignment": {`{"cells":[{"source":["key = jax.random.PRNGKey(0)\n"]}]}`, "***", 1}, - // Otherwise the string is kept as the scan left it. - "plain document": {`{ "b": "` + opaque + `", "a": 1 }`, `{ "b": "` + opaque + `", "a": 1 }`, 0}, - "a pair stays masked in place": {`{"password":"hunter2hunter2","n":1}`, `{"password":"hunt...","n":1}`, 1}, - "a letter folded once decoded": {`{"v":"\uFF21"}`, `{"v":"\uFF21"}`, 0}, + "object under credentials": {`{` + cred + `}`, `{"credentials":{"value":"***"}}`, 1}, + "array under password": {`{"password":["` + opaque + `"]}`, `{"password":["***"]}`, 1}, + "document that is an array": {`[{"token":{"v":"` + opaque + `"}}]`, `[{"token":{"v":"***"}}]`, 1}, + "key under credentials": {`{"credentials":{"` + opaque + `":"user"}}`, `{"credentials":{"***":"user"}}`, 1}, + "document in a document": {`{"inner":` + quote(`{"token":["`+opaque+`"]}`) + `}`, `{"inner":` + quote(`{"token":["***"]}`) + `}`, 1}, + // Decoded, each string is judged on its own. + "an assignment that opens a string": {`{"command":"API_KEY=` + opaque + `"}`, `{"command":"API_KEY=opaq..."}`, 1}, + "an assignment after an escaped break": {`{"c":"cd x\nAPI_KEY=` + opaque + ` run"}`, `{"c":"cd x\nAPI_KEY=opaq... run"}`, 1}, + "a token spelled with an escape": {`{"t":"ghp_\u0072` + token[5:] + `"}`, `{"t":"ghp_..."}`, 1}, + "tag characters written with escapes": {`{"t":"\uDB40\uDC67"}`, `{"t":"***"}`, 1}, + "digits in an escape under a secret name": {`{"auth":{"pin":"\u001b[31415926535m"}}`, `{"auth":{"pin":"***"}}`, 1}, + "an escape sequence around a token": {`{"n":"see \u001b]ghp_\u0072` + token[5:] + `\u0007"}`, `{"n":"***"}`, 1}, + // A document the normalizer changes at all is masked whole: folding + // can make a document, break one, or spell a pair into a key. + "folded into a document": {"\uFF5B" + cred + "}", "***", 1}, + "folding breaks a document": {"{\"note\":\"\uFF02\"," + cred + "}", "***", 1}, + "a BOM and folding that breaks a document": {"\uFEFF{\"note\":\"\uFF02\"," + cred + "}", "***", 1}, + "a fold that makes and breaks a document": {"\uFF5B\"note\":\"\uFF02\"," + cred + "}", "***", 1}, + "an escape before a document folding breaks": {"\x1b[0m{\"note\":\"\uFF02\"," + cred + "}", "***", 1}, + "a mask breaks a document": {`{"token":"abcdefgh\\\"x",` + cred + `}`, `{"credentials":{"value":"***"},"token":"***"}`, 2}, + // Folding that moves a value out of its secret-named member, or + // moves nothing, alike. + "folding that moves a value between members": {"{\"credentials\":{\"value\":\"\uFF02},\uFF02note\uFF02:{\uFF02value\uFF02:\uFF02" + opaque + "\"}}", "***", 1}, + "folding that moves nothing": {"{\"name\":\"Widget\u2122\"}", "***", 1}, + // Decoding keeps the last of two members of one name, and the + // record keeps what was decoded. + "a repeated name": {`{` + cred + `,"credentials":{}}`, `{"credentials":{}}`, 0}, + "a name repeated with an escape": {`{` + cred + `,"\u0063redentials":{}}`, `{"credentials":{}}`, 0}, + "a name repeated in an array item": {`[{"token":{"v":"` + opaque + `"},"token":{}}]`, `[{"token":{}}]`, 0}, + "a repeated name and no secret": {`{"a":1,"a":2}`, `{"a":2}`, 0}, + // A pattern's mask in a decoded string, and a false positive of + // one, stay in place. + "a mask that matches again": {`{"db":"postgres://u:hunter2hunter2@db/x"}`, `{"db":"postgres://u:hunt...@db/x"}`, 1}, + "a notebook assignment": {`{"cells":[{"source":["key = jax.random.PRNGKey(0)\n"]}]}`, `{"cells":[{"source":["key = jax....)\n"]}]}`, 1}, + // Otherwise the document is kept, encoded again: the encoder's key + // order, spacing and escapes. + "plain document": {`{ "b": "` + opaque + `", "a": 1 }`, `{"a":1,"b":"` + opaque + `"}`, 0}, + "a pair stays masked in place": {`{"password":"hunter2hunter2","n":1}`, `{"n":1,"password":"***"}`, 1}, + "a letter folded once decoded": {`{"v":"\uFF21"}`, `{"v":"A"}`, 1}, // Document-shaped text that does not parse cannot be judged either: // a raw control character, a trailing comma, a comment, an object // literal. Masked whole, secret or none. @@ -258,12 +259,11 @@ func TestContentCollector_JSONInAStringIsMaskedWholeWhenItsDocumentHoldsASecret( "an object literal": {`{ apiKey: "hunter2hunter2" }`, "***", 1}, // Shaped like one but quoting nothing: code, or a mask. "an object literal quoting nothing": {`{ apiKey: process.env.KEY }`, `{ apiKey: process.env.KEY }`, 0}, - "a mask": {"-----BEGIN RSA PRIVATE " + "KEY-----\nQQQQQQQQ\n-----END RSA PRIVATE " + "KEY-----", "[REDACTED PRIVATE KEY]", 1}, "a number past float64": {`{"maximum":1e999}`, `{"maximum":1e999}`, 0}, - "text like a document the scan changed": {`{'password': 'hunter2hunter2'}`, "***", 2}, + "text like a document the scan changed": {`{'password': 'hunter2hunter2'}`, "***", 1}, "a command that opens a brace": {`{ export API_KEY=` + opaque + `; } 2>/dev/null`, `{ export API_KEY=opaq...; } 2>/dev/null`, 1}, "a held string literal": {`"plain text"`, `"plain text"`, 0}, - "a value naming a sibling": {`{"name":"name","from":"to","to":"x"}`, `{"name":"name","from":"to","to":"x"}`, 0}, + "a value naming a sibling": {`{"name":"name","from":"to","to":"x"}`, `{"from":"to","name":"name","to":"x"}`, 0}, "names repeated in other objects": {`[{"a":1},{"a":2},{"x":{"a":1},"y":{"a":2}},{"a":{"x":1},"x":2}]`, `[{"a":1},{"a":2},{"x":{"a":1},"y":{"a":2}},{"a":{"x":1},"x":2}]`, 0}, } { t.Run(name, func(t *testing.T) { @@ -273,7 +273,7 @@ func TestContentCollector_JSONInAStringIsMaskedWholeWhenItsDocumentHoldsASecret( res := c.Result("stop") assert.Contains(t, res.OutputMessages, `"arguments":{"content":`+quote(tc.want)+`}`) assert.Len(t, res.Findings, tc.findings) - assert.Equal(t, tc.findings > 0 && tc.want != tc.content, !strings.Contains(res.OutputMessages, `"summary"`), "a secret drops the summary") + assert.Equal(t, strings.Contains(tc.want, "***") || strings.Contains(tc.want, "..."), !strings.Contains(res.OutputMessages, `"summary"`), "a secret drops the summary") }) } } @@ -281,6 +281,8 @@ func TestContentCollector_JSONInAStringIsMaskedWholeWhenItsDocumentHoldsASecret( func TestContentCollector_JSONInAStringIsAlsoScannedWhole(t *testing.T) { // A secret can span the document's strings: a notebook keeps each line // of a cell in its own string, and a runner env value can hold quotes. + // A private key over the lines drops the arguments; an exact value + // over the document's structure masks it whole. begin, end := "-----BEGIN RSA PRIVATE "+"KEY-----", "-----END RSA PRIVATE "+"KEY-----" body := strings.Repeat("Q", 40) notebook, _ := json.Marshal(map[string]any{"cells": []any{map[string]any{"source": []string{begin + "\n", body + "\n", end + "\n"}}}}) @@ -290,22 +292,22 @@ func TestContentCollector_JSONInAStringIsAlsoScannedWhole(t *testing.T) { content, gone string findings int }{ - "key over a cell's lines": {nil, string(notebook), body, 2}, + "key over a cell's lines": {nil, string(notebook), body, 1}, // Its strings in order, as written: the scan masks the block's last // line as a member's value before the block's pattern runs. "key from a member name into its value": {nil, `{"` + begin + `\n` + body[:20] + `":"` + body[20:] + `\n` + end + `"}`, body[:20], 2}, "a runtime value over two strings": {nil, `{"lines":["rtline1","rtline2x"]}`, "rtline2x", 1}, - // Its marker breaks the document, which cannot then be judged. - "runner env value with structure": {map[string]string{"DB_PASSWORD": `hunter22","port`}, `{"pw":"hunter22","port":5432}`, "hunter22", 2}, - "runner env value that ends the document": {map[string]string{"DB_PASSWORD": `hunter22"}`}, `{"pw":"hunter22"}`, "hunter22", 2}, - "and one behind a BOM": {map[string]string{"DB_PASSWORD": `hunter22"}`}, "\uFEFF{\"credentials\":{\"value\":\"opaque-credential-123\"},\"pw\":\"hunter22\"}", "opaque-credential-123", 3}, - "and one behind a space and a BOM": {map[string]string{"DB_PASSWORD": `hunter22"}`}, " \uFEFF{\"credentials\":{\"value\":\"opaque-credential-123\"},\"pw\":\"hunter22\"}", "opaque-credential-123", 3}, - "and one behind a zero-width space": {map[string]string{"DB_PASSWORD": `hunter22"}`}, "\u200B{\"credentials\":{\"value\":\"opaque-credential-123\"},\"pw\":\"hunter22\"}", "opaque-credential-123", 3}, + // An exact value over the document's structure. + "runner env value with structure": {map[string]string{"DB_PASSWORD": `hunter22","port`}, `{"pw":"hunter22","port":5432}`, "hunter22", 1}, + "runner env value that ends the document": {map[string]string{"DB_PASSWORD": `hunter22"}`}, `{"pw":"hunter22"}`, "hunter22", 1}, + "and one behind a BOM": {map[string]string{"DB_PASSWORD": `hunter22"}`}, "\uFEFF{\"credentials\":{\"value\":\"opaque-credential-123\"},\"pw\":\"hunter22\"}", "opaque-credential-123", 1}, + "and one behind a space and a BOM": {map[string]string{"DB_PASSWORD": `hunter22"}`}, " \uFEFF{\"credentials\":{\"value\":\"opaque-credential-123\"},\"pw\":\"hunter22\"}", "opaque-credential-123", 1}, + "and one behind a zero-width space": {map[string]string{"DB_PASSWORD": `hunter22"}`}, "\u200B{\"credentials\":{\"value\":\"opaque-credential-123\"},\"pw\":\"hunter22\"}", "opaque-credential-123", 1}, // Decoded, the literal pass sees it — in a document, or in a // string a held JSON string literal spells with escapes. "runner env value spelled with an escape": {map[string]string{"DB_PASSWORD": "hunter2hunter2"}, `{"pw":"\u0068unter2hunter2"}`, "unter2hunter2", 1}, "runner env value in a held string literal": {map[string]string{"DEPLOY_PASSWORD": "abcdefghijklmno"}, `"\u0061bcdefghijklmno"`, "bcdefghijklmno", 1}, - "and in one the scan breaks": {map[string]string{"DEPLOY_PASSWORD": "abcdefghijklmno"}, "\"\\u0061bcdefghijklmno\uFF02\"", "bcdefghijklmno", 2}, + "and in one the scan breaks": {map[string]string{"DEPLOY_PASSWORD": "abcdefghijklmno"}, "\"\\u0061bcdefghijklmno\uFF02\"", "bcdefghijklmno", 1}, } { t.Run(name, func(t *testing.T) { t.Setenv(telemetry.ContentCaptureEnvVar, "true") @@ -326,7 +328,8 @@ func TestContentCollector_SecretsSpanningTheArgumentsStrings(t *testing.T) { // the lines of an array or over a key and a value, a runtime value // over the lines of one. A secret found that way can be masked // nowhere but the level whole: the arguments are dropped and charged - // as encoded. (An assignment or a header whose value is the next + // as encoded; a private key block inside one string drops them as + // well. (An assignment or a header whose value is the next // string is not judged: its secret stops at white space.) begin, end := "-----BEGIN RSA PRIVATE "+"KEY-----", "-----END RSA PRIVATE "+"KEY-----" body := strings.Repeat("Q", 40) @@ -357,13 +360,16 @@ func TestContentCollector_SecretsSpanningTheArgumentsStrings(t *testing.T) { assert.Len(t, res.Findings, tc.findings) }) } - t.Run("a private key in one string is that string's own", func(t *testing.T) { + t.Run("a private key in one string drops the arguments too", func(t *testing.T) { c := newContentCollector(4096) c.Handle(agentruntime.ToolUseEvent{Name: "Write", Summary: "s", Arguments: `{"content":"` + begin + `\n` + body + `\n` + end + `","file_path":"k.pem"}`}) res := c.Result("stop") - assert.Contains(t, res.OutputMessages, `"arguments":{"content":"[REDACTED PRIVATE KEY]","file_path":"k.pem"}`) - assert.Len(t, res.Findings, 1) + part := partAt(t, decodeOutputMessages(t, res.OutputMessages), 0) + assert.NotContains(t, part, "arguments") + assert.Equal(t, true, part["fullsend.truncated"]) + assert.NotContains(t, res.OutputMessages, body) + assert.Len(t, res.Findings, 2) }) t.Run("dropped arguments are charged as encoded", func(t *testing.T) { c := newContentCollector(4096) @@ -484,8 +490,8 @@ func TestContentCollector_AKeyMaskedWholeNamesASecret(t *testing.T) { func TestContentCollector_ARunnerEnvValueOverTheLinesOfAnArray(t *testing.T) { // A runner env value can hold a line break; written over two strings // of an array, no string holds it whole and the literal pass sees no - // part of it. Judged across the strings it is found: the arguments - // are dropped, a held document masked whole. + // part of it. Judged across the strings it is found, the lines of a + // held document read as lines: the arguments are dropped. // Each line of a notebook cell keeps its own line break, a Windows // one or a trailing space: an exact value is sought in the strings in // order, white space aside. @@ -515,15 +521,17 @@ func TestContentCollector_ARunnerEnvValueOverTheLinesOfAnArray(t *testing.T) { c.Handle(agentruntime.ToolUseEvent{Name: "Write", Arguments: `{"content":` + string(content) + `}`}) res := c.Result("stop") - assert.Contains(t, res.OutputMessages, `"arguments":{"content":"***"}`) + part := partAt(t, decodeOutputMessages(t, res.OutputMessages), 0) + assert.NotContains(t, part, "arguments") + assert.Equal(t, true, part["fullsend.truncated"]) assert.NotContains(t, res.OutputMessages, "opaque-line") }) } } func TestContentCollector_JSONHeldPastTheBoundIsMaskedWhole(t *testing.T) { - // Each level is scanned once more, so documents are walked down to - // maxHeldDepth; one held deeper cannot be judged. + // Documents are walked down to maxHeldDepth, each encoded again with + // its masks; one held deeper is masked whole, the levels above kept. held := func(doc string, depth int) string { for range depth - 1 { b, _ := json.Marshal(map[string]string{"d": doc}) @@ -540,7 +548,7 @@ func TestContentCollector_JSONHeldPastTheBoundIsMaskedWhole(t *testing.T) { const secret, plain = `{"credentials":{"value":"opaque-credential-123"}}`, `{"v":"plain"}` res := run(held(secret, maxHeldDepth)) - assert.Contains(t, res.OutputMessages, `"arguments":{"content":"***"}`) + assert.Contains(t, res.OutputMessages, `"arguments":`+held(`{"credentials":{"value":"***"}}`, maxHeldDepth)) assert.Len(t, res.Findings, 1) res = run(held(plain, maxHeldDepth)) @@ -549,7 +557,8 @@ func TestContentCollector_JSONHeldPastTheBoundIsMaskedWhole(t *testing.T) { for _, doc := range []string{plain, secret} { res = run(held(doc, maxHeldDepth+1)) - assert.Contains(t, res.OutputMessages, `"arguments":{"content":"***"}`) + assert.Contains(t, res.OutputMessages, `"arguments":`+held("***", maxHeldDepth+1)) + assert.NotContains(t, res.OutputMessages, "opaque-credential") assert.Len(t, res.Findings, 1) } } @@ -563,7 +572,7 @@ func TestContentCollector_AKeyHoldingADocumentIsJudgedLikeAValue(t *testing.T) { }{ "key": {`{` + doc + `:"x"}`, `{"***":"x"}`, 1}, "key of a nested object": {`{"files":{` + doc + `:"x"}}`, `{"files":{"***":"x"}}`, 1}, - "key in a held document": {`{"content":` + quote(`{`+doc+`:"x"}`) + `}`, `{"content":"***"}`, 1}, + "key in a held document": {`{"content":` + quote(`{`+doc+`:"x"}`) + `}`, `{"content":` + quote(`{"***":"x"}`) + `}`, 1}, // Masked whole, the key still names a secret as scanned: its // value is judged against it. "a value under it": {`{` + doc + `:"opaque-credential-456"}`, `{"***":"***"}`, 2}, @@ -591,13 +600,13 @@ func TestContentCollector_AKeyHoldingADocumentIsJudgedLikeAValue(t *testing.T) { } func TestContentCollector_JSONInASecretNamedStringIsMaskedWholeOnce(t *testing.T) { - // The member-name pattern takes any string of eight bytes or more under - // the member, a document's encoding too: one finding, not one per value. + // A document under a secret-named member is walked under that member: + // its values are judged by the member's name, one finding each. c := newContentCollector(4096) c.Handle(agentruntime.ToolUseEvent{Name: "Write", Arguments: `{"credentials":"{\"value\":\"opaque-credential-123\"}"}`}) res := c.Result("stop") - assert.Contains(t, res.OutputMessages, `"arguments":{"credentials":"***"}`) + assert.Contains(t, res.OutputMessages, `"arguments":{"credentials":"{\"value\":\"***\"}"}`) assert.Len(t, res.Findings, 1) } From c8324dfd4cbf6d5b00eeec512c81187fac60853d Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Thu, 8 Oct 2026 13:53:24 -0400 Subject: [PATCH 27/40] refactor(cli): record only the listed members of a tool call's arguments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review found the arguments path accreting judgements without converging: a per-leaf scan, a member-name scan with a stand-in name, held documents walked to a depth, duplicate-name and normalizer-change detection, a fail-closed mask on document-shaped text that caught ordinary Edit fragments and cost them their summary, and a cross-string judgement in two renderings. Each round added a rule, and the known gaps stayed. The contract is now closed by construction. A tool_call part records the members of the call's input that name what was called — paths, patterns, commands, modes and bounds, the list in recordedArguments — each redacted as text, and nothing else of the input: a file body, an edit, a prompt, a notebook source, a todo list, and any member whose value is an object or an array is dropped, scanned first so a secret in it still counts, charged as the redacted text it was scanned as, and the part marked. No fail-closed mask is left: a dropped member raises no finding of its own, keeps the summary, and adds nothing to fullsend.content.redactions. A string the normalizer stripped an escape sequence or tag characters from is still masked whole. The raw bound goes with the walk whose cost it limited; the encoded bound stays. The redactor's match-location and runtime-secret helpers, added for the cross-string judgement, are gone with it; envLiterals keeps its one caller. The guides and the two ADR annotations state the list and the one-sentence guarantee. Signed-off-by: Dharit Shah --- ...050-distributed-tracing-instrumentation.md | 6 +- docs/ADRs/0108-tool-call-span-topology.md | 9 +- docs/guides/dev/tracing.md | 111 +-- .../infrastructure/distributed-tracing.md | 65 +- internal/cli/content_capture_record_test.go | 16 +- internal/cli/content_collector.go | 472 +++---------- internal/cli/content_collector_test.go | 659 +++--------------- internal/cli/run.go | 4 +- internal/security/redactor.go | 41 -- internal/security/redactor_matches_test.go | 42 -- 10 files changed, 247 insertions(+), 1178 deletions(-) delete mode 100644 internal/security/redactor_matches_test.go diff --git a/docs/ADRs/0050-distributed-tracing-instrumentation.md b/docs/ADRs/0050-distributed-tracing-instrumentation.md index 77eb580f11..55e3c63056 100644 --- a/docs/ADRs/0050-distributed-tracing-instrumentation.md +++ b/docs/ADRs/0050-distributed-tracing-instrumentation.md @@ -182,8 +182,10 @@ remains deferred. **2026-09-18 — Level 3 input message and tool arguments ([ADR 0108](0108-tool-call-span-topology.md)):** under the content gate, a retry iteration that carries validation feedback records the prompt the runner composed as `gen_ai.input.messages`, and -`tool_call` parts on the message record carry the call's arguments — Claude -Code today ([#7414](https://github.com/fullsend-ai/fullsend/issues/7414)). +`tool_call` parts on the message record carry the members of the call's +arguments that name what was called (a path, a pattern, a command), not a +file body or a prompt — Claude Code today +([#7414](https://github.com/fullsend-ai/fullsend/issues/7414)). The rest of the model's input is not recorded ([reference](../guides/infrastructure/distributed-tracing.md#content-capture-level-3)). The gate and the carrier are unchanged. diff --git a/docs/ADRs/0108-tool-call-span-topology.md b/docs/ADRs/0108-tool-call-span-topology.md index d12f59975c..47a9762ee1 100644 --- a/docs/ADRs/0108-tool-call-span-topology.md +++ b/docs/ADRs/0108-tool-call-span-topology.md @@ -125,9 +125,10 @@ convention bump. ## Annotations **2026-09-18 — Tool arguments on the message record:** the Decision's "full -arguments next" is in place up to a per-call bound: `tool_call` parts on the -`agent` span's `gen_ai.output.messages` record carry the call's redacted -arguments where the runtime's stream provides them, Claude Code today. The -bound and the drop rules are in the +arguments next" is in place for the members that name what was called: +`tool_call` parts on the `agent` span's `gen_ai.output.messages` record +carry the call's path, pattern, command and the like, redacted, where the +runtime's stream provides them, Claude Code today; a file body, an edit or +a prompt is not recorded. The list, the bound and the drop rules are in the [reference](../guides/infrastructure/distributed-tracing.md#content-capture-level-3). `execute_tool` spans still carry metadata only. diff --git a/docs/guides/dev/tracing.md b/docs/guides/dev/tracing.md index e9e728c61e..f15262da6f 100644 --- a/docs/guides/dev/tracing.md +++ b/docs/guides/dev/tracing.md @@ -238,88 +238,35 @@ content and its marker attributes on the span before either `finalizeAgentSpan` path can end it, so failed iterations keep their content. -Arguments that parse as JSON are not redacted as one serialized text. The -redactor's patterns are written for plain text, and over JSON they miss an -assignment that opens a string or follows an escaped newline, a value -behind escaped quotes, and JSON nested in a string; Unicode folding can -also turn a fullwidth quotation mark into one that closes the string. -`toolArguments` decodes the value, redacts each string and object key on -its own (a number as its digits; one that redacts becomes the redacted -string), and encodes the result again. A string value or key that holds -one JSON object, array or string literal is judged as the arguments are -(`redactHeld`): decoded from the string as written, walked the same way -under the same member, and encoded again, so the structure a reader -decodes from the exported string is the structure that was judged — a -fold or an escape is inside the encoder's quoting — at the cost of the -document's layout (the encoder's key order, spacing and escapes). A -document that cannot be judged is masked `***` whole, with one -`held_document` finding: one the normalizer changes at all, as written, -since folding can make a document of text that is none as written, break -one, or spell a member-name pair into a key (fullwidth braces and -quotation marks); one held more than `maxHeldDepth` (4) deep, each level -being decoded and encoded once more; one in which two keys redact alike; -text shaped like a document — it begins and ends like one, white space -and invisible characters aside, and quotes something — that does not -parse (a comment, a trailing comma, an object literal, a Python dict with -a quoted value); and one whose text holds an exact value (a runtime -secret, a runner environment value) with structure of its own, which can -run over the document's (`valueAcross` over the text alone). Of two -members of one name decoding keeps the last, and the record keeps what -was decoded. That masks whole text with no secret in it too — a document -with a compatibility character (™, a fullwidth letter), JSON with -comments or trailing commas, an `Edit` fragment shaped `{…}` with a -quoted value. A key holding a document is masked whole as well. A string -the normalizer stripped an escape sequence or tag characters from — a -value or a key — is masked `***` whole: a colour code ends at the next -letter, so the stripping can take a token's first letter and leave the -rest, and a title code's payload is text no pattern sees. A member is -named as written as well as as scanned, and a key masked whole names a -secret whatever it was called — its name is not to be had from its mask -— so the values under it are judged as a secret-named member's. The -arguments are also judged across their keys, strings and numbers -(`atoms`: decoded, in the order written, a held document's in its place, -so the lines of a notebook cell held in a `Write`'s content are read as -lines) for a secret that can hold a line break, which no one of them can -mask. The private key block is sought in them joined by a line break, as -decoded and as the normalizer renders them (`blockAcross`): over the -lines of an array, over a key and a value, and inside one string as -well, which drops the arguments rather than masking the block in place. -An exact value is sought in them in order with white space set aside — -however a line ends and wherever the value was cut — an occurrence -within one atom being the literal pass's to replace; one holding -structure of its own (a credentials document in an env value) is sought -in the text as written too (`valueAcross`). The arguments are then -dropped and charged as encoded. Every other pattern's secret stops at -white space or at a quote, so a match over the break has only run its -context into the next string, which is left unjudged. Not judged: -structure in text not shaped like a document (several documents, YAML, a -document inside code); an assignment, a header or a connection string -whose value begins in the next string or number; a token split in pieces -between strings, or an exact value whose pieces other strings separate -or a string's escape spells; a member-name pair in single quotes over -two strings' apostrophes, or one a string's own quote opens or closes. Each string and number is also -scanned once more, already redacted, beside the nearest enclosing key the -member-name pattern (`json_field`) names, and masked whole when that pattern -matches the pair — a pattern keyed on a member -name has no other way to see the pair; that scan's other findings are -discarded, since both strings were already scanned. It runs the pattern -stage alone: the normalizer is not idempotent over escape sequences, and a -second pass over the pair could strip the value or the key's keyword. A value in an -array or a nested object under a secret-named member is judged as that -member's own value would be — the pattern's eight-character floor included, -so a short count stays; booleans and nulls are left as they are. Keys under -such a member are judged the same way, since a credential can be the key: a -key of eight characters or more is masked `***`, field names included, and -two masked alike collide (below). Arguments that are not -one JSON value cannot be walked that way: they are scanned as text — with -the misses above — so their findings count, then dropped and charged. So -are arguments in which two keys of one object redact to the same string, -where keeping either member would misreport the call. A call the stream -reports without a name carries no arguments. This happens once, when the -event is handled; eviction and `Result` do not rescan it. A name that -redacts to nothing at `Result` takes the arguments with it: they are -charged to the dropped bytes, and the part — kept only when it has a -summary — is marked. +`toolArguments` records the members of a call's input that +`recordedArguments` names — paths, patterns, commands, modes and bounds: +`file_path`, `pattern`, `path`, `command`, `description`, `url`, the +notebook and bound members, the Grep flags — and nothing else: a file +body, an edit, a prompt, a notebook source or a todo list, and any member +whose value is an object or an array, is dropped, charged as the redacted +text it was scanned as, and the part marked `fullsend.truncated`. The +guarantee is one sentence: the record holds these members of a call, each +redacted as text, and nothing else of its input. The input is decoded +(numbers keep their digits) and each kept string goes through the text +pipeline on its own, since the redactor's patterns are written for plain +text and over serialised JSON miss an assignment that opens a string or +follows an escaped newline, and a value behind escaped quotes; the result +is encoded again, so key order, spacing and escapes are the encoder's. A +string the normalizer stripped an escape sequence or tag characters from +is masked `***` whole: a colour code ends at the next letter, so the +stripping can take a token's first letter and leave the rest, and a title +code's payload is text no pattern sees. Discarded content is redacted +first, here as everywhere: a dropped member is scanned as text — the +decoded string, or the compact encoding of an object or an array — so a +secret in a file body still counts, and costs the summary as a secret +found anywhere in the arguments does. Arguments that are not one JSON +object are scanned as text and dropped whole, charged; so are kept +members whose encoding exceeds `maxToolArgumentsBytes`, charged as +encoded. A call the stream reports without a name carries no arguments. +This happens once, when the event is handled; eviction and `Result` do +not rescan it. A name that redacts to nothing at `Result` takes the +arguments with it: they are charged to the dropped bytes, and the part — +kept only when it has a summary — is marked. The input message does not come from the stream. When `runAgent` composes a retry prompt (`buildFeedbackPrompt`, under `feedback_mode: append`), diff --git a/docs/guides/infrastructure/distributed-tracing.md b/docs/guides/infrastructure/distributed-tracing.md index b4a672184c..0be7e00066 100644 --- a/docs/guides/infrastructure/distributed-tracing.md +++ b/docs/guides/infrastructure/distributed-tracing.md @@ -91,10 +91,10 @@ then mask the marker as well and count again. Not covered: a value split or spelled that way which a pattern also recognises — in one of those contexts, in a connection string, or by its own prefix — is masked by that pattern, and shows what the pattern's mask shows. A tool call loses -its `summary` when redaction found a secret in its arguments, stripped an -escape sequence or tag characters from them, or masked a document there -it could not judge: the parser cut the summary out of them before -anything scanned it, so the secret could be there as a beginning. Where a runtime reports a summary and no +its `summary` when redaction found a secret in its arguments, the dropped +members included, or stripped an escape sequence or tag characters from +them: the parser cut the summary out of them before anything scanned it, +so the secret could be there as a beginning. Where a runtime reports a summary and no arguments (pi, codex, OpenCode), its parser has already run the patterns over the summary and, for pi and codex, cut it: a value that straddles a cut stays in part, and one a pattern recognised stays as that pattern's @@ -120,27 +120,19 @@ with a `finish_reason` of `stop` or `error`. Tool arguments, tool results, and t only when the runtime's stream provides them: Claude runs do; the pi and codex parsers emit none of the three yet ([#7414](https://github.com/fullsend-ai/fullsend/issues/7414)). -`arguments` is the call's input decoded, redacted string by string (a -number that redacts becomes the redacted string), and -encoded again, so key order, spacing and escapes are not the stream's. -A string value in it that holds one JSON object, array or string -literal — a JSON file passed to `Write`, say — is judged the same way, -decoded, and recorded encoded again, so its layout is the encoder's and a -secret nested in it is masked in place. It is masked `***` whole when it -cannot be judged: held more than four deep, shaped like a document but -not parsing (JSON with comments or trailing commas, an object literal), -holding a runner environment or runtime secret value across its -structure, or changed at all by Unicode normalization, since -folding can make or break a document or spell a secret-named pair into a -key — the [developer guide](../dev/tracing.md) has the list. A key that -holds a document is masked whole. A secret that runs across the strings -of the arguments, or of a held document, in the order written — a private -key over the lines of an array, a runner environment value over them — -can be masked nowhere but whole: the arguments are dropped. A private -key inside one string drops them as well. A string a terminal escape sequence was stripped from is masked -whole too, since the stripping can leave most of a token. A string -directly under a secret-named member is masked in place, as anywhere -else. +`arguments` holds the members of the call's input that name what was +called, each redacted as text: `file_path`, `notebook_path`, `path`, +`pattern`, `glob`, `command`, `description`, `url`, `output_mode`, +`model`, `subagent_type`, `skill`, `cell_id`, `cell_type`, `edit_mode`, +`replace_all`, `multiline`, `type`, `limit`, `offset`, `head_limit`, +`timeout`, `context`, `-A`, `-B`, `-C`, `-i`, `-n`. Nothing else of the +input is recorded: a file body, an edit, a prompt, a notebook source, a +todo list, and any member whose value is an object or an array are +dropped — scanned first, so a secret in them still counts — and the part +is marked `fullsend.truncated`. The kept members are decoded, redacted +one by one and encoded again, so key order, spacing and escapes are not +the stream's; a string a terminal escape sequence was stripped from is +masked whole, since the stripping can leave most of a token. On a retry iteration under `validation_loop.feedback_mode: append`, the prompt the runner composed — its fixed framing around the previous @@ -173,18 +165,17 @@ and up to six times on escape-dense content — a record over that is trimmed ag oldest first. Truncation is marked via `fullsend.content.truncated` on the span and `fullsend.truncated` on each cut part. A tool result over its bound keeps -its tail; arguments over theirs, not a complete JSON value, or with two -keys of one object that redact to the same string (two keys of eight -characters or more under a secret-named member, such as `username` and -`password` under `auth`, are both masked and so collide), are dropped -whole — a cut object is not JSON — and the marked `tool_call` part keeps -its `id`, name and summary — not the summary when redaction found a -secret in the arguments, stripped an escape sequence or tag characters -from them, or masked a document there it could not judge. Arguments also go, charged and marked, when -the call's name redacts to nothing; that part survives only if it has a -summary. So `fullsend.truncated` means different things by part type: on a -tool result, part of the response is kept; on a tool call, none of the -arguments is. Read it together with the part's `type`. The input message is not cut at this stage: +its tail; arguments over theirs, or not a complete JSON object, are +dropped whole — a cut object is not JSON — and the marked `tool_call` +part keeps its `id`, name and summary — not the summary when redaction +found a secret in the arguments or stripped an escape sequence or tag +characters from them. A member outside the recorded list (above) is +dropped and the part marked the same way, its listed members kept. +Arguments also go, charged and marked, when the call's name redacts to +nothing; that part survives only if it has a summary. So +`fullsend.truncated` means different things by part type: on a tool +result, part of the response is kept; on a tool call, some or all of the +input is not. Read it together with the part's `type`. The input message is not cut at this stage: the validation output inside it is cut at 10 KiB when the prompt is composed (the prompt then says `[truncated]`), and the two markers above do not describe it. The recorded copy is the redacted one, so it can diff --git a/internal/cli/content_capture_record_test.go b/internal/cli/content_capture_record_test.go index fa75f8c3cd..f80c85ad12 100644 --- a/internal/cli/content_capture_record_test.go +++ b/internal/cli/content_capture_record_test.go @@ -76,10 +76,11 @@ func TestContentCapture_SecretsAcrossArgumentsNeverReachTheRecord(t *testing.T) // The record that leaves the runner, run-telemetry.jsonl, written // through the real event handler, collector, tool span tracker and // file sink with Level 3 on: a private key over the lines of a - // notebook cell passed as arguments, a credential that folding moves - // between members of a held document, and a token a colour code hides - // the first letter of are not in it; the clean call's arguments and - // every call's name are; and the markers say what was dropped. + // notebook cell passed as arguments, a credential in a file body + // folding would move between members, and a token a colour code + // hides the first letter of are not in it; the listed members of + // every call and every call's name are; and the markers say what was + // dropped. for _, k := range []string{"OTEL_SDK_DISABLED", "OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT", "OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT"} { t.Setenv(k, "") } @@ -132,10 +133,11 @@ func TestContentCapture_SecretsAcrossArgumentsNeverReachTheRecord(t *testing.T) } } require.Len(t, calls, 4, "every call is in the record, arguments or not") - assert.NotContains(t, calls["NotebookEdit"], "arguments", "a private key over the cell's lines drops the arguments") + assert.Equal(t, map[string]any{"notebook_path": "notes.ipynb"}, calls["NotebookEdit"]["arguments"], "the cells are not recorded; the path is") assert.Equal(t, true, calls["NotebookEdit"]["fullsend.truncated"]) - assert.NotContains(t, calls["NotebookEdit"], "summary", "a secret in the arguments costs the summary") - assert.Equal(t, map[string]any{"file_path": "cfg.json", "content": "***"}, calls["Write"]["arguments"], "the held document folding changes is masked whole") + assert.NotContains(t, calls["NotebookEdit"], "summary", "a secret in the dropped cells still costs the summary") + assert.Equal(t, map[string]any{"file_path": "cfg.json"}, calls["Write"]["arguments"], "the file body is not recorded; the path is") + assert.Equal(t, "cfg.json", calls["Write"]["summary"], "no secret was found in it") assert.Equal(t, map[string]any{"command": "***"}, calls["Bash"]["arguments"], "the string the colour code was stripped from is masked whole") assert.Equal(t, map[string]any{"file_path": "README.md"}, calls["Read"]["arguments"], "the clean call's arguments are recorded") assert.Equal(t, "README.md", calls["Read"]["summary"]) diff --git a/internal/cli/content_collector.go b/internal/cli/content_collector.go index b0df97b683..c3826aff7c 100644 --- a/internal/cli/content_collector.go +++ b/internal/cli/content_collector.go @@ -6,7 +6,6 @@ import ( "slices" "sort" "strings" - "unicode" "unicode/utf8" "go.opentelemetry.io/otel/attribute" @@ -98,19 +97,6 @@ const maxToolResultBytes = 8 * 1024 // measured; a Write call carries the file body as an argument. const maxToolArgumentsBytes = 8 * 1024 -// maxRawToolArgumentsBytes bounds the arguments as written, before they -// are decoded. Redaction can bring them under maxToolArgumentsBytes (a -// token masks to seven bytes), but not from far over it, and the walk -// must do no work that bound does not limit. Over it they are scanned -// as text, so their findings count, then dropped and charged. -const maxRawToolArgumentsBytes = 4 * maxToolArgumentsBytes - -// maxHeldDepth bounds how deep redactHeld walks documents held in -// strings: each level is decoded and encoded once more, and with \u -// escapes a document can nest about the square root of its length deep. -// A string holding one deeper is masked whole. -const maxHeldDepth = 4 - // newContentCollectorIfEnabled returns a live collector when the Level 3 // gate is on and nil otherwise — nil is the off state and is inert at // every call site, so the gate needs no second check. @@ -376,11 +362,6 @@ type contentCollector struct { // maxEncoded bounds the marshaled attribute; see maxEncodedContentBytes. maxEncoded int pipeline *security.Pipeline - // secrets is the pipeline's pattern stage alone, for secretNamed and - // blockAcross; normalizer its first stage alone, for the detection - // copies holdsDocument and blockAcross judge. - secrets *security.SecretRedactor - normalizer *security.UnicodeNormalizer // runnerEnv feeds redact's literal pass; see newContentCollectorIfEnabled. runnerEnv map[string]string parts []contentPart @@ -401,13 +382,10 @@ type contentCollector struct { // Merged into the contentResult at Result so they count exactly like // assembly-time ones. findings []security.Finding - // held counts the documents held in strings that redactHeld is - // walking; see maxHeldDepth. - held int } func newContentCollector(maxBytes int) *contentCollector { - return &contentCollector{maxBytes: maxBytes, maxEncoded: maxEncodedContentBytes, pipeline: security.OutputPipeline(), secrets: security.NewSecretRedactor(), normalizer: security.NewUnicodeNormalizer()} + return &contentCollector{maxBytes: maxBytes, maxEncoded: maxEncodedContentBytes, pipeline: security.OutputPipeline()} } // Handle consumes one normalized event. Contiguous text and reasoning @@ -434,11 +412,10 @@ func (c *contentCollector) Handle(evt agentruntime.AgentEvent) { if slices.ContainsFunc(c.findings[scanned:], revealing) { // The parser cut the summary out of these arguments // before anything scanned it, so a secret found in them, - // stripped with an escape sequence or tag characters, or - // in a document masked unjudged, can be in the summary as - // a beginning that neither the literal pass nor a pattern - // matches, nor the normalizer strips once its terminator - // is cut. + // or stripped with an escape sequence or tag characters, + // can be in the summary as a beginning that neither the + // literal pass nor a pattern matches, nor the normalizer + // strips once its terminator is cut. p.Summary = "" } } @@ -779,56 +756,96 @@ func redactText(pipeline *security.Pipeline, runnerEnv map[string]string, text s return text } -// toolArguments returns a tool call's arguments redacted and re-encoded, -// and whether they were dropped instead. There are none to return when -// the event carries none or JSON null. +// recordedArguments names the members of a tool call's arguments that +// the record keeps: paths, patterns, commands, modes and bounds — what a +// trace needs to say which call was made — and never a file body, an +// edit, a prompt, a notebook source or a todo list, the members that +// carry what the agent read or wrote. A member outside the list, or one +// whose value is an object or an array, is dropped: scanned as text so +// its findings count, charged and marked, like any other discarded +// content. Each kept string goes through the text pipeline on its own. +// The guarantee is then one sentence: the record holds these members of +// a call, each redacted as text, and nothing else of its input. Claude +// Code's tools today: Bash (command, description, timeout), Read +// (file_path, limit, offset), Write, Edit and NotebookEdit (file_path, +// notebook_path, cell_id, replace_all), Grep and Glob (pattern, path, +// glob, output_mode, the context flags), WebFetch (url), Agent +// (description, model, subagent_type), Skill (skill). The reference +// lists them; a tool that names its path or pattern otherwise records +// nothing of its input, which is the safe default. +var recordedArguments = map[string]bool{ + "-A": true, "-B": true, "-C": true, "-i": true, "-n": true, + "cell_id": true, "cell_type": true, "command": true, "context": true, + "description": true, "edit_mode": true, "file_path": true, "glob": true, + "head_limit": true, "limit": true, "model": true, "multiline": true, + "notebook_path": true, "offset": true, "output_mode": true, "path": true, + "pattern": true, "replace_all": true, "skill": true, "subagent_type": true, + "timeout": true, "type": true, "url": true, +} + +// toolArguments returns the recorded members of a tool call's arguments +// (recordedArguments), redacted and encoded again, and whether any of +// the input was dropped instead. There are none to return when the event +// carries none or JSON null. // -// The redactor's patterns are written for plain text: run over serialised +// The input is decoded and each kept string is redacted on its own: the +// redactor's patterns are written for plain text, and run over serialised // JSON they miss an assignment that opens a string or follows an escaped -// newline, a value behind escaped quotes, and JSON nested in a string, and -// Unicode folding can turn a fullwidth quotation mark into one that closes -// the string. So the value is decoded, each string, number and object key -// is redacted on its own (redactValue), and the result is encoded again — -// key order, spacing and escapes are the encoder's, not the wire's. -// -// Text that is not one JSON value (see ToolUseEvent.Arguments) cannot be -// redacted that way, and text over maxRawToolArgumentsBytes is not -// decoded at all. Either is scanned as text so its findings count, then -// dropped and charged like any other discarded content. Arguments whose -// redacted encoding exceeds maxToolArgumentsBytes, in which two keys of -// one object redact to the same string, or across whose keys, strings -// and numbers a secret runs — a private key block over the lines of an -// array (blockAcross), an exact value over them (valueAcross) — which -// no one of them can mask, are dropped the same way and charged as -// encoded. On a collision that is the encoding after the later key (in -// sorted order) replaced the earlier member, so the replaced member is -// not counted. +// newline, and a value behind escaped quotes. The result is encoded +// again — key order, spacing and escapes are the encoder's, not the +// wire's — and a number keeps its digits. Text that is not one JSON +// object (see ToolUseEvent.Arguments) cannot be walked that way: it is +// scanned as text so its findings count, then dropped whole and charged. +// So are arguments whose encoding exceeds maxToolArgumentsBytes, charged +// as encoded. A dropped member is charged as the redacted text it was +// scanned as. func (c *contentCollector) toolArguments(args string) (json.RawMessage, bool) { if args == "" { return nil, false } - if len(args) > maxRawToolArgumentsBytes || !json.Valid([]byte(args)) { + v, err := decodeJSON(args) + if err == nil && v == nil { + return nil, false + } + obj, ok := v.(map[string]any) + if err != nil || !ok { c.evicted += len(c.redact(args, &c.findings)) return nil, true } - v, err := decodeJSON(args) - if err != nil || v == nil { - return nil, false + out, dropped := make(map[string]any, len(obj)), false + for _, k := range slices.Sorted(maps.Keys(obj)) { + switch t := obj[k].(type) { + case string: + if recordedArguments[k] { + out[k] = c.scanLeaf(t) + continue + } + c.evicted += len(c.redact(t, &c.findings)) + case map[string]any, []any: + enc, _ := json.Marshal(t) // decoded JSON values marshal unconditionally + c.evicted += len(c.redact(string(enc), &c.findings)) + default: // a number, a boolean, null + if recordedArguments[k] { + out[k] = t + continue + } + enc, _ := json.Marshal(t) + c.evicted += len(c.redact(string(enc), &c.findings)) + } + dropped = true + } + if len(out) == 0 && dropped { + return nil, true } - collided := false - out, err := json.Marshal(c.redactValue(v, "", &collided)) + enc, err := json.Marshal(out) if err != nil { return nil, false // decoded JSON values marshal unconditionally } - // Judged across its atoms whatever else drops it: the secret found - // there is a finding of the call, and costs it its summary. - as := atoms(args, 0) - across := c.blockAcross(as) || c.valueAcross(as, args) - if collided || across || len(out) > maxToolArgumentsBytes { - c.evicted += len(out) + if len(enc) > maxToolArgumentsBytes { + c.evicted += len(enc) return nil, true } - return out, false + return enc, dropped } // decodeJSON decodes one JSON value; float64 would rewrite integers past @@ -841,81 +858,12 @@ func decodeJSON(s string) (any, error) { return v, err } -// jsonDocument returns s trimmed when it is one JSON object, array or -// string literal: a string literal can spell a secret with escapes too. -func jsonDocument(s string) (string, bool) { - s = strings.TrimSpace(s) - return s, s != "" && strings.IndexByte(`{["`, s[0]) >= 0 && json.Valid([]byte(s)) -} - -// documentLike reports text that begins and ends as a JSON object, array -// or string literal does and holds a quoted token, white space, control -// and format characters (a byte order mark, a zero-width space) aside: -// the normalizer strips the latter from the scanned text, not from the -// written one. A mask is bracketed too — [REDACTED PRIVATE KEY], the -// runner env marker — but quotes nothing. -func documentLike(s string) bool { - s = strings.TrimFunc(s, func(r rune) bool { return unicode.IsSpace(r) || unicode.IsControl(r) || unicode.Is(unicode.Cf, r) }) - return len(s) >= 2 && (s[0] == '{' && s[len(s)-1] == '}' || s[0] == '[' && s[len(s)-1] == ']' || s[0] == '"' && s[len(s)-1] == '"') && strings.ContainsAny(s, `"'`) -} - -// heldFinding is the finding for a string holding a JSON document that -// is masked whole, for reason. -func heldFinding(reason string) security.Finding { - return security.Finding{Scanner: "held_document", Name: reason, Severity: "high", Detail: "string holding a JSON document masked whole", Position: -1} -} - -// redactHeld redacts the JSON object, array or string literal doc that a -// string of the arguments holds, as the arguments are redacted: decoded, -// walked (redactValue, under the same member), and encoded again. The -// structure a reader decodes from the string is then the structure that -// was judged — a fold or an escape is inside the encoder's quoting, and -// moves no value between members — at the cost of the document's layout. -// Of two members of one name decoding keeps the last, and the record -// keeps what was decoded. A document that cannot be judged is masked -// whole: one held deeper than maxHeldDepth, and one in which two keys -// redact alike. -func (c *contentCollector) redactHeld(doc, under string) string { - if c.held == maxHeldDepth { - c.findings = append(c.findings, heldFinding("uninspected")) - return "***" - } - v, _ := decodeJSON(doc) // a valid document decodes - collided := false - c.held++ - e := c.redactValue(v, under, &collided) - c.held-- - if collided { - c.findings = append(c.findings, heldFinding("collision")) - return "***" - } - b, _ := json.Marshal(e) // decoded JSON values marshal unconditionally - return string(b) -} - // revealing reports a finding that shows a secret was there: any but the // normalizer's, and the normalizer's stripping of content. func revealing(f security.Finding) bool { return f.Scanner != "unicode_normalizer" || stripped(f) } -// holdsDocument returns the JSON document s holds and can be judged, -// or whether s is shaped like one that cannot (like): a document the -// normalizer changes at all — folding can spell a member-name pair -// into a key, or make a document of text that is none as written (a -// fullwidth brace or quotation mark) — and text shaped like a document -// (documentLike) as written or as rendered. -func (c *contentCollector) holdsDocument(s string) (doc string, ok, like bool) { - n := c.normalized(s) - if doc, ok = jsonDocument(s); ok { - return doc, n == s, n != s - } - if _, ok = jsonDocument(n); ok { - return "", false, true - } - return "", false, documentLike(s) || documentLike(n) -} - // stripped reports the normalizer's removal of an escape sequence or of // tag characters, which can carry text the patterns never see — digits // as a CSI parameter, a token inside an OSC sequence — or take the first @@ -924,10 +872,10 @@ func stripped(f security.Finding) bool { return f.Scanner == "unicode_normalizer" && (f.Name == "ansi_escape" || f.Name == "osc_escape" || f.Name == "tag_char") } -// scanLeaf redacts one string of the arguments, a value or a key: through -// the pipeline, which masks in place, and masked whole when the -// normalizer stripped an escape sequence or tag characters from it -// (stripped), since what is left can be most of a token. +// scanLeaf redacts one string of the arguments: through the pipeline, +// which masks in place, and masked whole when the normalizer stripped an +// escape sequence or tag characters from it (stripped), since what is +// left can be most of a token. func (c *contentCollector) scanLeaf(s string) string { mark := len(c.findings) r := c.redact(s, &c.findings) @@ -937,258 +885,6 @@ func (c *contentCollector) scanLeaf(s string) string { return r } -// normalized returns the normalizer's rendering of s: a detection copy, -// which nothing exports. -func (c *contentCollector) normalized(s string) string { - return c.normalizer.Scan(s).Sanitized -} - -// atoms lists the strings and numbers of raw, one valid JSON value, -// decoded, in the order written — as a reader of the arguments reads -// them, whatever order the encoder gives the keys. A string holding a -// document (to maxHeldDepth) gives the document's atoms in its place, so -// the lines of a notebook cell held in a Write's content are read as -// lines. -func atoms(raw string, depth int) []string { - dec := json.NewDecoder(strings.NewReader(raw)) - dec.UseNumber() - var out []string - for { - tok, err := dec.Token() - if err != nil { - return out // io.EOF: raw is valid - } - switch t := tok.(type) { - case string: - if doc, ok := jsonDocument(t); ok && depth < maxHeldDepth { - out = append(out, atoms(doc, depth+1)...) - } else { - out = append(out, t) - } - case json.Number: - out = append(out, t.String()) - } - } -} - -// blockAcross reports, and records as one critical finding, a private -// key block in the atoms of the arguments — over the lines of an array, -// over a key and a value, over the lines of a held document — which no -// one of them can mask: its pattern reads over a line break, so the -// atoms are judged joined by one, as decoded and as the normalizer -// renders them, since a marker can be folded. A block inside one atom -// drops the arguments as well: coarser than the mask in place the atom -// gets, and nothing left to judge. -func (c *contentCollector) blockAcross(atoms []string) bool { - lines := strings.Join(atoms, "\n") - for _, text := range []string{lines, c.normalized(lines)} { - for _, m := range c.secrets.Matches(text) { - if m.Name == "private_key" { - c.findings = append(c.findings, security.Finding{Scanner: "secret_redactor", Name: m.Name, Severity: "critical", Detail: "secret spanning keys, strings or numbers", Position: -1}) - return true - } - } - } - return false -} - -// valueAcross reports, and records as one critical finding, an exact -// value — a runtime secret, a runner env value the literal pass replaces -// — that runs across the atoms of the arguments, in the order written, -// with white space set aside: however a line ends (its own line break, -// a Windows one, a space) and wherever the value was cut, it is read as -// a reader of the lines reads it. An occurrence within one atom is the -// literal pass's to replace, and is set aside first. A value holding -// structure of its own — a quote, a comma, a colon, a bracket, as a -// credentials document in an env value does — is sought in the text as -// written as well, where it can run over the structure between the -// atoms. -func (c *contentCollector) valueAcross(atoms []string, raw string) bool { - for _, l := range exactValues(c.runnerEnv) { - v := unspaced(l.value) - if v == "" { - continue - } - var b strings.Builder - for _, a := range atoms { - b.WriteString(strings.ReplaceAll(unspaced(a), v, "")) - } - if strings.Contains(b.String(), v) || strings.ContainsAny(v, `",:{}[]`) && strings.Contains(unspaced(raw), v) { - c.findings = append(c.findings, security.Finding{Scanner: l.scanner, Name: l.key, Severity: "critical", Detail: "secret spanning keys, strings or numbers", Position: -1}) - return true - } - } - return false -} - -// exactValue is a value sought exactly, and how a finding names it. -type exactValue struct{ scanner, key, value string } - -// exactValues lists the runtime secrets and the runner env values the -// literal pass replaces. -func exactValues(runnerEnv map[string]string) []exactValue { - var out []exactValue - for _, v := range security.RuntimeSecrets() { - out = append(out, exactValue{"secret_redactor", "runtime_secret", v}) - } - for _, l := range envLiterals(runnerEnv) { - out = append(out, exactValue{"runner_env", l.key, l.value}) - } - return out -} - -// unspaced renders s without its white space. -func unspaced(s string) string { - return strings.Map(func(r rune) rune { - if unicode.IsSpace(r) { - return -1 - } - return r - }, s) -} - -// redactValue redacts every string, number and object key under v; a -// number that redacts becomes the redacted string, and a string holding -// a JSON document is judged as the arguments are and encoded again -// (redactHeld), or masked whole when it cannot be judged. A pattern -// keyed on a member name ("password": "...") cannot see the pair that -// way, so a string or number under a key that pattern names — the nearest -// such enclosing key, under, which a stand-in name represents — is scanned -// once more, already redacted, beside it and masked whole on a match -// (secretNamed); a direct string member is also scanned beside its own -// key. A value in -// an array or a nested object under a secret-named member is judged as -// that member's own value would be, and so is a key there. -// Keys are walked in sorted order, so neither the findings nor a dropped -// value's charge follow map order. *collided reports two keys of one -// object that redact to the same string: keeping either member would -// show a call the agent did not make. -func (c *contentCollector) redactValue(v any, under string, collided *bool) any { - switch t := v.(type) { - case string: - doc, ok, like := c.holdsDocument(t) - if ok { - if c.valueAcross(nil, t) { - // An exact value holding structure of its own can run - // over the document's, which no leaf of the walk holds. - return "***" - } - return c.redactHeld(doc, under) - } - if like { - // A document that cannot be judged: one the normalizer - // changes, or text shaped like one that does not parse — a - // comment, a trailing comma, an object literal — holding - // whatever it nests under a secret-named member, which - // neither the patterns nor a walk can see. - c.findings = append(c.findings, heldFinding("uninspected")) - return "***" - } - s := c.scanLeaf(t) - if under != "" && c.secretNamed(under, s) { - return "***" - } - return s - case json.Number: - // Digits can be a credential too; a number that redacts is - // recorded as the redacted string. - s := c.redact(t.String(), &c.findings) - if under != "" && c.secretNamed(under, s) { - return "***" - } - if s != t.String() { - return s - } - case []any: - for i := range t { - t[i] = c.redactValue(t[i], under, collided) - } - case map[string]any: - out := make(map[string]any, len(t)) - for _, k := range slices.Sorted(maps.Keys(t)) { - // The key as scanned names the secret, masked whole or not. - sk := c.scanLeaf(k) - rk := sk - if _, ok, like := c.holdsDocument(k); ok || like { - // A key holding a document is masked whole: a reader - // decodes nothing from a key, and a document can name a - // secret in a spelling only decoding reads. - c.findings = append(c.findings, heldFinding("key")) - rk = "***" - } - member := under - // Named as scanned, or as written: the normalizer can strip - // the name's first letter with a colour code, and the value - // under it is still that name's. A key masked whole — by the - // scan, or as a document it holds — names a secret whatever - // it was called: its name is not to be had from its mask. - named := rk == "***" || c.namesASecret(sk) || c.namesASecret(k) - if named { - // Decided once for the member: the pattern reads any - // name it matches alike, so a stand-in judges the values - // under it at a cost the key's length does not multiply. - member = "secret" - } - e := c.redactValue(t[k], member, collided) - if s, ok := e.(string); ok && !named && c.secretNamed(sk, s) { - // A single quote in the key can let the pattern match - // the pair even when the key names no secret. - e = "***" - } - if under != "" && c.secretNamed(under, sk) { - // A key under a secret-named member is judged like its - // values: a credential can be the key. Two masked alike - // take the collision path below. - rk = "***" - } - if _, dup := out[rk]; dup { - *collided = true - } - out[rk] = e - } - return out - } - return v -} - -// namesASecret reports whether the member-name pattern matches key with -// any value long enough: a value placed under such a key is then judged -// against it. It records no finding; secretNamed does, for a value that -// matches. -func (c *contentCollector) namesASecret(key string) bool { - return slices.ContainsFunc(c.secrets.Scan(memberPair(key, strings.Repeat("x", 8))).Findings, - func(f security.Finding) bool { return f.Name == "json_field" }) -} - -// secretNamed reports whether the redactor's member-name pattern -// (json_field) matches the pair, and records that finding. Any other -// finding of this second scan is discarded: both strings were scanned -// already, and a connection-string mask of the "abcd..." form (a password -// of ten or more bytes) matches its own pattern again. The scanned text -// is never exported, and it goes to the pattern stage alone: both strings -// are normalized already, and the normalizer is not idempotent over -// escape sequences — run again over the pair it can strip one the first -// pass left open, and the value or the key's keyword with it. A double -// quote would end the pattern's quoted run early, so ',' stands in for it: like the quote it is outside every -// pattern's token class, so it joins no two runs into a token that a -// prefix pattern would mask ahead of the member-name pattern. -func (c *contentCollector) secretNamed(key, value string) bool { - for _, f := range c.secrets.Scan(memberPair(key, value)).Findings { - if f.Name == "json_field" { - c.findings = append(c.findings, f) - return true - } - } - return false -} - -// memberPair writes key and value as the JSON member secretNamed scans, -// ',' standing in for a double quote. -func memberPair(key, value string) string { - unquote := strings.NewReplacer(`"`, ",") - return `"` + unquote.Replace(key) + `":"` + unquote.Replace(value) + `"` -} - // replaceEnv is redactText's literal pass, with one finding for each key // it replaced. func replaceEnv(runnerEnv map[string]string, text string, findings *[]security.Finding) string { diff --git a/internal/cli/content_collector_test.go b/internal/cli/content_collector_test.go index 966228e6e5..baaf8c9dbd 100644 --- a/internal/cli/content_collector_test.go +++ b/internal/cli/content_collector_test.go @@ -4,10 +4,8 @@ import ( "encoding/json" "fmt" "math/rand" - "slices" "strings" "testing" - "time" "unicode/utf8" "github.com/stretchr/testify/assert" @@ -16,7 +14,6 @@ import ( "go.opentelemetry.io/otel/sdk/trace/tracetest" agentruntime "github.com/fullsend-ai/fullsend/internal/runtime" - "github.com/fullsend-ai/fullsend/internal/security" "github.com/fullsend-ai/fullsend/internal/telemetry" ) @@ -116,33 +113,26 @@ func TestContentCollector_ToolCallPartCarriesArguments(t *testing.T) { } func TestContentCollector_RedactsSecretsInArguments(t *testing.T) { + // A kept string goes through the text pipeline on its own, decoded: + // the patterns are written for plain text, and over serialised JSON + // they miss an assignment that opens a string or follows an escaped + // newline, and a value behind escaped quotes. const secret = "s3cr3tvalue99xyz" pat := "ghp_" + strings.Repeat("m", 36) for name, args := range map[string]string{ "leading assignment": `{"command":"DEPLOY_TOKEN=` + secret + ` make"}`, "assignment after newline": `{"command":"cd x\nAPI_KEY=` + secret + ` run"}`, "assignment in quotes": `{"command":"export GH_TOKEN=\"` + secret + `\""}`, - "json inside a string": `{"content":"{\"password\": \"` + secret + `\"}"}`, - "secret-named member": `{"password":"` + secret + `"}`, - "nested member": `{"edits":[{"headers":{"api_key":"` + secret + `"}}]}`, "token prefix": `{"command":"curl -u x:` + pat + ` h"}`, - "token as a key": `{"` + pat + `":"v"}`, - // A secret-named member whose value raises a finding of its own - // (Unicode or another secret) is still scanned beside its key. - "secret-named, zero width": `{"password":"s3cr3tva\u200Blue99xyz"}`, - "secret-named, folded": `{"password":"` + secret + `\uFF01"}`, - "secret-named, two secrets": `{"password":"` + secret + ` ` + pat + `"}`, - "secret-named, early quote": `{"password":"ab\"` + secret + `"}`, - "quote in a secret-named key": `{"api\"key":"` + secret + `"}`, - // Joined by the probe's stand-in for the quote, the two runs must - // not become one token a prefix pattern masks ahead of the - // member-name pattern. - "quote inside a token run": `{"api_key":"sk-abc\"` + secret + `"}`, - // The normalizer is not idempotent over escape sequences: run a - // second time over the pair, it strips one the first pass left - // open and takes the value, or the key's keyword, with it. - "escape sequence closed by the pair, value": `{"password":"\u001b]\u001b]x\u0007` + secret + `\u0007"}`, - "escape sequence closed by the pair, key": `{"password\u001b]":"ab\u0007` + secret + `"}`, + "two secrets": `{"command":"TOKEN=` + secret + ` ` + pat + `"}`, + // The normalizer joins what a zero-width character split and + // folds what was spelled in compatibility forms before the + // patterns run. + "zero width": `{"command":"TOKEN=s3cr3tva​lue99xyz"}`, + "folded": `{"command":"TOKEN=` + secret + `！"}`, + // An escape sequence the normalizer strips can carry the value + // the patterns never see: such a string is masked whole. + "inside an escape sequence": `{"command":"\u001b]\u001b]x\u0007` + secret + `\u0007"}`, } { t.Run(name, func(t *testing.T) { c := newContentCollector(4096) @@ -158,251 +148,85 @@ func TestContentCollector_RedactsSecretsInArguments(t *testing.T) { } } -func TestContentCollector_SecretNamedMembersReachTheirNestedValues(t *testing.T) { - // A value at any depth under a secret-named member is judged as if it - // were that member's own string value. +func TestContentCollector_RecordsOnlyListedArguments(t *testing.T) { + // The record keeps the members recordedArguments names — paths, + // patterns, commands, modes and bounds — and nothing else of a call's + // input: a file body, an edit, a prompt, a notebook source or a todo + // list is dropped, charged and marked, with no finding and the + // summary kept, since nothing found a secret. const opaque = "opaque-credential-123" - for name, tc := range map[string]struct{ args, want string }{ - "array": {`{"api_keys":["` + opaque + `"]}`, `{"api_keys":["***"]}`}, - "object": {`{"credentials":{"value":"` + opaque + `"}}`, `{"credentials":{"value":"***"}}`}, - "array of object": {`{"tokens":[{"v":"` + opaque + `"}]}`, `{"tokens":[{"v":"***"}]}`}, - "number": {`{"password":31415926535}`, `{"password":"***"}`}, - "nested number": {`{"secrets":{"pin":31415926535}}`, `{"secrets":{"pin":"***"}}`}, - // The member-name pattern wants eight bytes or more, as it does of - // a string member: short values and a token count stay. - "short values stay": {`{"max_tokens":4096,"api_keys":["ab",true,null]}`, `{"api_keys":["ab",true,null],"max_tokens":4096}`}, - // No secret-named member above: nothing to judge against. - "plain nesting stays": {`{"edits":[{"value":"` + opaque + `"}]}`, `{"edits":[{"value":"` + opaque + `"}]}`}, - // A quote in the key lets the pattern's single-quote form match - // across the pair: the member's own key is still checked. - "quote in a plain key": {`{"it's":"password': 'hunter2hunter2'"}`, `{"it's":"***"}`}, - // Keys are judged too: a credential can be the key. - "key under a secret-named member": {`{"credentials":{"` + opaque + `":"user"}}`, `{"credentials":{"***":"user"}}`}, - "key under a secret-named array": {`{"api_keys":[{"` + opaque + `":"prod"}]}`, `{"api_keys":[{"***":"prod"}]}`}, - "short keys stay": {`{"auth":{"user":"x"}}`, `{"auth":{"user":"x"}}`}, - "keys with no secret-named ancestor": {`{"edits":{"` + opaque + `":"x"}}`, `{"edits":{"` + opaque + `":"x"}}`}, - } { - t.Run(name, func(t *testing.T) { - c := newContentCollector(4096) - c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Arguments: tc.args}) - - res := c.Result("stop") - assert.Contains(t, res.OutputMessages, `"arguments":`+tc.want) - if tc.want == tc.args || strings.Contains(name, "stay") { - assert.Empty(t, res.Findings) - } else { - assert.Len(t, res.Findings, 1, "one json_field finding") - } - }) - } -} - -func TestContentCollector_JSONInAStringIsMaskedWholeWhenItsDocumentHoldsASecret(t *testing.T) { - // A string holding a document is judged as the arguments are: decoded, - // walked and encoded again, so a secret-named member nested in it, or - // a secret only decoding spells out, is masked in place; what cannot - // be judged is masked whole. - const opaque = "opaque-credential-123" - cred := `"credentials":{"value":"` + opaque + `"}` - token := "ghp_" + strings.Repeat("r", 36) - quote := func(s string) string { b, _ := json.Marshal(s); return string(b) } - for name, tc := range map[string]struct { - content, want string - findings int - }{ - "object under credentials": {`{` + cred + `}`, `{"credentials":{"value":"***"}}`, 1}, - "array under password": {`{"password":["` + opaque + `"]}`, `{"password":["***"]}`, 1}, - "document that is an array": {`[{"token":{"v":"` + opaque + `"}}]`, `[{"token":{"v":"***"}}]`, 1}, - "key under credentials": {`{"credentials":{"` + opaque + `":"user"}}`, `{"credentials":{"***":"user"}}`, 1}, - "document in a document": {`{"inner":` + quote(`{"token":["`+opaque+`"]}`) + `}`, `{"inner":` + quote(`{"token":["***"]}`) + `}`, 1}, - // Decoded, each string is judged on its own. - "an assignment that opens a string": {`{"command":"API_KEY=` + opaque + `"}`, `{"command":"API_KEY=opaq..."}`, 1}, - "an assignment after an escaped break": {`{"c":"cd x\nAPI_KEY=` + opaque + ` run"}`, `{"c":"cd x\nAPI_KEY=opaq... run"}`, 1}, - "a token spelled with an escape": {`{"t":"ghp_\u0072` + token[5:] + `"}`, `{"t":"ghp_..."}`, 1}, - "tag characters written with escapes": {`{"t":"\uDB40\uDC67"}`, `{"t":"***"}`, 1}, - "digits in an escape under a secret name": {`{"auth":{"pin":"\u001b[31415926535m"}}`, `{"auth":{"pin":"***"}}`, 1}, - "an escape sequence around a token": {`{"n":"see \u001b]ghp_\u0072` + token[5:] + `\u0007"}`, `{"n":"***"}`, 1}, - // A document the normalizer changes at all is masked whole: folding - // can make a document, break one, or spell a pair into a key. - "folded into a document": {"\uFF5B" + cred + "}", "***", 1}, - "folding breaks a document": {"{\"note\":\"\uFF02\"," + cred + "}", "***", 1}, - "a BOM and folding that breaks a document": {"\uFEFF{\"note\":\"\uFF02\"," + cred + "}", "***", 1}, - "a fold that makes and breaks a document": {"\uFF5B\"note\":\"\uFF02\"," + cred + "}", "***", 1}, - "an escape before a document folding breaks": {"\x1b[0m{\"note\":\"\uFF02\"," + cred + "}", "***", 1}, - "a mask breaks a document": {`{"token":"abcdefgh\\\"x",` + cred + `}`, `{"credentials":{"value":"***"},"token":"***"}`, 2}, - // Folding that moves a value out of its secret-named member, or - // moves nothing, alike. - "folding that moves a value between members": {"{\"credentials\":{\"value\":\"\uFF02},\uFF02note\uFF02:{\uFF02value\uFF02:\uFF02" + opaque + "\"}}", "***", 1}, - "folding that moves nothing": {"{\"name\":\"Widget\u2122\"}", "***", 1}, - // Decoding keeps the last of two members of one name, and the - // record keeps what was decoded. - "a repeated name": {`{` + cred + `,"credentials":{}}`, `{"credentials":{}}`, 0}, - "a name repeated with an escape": {`{` + cred + `,"\u0063redentials":{}}`, `{"credentials":{}}`, 0}, - "a name repeated in an array item": {`[{"token":{"v":"` + opaque + `"},"token":{}}]`, `[{"token":{}}]`, 0}, - "a repeated name and no secret": {`{"a":1,"a":2}`, `{"a":2}`, 0}, - // A pattern's mask in a decoded string, and a false positive of - // one, stay in place. - "a mask that matches again": {`{"db":"postgres://u:hunter2hunter2@db/x"}`, `{"db":"postgres://u:hunt...@db/x"}`, 1}, - "a notebook assignment": {`{"cells":[{"source":["key = jax.random.PRNGKey(0)\n"]}]}`, `{"cells":[{"source":["key = jax....)\n"]}]}`, 1}, - // Otherwise the document is kept, encoded again: the encoder's key - // order, spacing and escapes. - "plain document": {`{ "b": "` + opaque + `", "a": 1 }`, `{"a":1,"b":"` + opaque + `"}`, 0}, - "a pair stays masked in place": {`{"password":"hunter2hunter2","n":1}`, `{"n":1,"password":"***"}`, 1}, - "a letter folded once decoded": {`{"v":"\uFF21"}`, `{"v":"A"}`, 1}, - // Document-shaped text that does not parse cannot be judged either: - // a raw control character, a trailing comma, a comment, an object - // literal. Masked whole, secret or none. - "a control character the scan keeps": {"{\"a\":\"x\x01y\"}", "***", 1}, - "a trailing comma": {`{` + cred + `,}`, "***", 1}, - "a trailing comma and no secret": {`{ "a": 1, }`, "***", 1}, - "a comment": {"{\n // dev \n " + cred + "\n}", "***", 1}, - "an object literal": {`{ apiKey: "hunter2hunter2" }`, "***", 1}, - // Shaped like one but quoting nothing: code, or a mask. - "an object literal quoting nothing": {`{ apiKey: process.env.KEY }`, `{ apiKey: process.env.KEY }`, 0}, - "a number past float64": {`{"maximum":1e999}`, `{"maximum":1e999}`, 0}, - "text like a document the scan changed": {`{'password': 'hunter2hunter2'}`, "***", 1}, - "a command that opens a brace": {`{ export API_KEY=` + opaque + `; } 2>/dev/null`, `{ export API_KEY=opaq...; } 2>/dev/null`, 1}, - "a held string literal": {`"plain text"`, `"plain text"`, 0}, - "a value naming a sibling": {`{"name":"name","from":"to","to":"x"}`, `{"from":"to","name":"name","to":"x"}`, 0}, - "names repeated in other objects": {`[{"a":1},{"a":2},{"x":{"a":1},"y":{"a":2}},{"a":{"x":1},"x":2}]`, `[{"a":1},{"a":2},{"x":{"a":1},"y":{"a":2}},{"a":{"x":1},"x":2}]`, 0}, - } { - t.Run(name, func(t *testing.T) { - c := newContentCollector(4096) - c.Handle(agentruntime.ToolUseEvent{Name: "Write", Summary: "kept", Arguments: `{"content":` + quote(tc.content) + `}`}) - - res := c.Result("stop") - assert.Contains(t, res.OutputMessages, `"arguments":{"content":`+quote(tc.want)+`}`) - assert.Len(t, res.Findings, tc.findings) - assert.Equal(t, strings.Contains(tc.want, "***") || strings.Contains(tc.want, "..."), !strings.Contains(res.OutputMessages, `"summary"`), "a secret drops the summary") - }) - } -} - -func TestContentCollector_JSONInAStringIsAlsoScannedWhole(t *testing.T) { - // A secret can span the document's strings: a notebook keeps each line - // of a cell in its own string, and a runner env value can hold quotes. - // A private key over the lines drops the arguments; an exact value - // over the document's structure masks it whole. - begin, end := "-----BEGIN RSA PRIVATE "+"KEY-----", "-----END RSA PRIVATE "+"KEY-----" - body := strings.Repeat("Q", 40) - notebook, _ := json.Marshal(map[string]any{"cells": []any{map[string]any{"source": []string{begin + "\n", body + "\n", end + "\n"}}}}) - security.RegisterRuntimeSecret("rtline1\nrtline2x") for name, tc := range map[string]struct { - env map[string]string - content, gone string - findings int - }{ - "key over a cell's lines": {nil, string(notebook), body, 1}, - // Its strings in order, as written: the scan masks the block's last - // line as a member's value before the block's pattern runs. - "key from a member name into its value": {nil, `{"` + begin + `\n` + body[:20] + `":"` + body[20:] + `\n` + end + `"}`, body[:20], 2}, - "a runtime value over two strings": {nil, `{"lines":["rtline1","rtline2x"]}`, "rtline2x", 1}, - // An exact value over the document's structure. - "runner env value with structure": {map[string]string{"DB_PASSWORD": `hunter22","port`}, `{"pw":"hunter22","port":5432}`, "hunter22", 1}, - "runner env value that ends the document": {map[string]string{"DB_PASSWORD": `hunter22"}`}, `{"pw":"hunter22"}`, "hunter22", 1}, - "and one behind a BOM": {map[string]string{"DB_PASSWORD": `hunter22"}`}, "\uFEFF{\"credentials\":{\"value\":\"opaque-credential-123\"},\"pw\":\"hunter22\"}", "opaque-credential-123", 1}, - "and one behind a space and a BOM": {map[string]string{"DB_PASSWORD": `hunter22"}`}, " \uFEFF{\"credentials\":{\"value\":\"opaque-credential-123\"},\"pw\":\"hunter22\"}", "opaque-credential-123", 1}, - "and one behind a zero-width space": {map[string]string{"DB_PASSWORD": `hunter22"}`}, "\u200B{\"credentials\":{\"value\":\"opaque-credential-123\"},\"pw\":\"hunter22\"}", "opaque-credential-123", 1}, - // Decoded, the literal pass sees it — in a document, or in a - // string a held JSON string literal spells with escapes. - "runner env value spelled with an escape": {map[string]string{"DB_PASSWORD": "hunter2hunter2"}, `{"pw":"\u0068unter2hunter2"}`, "unter2hunter2", 1}, - "runner env value in a held string literal": {map[string]string{"DEPLOY_PASSWORD": "abcdefghijklmno"}, `"\u0061bcdefghijklmno"`, "bcdefghijklmno", 1}, - "and in one the scan breaks": {map[string]string{"DEPLOY_PASSWORD": "abcdefghijklmno"}, "\"\\u0061bcdefghijklmno\uFF02\"", "bcdefghijklmno", 1}, - } { - t.Run(name, func(t *testing.T) { - t.Setenv(telemetry.ContentCaptureEnvVar, "true") - c := newContentCollectorIfEnabled(tc.env) - args, _ := json.Marshal(map[string]string{"content": tc.content}) - c.Handle(agentruntime.ToolUseEvent{Name: "Write", Arguments: string(args)}) - - res := c.Result("stop") - assert.NotContains(t, res.OutputMessages, tc.gone) - assert.Len(t, res.Findings, tc.findings) - }) - } -} - -func TestContentCollector_SecretsSpanningTheArgumentsStrings(t *testing.T) { - // Each key, string and number is judged on its own, and the level as a - // reader of them in the order written sees them: a private key over - // the lines of an array or over a key and a value, a runtime value - // over the lines of one. A secret found that way can be masked - // nowhere but the level whole: the arguments are dropped and charged - // as encoded; a private key block inside one string drops them as - // well. (An assignment or a header whose value is the next - // string is not judged: its secret stops at white space.) - begin, end := "-----BEGIN RSA PRIVATE "+"KEY-----", "-----END RSA PRIVATE "+"KEY-----" - body := strings.Repeat("Q", 40) - security.RegisterRuntimeSecret("rtline1\nrtline2x") - for name, tc := range map[string]struct { - args, gone string - findings int + args, want string + dropped bool }{ - "a private key over the lines of a cell": {`{"cells":[{"source":["` + begin + `\n","` + body + `\n","` + end + `\n"]}]}`, body, 1}, - "a private key over members against key order": {`{"b":"` + begin + `\n` + body[:20] + `","a":"` + body[20:] + `\n` + end + `"}`, body[:20], 1}, - "a private key over a key and a value": {`{"` + begin + `\n` + body[:20] + `":"` + body[20:] + `\n` + end + `"}`, body[:20], 2}, - "a runtime value over two strings": {`{"lines":["rtline1","rtline2x"]}`, "rtline2x", 1}, - "and over two lines keeping their breaks": {`{"lines":["rtline1\n","rtline2x"]}`, "rtline2x", 1}, - "and over two lines keeping a Windows break": {`{"lines":["rtline1\r\n","rtline2x"]}`, "rtline2x", 1}, - "a marker inside an escape the normalizer strips": {`{"lines":["\u001b]0;` + begin + `\u0007\n","` + body + `\n","` + end + `\n"]}`, body, 2}, - "a marker folded": {`{"lines":["\uFF0D\uFF0D\uFF0D\uFF0D\uFF0DBEGIN RSA PRIVATE ` + `KEY-----\n","` + body + `\n","` + end + `\n"]}`, body, 2}, + "Write keeps the path": {`{"file_path":"/x/a.json","content":"{\"credentials\":{\"value\":\"` + opaque + `\"}}"}`, `{"file_path":"/x/a.json"}`, true}, + "Edit keeps the path": {`{"file_path":"/x/a.go","old_string":"` + opaque + `","new_string":"x","replace_all":true}`, `{"file_path":"/x/a.go","replace_all":true}`, true}, + "MultiEdit keeps the path": {`{"file_path":"/x/a.go","edits":[{"old_string":"` + opaque + `","new_string":"x"}]}`, `{"file_path":"/x/a.go"}`, true}, + "NotebookEdit keeps the path": {`{"notebook_path":"n.ipynb","cell_id":"c1","new_source":"print('` + opaque + `')"}`, `{"cell_id":"c1","notebook_path":"n.ipynb"}`, true}, + "Agent keeps its description": {`{"description":"scan","prompt":"the token is ` + opaque + `","model":"sonnet","subagent_type":"Explore"}`, `{"description":"scan","model":"sonnet","subagent_type":"Explore"}`, true}, + "Grep keeps everything": {`{"pattern":"TODO","path":"src","output_mode":"content","-n":true,"context":2}`, `{"-n":true,"context":2,"output_mode":"content","path":"src","pattern":"TODO"}`, false}, + "Read keeps everything": {`{"file_path":"/x","offset":10,"limit":50}`, `{"file_path":"/x","limit":50,"offset":10}`, false}, + "a listed member holding an object": {`{"path":{"dir":"` + opaque + `"},"pattern":"x"}`, `{"pattern":"x"}`, true}, + "a listed member holding an array": {`{"pattern":["` + opaque + `"],"path":"src"}`, `{"path":"src"}`, true}, + "no members": {`{}`, `{}`, false}, } { t.Run(name, func(t *testing.T) { c := newContentCollector(4096) - c.Handle(agentruntime.ToolUseEvent{Name: "Write", Summary: "s", Arguments: tc.args}) + c.Handle(agentruntime.ToolUseEvent{Name: "Tool", Summary: "s", Arguments: tc.args}) res := c.Result("stop") - part := partAt(t, decodeOutputMessages(t, res.OutputMessages), 0) - assert.NotContains(t, part, "arguments") - assert.NotContains(t, part, "summary") - assert.Equal(t, true, part["fullsend.truncated"]) - assert.NotContains(t, res.OutputMessages, tc.gone) - assert.Len(t, res.Findings, tc.findings) + assert.Contains(t, res.OutputMessages, `"arguments":`+tc.want) + assert.NotContains(t, res.OutputMessages, opaque) + assert.Contains(t, res.OutputMessages, `"summary":"s"`, "nothing found a secret") + assert.Empty(t, res.Findings) + assert.Equal(t, tc.dropped, res.Truncated) + assert.Equal(t, tc.dropped, strings.Contains(res.OutputMessages, `"fullsend.truncated":true`)) + assert.Equal(t, tc.dropped, res.DroppedBytes > 0, "a dropped member is charged") }) } - t.Run("a private key in one string drops the arguments too", func(t *testing.T) { + t.Run("nothing listed records no arguments", func(t *testing.T) { c := newContentCollector(4096) - c.Handle(agentruntime.ToolUseEvent{Name: "Write", Summary: "s", Arguments: `{"content":"` + begin + `\n` + body + `\n` + end + `","file_path":"k.pem"}`}) + c.Handle(agentruntime.ToolUseEvent{Name: "TodoWrite", Summary: "s", Arguments: `{"todos":[{"content":"` + opaque + `","status":"pending"}]}`}) res := c.Result("stop") part := partAt(t, decodeOutputMessages(t, res.OutputMessages), 0) assert.NotContains(t, part, "arguments") + assert.Equal(t, "s", part["summary"]) assert.Equal(t, true, part["fullsend.truncated"]) - assert.NotContains(t, res.OutputMessages, body) - assert.Len(t, res.Findings, 2) - }) - t.Run("dropped arguments are charged as encoded", func(t *testing.T) { - c := newContentCollector(4096) - args := `{"lines": ["` + begin + `", "` + body + `", "` + end + `"]}` - c.Handle(agentruntime.ToolUseEvent{Name: "Write", Arguments: args}) - assert.Equal(t, len(args)-3*len(" "), c.Result("stop").DroppedBytes) + assert.NotContains(t, res.OutputMessages, opaque) }) } -func TestContentCollector_JoinedNeighboursMakeNoSecret(t *testing.T) { - // Judged in order, two strings with no secret between them must not - // make one: a token stops at the end of its string, and so does a - // connection string's user. - for name, args := range map[string]string{ - "a database URL beside an address": `{"DATABASE_URL":"postgres://localhost:5432/dev","SMTP_FROM":"noreply@acme.io"}`, - "an empty assignment beside a description": `{"command":"export NPM_TOKEN=","description":"Configure npm auth"}`, - "names that run into a token's prefix": `{"service":"task-scheduler","namespace":"production_workloads"}`, - "a command's flags": `{"command":["flask","--app","hello","run","--no-reload"]}`, - "apostrophes of two strings": `{"old_string":"print(key + ': ' + str(cfg[key]))","new_string":"print(key + ': ' + str(cfg[key]))\nlog.debug('dumped')"}`, - "a connection string with no host": `{"pattern":"postgres://user:pass@","path":"src","output_mode":"files_with_matches"}`, - "a database URL before a remote": `{"database":"postgres://localhost/app","remote":"git+ssh://git@github.com/org/app.git"}`, - "and before a registry, nested": `{"environment":{"REDIS_URL":"redis://redis","NPM_CONFIG_REGISTRY":"https://npm.pkg.github.com/@acme"}}`, +func TestContentCollector_DroppedArgumentsAreScannedAndCharged(t *testing.T) { + // Discarded content is redacted first, so a secret in a member that + // is not recorded still counts — and costs the summary, as a secret + // found anywhere in the arguments does — and the member is charged as + // the redacted text it was scanned as. + token := "ghp_" + strings.Repeat("r", 36) + for name, tc := range map[string]struct { + args string + charged int + }{ + "a file body": {`{"file_path":"/x","content":"echo ` + token + `"}`, len("echo ghp_...")}, + "a listed member holding an object": {`{"file_path":"/x","path":{"t":"` + token + `"}}`, len(`{"t":"ghp_..."}`)}, + "a number outside the list": {`{"file_path":"/x","pin":12345678}`, len("12345678")}, } { t.Run(name, func(t *testing.T) { c := newContentCollector(4096) - c.Handle(agentruntime.ToolUseEvent{Name: "Write", Summary: "s", Arguments: args}) + c.Handle(agentruntime.ToolUseEvent{Name: "Write", Summary: "/x", Arguments: tc.args}) res := c.Result("stop") - var want, got any - require.NoError(t, json.Unmarshal([]byte(args), &want)) - got = partAt(t, decodeOutputMessages(t, res.OutputMessages), 0)["arguments"] - assert.Equal(t, want, got) - assert.Empty(t, res.Findings) + assert.NotContains(t, res.OutputMessages, token[4:]) + assert.Contains(t, res.OutputMessages, `"arguments":{"file_path":"/x"}`) + assert.Equal(t, tc.charged, res.DroppedBytes) + assert.True(t, res.Truncated) + if strings.Contains(name, "number") { + assert.Empty(t, res.Findings) + assert.Contains(t, res.OutputMessages, `"summary":"/x"`) + } else { + assert.Len(t, res.Findings, 1) + assert.NotContains(t, res.OutputMessages, `"summary"`) + } }) } } @@ -411,16 +235,15 @@ func TestContentCollector_AStringTheNormalizerStrippedIsMaskedWhole(t *testing.T // The normalizer strips a terminal escape sequence or tag characters // before the patterns run, and the stripping can take the first // letter of a token (a CSI sequence ends at the next letter) or a - // whole payload the patterns never see: such a string, a value or a - // key, is masked whole. + // whole payload the patterns never see: such a string is masked + // whole, and the summary goes with it. token := "ghp_" + strings.Repeat("r", 36) for name, tc := range map[string]struct { args, want, gone string }{ - "a colour code that eats a token's first letter": {`{"n":"\u001b[` + token + `"}`, `{"n":"***"}`, token[1:]}, + "a colour code that eats a token's first letter": {`{"command":"\u001b[` + token + `"}`, `{"command":"***"}`, token[1:]}, "a token inside a title code": {`{"command":"echo \u001b]` + token + `\u0007"}`, `{"command":"***"}`, token[4:]}, - "tag characters": {`{"t":"x\uDB40\uDC67y"}`, `{"t":"***"}`, "xy"}, - "a key": {`{"\u001b[0mpassword":"hunter2hunter2"}`, `{"***":"***"}`, "hunter2"}, + "tag characters": {`{"command":"x󠁧y"}`, `{"command":"***"}`, "xy"}, } { t.Run(name, func(t *testing.T) { c := newContentCollector(4096) @@ -432,220 +255,6 @@ func TestContentCollector_AStringTheNormalizerStrippedIsMaskedWhole(t *testing.T assert.NotContains(t, res.OutputMessages, `"summary"`) }) } - t.Run("two keys masked alike collide", func(t *testing.T) { - c := newContentCollector(4096) - c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Arguments: `{"\u001b[0ma":1,"\u001b[1mb":2}`}) - assert.NotContains(t, c.Result("stop").OutputMessages, `"arguments"`) - }) -} - -func TestContentCollector_ACollisionStillCountsASecretAcrossTheStrings(t *testing.T) { - // Two keys folding alike drop the arguments on their own; the level is - // still judged across its strings, since the secret found there is - // what costs the call its summary — the parser cut it from these - // arguments, and a private key's first line can be in it. - begin, end := "-----BEGIN RSA PRIVATE "+"KEY-----", "-----END RSA PRIVATE "+"KEY-----" - body := strings.Repeat("Q", 40) - c := newContentCollector(4096) - c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Summary: begin + "\n" + body, Arguments: `{"a":"x","\uFF41":"y","command":"` + begin + `\n` + body + `","description":"` + end + `"}`}) - - res := c.Result("stop") - part := partAt(t, decodeOutputMessages(t, res.OutputMessages), 0) - assert.NotContains(t, part, "arguments") - assert.NotContains(t, part, "summary", "the secret across the strings costs the summary") - assert.NotContains(t, res.OutputMessages, body) - assert.True(t, slices.ContainsFunc(res.Findings, func(f security.Finding) bool { return f.Name == "private_key" }), "the spanning secret is recorded") -} - -func TestContentCollector_AKeyMaskedWholeNamesASecret(t *testing.T) { - // A key the normalizer stripped an escape sequence from is masked - // whole, and its name is then not to be had from its mask, nor from - // its spelling when that is in compatibility forms the pattern does - // not read, nor from the normalizer's rendering when the stripping ate - // the name's first letter: the values under such a key are judged as - // a secret-named member's, whatever it was called. - const opaque = "opaque-credential-123" - password := "\uFF50\uFF41\uFF53\uFF53\uFF57\uFF4F\uFF52\uFF44" // fullwidth - for name, tc := range map[string]struct{ args, want string }{ - "a colour code before a fullwidth name": {`{"\u001b[0m` + password + `":"` + opaque + `"}`, `{"***":"***"}`}, - "an unfinished colour code the fold completes": {`{"\u001b[` + password + `":"` + opaque + `"}`, `{"***":"***"}`}, - "and the values nested under it": {`{"\u001b[0m` + password + `":{"v":["` + opaque + `"]}}`, `{"***":{"v":["***"]}}`}, - "a colour code alone": {`{"\u001b[0mnote":"` + opaque + `"}`, `{"***":"***"}`}, - // Masked whole as a document it holds, with a name only decoding - // spells: no keyword to read in the key itself. - "a key holding a document naming a secret": {`{"{\"\\u0070assword\":{\"v\":\"opaque-inner-value\"}}":"` + opaque + `"}`, `{"***":"***"}`}, - "and the values nested under that": {`{"{\"\\u0070assword\":{\"v\":\"opaque-inner-value\"}}":{"v":["` + opaque + `"]}}`, `{"***":{"v":["***"]}}`}, - } { - t.Run(name, func(t *testing.T) { - c := newContentCollector(4096) - c.Handle(agentruntime.ToolUseEvent{Name: "Write", Summary: "s", Arguments: tc.args}) - - res := c.Result("stop") - assert.Contains(t, res.OutputMessages, `"arguments":`+tc.want) - assert.NotContains(t, res.OutputMessages, opaque) - }) - } -} - -func TestContentCollector_ARunnerEnvValueOverTheLinesOfAnArray(t *testing.T) { - // A runner env value can hold a line break; written over two strings - // of an array, no string holds it whole and the literal pass sees no - // part of it. Judged across the strings it is found, the lines of a - // held document read as lines: the arguments are dropped. - // Each line of a notebook cell keeps its own line break, a Windows - // one or a trailing space: an exact value is sought in the strings in - // order, white space aside. - env := map[string]string{"DEPLOY_SECRET": "opaque-line-one\nopaque-line-two"} - t.Setenv(telemetry.ContentCaptureEnvVar, "true") - for name, lines := range map[string]string{ - "lines": `["opaque-line-one","opaque-line-two"]`, - "lines keeping the break": `["opaque-line-one\n","opaque-line-two"]`, - "lines keeping a Windows break": `["opaque-line-one\r\n","opaque-line-two"]`, - "a line ending in a space": `["opaque-line-one \n","opaque-line-two"]`, - "a value split mid-word": `["opaque-li","ne-one\nopaque-line-two"]`, - } { - t.Run(name+", arguments", func(t *testing.T) { - c := newContentCollectorIfEnabled(env) - c.Handle(agentruntime.ToolUseEvent{Name: "Write", Summary: "s", Arguments: `{"lines":` + lines + `}`}) - - res := c.Result("stop") - part := partAt(t, decodeOutputMessages(t, res.OutputMessages), 0) - assert.NotContains(t, part, "arguments") - assert.NotContains(t, part, "summary") - assert.NotContains(t, res.OutputMessages, "opaque-line") - assert.True(t, slices.ContainsFunc(res.Findings, func(f security.Finding) bool { return f.Scanner == "runner_env" && f.Name == "DEPLOY_SECRET" })) - }) - t.Run(name+", held", func(t *testing.T) { - c := newContentCollectorIfEnabled(env) - content, _ := json.Marshal(`{"lines":` + lines + `}`) - c.Handle(agentruntime.ToolUseEvent{Name: "Write", Arguments: `{"content":` + string(content) + `}`}) - - res := c.Result("stop") - part := partAt(t, decodeOutputMessages(t, res.OutputMessages), 0) - assert.NotContains(t, part, "arguments") - assert.Equal(t, true, part["fullsend.truncated"]) - assert.NotContains(t, res.OutputMessages, "opaque-line") - }) - } -} - -func TestContentCollector_JSONHeldPastTheBoundIsMaskedWhole(t *testing.T) { - // Documents are walked down to maxHeldDepth, each encoded again with - // its masks; one held deeper is masked whole, the levels above kept. - held := func(doc string, depth int) string { - for range depth - 1 { - b, _ := json.Marshal(map[string]string{"d": doc}) - doc = string(b) - } - b, _ := json.Marshal(map[string]string{"content": doc}) - return string(b) - } - run := func(args string) contentResult { - c := newContentCollector(4096) - c.Handle(agentruntime.ToolUseEvent{Name: "Write", Arguments: args}) - return c.Result("stop") - } - const secret, plain = `{"credentials":{"value":"opaque-credential-123"}}`, `{"v":"plain"}` - - res := run(held(secret, maxHeldDepth)) - assert.Contains(t, res.OutputMessages, `"arguments":`+held(`{"credentials":{"value":"***"}}`, maxHeldDepth)) - assert.Len(t, res.Findings, 1) - - res = run(held(plain, maxHeldDepth)) - assert.Contains(t, res.OutputMessages, `"arguments":`+held(plain, maxHeldDepth)) - assert.Empty(t, res.Findings) - - for _, doc := range []string{plain, secret} { - res = run(held(doc, maxHeldDepth+1)) - assert.Contains(t, res.OutputMessages, `"arguments":`+held("***", maxHeldDepth+1)) - assert.NotContains(t, res.OutputMessages, "opaque-credential") - assert.Len(t, res.Findings, 1) - } -} - -func TestContentCollector_AKeyHoldingADocumentIsJudgedLikeAValue(t *testing.T) { - quote := func(s string) string { b, _ := json.Marshal(s); return string(b) } - doc := quote(`{"credentials":{"value":"opaque-credential-123"}}`) - for name, tc := range map[string]struct { - args, want string - findings int - }{ - "key": {`{` + doc + `:"x"}`, `{"***":"x"}`, 1}, - "key of a nested object": {`{"files":{` + doc + `:"x"}}`, `{"files":{"***":"x"}}`, 1}, - "key in a held document": {`{"content":` + quote(`{`+doc+`:"x"}`) + `}`, `{"content":` + quote(`{"***":"x"}`) + `}`, 1}, - // Masked whole, the key still names a secret as scanned: its - // value is judged against it. - "a value under it": {`{` + doc + `:"opaque-credential-456"}`, `{"***":"***"}`, 2}, - "an object under it": {`{` + doc + `:{"u":"opaque-credential-456"}}`, `{"***":{"u":"***"}}`, 2}, - "a value under a repeated name": {`{` + quote(`{"auth":1,"auth":2}`) + `:"opaque-credential-456"}`, `{"***":"***"}`, 2}, - } { - t.Run(name, func(t *testing.T) { - c := newContentCollector(4096) - c.Handle(agentruntime.ToolUseEvent{Name: "Write", Arguments: tc.args}) - - res := c.Result("stop") - assert.Contains(t, res.OutputMessages, `"arguments":`+tc.want) - assert.NotContains(t, res.OutputMessages, "opaque-credential-") - assert.Len(t, res.Findings, tc.findings) - }) - } - - // Two such keys are masked alike: the arguments go, as for any collision. - c := newContentCollector(4096) - c.Handle(agentruntime.ToolUseEvent{Name: "Write", Arguments: `{` + doc + `:"x",` + quote(`{"token":["opaque-credential-456"]}`) + `:"y"}`}) - res := c.Result("stop") - part := partAt(t, decodeOutputMessages(t, res.OutputMessages), 0) - assert.NotContains(t, part, "arguments") - assert.Equal(t, true, part["fullsend.truncated"]) -} - -func TestContentCollector_JSONInASecretNamedStringIsMaskedWholeOnce(t *testing.T) { - // A document under a secret-named member is walked under that member: - // its values are judged by the member's name, one finding each. - c := newContentCollector(4096) - c.Handle(agentruntime.ToolUseEvent{Name: "Write", Arguments: `{"credentials":"{\"value\":\"opaque-credential-123\"}"}`}) - - res := c.Result("stop") - assert.Contains(t, res.OutputMessages, `"arguments":{"credentials":"{\"value\":\"***\"}"}`) - assert.Len(t, res.Findings, 1) -} - -func TestContentCollector_AKeyFindingLeavesTheValueAndCountsOnce(t *testing.T) { - for name, tc := range map[string]struct{ args, want string }{ - "token": {`{"ghp_` + strings.Repeat("r", 36) + `":"hello"}`, `{"ghp_...":"hello"}`}, - "fullwidth": {`{"\uFF50ath":"/tmp/x"}`, `{"path":"/tmp/x"}`}, - } { - t.Run(name, func(t *testing.T) { - c := newContentCollector(4096) - c.Handle(agentruntime.ToolUseEvent{Name: "Read", Arguments: tc.args}) - - res := c.Result("stop") - assert.Contains(t, res.OutputMessages, `"arguments":`+tc.want) - assert.Len(t, res.Findings, 1) - }) - } -} - -func TestContentCollector_KeysThatRedactAlikeDropTheArguments(t *testing.T) { - // Folding makes the two keys one; keeping either member would show a - // call the agent did not make, and which one would follow map order. - const args = `{"command":"rm -rf /","\uFF43ommand":"ls"}` - var first contentResult - for i := 0; i < 100; i++ { - c := newContentCollector(4096) - c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Summary: "rm -rf /", Arguments: args}) - res := c.Result("stop") - if i == 0 { - first = res - } - require.Equal(t, first.OutputMessages, res.OutputMessages) - require.Equal(t, first.DroppedBytes, res.DroppedBytes) - } - part := partAt(t, decodeOutputMessages(t, first.OutputMessages), 0) - assert.NotContains(t, part, "arguments") - assert.Equal(t, true, part["fullsend.truncated"]) - assert.True(t, first.Truncated) - assert.Positive(t, first.DroppedBytes) } func TestContentCollector_NamelessCallCarriesNoArgumentsAndNoCharge(t *testing.T) { @@ -854,45 +463,6 @@ func TestContentCollector_ReplacesProviderOnlyValuesFromTheProcessEnv(t *testing assert.Equal(t, 2, runnerEnvFindings(res), "counted like a runner env value") } -func TestContentCollector_KeysMaskedAlikeDropTheArguments(t *testing.T) { - // Masked keys go the way of keys that redact alike: keeping either - // member would show a call the agent did not make. Field names of eight - // characters or more under a secret-named member are masked too, so an - // ordinary credentials object can cost the arguments. - for name, args := range map[string]string{ - "two opaque keys": `{"tokens":{"opaque-credential-123":"a","opaque-credential-456":"b"}}`, - "username and password": `{"auth":{"username":"alice","password":"hunter2hunter2"}}`, - } { - t.Run(name, func(t *testing.T) { - c := newContentCollector(4096) - c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Summary: "s", Arguments: args}) - - res := c.Result("stop") - assert.NotContains(t, res.OutputMessages, "arguments") - assert.NotContains(t, res.OutputMessages, "opaque-credential") - assert.NotContains(t, res.OutputMessages, "hunter2") - assert.True(t, res.Truncated) - assert.Positive(t, res.DroppedBytes) - }) - } -} - -func TestContentCollector_SecretNamedChecksSeeNormalizedText(t *testing.T) { - // The key and the value are judged as the normalizer writes them: a - // fullwidth key names a secret once folded, and four ligatures are - // eight characters once folded. - for name, tc := range map[string]struct{ args, want string }{ - "fullwidth key over an array": {`{"\uFF50\uFF41\uFF53\uFF53\uFF57\uFF4F\uFF52\uFF44":["opaque-credential-123"]}`, `{"password":["***"]}`}, - "value folded past the floor": {`{"password":"\uFB01\uFB01\uFB01\uFB01"}`, `{"password":"***"}`}, - } { - t.Run(name, func(t *testing.T) { - c := newContentCollector(4096) - c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Arguments: tc.args}) - assert.Contains(t, c.Result("stop").OutputMessages, `"arguments":`+tc.want) - }) - } -} - func TestContentCollector_ReplacesBothValuesWhenRunnerEnvReusesAProviderOnlyName(t *testing.T) { // Nothing refuses GH_WORKFLOW_TOKEN as a runner env key; the process // value is still the one redactFeedback replaces. @@ -906,17 +476,22 @@ func TestContentCollector_ReplacesBothValuesWhenRunnerEnvReusesAProviderOnlyName } func TestContentCollector_ReplacesARunnerEnvValueSentAsANumber(t *testing.T) { + // A number outside the list is scanned as its digits before it is + // dropped, so the literal pass still counts it. digits := strings.Repeat("7", minRedactableSecretLen) t.Setenv(telemetry.ContentCaptureEnvVar, "true") c := newContentCollectorIfEnabled(map[string]string{"DEPLOY_PASSWORD": digits}) - c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Arguments: `{"pin":` + digits + `,"n":42}`}) + c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Arguments: `{"pin":` + digits + `,"timeout":42}`}) res := c.Result("stop") - assert.Contains(t, res.OutputMessages, `"arguments":{"n":42,"pin":"[REDACTED:DEPLOY_PASSWORD]"}`) + assert.Contains(t, res.OutputMessages, `"arguments":{"timeout":42}`) + assert.NotContains(t, res.OutputMessages, digits) + assert.Equal(t, 1, runnerEnvFindings(res)) + assert.Equal(t, len("[REDACTED:DEPLOY_PASSWORD]"), res.DroppedBytes) } func TestContentCollector_ANameRedactedAwayTakesItsArgumentsAlong(t *testing.T) { - const args = `{"a":1}` + const args = `{"command":"ls"}` c := newContentCollector(4096) c.Handle(agentruntime.ToolUseEvent{ID: "toolu_01", Name: "\u200B", Arguments: args}) @@ -987,10 +562,10 @@ func TestContentCollector_FullwidthQuotesCannotAddAMember(t *testing.T) { // NFKC folds a fullwidth quotation mark to '"'. Folded inside the // serialised text it would close the string and add a member. c := newContentCollector(4096) - c.Handle(agentruntime.ToolUseEvent{Name: "Write", Arguments: `{"content":"a\uFF02,\uFF02injected\uFF02:\uFF02b"}`}) + c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Arguments: `{"command":"a\uFF02,\uFF02injected\uFF02:\uFF02b"}`}) part := partAt(t, decodeOutputMessages(t, c.Result("stop").OutputMessages), 0) - assert.Equal(t, map[string]any{"content": `a","injected":"b`}, part["arguments"]) + assert.Equal(t, map[string]any{"command": `a","injected":"b`}, part["arguments"]) } func TestContentCollector_IncompleteArgumentsAreDroppedAndMarked(t *testing.T) { @@ -1018,7 +593,7 @@ func TestContentCollector_IncompleteArgumentsAreDroppedAndMarked(t *testing.T) { func TestContentCollector_OverBoundArgumentsAreDroppedWholeAndMarked(t *testing.T) { body := strings.Repeat("x", maxToolArgumentsBytes) c := newContentCollector(maxContentBytes) - c.Handle(agentruntime.ToolUseEvent{ID: "toolu_01", Name: "Write", Summary: "/x", Arguments: `{"content": "` + body + `"}`}) + c.Handle(agentruntime.ToolUseEvent{ID: "toolu_01", Name: "Bash", Summary: "/x", Arguments: `{"command": "` + body + `"}`}) res := c.Result("stop") part := partAt(t, decodeOutputMessages(t, res.OutputMessages), 0) @@ -1027,14 +602,14 @@ func TestContentCollector_OverBoundArgumentsAreDroppedWholeAndMarked(t *testing. assert.Equal(t, "toolu_01", part["id"]) assert.Equal(t, true, part["fullsend.truncated"]) assert.True(t, res.Truncated) - assert.Equal(t, len(`{"content":"`+body+`"}`), res.DroppedBytes, "charged as re-encoded") + assert.Equal(t, len(`{"command":"`+body+`"}`), res.DroppedBytes, "charged as re-encoded") } func TestContentCollector_ArgumentsAtTheBoundAreKept(t *testing.T) { - args := `{"content":"` + strings.Repeat("x", maxToolArgumentsBytes-len(`{"content":""}`)) + `"}` + args := `{"command":"` + strings.Repeat("x", maxToolArgumentsBytes-len(`{"command":""}`)) + `"}` require.Len(t, args, maxToolArgumentsBytes) c := newContentCollector(maxContentBytes) - c.Handle(agentruntime.ToolUseEvent{Name: "Write", Arguments: args}) + c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Arguments: args}) res := c.Result("stop") assert.Contains(t, res.OutputMessages, args) @@ -1057,68 +632,6 @@ func TestContentCollector_ArgumentsBoundAppliesAfterRedaction(t *testing.T) { assert.NotContains(t, res.OutputMessages, "qqqq") } -func TestContentCollector_ArgumentsFarOverTheBoundAreDroppedUnjudged(t *testing.T) { - // Redaction can bring arguments under the bound, but not from far - // over it, and the walk must do no work the bound does not limit: a - // secret-named key over an array once cost the key's length for each - // element. Over four times the bound as written, the arguments are - // scanned as text, so a secret still counts, then dropped and charged. - for name, tc := range map[string]struct { - args string - findings int - summary bool - }{ - "a long secret-named key over an array": {`{"password` + strings.Repeat("a", 32<<10) + `":[` + strings.Repeat("0,", 1999) + `0]}`, 0, true}, - "a token in a value": {`{"command":"echo ghp_` + strings.Repeat("q", 36) + ` ` + strings.Repeat("x", maxRawToolArgumentsBytes) + `"}`, 1, false}, - "spacing the encoder would drop": {`{"content": "` + strings.Repeat("x", maxRawToolArgumentsBytes) + `"}`, 0, true}, - } { - t.Run(name, func(t *testing.T) { - c := newContentCollector(maxContentBytes) - c.Handle(agentruntime.ToolUseEvent{Name: "Write", Summary: "/x", Arguments: tc.args}) - - res := c.Result("stop") - part := partAt(t, decodeOutputMessages(t, res.OutputMessages), 0) - assert.NotContains(t, part, "arguments") - assert.Equal(t, tc.summary, part["summary"] != nil) - assert.Equal(t, true, part["fullsend.truncated"]) - assert.Len(t, res.Findings, tc.findings) - assert.NotContains(t, res.OutputMessages, "qqqq") - assert.Equal(t, len(c.redact(tc.args, new([]security.Finding))), res.DroppedBytes, "charged as redacted text, like arguments that are not JSON") - }) - } - t.Run("at four times the bound, redaction still brings them under it", func(t *testing.T) { - args := `{"command":"echo ghp_` + strings.Repeat("q", maxRawToolArgumentsBytes-len(`{"command":"echo ghp_"}`)) + `"}` - require.Len(t, args, maxRawToolArgumentsBytes) - c := newContentCollector(maxContentBytes) - c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Arguments: args}) - - part := partAt(t, decodeOutputMessages(t, c.Result("stop").OutputMessages), 0) - assert.Equal(t, map[string]any{"command": "echo ghp_..."}, part["arguments"]) - }) -} - -func TestContentCollector_ASecretNamedKeyIsJudgedOnceForItsValues(t *testing.T) { - // The values under a secret-named member are judged beside a stand-in - // name decided once for the member, so each scan costs the value's - // length, not the key's: a long key over a thousand values must cost - // about what a short one does. A ratio, not an absolute time, so load - // cannot flake it. - values := `[` + strings.Repeat(`"12345678",`, 999) + `"12345678"]` - handle := func(key string) (time.Duration, contentResult) { - c := newContentCollector(maxContentBytes) - entry := time.Now() - c.Handle(agentruntime.ToolUseEvent{Name: "Write", Arguments: `{"` + key + `":` + values + `}`}) - return time.Since(entry), c.Result("stop") - } - short, res := handle("password") - require.Len(t, res.Findings, 1000, "every value is judged beside the name") - assert.NotContains(t, res.OutputMessages, "12345678") - long, res := handle("password" + strings.Repeat("a", 16<<10)) - require.Len(t, res.Findings, 1000) - assert.NotContains(t, res.OutputMessages, "12345678") - assert.Less(t, long, 20*short, "the key's length must not multiply the cost of its values") -} - func TestContentCollector_NullArgumentsAreOmitted(t *testing.T) { c := newContentCollector(4096) c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Arguments: `null`}) diff --git a/internal/cli/run.go b/internal/cli/run.go index 657253b423..eca5566209 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -3665,8 +3665,8 @@ func replaceEnvSecrets(text string, runnerEnv map[string]string) (string, []stri type envLiteral struct{ key, value string } // envLiterals lists the values replaceEnvSecrets replaces — longer values -// first, then key order — for it and for a caller that must find one -// where no replacement can mask it (the collector's spanning). +// first, then key order, so a value that contains another is replaced +// whole whichever source holds it. func envLiterals(runnerEnv map[string]string) []envLiteral { var literals []envLiteral for key, value := range runnerEnv { diff --git a/internal/security/redactor.go b/internal/security/redactor.go index 482e58a2c2..6c29e01c09 100644 --- a/internal/security/redactor.go +++ b/internal/security/redactor.go @@ -39,10 +39,6 @@ func RegisterRuntimeSecret(value string) bool { return true } -// RuntimeSecrets returns the registered values, for a caller that must -// find one where no scan can mask it. -func RuntimeSecrets() []string { return runtimeSecretSnapshot() } - func runtimeSecretSnapshot() []string { runtimeSecretsMu.RLock() defer runtimeSecretsMu.RUnlock() @@ -187,43 +183,6 @@ func (s *SecretRedactor) Scan(text string) ScanResult { return result } -// Match locates one secret in a text — a runtime value, or a pattern's -// whole match, the pair for a member-name one. Matches reports them -// without masking, over the text as given, for a caller that judges -// text it does not export: a scan would mask one match ahead of the -// next, and a mask can match a pattern again. -type Match struct { - Start, End int - Name string -} - -// Matches reports every secret in text; see Match. -func (s *SecretRedactor) Matches(text string) []Match { - var out []Match - for _, secret := range runtimeSecretSnapshot() { - for i := 0; ; { - j := strings.Index(text[i:], secret) - if j < 0 { - break - } - out = append(out, Match{i + j, i + j + len(secret), "runtime_secret"}) - i += j + len(secret) - } - } - add := func(p secretPattern) { - for _, loc := range p.regex.FindAllStringIndex(text, -1) { - out = append(out, Match{loc[0], loc[1], p.name}) - } - } - for _, p := range s.prefixPatterns { - add(p) - } - for _, p := range s.structuralPatterns { - add(p) - } - return out -} - func mask(value string) string { if len(value) < 10 { return "***" diff --git a/internal/security/redactor_matches_test.go b/internal/security/redactor_matches_test.go deleted file mode 100644 index 74a9307bb6..0000000000 --- a/internal/security/redactor_matches_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package security - -import ( - "strings" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestSecretRedactor_Matches(t *testing.T) { - // Matches locates secrets without masking, over the text as given: a - // runtime value, a prefix token and a structural pattern's whole match, - // each by its bounds, so a caller can judge text it does not export. - RegisterRuntimeSecret("runtime-opaque-value-42") - t.Cleanup(resetRuntimeSecrets) - token := "ghp_" + strings.Repeat("a", 36) - text := "x runtime-opaque-value-42 " + token + " API_KEY=s3cr3tvalue99 end" - - got := map[string]string{} - for _, m := range NewSecretRedactor().Matches(text) { - got[m.Name] = text[m.Start:m.End] - } - assert.Equal(t, map[string]string{ - "runtime_secret": "runtime-opaque-value-42", - "github_pat": token, - "env_assignment": " API_KEY=s3cr3tvalue99", - }, got) - assert.Empty(t, NewSecretRedactor().Matches("nothing here")) -} - -func TestRuntimeSecrets(t *testing.T) { - // The registered values, for a caller that must find one where no scan - // can mask it; a copy, so the caller cannot change the registry. - t.Cleanup(resetRuntimeSecrets) - resetRuntimeSecrets() - assert.Empty(t, RuntimeSecrets()) - RegisterRuntimeSecret("runtime-opaque-value-42") - got := RuntimeSecrets() - assert.Equal(t, []string{"runtime-opaque-value-42"}, got) - got[0] = "changed" - assert.Equal(t, []string{"runtime-opaque-value-42"}, RuntimeSecrets()) -} From cbfe80e06889be32e8def021b8c58640e0f64ead Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Thu, 8 Oct 2026 14:54:52 -0400 Subject: [PATCH 28/40] fix(cli): scan listed numbers, reject trailing input, keep the record private A number under a listed member of a tool call's arguments was copied without the scan the previous head gave it: a runner env value sent as a number was exported whole. It is now scanned as its digits like every kept string, and the mask is exported as the string it is; a clean number stays a number. decodeJSON accepted anything after the first JSON value once the raw bound and its json.Valid gate went. It now requires white space at most after the value; arguments with trailing input take the existing path: scanned as text so a secret in the tail counts, dropped whole, charged. The telemetry file is created 0600: under the content gate it holds prompts and tool arguments, like the feedback audit file the runner keeps at 0600. Docs: the user guide no longer says tool arguments include file contents; the dev guide drops the two sentences describing the deleted raw bound and key-collision rule and says kept numbers are scanned. Review findings on c8324dfd4: 4222621667 (high), 4222621676 (medium), 4222621683 (low), and the telemetry file mode from the summary. Signed-off-by: Dharit Shah --- docs/guides/dev/tracing.md | 13 +++--- docs/guides/user/how-to-emit-traces.md | 5 ++- internal/cli/content_collector.go | 38 ++++++++++++---- internal/cli/content_collector_test.go | 60 ++++++++++++++++++++++++++ internal/telemetry/telemetry.go | 2 +- internal/telemetry/telemetry_test.go | 14 ++++++ 6 files changed, 114 insertions(+), 18 deletions(-) diff --git a/docs/guides/dev/tracing.md b/docs/guides/dev/tracing.md index f15262da6f..33d4ee5bef 100644 --- a/docs/guides/dev/tracing.md +++ b/docs/guides/dev/tracing.md @@ -217,13 +217,11 @@ final answer is what consumers judge) plus an 8 KiB per-tool-result bound (tail-kept, redacted before the cut, the part marked `fullsend.truncated`) and an 8 KiB per-call arguments bound (over it the arguments are dropped whole and the part marked, since a cut object is not -JSON; over four times it as written they are dropped before they are -decoded, so no walk does work the bound does not limit), with exact dropped-byte +JSON), with exact dropped-byte accounting across content, tool names, summaries, responses, part -ids, and dropped arguments (counted as re-encoded JSON, or as redacted -text when they were not a JSON value or were dropped before they were -decoded; on a key collision each member a -later key (in sorted order) replaced is not counted), then holds the marshaled string to `maxEncodedContentBytes` +ids, and dropped arguments (charged as the kept encoding when the bound +drops them, otherwise as the redacted text they were scanned as, and as +written when the call's name redacts away), then holds the marshaled string to `maxEncodedContentBytes` (255,000 — just under the one size the pilot backend is proven to accept) by trimming the oldest content again, measured on the encoding itself and still charged in raw bytes (a separate, earlier boundary — the parser's 1 MiB @@ -247,7 +245,8 @@ whose value is an object or an array, is dropped, charged as the redacted text it was scanned as, and the part marked `fullsend.truncated`. The guarantee is one sentence: the record holds these members of a call, each redacted as text, and nothing else of its input. The input is decoded -(numbers keep their digits) and each kept string goes through the text +(one JSON object, white space at most after it) and each kept string, and +each kept number as its digits, goes through the text pipeline on its own, since the redactor's patterns are written for plain text and over serialised JSON miss an assignment that opens a string or follows an escaped newline, and a value behind escaped quotes; the result diff --git a/docs/guides/user/how-to-emit-traces.md b/docs/guides/user/how-to-emit-traces.md index dd7d2906c8..36883b6a9d 100644 --- a/docs/guides/user/how-to-emit-traces.md +++ b/docs/guides/user/how-to-emit-traces.md @@ -125,8 +125,9 @@ runtime's stream provides them; Claude runs do — to each `agent` span, in the local file and at the endpoint. A retry that carries validation feedback also records the prompt the runner composed. Content is redacted for secrets and bounded per iteration, -but may still contain proprietary code or PII — tool arguments include the -commands an agent ran and the file contents it wrote — so make sure your backend's +but may still contain proprietary code or PII — tool arguments carry the +commands an agent ran and the paths and patterns it used, never a file body +or an edit — so make sure your backend's access controls fit before enabling it. ```bash diff --git a/internal/cli/content_collector.go b/internal/cli/content_collector.go index c3826aff7c..d2053724b1 100644 --- a/internal/cli/content_collector.go +++ b/internal/cli/content_collector.go @@ -2,6 +2,8 @@ package cli import ( "encoding/json" + "errors" + "io" "maps" "slices" "sort" @@ -793,9 +795,12 @@ var recordedArguments = map[string]bool{ // JSON they miss an assignment that opens a string or follows an escaped // newline, and a value behind escaped quotes. The result is encoded // again — key order, spacing and escapes are the encoder's, not the -// wire's — and a number keeps its digits. Text that is not one JSON -// object (see ToolUseEvent.Arguments) cannot be walked that way: it is -// scanned as text so its findings count, then dropped whole and charged. +// wire's. A kept number is scanned as its digits — a runner env value +// can be sent as one — and stays a number when nothing was replaced. +// Text that is not one JSON object (see ToolUseEvent.Arguments), or +// carries anything but white space after it, cannot be walked that way: +// it is scanned as text so its findings count, then dropped whole and +// charged. // So are arguments whose encoding exceeds maxToolArgumentsBytes, charged // as encoded. A dropped member is charged as the redacted text it was // scanned as. @@ -824,7 +829,17 @@ func (c *contentCollector) toolArguments(args string) (json.RawMessage, bool) { case map[string]any, []any: enc, _ := json.Marshal(t) // decoded JSON values marshal unconditionally c.evicted += len(c.redact(string(enc), &c.findings)) - default: // a number, a boolean, null + case json.Number: + if recordedArguments[k] { + if r := c.scanLeaf(string(t)); r != string(t) { + out[k] = r // the mask, as the string it is + } else { + out[k] = t + } + continue + } + c.evicted += len(c.redact(string(t), &c.findings)) + default: // a boolean, null if recordedArguments[k] { out[k] = t continue @@ -848,14 +863,21 @@ func (c *contentCollector) toolArguments(args string) (json.RawMessage, bool) { return enc, dropped } -// decodeJSON decodes one JSON value; float64 would rewrite integers past -// 2^53, so numbers keep their digits. +var errTrailingInput = errors.New("trailing input after the JSON value") + +// decodeJSON decodes one JSON value, with white space at most after it; +// float64 would rewrite integers past 2^53, so numbers keep their digits. func decodeJSON(s string) (any, error) { dec := json.NewDecoder(strings.NewReader(s)) dec.UseNumber() var v any - err := dec.Decode(&v) - return v, err + if err := dec.Decode(&v); err != nil { + return nil, err + } + if _, err := dec.Token(); !errors.Is(err, io.EOF) { + return nil, errTrailingInput + } + return v, nil } // revealing reports a finding that shows a secret was there: any but the diff --git a/internal/cli/content_collector_test.go b/internal/cli/content_collector_test.go index baaf8c9dbd..322083816d 100644 --- a/internal/cli/content_collector_test.go +++ b/internal/cli/content_collector_test.go @@ -257,6 +257,66 @@ func TestContentCollector_AStringTheNormalizerStrippedIsMaskedWhole(t *testing.T } } +func TestContentCollector_ANumberUnderAListedMemberIsScanned(t *testing.T) { + // A number under a listed member is kept, and scanned as its digits + // first like every kept string: a runner env value sent as a number + // is replaced, and the mask is exported as a string, so a reader that + // gets an integer back got the real one. A clean number stays a + // number. + digits := strings.Repeat("7", minRedactableSecretLen) + t.Setenv(telemetry.ContentCaptureEnvVar, "true") + c := newContentCollectorIfEnabled(map[string]string{"DEPLOY_PASSWORD": digits}) + c.Handle(agentruntime.ToolUseEvent{Name: "Read", Arguments: `{"file_path":"/x","limit":50,"offset":` + digits + `}`}) + + res := c.Result("stop") + assert.Contains(t, res.OutputMessages, `"arguments":{"file_path":"/x","limit":50,"offset":"[REDACTED:DEPLOY_PASSWORD]"}`) + assert.NotContains(t, res.OutputMessages, digits) + assert.Equal(t, 1, runnerEnvFindings(res)) + assert.False(t, res.Truncated) + assert.Zero(t, res.DroppedBytes) +} + +func TestContentCollector_ArgumentsWithTrailingInputAreDropped(t *testing.T) { + // One JSON value, then white space at most. Anything after it is not + // the object the record would describe, so the text is scanned — a + // secret in the tail counts, and costs the summary — then dropped + // whole and charged as the redacted text. + token := "ghp_" + strings.Repeat("r", 36) + for name, tc := range map[string]struct { + args string + charged int + found int + }{ + "text after the object": {`{"command":"ls"} garbage`, len(`{"command":"ls"} garbage`), 0}, + "a second value": {`{"command":"ls"}{"command":"rm"}`, len(`{"command":"ls"}{"command":"rm"}`), 0}, + "text after null": {`null x`, len(`null x`), 0}, + "a token in the tail": {`{"command":"ls"} ` + token, len(`{"command":"ls"} ghp_...`), 1}, + } { + t.Run(name, func(t *testing.T) { + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Summary: "ls", Arguments: tc.args}) + + res := c.Result("stop") + assert.NotContains(t, res.OutputMessages, "arguments") + assert.NotContains(t, res.OutputMessages, token[4:]) + assert.True(t, res.Truncated) + assert.Equal(t, tc.charged, res.DroppedBytes) + assert.Len(t, res.Findings, tc.found) + if tc.found == 0 { + assert.Contains(t, res.OutputMessages, `"summary":"ls"`) + } else { + assert.NotContains(t, res.OutputMessages, `"summary"`) + } + }) + } + + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Arguments: "{\"command\":\"ls\"} \n"}) + res := c.Result("stop") + assert.Contains(t, res.OutputMessages, `"arguments":{"command":"ls"}`, "white space after the value is not trailing input") + assert.False(t, res.Truncated) +} + func TestContentCollector_NamelessCallCarriesNoArgumentsAndNoCharge(t *testing.T) { // The schema requires a name on a tool_call part. A nameless call is // refused; its arguments must not survive alone, nor charge a part diff --git a/internal/telemetry/telemetry.go b/internal/telemetry/telemetry.go index 5734db6f42..a132f7af78 100644 --- a/internal/telemetry/telemetry.go +++ b/internal/telemetry/telemetry.go @@ -248,7 +248,7 @@ func Setup(dir string, serviceVersion string) (trace.Tracer, func(context.Contex return tracenoop.NewTracerProvider().Tracer(""), noop } - f, err := os.OpenFile(filepath.Join(dir, TelemetryFile), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + f, err := os.OpenFile(filepath.Join(dir, TelemetryFile), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600) if err != nil { return tracenoop.NewTracerProvider().Tracer(""), noop } diff --git a/internal/telemetry/telemetry_test.go b/internal/telemetry/telemetry_test.go index 1697c6c0ce..95bb57255d 100644 --- a/internal/telemetry/telemetry_test.go +++ b/internal/telemetry/telemetry_test.go @@ -75,6 +75,20 @@ func TestSetup_FileExporter(t *testing.T) { assert.Equal(t, "test-span", td.ResourceSpans[0].ScopeSpans[0].Spans[0].Name) } +// TestSetup_TelemetryFileIsPrivate pins the record's mode: under the +// content gate it holds prompts and tool arguments, like the feedback +// audit file the runner keeps at 0600. +func TestSetup_TelemetryFileIsPrivate(t *testing.T) { + pinOTELEnv(t) + dir := t.TempDir() + _, cleanup := Setup(dir, "1.0.0-test") + cleanup(context.Background()) + + info, err := os.Stat(filepath.Join(dir, TelemetryFile)) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o600), info.Mode().Perm()) +} + // TestSetup_SpanAttributeValueLengthLimit pins the provider-level bound on // attribute values: a free-text attribute (model name, skip reason) cannot // ride an export at arbitrary size. From efa0465e9e35b8248a4f1bf1053a55c4714d2641 Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Tue, 13 Oct 2026 13:13:11 -0400 Subject: [PATCH 29/40] fix(cli): scan a string the normalizer stripped as written, once for every caller The normalizer runs before the patterns and strips a CSI sequence up to its final byte, which is a letter: ESC [ g eats the g of ghp_, and an OSC payload is text no pattern sees. Every Level 3 string and every execute_tool name and id goes through redactText, but the guard against that lived in one caller (scanLeaf, for argument leaves) and the review asked for a second (safeName). It now lives in redactText: a string the normalizer stripped something from is scanned as written too, and masked *** whole when that pass finds a secret the stripping hid. Results, summaries, names and ids get it at once; the finding count takes the patterns' findings from the pass that saw them all. Docs: the infrastructure guide no longer describes the raw bound or the key-collision rule the allowlist commit removed; both guides name the as-written pass. Review findings on 31f363de6: 4261579036 (medium, safeName) and 4261579042 (medium, incorrect doc). Signed-off-by: Dharit Shah --- docs/guides/dev/tracing.md | 6 ++- .../infrastructure/distributed-tracing.md | 11 +++--- internal/cli/content_collector.go | 35 +++++++++++++++++ internal/cli/content_collector_test.go | 39 +++++++++++++++++++ internal/cli/tool_spans_test.go | 23 +++++++++++ 5 files changed, 108 insertions(+), 6 deletions(-) diff --git a/docs/guides/dev/tracing.md b/docs/guides/dev/tracing.md index bc91905b65..369f1b4b16 100644 --- a/docs/guides/dev/tracing.md +++ b/docs/guides/dev/tracing.md @@ -231,7 +231,11 @@ codex emit none of them, required result field is `response`), redacts every part — `security.OutputPipeline()`, with `replaceEnvSecrets` for the values of sensitive runner env keys and of `providerOnlyKeys`, in one pass, on both -sides of it (`redactText`) — at assembly (redaction runs +sides of it (`redactText`; a string the normalizer stripped an escape +sequence or tag characters from is scanned as written too, and masked +`***` whole when that pass finds a secret the stripping hid — a colour +code ends at the next letter, so it can take a token's first letter, and +a title code's payload is text no pattern sees) — at assembly (redaction runs before the size budget — truncating first could split a secret past recognition), enforces a 256 KiB ordered-suffix budget (the ending survives — the final answer is what consumers judge) plus an 8 KiB per-tool-result diff --git a/docs/guides/infrastructure/distributed-tracing.md b/docs/guides/infrastructure/distributed-tracing.md index f066ee789e..0a9ff9f206 100644 --- a/docs/guides/infrastructure/distributed-tracing.md +++ b/docs/guides/infrastructure/distributed-tracing.md @@ -154,14 +154,15 @@ which sees the runtime's stream, not its API requests. **Redaction and size:** every part passes through security redaction (runner environment values, Unicode normalization, secret masking, then -runner environment values again) before reaching the span. +runner environment values again; a string the normalizer stripped an +escape sequence from is also scanned as written, and masked whole when +that finds a secret the stripping hid) before reaching the span. Content is bounded at 256 KiB per iteration — each tool result at 8 KiB, -each call's arguments at 8 KiB, and at four times that as written, before -they are decoded — +each call's arguments at 8 KiB — kept as an ordered suffix; overflow drops the oldest content first. Those bounds count raw bytes, except that arguments count as the re-encoded JSON described above, against their own bound and against the -total (as redacted text when they were dropped before they were decoded). That encoding writes `<`, `>` and `&` as six bytes each, so +total (as redacted text when a member was dropped by name or the arguments were not one JSON object). That encoding writes `<`, `>` and `&` as six bytes each, so arguments dense in those characters reach their bound at about a sixth of that size on the stream. The exported JSON string is bounded as well, at 255,000 bytes, because encoding adds 8–10% at these bounds on real runs @@ -331,7 +332,7 @@ The `agent` span's provider identity reflects only the parent run's serving endp | `gen_ai.input.messages` | `agent` | Level 3 only, on a retry iteration that carries validation feedback: the prompt the runner composed, as a JSON string (see Content capture) | | `gen_ai.output.messages` | `agent` | Level 3 only: the iteration's conversation content as a JSON string (see Content capture) | | `fullsend.content.truncated` | `agent` | Level 3 only: present (`true`) when the size budget cut or dropped content, a tool call lost its arguments, or a kept tool result is a parser-side fragment or an oversized line's empty stand-in (`fullsend.truncated` on the part; no byte count for the last two) | -| `fullsend.content.dropped_bytes` | `agent` | Level 3 only: exact part bytes removed by the size budget — content, ids, dropped arguments (counted as re-encoded JSON — on a key collision, without each member a later key (in sorted key order, not stream order) replaced — or as redacted text when they were not a JSON value), and the fixed footprint of an errored-empty or oversized stand-in part — otherwise in raw bytes, whichever bound made the cut; the bytes of a skipped oversized line were never decoded and are not counted | +| `fullsend.content.dropped_bytes` | `agent` | Level 3 only: exact part bytes removed by the size budget — content, ids, dropped arguments (counted as re-encoded JSON when the bound drops them, or as redacted text when a member was dropped by name or the arguments were not one JSON object), and the fixed footprint of an errored-empty or oversized stand-in part — otherwise in raw bytes, whichever bound made the cut; the bytes of a skipped oversized line were never decoded and are not counted | | `fullsend.content.redactions` | `agent` | Level 3 only: number of security findings raised while redacting content at assembly, the input message included (and findings from parts the size budget later dropped) | | `fullsend.tool.unmatched` | `execute_tool` | Present (`true`) when a result arrived for a call the stream never reported; the span has near-zero duration | | `fullsend.tool.result_oversized` | `execute_tool` | Present (`true`) when the result's stream line exceeded the parser's 1 MiB bound: the call was answered but nothing of the result was decoded, so the span has no status and no `error.type` | diff --git a/internal/cli/content_collector.go b/internal/cli/content_collector.go index d2053724b1..d05b6e2850 100644 --- a/internal/cli/content_collector.go +++ b/internal/cli/content_collector.go @@ -748,6 +748,25 @@ func redactText(pipeline *security.Pipeline, runnerEnv map[string]string, text s } text = replaceEnv(runnerEnv, text, findings) scanned := pipeline.Scan(text) + if slices.ContainsFunc(scanned.Findings, stripped) { + // The normalizer strips a terminal escape sequence before the + // patterns run, and a colour code ends at the next letter — so the + // stripping can take a token's first letter and leave a body no + // pattern matches — and a title code's payload is text no pattern + // sees. So the text is scanned as written too; a secret only that + // pass finds was hidden by the stripping, and the string is masked + // whole: nothing cut from it can be trusted. The normalizer's + // findings stay; the patterns' come from the pass that saw them all. + if raw := rawPatterns.Scan(text); patternFindings(raw.Findings) > patternFindings(scanned.Findings) { + for _, f := range scanned.Findings { + if f.Scanner == "unicode_normalizer" { + *findings = append(*findings, f) + } + } + *findings = append(*findings, raw.Findings...) + return "***" + } + } *findings = append(*findings, scanned.Findings...) if scanned.Sanitized != "" { return replaceEnv(runnerEnv, scanned.Sanitized, findings) @@ -758,6 +777,22 @@ func redactText(pipeline *security.Pipeline, runnerEnv map[string]string, text s return text } +// rawPatterns is the secret redactor alone, for the pass over text as the +// stream wrote it (redactText). +var rawPatterns = security.NewPipeline(security.NewSecretRedactor()) + +// patternFindings counts the findings the patterns made, as opposed to +// the normalizer's. +func patternFindings(fs []security.Finding) int { + n := 0 + for _, f := range fs { + if f.Scanner != "unicode_normalizer" { + n++ + } + } + return n +} + // recordedArguments names the members of a tool call's arguments that // the record keeps: paths, patterns, commands, modes and bounds — what a // trace needs to say which call was made — and never a file body, an diff --git a/internal/cli/content_collector_test.go b/internal/cli/content_collector_test.go index 322083816d..cd583f6606 100644 --- a/internal/cli/content_collector_test.go +++ b/internal/cli/content_collector_test.go @@ -317,6 +317,45 @@ func TestContentCollector_ArgumentsWithTrailingInputAreDropped(t *testing.T) { assert.False(t, res.Truncated) } +func TestContentCollector_AStringTheNormalizerStrippedIsScannedAsWritten(t *testing.T) { + // The normalizer strips a terminal escape sequence before the patterns + // run, and a colour code ends at the next letter: ESC [ g eats the g + // of ghp_, and a title code's payload is text no pattern sees. Every + // Level 3 string goes through redactText, so the rule lives there: a + // string the normalizer stripped something from is scanned as written + // too, and masked whole when that pass finds a secret the stripping + // hid. A coloured result with no secret, or with one the normalized + // pass also finds, keeps its text. + token := "ghp_" + strings.Repeat("r", 36) + for name, tc := range map[string]struct { + result, want string + found int + }{ + "a colour code that eats a token's first letter": {"\x1b[" + token, `"response":"***"`, 2}, + "a token inside a title code": {"ok \x1b]" + token + "\x07", `"response":"***"`, 2}, + "a coloured line with no secret": {"\x1b[31mok\x1b[0m", `"response":"ok"`, 1}, + "a coloured line with a token both passes see": {"\x1b[31m" + token + "\x1b[0m", `"response":"ghp_..."`, 2}, + } { + t.Run(name, func(t *testing.T) { + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{ID: "toolu_01", Name: "Bash"}) + c.Handle(agentruntime.ToolResultEvent{ID: "toolu_01", Result: tc.result}) + + res := c.Result("stop") + assert.Contains(t, res.OutputMessages, tc.want) + assert.NotContains(t, res.OutputMessages, token[4:]) + assert.Len(t, res.Findings, tc.found, "the normalizer once, and the secret once") + }) + } + + // The same rule covers a call's name on the record. + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{ID: "toolu_02", Name: "\x1b[" + token}) + res := c.Result("stop") + assert.Contains(t, res.OutputMessages, `"name":"***"`) + assert.NotContains(t, res.OutputMessages, token[4:]) +} + func TestContentCollector_NamelessCallCarriesNoArgumentsAndNoCharge(t *testing.T) { // The schema requires a name on a tool_call part. A nameless call is // refused; its arguments must not survive alone, nor charge a part diff --git a/internal/cli/tool_spans_test.go b/internal/cli/tool_spans_test.go index 1cb992e85d..0d15d52c5f 100644 --- a/internal/cli/tool_spans_test.go +++ b/internal/cli/tool_spans_test.go @@ -408,6 +408,29 @@ func TestToolSpanTracker_NameGoesThroughTheOutputSanitizer(t *testing.T) { assert.NotContains(t, toolSpanAttrs(spans[3]), attribute.Key("gen_ai.tool.name")) } +func TestToolSpanTracker_ANameTheNormalizerStrippedIsScannedAsWritten(t *testing.T) { + // A colour code ends at the next letter, so stripping ESC [ g takes the + // g of ghp_ and leaves a body no pattern matches, and a title code's + // payload is text no pattern sees. The name is scanned as written too + // (redactText) and masked whole. The token is not in the runner env, + // so only the pattern scan could have caught it. + token := "ghp_" + strings.Repeat("r", 36) + tr, rec, _ := toolSpanFixture(t) + for id, name := range map[string]string{ + "toolu_15": "\x1b[" + token, + "toolu_16": "Bash\x1b]" + token + "\x07", + } { + tr.Handle(agentruntime.ToolUseEvent{ID: id, Name: name}) + tr.Handle(agentruntime.ToolResultEvent{ID: id}) + } + spans := endedToolSpans(rec) + require.Len(t, spans, 2) + for _, s := range spans { + assert.NotContains(t, s.Name, token[4:]) + assert.Equal(t, "***", toolSpanAttrs(s)["gen_ai.tool.name"].AsString()) + } +} + func TestRecordToolSpanOverflow_MarksAgentSpanOnlyWhenDropped(t *testing.T) { rec := tracetest.NewSpanRecorder() tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(rec)) From 16c1e13f8a9b706a20711cfdbe4764d5f09f1d38 Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Tue, 13 Oct 2026 13:37:34 -0400 Subject: [PATCH 30/40] fix(cli): mask a string the normalizer stripped from whole, no count to second-guess it efa0465e9 scanned such a string as written and masked it whole only when that pass found more pattern findings than the normalized one. Review showed the count cannot tell: the redactor reports one finding for repeated matches, so an escape that eats a token's first letter, followed by the same token again, gives equal counts and the eaten copy is exported; and a second value the normalizer joins can hide a first one it cut. The rule is now the coarse one scanLeaf had: a string the normalizer stripped an escape sequence or tag characters from is masked *** whole, in redactText, for every caller. The as-written pass and the count are gone, and scanLeaf with them, since redactText now does what it did. Tests: tool results and span names with the eaten token repeated, with a second token the normalizer joins, and coloured text with and without a secret, all masked whole; the sanitizer test's colour-code row moves to the whole-mask test. Review findings on efa0465e9: 4261862948 (high, logic error) and 4261862956 (medium, safeName). Signed-off-by: Dharit Shah --- docs/guides/dev/tracing.md | 14 ++--- .../infrastructure/distributed-tracing.md | 3 +- internal/cli/content_collector.go | 57 ++++--------------- internal/cli/content_collector_test.go | 37 ++++++------ internal/cli/tool_spans_test.go | 25 ++++---- 5 files changed, 51 insertions(+), 85 deletions(-) diff --git a/docs/guides/dev/tracing.md b/docs/guides/dev/tracing.md index 369f1b4b16..eeec8d6386 100644 --- a/docs/guides/dev/tracing.md +++ b/docs/guides/dev/tracing.md @@ -232,10 +232,10 @@ required result field is `response`), redacts every part — `security.OutputPipeline()`, with `replaceEnvSecrets` for the values of sensitive runner env keys and of `providerOnlyKeys`, in one pass, on both sides of it (`redactText`; a string the normalizer stripped an escape -sequence or tag characters from is scanned as written too, and masked -`***` whole when that pass finds a secret the stripping hid — a colour -code ends at the next letter, so it can take a token's first letter, and -a title code's payload is text no pattern sees) — at assembly (redaction runs +sequence or tag characters from is masked `***` whole — a colour code +ends at the next letter, so it can take a token's first letter, and a +title code's payload is text no pattern sees, so nothing cut from such a +string can be trusted, and no scan of it as written can stand in) — at assembly (redaction runs before the size budget — truncating first could split a secret past recognition), enforces a 256 KiB ordered-suffix budget (the ending survives — the final answer is what consumers judge) plus an 8 KiB per-tool-result @@ -276,10 +276,8 @@ pipeline on its own, since the redactor's patterns are written for plain text and over serialised JSON miss an assignment that opens a string or follows an escaped newline, and a value behind escaped quotes; the result is encoded again, so key order, spacing and escapes are the encoder's. A -string the normalizer stripped an escape sequence or tag characters from -is masked `***` whole: a colour code ends at the next letter, so the -stripping can take a token's first letter and leave the rest, and a title -code's payload is text no pattern sees. Discarded content is redacted +string the normalizer stripped from is masked `***` whole, as every +string is (above). Discarded content is redacted first, here as everywhere: a dropped member is scanned as text — the decoded string, or the compact encoding of an object or an array — so a secret in a file body still counts, and costs the summary as a secret diff --git a/docs/guides/infrastructure/distributed-tracing.md b/docs/guides/infrastructure/distributed-tracing.md index 0a9ff9f206..b79ab5b157 100644 --- a/docs/guides/infrastructure/distributed-tracing.md +++ b/docs/guides/infrastructure/distributed-tracing.md @@ -155,8 +155,7 @@ which sees the runtime's stream, not its API requests. **Redaction and size:** every part passes through security redaction (runner environment values, Unicode normalization, secret masking, then runner environment values again; a string the normalizer stripped an -escape sequence from is also scanned as written, and masked whole when -that finds a secret the stripping hid) before reaching the span. +escape sequence from is masked whole) before reaching the span. Content is bounded at 256 KiB per iteration — each tool result at 8 KiB, each call's arguments at 8 KiB — kept as an ordered suffix; overflow drops the oldest content first. Those diff --git a/internal/cli/content_collector.go b/internal/cli/content_collector.go index d05b6e2850..c5fce65c12 100644 --- a/internal/cli/content_collector.go +++ b/internal/cli/content_collector.go @@ -741,33 +741,27 @@ func (c *contentCollector) redact(text string, findings *[]security.Finding) str // assignment, an auth header, a secret-named field or a connection string, // or by its own prefix — is masked by that pattern, which shows what its // mask shows; and a value the normalizer itself rewrites is matched only as -// the env has it. An empty result with findings means fully redacted. +// the env has it. An empty result with findings means fully redacted. A +// string the normalizer stripped an escape sequence or tag characters +// from (stripped) is masked *** whole. func redactText(pipeline *security.Pipeline, runnerEnv map[string]string, text string, findings *[]security.Finding) string { if text == "" { return text } text = replaceEnv(runnerEnv, text, findings) scanned := pipeline.Scan(text) + *findings = append(*findings, scanned.Findings...) if slices.ContainsFunc(scanned.Findings, stripped) { // The normalizer strips a terminal escape sequence before the // patterns run, and a colour code ends at the next letter — so the // stripping can take a token's first letter and leave a body no // pattern matches — and a title code's payload is text no pattern - // sees. So the text is scanned as written too; a secret only that - // pass finds was hidden by the stripping, and the string is masked - // whole: nothing cut from it can be trusted. The normalizer's - // findings stay; the patterns' come from the pass that saw them all. - if raw := rawPatterns.Scan(text); patternFindings(raw.Findings) > patternFindings(scanned.Findings) { - for _, f := range scanned.Findings { - if f.Scanner == "unicode_normalizer" { - *findings = append(*findings, f) - } - } - *findings = append(*findings, raw.Findings...) - return "***" - } + // sees. Nothing cut from such a string can be trusted, so it is + // masked whole. No scan of the text as written can stand in: the + // redactor reports one finding for repeated matches, and a value + // the normalizer joins can hide one it cut. + return "***" } - *findings = append(*findings, scanned.Findings...) if scanned.Sanitized != "" { return replaceEnv(runnerEnv, scanned.Sanitized, findings) } @@ -777,22 +771,6 @@ func redactText(pipeline *security.Pipeline, runnerEnv map[string]string, text s return text } -// rawPatterns is the secret redactor alone, for the pass over text as the -// stream wrote it (redactText). -var rawPatterns = security.NewPipeline(security.NewSecretRedactor()) - -// patternFindings counts the findings the patterns made, as opposed to -// the normalizer's. -func patternFindings(fs []security.Finding) int { - n := 0 - for _, f := range fs { - if f.Scanner != "unicode_normalizer" { - n++ - } - } - return n -} - // recordedArguments names the members of a tool call's arguments that // the record keeps: paths, patterns, commands, modes and bounds — what a // trace needs to say which call was made — and never a file body, an @@ -857,7 +835,7 @@ func (c *contentCollector) toolArguments(args string) (json.RawMessage, bool) { switch t := obj[k].(type) { case string: if recordedArguments[k] { - out[k] = c.scanLeaf(t) + out[k] = c.redact(t, &c.findings) continue } c.evicted += len(c.redact(t, &c.findings)) @@ -866,7 +844,7 @@ func (c *contentCollector) toolArguments(args string) (json.RawMessage, bool) { c.evicted += len(c.redact(string(enc), &c.findings)) case json.Number: if recordedArguments[k] { - if r := c.scanLeaf(string(t)); r != string(t) { + if r := c.redact(string(t), &c.findings); r != string(t) { out[k] = r // the mask, as the string it is } else { out[k] = t @@ -929,19 +907,6 @@ func stripped(f security.Finding) bool { return f.Scanner == "unicode_normalizer" && (f.Name == "ansi_escape" || f.Name == "osc_escape" || f.Name == "tag_char") } -// scanLeaf redacts one string of the arguments: through the pipeline, -// which masks in place, and masked whole when the normalizer stripped an -// escape sequence or tag characters from it (stripped), since what is -// left can be most of a token. -func (c *contentCollector) scanLeaf(s string) string { - mark := len(c.findings) - r := c.redact(s, &c.findings) - if slices.ContainsFunc(c.findings[mark:], stripped) { - return "***" - } - return r -} - // replaceEnv is redactText's literal pass, with one finding for each key // it replaced. func replaceEnv(runnerEnv map[string]string, text string, findings *[]security.Finding) string { diff --git a/internal/cli/content_collector_test.go b/internal/cli/content_collector_test.go index cd583f6606..f99fcbe74b 100644 --- a/internal/cli/content_collector_test.go +++ b/internal/cli/content_collector_test.go @@ -317,40 +317,41 @@ func TestContentCollector_ArgumentsWithTrailingInputAreDropped(t *testing.T) { assert.False(t, res.Truncated) } -func TestContentCollector_AStringTheNormalizerStrippedIsScannedAsWritten(t *testing.T) { +func TestContentCollector_AResultTheNormalizerStrippedIsMaskedWhole(t *testing.T) { // The normalizer strips a terminal escape sequence before the patterns // run, and a colour code ends at the next letter: ESC [ g eats the g // of ghp_, and a title code's payload is text no pattern sees. Every - // Level 3 string goes through redactText, so the rule lives there: a - // string the normalizer stripped something from is scanned as written - // too, and masked whole when that pass finds a secret the stripping - // hid. A coloured result with no secret, or with one the normalized - // pass also finds, keeps its text. + // Level 3 string goes through redactText, so the rule lives there and + // is coarse on purpose: a string the normalizer stripped anything from + // is masked whole. No scan of the text as written can stand in — the + // redactor reports one finding for repeated matches, and a second + // value the normalizer joins can hide a first one it cut. token := "ghp_" + strings.Repeat("r", 36) - for name, tc := range map[string]struct { - result, want string - found int - }{ - "a colour code that eats a token's first letter": {"\x1b[" + token, `"response":"***"`, 2}, - "a token inside a title code": {"ok \x1b]" + token + "\x07", `"response":"***"`, 2}, - "a coloured line with no secret": {"\x1b[31mok\x1b[0m", `"response":"ok"`, 1}, - "a coloured line with a token both passes see": {"\x1b[31m" + token + "\x1b[0m", `"response":"ghp_..."`, 2}, + other := "ghp_" + strings.Repeat("s", 36) + for name, result := range map[string]string{ + "a colour code that eats a token's first letter": "\x1b[" + token, + "a token inside a title code": "ok \x1b]" + token + "\x07", + "the eaten token repeated": "\x1b[" + token + " " + token, + "a second token the normalizer joins": "\x1b[" + token + " " + other[:6] + "\u200b" + other[6:], + "a coloured line with no secret": "\x1b[31mok\x1b[0m", + "a coloured line with a token": "\x1b[31m" + token + "\x1b[0m", } { t.Run(name, func(t *testing.T) { c := newContentCollector(4096) c.Handle(agentruntime.ToolUseEvent{ID: "toolu_01", Name: "Bash"}) - c.Handle(agentruntime.ToolResultEvent{ID: "toolu_01", Result: tc.result}) + c.Handle(agentruntime.ToolResultEvent{ID: "toolu_01", Result: result}) res := c.Result("stop") - assert.Contains(t, res.OutputMessages, tc.want) + assert.Contains(t, res.OutputMessages, `"response":"***"`) assert.NotContains(t, res.OutputMessages, token[4:]) - assert.Len(t, res.Findings, tc.found, "the normalizer once, and the secret once") + assert.NotContains(t, res.OutputMessages, other[10:]) + assert.NotEmpty(t, res.Findings) }) } // The same rule covers a call's name on the record. c := newContentCollector(4096) - c.Handle(agentruntime.ToolUseEvent{ID: "toolu_02", Name: "\x1b[" + token}) + c.Handle(agentruntime.ToolUseEvent{ID: "toolu_02", Name: "\x1b[" + token + " " + token}) res := c.Result("stop") assert.Contains(t, res.OutputMessages, `"name":"***"`) assert.NotContains(t, res.OutputMessages, token[4:]) diff --git a/internal/cli/tool_spans_test.go b/internal/cli/tool_spans_test.go index 0d15d52c5f..562741d666 100644 --- a/internal/cli/tool_spans_test.go +++ b/internal/cli/tool_spans_test.go @@ -378,8 +378,9 @@ func TestToolSpanTracker_NameGoesThroughTheOutputSanitizer(t *testing.T) { // Unicode normalization (escapes, NUL, zero-width and bidi overrides) // and then secret redaction — not the secret pass alone. tr, rec, _ := toolSpanFixture(t) + // (A name with a terminal escape sequence is masked whole instead: + // TestToolSpanTracker_ANameTheNormalizerStrippedIsMaskedWhole.) cases := map[string]string{ - "toolu_11": "Bash\x1b[31mred\x1b[0m", "toolu_12": "Bash\u200b\u202eevil", "toolu_13": "Bash\x00nul", } @@ -388,10 +389,10 @@ func TestToolSpanTracker_NameGoesThroughTheOutputSanitizer(t *testing.T) { tr.Handle(agentruntime.ToolResultEvent{ID: id}) } spans := endedToolSpans(rec) - require.Len(t, spans, 3) + require.Len(t, spans, 2) for _, s := range spans { name := toolSpanAttrs(s)["gen_ai.tool.name"].AsString() - for _, bad := range []string{"\x1b", "\u200b", "\u202e", "\x00"} { + for _, bad := range []string{"\u200b", "\u202e", "\x00"} { assert.NotContains(t, name, bad, "attribute must be normalized") assert.NotContains(t, s.Name, bad, "span name must be normalized") } @@ -403,28 +404,30 @@ func TestToolSpanTracker_NameGoesThroughTheOutputSanitizer(t *testing.T) { tr.Handle(agentruntime.ToolUseEvent{ID: "toolu_14", Name: "\u200b\u202e"}) tr.Handle(agentruntime.ToolResultEvent{ID: "toolu_14"}) spans = endedToolSpans(rec) - require.Len(t, spans, 4) - assert.Equal(t, "execute_tool", spans[3].Name) - assert.NotContains(t, toolSpanAttrs(spans[3]), attribute.Key("gen_ai.tool.name")) + require.Len(t, spans, 3) + assert.Equal(t, "execute_tool", spans[2].Name) + assert.NotContains(t, toolSpanAttrs(spans[2]), attribute.Key("gen_ai.tool.name")) } -func TestToolSpanTracker_ANameTheNormalizerStrippedIsScannedAsWritten(t *testing.T) { +func TestToolSpanTracker_ANameTheNormalizerStrippedIsMaskedWhole(t *testing.T) { // A colour code ends at the next letter, so stripping ESC [ g takes the // g of ghp_ and leaves a body no pattern matches, and a title code's - // payload is text no pattern sees. The name is scanned as written too - // (redactText) and masked whole. The token is not in the runner env, - // so only the pattern scan could have caught it. + // payload is text no pattern sees. Such a name is masked whole + // (redactText), with no scan of the text as written to second-guess it. + // The token is not in the runner env, so only a pattern could have + // caught it. token := "ghp_" + strings.Repeat("r", 36) tr, rec, _ := toolSpanFixture(t) for id, name := range map[string]string{ "toolu_15": "\x1b[" + token, "toolu_16": "Bash\x1b]" + token + "\x07", + "toolu_17": "\x1b[" + token + " " + token, // one finding either way: no count can tell } { tr.Handle(agentruntime.ToolUseEvent{ID: id, Name: name}) tr.Handle(agentruntime.ToolResultEvent{ID: id}) } spans := endedToolSpans(rec) - require.Len(t, spans, 2) + require.Len(t, spans, 3) for _, s := range spans { assert.NotContains(t, s.Name, token[4:]) assert.Equal(t, "***", toolSpanAttrs(s)["gen_ai.tool.name"].AsString()) From 28055ba619bac3fc10073bd5e4394af5a39cfb0f Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Tue, 13 Oct 2026 14:24:06 -0400 Subject: [PATCH 31/40] test(cli): state the redaction guarantee as one invariant; mask what no pass can see whole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TestRedaction_NoSecretSurvivesAnyWrapping states what the Level 3 record and the execute_tool spans guarantee — no 12-byte run of a known secret's body in any exported string, read raw, normalized, or JSON-decoded — and checks it over nine secrets, twenty-one wrappings and thirteen surfaces, with a control that the detector sees the secret in every input. Four limits are named there and in the dev guide: an env value in compatibility forms, a secret split across parts, a secret no pattern or env knows, a summary a pi or codex parser cut before the collector saw it. Running it, and a pre-push review with the review agent's own security lens, found three defects, each fixed by masking whole in the shared function: - The retry prompt is sanitized and cut before it is recorded, so the recorder's pass cannot see what the sanitizer stripped: a colour code that ate a token's first letter plus a zero-width character inside it left "hp_" and the body (the review's finding on 16c1e13f8), and a token in compatibility forms straddling the 10 KiB feedback cut was split before anything could fold it. promptHidesAlterations tells attachInput when the composition altered the feedback in a way its own pass cannot see; the record is then "***" whole. Plain feedback over the cut is still recorded. - A secret holding an invalid UTF-8 byte passed every scan unjoined, and the span and attribute writers' ToValidUTF8 then joined the halves — the whole token on a span name, gen_ai.tool.name and gen_ai.tool.call.id (an oversized result's id reaches the tracker as raw line bytes). redactText masks a string holding invalid UTF-8 whole; toolArguments drops such arguments before the decoder could coerce the byte to a visible U+FFFD. - Arguments of white space only were charged and marked as dropped; they carry nothing, like "". Docs: the guarantee and its limits in the dev guide; the infrastructure guide names tag characters and invalid UTF-8 beside escape sequences; the user guide says what a user sees first — a coloured string recorded as "***" whole. Review finding on 16c1e13f8: 4262112732 (low, the upstream-sanitized prompt). Signed-off-by: Dharit Shah --- docs/guides/dev/tracing.md | 31 +- .../infrastructure/distributed-tracing.md | 11 +- docs/guides/user/how-to-emit-traces.md | 6 +- internal/cli/content_collector.go | 48 ++- internal/cli/content_collector_test.go | 74 +++- internal/cli/redaction_invariant_test.go | 339 ++++++++++++++++++ internal/cli/run.go | 22 +- internal/cli/telemetry_run_test.go | 4 +- internal/cli/tool_spans_test.go | 17 + 9 files changed, 517 insertions(+), 35 deletions(-) create mode 100644 internal/cli/redaction_invariant_test.go diff --git a/docs/guides/dev/tracing.md b/docs/guides/dev/tracing.md index eeec8d6386..a6151b70c6 100644 --- a/docs/guides/dev/tracing.md +++ b/docs/guides/dev/tracing.md @@ -193,7 +193,7 @@ result never arrives as a `tool_result`), so it appears in neither count. The na — the runner env literal pass on both sides of `security.OutputPipeline()`, the redaction span content gets — and is bounded to 256 bytes before it becomes the attribute; the span name keeps at most 128 bytes of it. The call id goes through the -same `redactText` and dropped from the span on any finding — never substituted, +same `redactText` and is dropped from the span on any finding — never substituted, since a masked id could collide with another call's — while the raw bounded id still keys the open-call map, so correlation is unaffected. The tracker records at most `maxToolSpansPerIteration` (1,024) spans per @@ -235,7 +235,9 @@ sides of it (`redactText`; a string the normalizer stripped an escape sequence or tag characters from is masked `***` whole — a colour code ends at the next letter, so it can take a token's first letter, and a title code's payload is text no pattern sees, so nothing cut from such a -string can be trusted, and no scan of it as written can stand in) — at assembly (redaction runs +string can be trusted, and no scan of it as written can stand in; so is a +string holding invalid UTF-8, since the span and attribute writers would +join the halves of a value such a byte splits) — at assembly (redaction runs before the size budget — truncating first could split a secret past recognition), enforces a 256 KiB ordered-suffix budget (the ending survives — the final answer is what consumers judge) plus an 8 KiB per-tool-result @@ -261,6 +263,26 @@ content and its marker attributes on the span before either `finalizeAgentSpan` path can end it, so failed iterations keep their content. +**Guarantee and limits.** For every string the record or an `execute_tool` +span exports, no 12-byte run of a known secret's body appears, read as raw +bytes, after the normalizer, or after decoding the record's JSON with every +string walked; a known secret is one a `SecretRedactor` pattern matches, or +a runner env or provider-only env value the literal pass holds. +`TestRedaction_NoSecretSurvivesAnyWrapping` (`internal/cli`) states it and +checks it over every surface (text, names, summaries, arguments on each +path, results, the input message, span names and ids) and every wrapping a +review has constructed so far — terminal escapes before, around and inside +a secret, repeated and paired secrets, zero-width, bidi, tag, NUL and +variation-selector characters inside it, compatibility forms, and noise. +Four limits are named there and here, and nowhere else: an env value +written in compatibility (fullwidth) forms, since the normalizer rewrites +it and the literal pass matches only what the env has; a secret split +across two parts or events; a secret that matches no pattern and is in no +env; a summary a pi or codex parser cut before the collector saw it, since +that cut can split a value the parser's own scan did not see joined (a +Claude summary is cut from a member the collector scans whole). A finding +inside the guarantee is a defect; one on this list is a documented limit. + `toolArguments` records the members of a call's input that `recordedArguments` names — paths, patterns, commands, modes and bounds: `file_path`, `pattern`, `path`, `command`, `description`, `url`, the @@ -295,7 +317,10 @@ a retry prompt (`buildFeedbackPrompt`, under `feedback_mode: append`), `attachInput` redacts it with the collector's pipeline — `redactFeedback` ran before the feedback was sanitized and framed, and sanitizing can join a token that scan saw split, so the recorded copy is pattern-scanned -again; the prompt the agent is sent is not changed, and the mask the first +again; a prompt the composition altered in a way that pass cannot see — +sanitized, or cut at `maxFeedbackBytes` while holding compatibility +characters (`promptHidesAlterations`) — is recorded `***` whole; the +prompt the agent is sent is not changed, and the mask the first scan leaves for a connection-string password of ten or more bytes matches its own pattern and counts a finding — and sets `gen_ai.input.messages` on the span diff --git a/docs/guides/infrastructure/distributed-tracing.md b/docs/guides/infrastructure/distributed-tracing.md index b79ab5b157..36042d6213 100644 --- a/docs/guides/infrastructure/distributed-tracing.md +++ b/docs/guides/infrastructure/distributed-tracing.md @@ -135,8 +135,10 @@ todo list, and any member whose value is an object or an array are dropped — scanned first, so a secret in them still counts — and the part is marked `fullsend.truncated`. The kept members are decoded, redacted one by one and encoded again, so key order, spacing and escapes are not -the stream's; a string a terminal escape sequence was stripped from is -masked whole, since the stripping can leave most of a token. +the stream's; a string a terminal escape sequence or tag characters were +stripped from, or that holds invalid UTF-8, is masked whole, since the +stripping can leave most of a token and a later write would join the +halves of a value an invalid byte splits. On a retry iteration under `validation_loop.feedback_mode: append`, the prompt the runner composed — its fixed framing around the previous @@ -155,7 +157,10 @@ which sees the runtime's stream, not its API requests. **Redaction and size:** every part passes through security redaction (runner environment values, Unicode normalization, secret masking, then runner environment values again; a string the normalizer stripped an -escape sequence from is masked whole) before reaching the span. +escape sequence or tag characters from, or that holds invalid UTF-8, is +masked whole) before reaching the span. What +that guarantees, and its four named limits, is stated in the +[developer guide](../dev/tracing.md) beside the test that checks it. Content is bounded at 256 KiB per iteration — each tool result at 8 KiB, each call's arguments at 8 KiB — kept as an ordered suffix; overflow drops the oldest content first. Those diff --git a/docs/guides/user/how-to-emit-traces.md b/docs/guides/user/how-to-emit-traces.md index c7c979f3a4..cb8a6c6a86 100644 --- a/docs/guides/user/how-to-emit-traces.md +++ b/docs/guides/user/how-to-emit-traces.md @@ -128,8 +128,10 @@ text, reasoning, tool calls, and their arguments and results — when the runtime's stream provides them; Claude runs do — to each `agent` span, in the local file and at the endpoint. A retry that carries validation feedback also records the -prompt the runner composed. Content is redacted for secrets and bounded per iteration, -but may still contain proprietary code or PII — tool arguments carry the +prompt the runner composed. Content is redacted for secrets and bounded per iteration; +a string that held a terminal colour or title code or tag characters, as most +coloured CLI output does, is recorded as `***` whole, and so is a retry +prompt the runner sanitized or cut. Content may still contain proprietary code or PII — tool arguments carry the commands an agent ran and the paths and patterns it used, never a file body or an edit — so make sure your backend's access controls fit before enabling it. diff --git a/internal/cli/content_collector.go b/internal/cli/content_collector.go index c5fce65c12..97ac22bafe 100644 --- a/internal/cli/content_collector.go +++ b/internal/cli/content_collector.go @@ -184,12 +184,24 @@ func attachContent(span trace.Span, res contentResult) { // The prompt is cut upstream (maxFeedbackBytes), not here. Folding can // grow the copy about elevenfold, so its encoded size is charged to // maxEncoded: the two content attributes of one span together stay -// within the size maxEncodedContentBytes is proven for. -func (c *contentCollector) attachInput(span trace.Span, prompt string) { - if c == nil { +// within the size maxEncodedContentBytes is proven for. A prompt the +// composition altered in a way this pass cannot see — sanitized, or cut +// while holding compatibility characters (promptHidesAlterations) — is +// recorded "***" whole. +func (c *contentCollector) attachInput(span trace.Span, prompt string, alteredUpstream bool) { + if c == nil || prompt == "" { return } text := c.redact(prompt, &c.findings) + if alteredUpstream { + // The composition stripped or cut the feedback before this pass + // could see it (promptHidesAlterations), so what is left can be + // most of a token that no longer matches: masked whole, as every + // string the normalizer stripped something from is (redactText). + c.findings = append(c.findings, security.Finding{Scanner: "unicode_normalizer", Name: "altered_upstream", Severity: "medium", + Detail: "the retry prompt was sanitized or cut before it was recorded; recorded masked whole", Position: -1}) + text = "***" + } if text == "" { return } @@ -743,11 +755,22 @@ func (c *contentCollector) redact(text string, findings *[]security.Finding) str // mask shows; and a value the normalizer itself rewrites is matched only as // the env has it. An empty result with findings means fully redacted. A // string the normalizer stripped an escape sequence or tag characters -// from (stripped) is masked *** whole. +// from (stripped), or that holds invalid UTF-8, is masked *** whole. func redactText(pipeline *security.Pipeline, runnerEnv map[string]string, text string, findings *[]security.Finding) string { if text == "" { return text } + if !utf8.ValidString(text) { + // An invalid byte splits a value as a zero-width character does, + // but nothing here joins it: the normalizer leaves it, no pattern + // spans it, the literal pass needs the value whole — and the span + // and attribute writers' ToValidUTF8 would join the halves after + // this pass. Masked whole, as every string the normalizer stripped + // something from is. + *findings = append(*findings, security.Finding{Scanner: "unicode_normalizer", Name: "invalid_utf8", Severity: "medium", + Detail: "invalid UTF-8 inside the string; masked whole", Position: -1}) + return "***" + } text = replaceEnv(runnerEnv, text, findings) scanned := pipeline.Scan(text) *findings = append(*findings, scanned.Findings...) @@ -801,7 +824,7 @@ var recordedArguments = map[string]bool{ // toolArguments returns the recorded members of a tool call's arguments // (recordedArguments), redacted and encoded again, and whether any of // the input was dropped instead. There are none to return when the event -// carries none or JSON null. +// carries none, white space only, or JSON null. // // The input is decoded and each kept string is redacted on its own: the // redactor's patterns are written for plain text, and run over serialised @@ -811,16 +834,23 @@ var recordedArguments = map[string]bool{ // wire's. A kept number is scanned as its digits — a runner env value // can be sent as one — and stays a number when nothing was replaced. // Text that is not one JSON object (see ToolUseEvent.Arguments), or -// carries anything but white space after it, cannot be walked that way: -// it is scanned as text so its findings count, then dropped whole and -// charged. +// carries anything but white space after it, or holds invalid UTF-8, +// cannot be walked that way: it is scanned as text so its findings count, +// then dropped whole and charged. // So are arguments whose encoding exceeds maxToolArgumentsBytes, charged // as encoded. A dropped member is charged as the redacted text it was // scanned as. func (c *contentCollector) toolArguments(args string) (json.RawMessage, bool) { - if args == "" { + if strings.TrimSpace(args) == "" { return nil, false } + if !utf8.ValidString(args) { + // The decoder would coerce an invalid byte to a visible U+FFFD, + // and a value it splits would reach the record in halves no pattern + // spans. Scanned as text — which masks it whole — and dropped. + c.evicted += len(c.redact(args, &c.findings)) + return nil, true + } v, err := decodeJSON(args) if err == nil && v == nil { return nil, false diff --git a/internal/cli/content_collector_test.go b/internal/cli/content_collector_test.go index f99fcbe74b..03b857bf83 100644 --- a/internal/cli/content_collector_test.go +++ b/internal/cli/content_collector_test.go @@ -357,6 +357,56 @@ func TestContentCollector_AResultTheNormalizerStrippedIsMaskedWhole(t *testing.T assert.NotContains(t, res.OutputMessages, token[4:]) } +func TestAttachInput_APromptAlteredUpstreamIsMaskedWhole(t *testing.T) { + // The retry prompt is sanitized and cut before it is recorded + // (buildFeedbackPrompt), so the recorder's own pass cannot see what + // the sanitizer stripped: a colour code that ate a token's first + // letter plus a zero-width character that kept the token from the + // pattern scan leave "hp_" and the body; and a token in compatibility + // forms straddling the cut is split before anything can fold it. Both + // are recorded masked whole (promptHidesAlterations). Big plain + // feedback is not. + token := "ghp_" + strings.Repeat("r", 36) + var fw strings.Builder + for _, r := range token { + fw.WriteRune(r + 0xfee0) + } + plain := strings.Repeat("the validator said no. ", 600) // over maxFeedbackBytes + for name, tc := range map[string]struct { + feedback string + masked bool + }{ + "a colour code ate the first letter of a zero-width-split token": {"\x1b[" + token[:20] + "\u200b" + token[20:], true}, + "a fullwidth token straddling the cut": {strings.Repeat("x", maxFeedbackBytes-48) + " " + fw.String(), true}, + "plain feedback over the cut": {plain, false}, + } { + t.Run(name, func(t *testing.T) { + fb := redactFeedback(tc.feedback, nil) + prompt, sanitized := buildFeedbackPrompt(fb) + c := newContentCollector(maxContentBytes) + in := recordedInput(t, c, prompt, promptHidesAlterations(fb, sanitized))["gen_ai.input.messages"].AsString() + assert.NotContains(t, in, token[4:16]) + if tc.masked { + assert.Contains(t, in, `"content":"***"`) + } else { + assert.Contains(t, in, "the validator said no.") + } + }) + } +} + +func TestContentCollector_WhiteSpaceArgumentsAreNone(t *testing.T) { + // A call whose arguments are white space only carried nothing: no + // arguments, nothing charged, nothing marked — as for "". + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Summary: "ls", Arguments: " \n\t"}) + res := c.Result("stop") + assert.Contains(t, res.OutputMessages, `"summary":"ls"}`) + assert.NotContains(t, res.OutputMessages, "arguments") + assert.False(t, res.Truncated) + assert.Zero(t, res.DroppedBytes) +} + func TestContentCollector_NamelessCallCarriesNoArgumentsAndNoCharge(t *testing.T) { // The schema requires a name on a tool_call part. A nameless call is // refused; its arguments must not survive alone, nor charge a part @@ -458,7 +508,7 @@ func TestContentCollector_ReplacesRunnerEnvValuesTheNormalizerSpellsOut(t *testi assert.Equal(t, 2, strings.Count(res.OutputMessages, "[REDACTED:PUSH_TOKEN]")) assert.Equal(t, 2, runnerEnvFindings(res), "a value found after the pipeline counts like any other") - input := recordedInput(t, newContentCollectorIfEnabled(env), "validator said: "+fullwidth(secret))["gen_ai.input.messages"].AsString() + input := recordedInput(t, newContentCollectorIfEnabled(env), "validator said: "+fullwidth(secret), false)["gen_ai.input.messages"].AsString() assert.Contains(t, input, "validator said: [REDACTED:PUSH_TOKEN]") } @@ -1446,9 +1496,11 @@ func TestContentCollector_EvictsWholeOldPartsExactly(t *testing.T) { func TestContentCollector_EncodedCeilingBoundsEscapeDenseContent(t *testing.T) { // The size budget counts raw bytes; JSON encoding spends six on every - // '<', control byte and invalid byte. The exported attribute must - // still fit the encoded ceiling, keeping as much of the tail as fits. - for _, unit := range []string{"<", "\x01", "\xff", "€\xe2\x82"} { + // '<' and control byte. The exported attribute must still fit the + // encoded ceiling, keeping as much of the tail as fits. (An invalid + // byte never reaches the encoder: a string holding one is masked whole + // by redactText.) + for _, unit := range []string{"<", "\x01"} { t.Run(fmt.Sprintf("%q", unit), func(t *testing.T) { raw := strings.Repeat(unit, 100*1024/len(unit)) c := newContentCollector(200_000) @@ -1749,10 +1801,10 @@ func TestTailToRuneBoundary(t *testing.T) { // recordedInput runs attachInput on a recorded agent span and returns the // attributes the span ended with. -func recordedInput(t *testing.T, c *contentCollector, prompt string) map[attribute.Key]attribute.Value { +func recordedInput(t *testing.T, c *contentCollector, prompt string, alteredUpstream bool) map[attribute.Key]attribute.Value { t.Helper() _, rec, span := toolSpanFixture(t) - c.attachInput(span, prompt) + c.attachInput(span, prompt, alteredUpstream) span.End() ended := rec.Ended() require.Len(t, ended, 1) @@ -1761,7 +1813,7 @@ func recordedInput(t *testing.T, c *contentCollector, prompt string) map[attribu func TestAttachInput_RecordsTheRetryPromptAsOneUserMessage(t *testing.T) { prompt, _ := buildFeedbackPrompt("lint: main.go:3 unused variable ") - attrs := recordedInput(t, newContentCollector(maxContentBytes), prompt) + attrs := recordedInput(t, newContentCollector(maxContentBytes), prompt, false) var msgs []map[string]any require.NoError(t, json.Unmarshal([]byte(attrs["gen_ai.input.messages"].AsString()), &msgs)) @@ -1774,15 +1826,15 @@ func TestAttachInput_RecordsTheRetryPromptAsOneUserMessage(t *testing.T) { } func TestAttachInput_NoPromptOrNoCollectorAddsNothing(t *testing.T) { - assert.Empty(t, recordedInput(t, newContentCollector(maxContentBytes), ""), + assert.Empty(t, recordedInput(t, newContentCollector(maxContentBytes), "", false), "no composed prompt (the first iteration, or feedback_mode off)") - assert.Empty(t, recordedInput(t, nil, "would-be prompt"), "gate off") + assert.Empty(t, recordedInput(t, nil, "would-be prompt", false), "gate off") } func TestAttachInput_RedactsAndCountsFindings(t *testing.T) { secret := "ghp_" + strings.Repeat("k", 36) c := newContentCollector(maxContentBytes) - in := recordedInput(t, c, "token "+secret+" leaked")["gen_ai.input.messages"].AsString() + in := recordedInput(t, c, "token "+secret+" leaked", false)["gen_ai.input.messages"].AsString() assert.NotContains(t, in, secret) assert.Contains(t, in, "leaked") @@ -1797,7 +1849,7 @@ func TestAttachInput_WorstCaseFeedbackSharesTheEncodedCeiling(t *testing.T) { // order of magnitude larger than the prompt the agent was sent. prompt, _ := buildFeedbackPrompt(strings.Repeat("\uFDFA", maxFeedbackBytes)) c := newContentCollector(maxContentBytes) - in := recordedInput(t, c, prompt)["gen_ai.input.messages"].AsString() + in := recordedInput(t, c, prompt, false)["gen_ai.input.messages"].AsString() require.Greater(t, len(in), 10*maxFeedbackBytes, "fixture must expand for this test to prove anything") c.Handle(agentruntime.TextEvent{Text: strings.Repeat("<", maxContentBytes)}) // encodes sixfold diff --git a/internal/cli/redaction_invariant_test.go b/internal/cli/redaction_invariant_test.go new file mode 100644 index 0000000000..9c2245e2d4 --- /dev/null +++ b/internal/cli/redaction_invariant_test.go @@ -0,0 +1,339 @@ +package cli + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + agentruntime "github.com/fullsend-ai/fullsend/internal/runtime" + "github.com/fullsend-ai/fullsend/internal/security" +) + +// TestRedaction_NoSecretSurvivesAnyWrapping states what the Level 3 +// record and the execute_tool spans guarantee, and checks it over every +// hostile wrapping a reviewer has constructed so far, enumerated — not +// one example per finding. The invariant: for every string the record or +// a span exports, no 12-byte run of a known secret's body appears, read +// three ways — as raw bytes, after the normalizer, and after decoding the +// record's JSON with every string walked. A known secret is one a +// SecretRedactor pattern matches, or a runner env / provider-only env +// value the literal pass holds. +// +// Named exclusions, skipped below with the reason beside them and listed +// in docs/guides/dev/tracing.md ("Guarantee and limits"): an env value +// written in compatibility (fullwidth) forms, since the normalizer +// rewrites it and the literal pass matches only what the env has; a +// secret split across two parts or events; a secret matching no pattern +// and in no env; a summary a pi or codex parser cut before the collector +// saw it, since that cut can split a value the parser's own scan did not +// see joined (a Claude summary is cut from a member the collector scans +// whole). Nothing else is excluded. +func TestRedaction_NoSecretSurvivesAnyWrapping(t *testing.T) { + const window = 12 + + // Deterministic bodies: lowercase letters and digits, so no 12-byte + // window can occur in the noise (lowercase letters only) or in a mask. + alnum := func(seed uint32, n int, alphabet string) string { + var b strings.Builder + x := seed + for i := 0; i < n; i++ { + x = x*1664525 + 1013904223 + b.WriteByte(alphabet[int(x>>24)%len(alphabet)]) + } + return b.String() + } + const lowerDigits = "abcdefghijklmnopqrstuvwxyz0123456789" + const upperDigits = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + noise := func(seed uint32, n int) string { return alnum(seed, n, "abcdefghijklmnopqrstuvwxyz") } + + type secret struct { + name string + text string // the whole secret as a stream would carry it + body string // the part whose 12-byte windows must not survive + env bool // held by the literal pass, not a pattern + } + envValue := alnum(11, 32, lowerDigits) + providerValue := alnum(12, 32, lowerDigits) + runnerEnv := map[string]string{"DEPLOY_PASSWORD": envValue} + t.Setenv(workflowTokenEnv, providerValue) + pemBody := alnum(13, 64, lowerDigits) + "\n" + alnum(14, 64, lowerDigits) + pemHeader := "-----BEGIN " + "PRIVATE KEY-----" // split so the private-key lint stays quiet + pemFooter := "-----END " + "PRIVATE KEY-----" + mk := func(name string, seed uint32) [2]secret { // two instances, for the two-secrets wrapping + var out [2]secret + for i := range out { + b := alnum(seed+uint32(i), 36, lowerDigits) + switch name { + case "github_pat": + out[i] = secret{name, "ghp_" + b, b, false} + case "aws_access_key": + u := alnum(seed+uint32(i), 16, upperDigits) + out[i] = secret{name, "AKIA" + u, u, false} + case "runner_env": + out[i] = secret{name, envValue, envValue, true} + case "provider_env": + out[i] = secret{name, providerValue, providerValue, true} + case "private_key": + out[i] = secret{name, pemHeader + "\n" + pemBody + "\n" + pemFooter, pemBody[:64], false} + case "db_connection_password": + p := b[:16] + out[i] = secret{name, "postgres://user:" + p + "@db.example.com/app", p, false} + case "json_field": + p := b[:16] + out[i] = secret{name, `"password":"` + p + `"`, p, false} + case "env_assignment": + out[i] = secret{name, "AWS_SECRET_ACCESS_KEY=" + b, b, false} + case "auth_header": + out[i] = secret{name, "Authorization: Bearer " + b[:32], b[:32], false} + } + } + return out + } + secrets := [][2]secret{ + mk("github_pat", 100), mk("aws_access_key", 200), mk("runner_env", 300), mk("provider_env", 400), + mk("private_key", 500), mk("db_connection_password", 600), mk("json_field", 700), + mk("env_assignment", 800), mk("auth_header", 900), + } + + // Wrappings over the secret text; the ones that insert a character + // do so at an offset into the body. + insert := func(s secret, off int, ins string) string { + i := strings.Index(s.text, s.body) + off + return s.text[:i] + ins + s.text[i:] + } + zwSplit := func(s secret) string { return insert(s, len(s.body)/2, "\u200b") } + fullwidth := func(s secret) string { + var b strings.Builder + for _, r := range s.body { + if r >= 0x21 && r <= 0x7e { + r += 0xfee0 + } + b.WriteRune(r) + } + return strings.Replace(s.text, s.body, b.String(), 1) + } + tags := func(s secret) string { + var b strings.Builder + for i, r := range s.body { + if i > 0 && i%4 == 0 { + b.WriteRune(0xe0041) // TAG LATIN CAPITAL LETTER A + } + b.WriteRune(r) + } + return strings.Replace(s.text, s.body, b.String(), 1) + } + type wrapping struct { + name string + wrap func(a, b secret) string + excluded func(s secret) string // non-empty = the named exclusion's reason + twoBodies bool + } + envExcluded := func(s secret) string { + if s.env { + return "an env value in compatibility forms: the normalizer rewrites it and the literal pass matches only what the env has" + } + return "" + } + none := func(secret) string { return "" } + wrappings := []wrapping{ + {"plain", func(a, _ secret) string { return "see " + a.text + " here" }, none, false}, + {"csi eats the first letter", func(a, _ secret) string { return "\x1b[" + a.text }, none, false}, + {"csi eats the first letter, zero-width inside", func(a, _ secret) string { return "\x1b[" + zwSplit(a) }, none, false}, + {"csi colour around", func(a, _ secret) string { return "\x1b[31m" + a.text + "\x1b[0m" }, none, false}, + {"csi inside", func(a, _ secret) string { return insert(a, 3, "\x1b[0m") }, none, false}, + {"osc bel around", func(a, _ secret) string { return "\x1b]" + a.text + "\x07" }, none, false}, + {"osc st before", func(a, _ secret) string { return "\x1b]8;;x\x1b\\" + a.text }, none, false}, + {"repeated after csi", func(a, _ secret) string { return "\x1b[" + a.text + " " + a.text }, none, false}, + {"two secrets, second zero-width split", func(a, b secret) string { return a.text + " " + zwSplit(b) }, none, true}, + {"two secrets after csi, second zero-width split", func(a, b secret) string { return "\x1b[" + a.text + " " + zwSplit(b) }, none, true}, + {"U+200B at body+1", func(a, _ secret) string { return insert(a, 1, "\u200b") }, none, false}, + {"U+FEFF at body+4", func(a, _ secret) string { return insert(a, 4, "\ufeff") }, none, false}, + {"U+2060 at body middle", func(a, _ secret) string { return insert(a, len(a.body)/2, "\u2060") }, none, false}, + {"bidi override around", func(a, _ secret) string { return "\u202e" + a.text + "\u202c" }, none, false}, + {"tag characters every four bytes", func(a, _ secret) string { return tags(a) }, none, false}, + {"NUL inside", func(a, _ secret) string { return insert(a, 5, "\x00") }, none, false}, + {"invalid byte inside the body", func(a, _ secret) string { return insert(a, 1, "\xff") }, none, false}, + {"variation selector inside", func(a, _ secret) string { return insert(a, 5, "\ufe0f") }, none, false}, + {"fullwidth body", func(a, _ secret) string { return fullwidth(a) }, envExcluded, false}, + {"inside 2 KiB of noise each side", func(a, _ secret) string { return noise(1, 2048) + " " + a.text + " " + noise(2, 2048) }, none, false}, + {"fullwidth body straddling the feedback cut", func(a, _ secret) string { return noise(3, maxFeedbackBytes-48) + " " + fullwidth(a) }, envExcluded, false}, + } + + // Views of one exported string. + normalizer := security.NewUnicodeNormalizer() + views := func(out string) []string { + vs := []string{out} + if n := normalizer.Scan(out); n.Sanitized != "" { + vs = append(vs, n.Sanitized) + } + if json.Valid([]byte(out)) { + var v any + if json.Unmarshal([]byte(out), &v) == nil { + var walk func(any) + walk = func(x any) { + switch t := x.(type) { + case string: + vs = append(vs, t) + if n := normalizer.Scan(t); n.Sanitized != "" { + vs = append(vs, n.Sanitized) + } + case []any: + for _, e := range t { + walk(e) + } + case map[string]any: + for _, e := range t { + walk(e) + } + } + } + walk(v) + } + } + return vs + } + survives := func(out, body string) string { + for _, v := range views(out) { + for i := 0; i+window <= len(body); i++ { + if strings.Contains(v, body[i:i+window]) { + return body[i : i+window] + } + } + } + return "" + } + + // Surfaces: each takes the wrapped text and returns what was exported. + collector := func() *contentCollector { + c := newContentCollector(maxContentBytes) + c.runnerEnv = runnerEnv + return c + } + // jsonStr quotes text as a JSON string while keeping its bytes as the + // stream would carry them: json.Marshal would coerce an invalid byte to + // U+FFFD before the collector could see it. + jsonStr := func(text string) string { + var b strings.Builder + b.WriteByte('"') + for i := 0; i < len(text); i++ { + switch c := text[i]; { + case c == '"' || c == '\\': + b.WriteByte('\\') + b.WriteByte(c) + case c < 0x20: + fmt.Fprintf(&b, "\\u%04x", c) + default: + b.WriteByte(c) + } + } + b.WriteByte('"') + return b.String() + } + + type surface struct { + name string + run func(t *testing.T, text string) []string + } + viaCollector := func(f func(c *contentCollector, text string)) func(*testing.T, string) []string { + return func(_ *testing.T, text string) []string { + c := collector() + f(c, text) + return []string{c.Result("stop").OutputMessages} + } + } + viaTracker := func(f func(tr *toolSpanTracker, text string)) func(*testing.T, string) []string { + return func(t *testing.T, text string) []string { + tr, rec, _ := toolSpanFixture(t) + tr.runnerEnv = runnerEnv + f(tr, text) + var out []string + for _, s := range endedToolSpans(rec) { + out = append(out, s.Name) + for _, kv := range s.Attributes { + out = append(out, kv.Value.Emit()) + } + } + return out + } + } + surfaces := []surface{ + {"redactText", func(_ *testing.T, text string) []string { + var findings []security.Finding + return []string{redactText(security.OutputPipeline(), runnerEnv, text, &findings)} + }}, + {"text part", viaCollector(func(c *contentCollector, text string) { c.Handle(agentruntime.TextEvent{Text: text}) })}, + {"call name", viaCollector(func(c *contentCollector, text string) { + c.Handle(agentruntime.ToolUseEvent{ID: "toolu_01", Name: text}) + })}, + {"call summary", viaCollector(func(c *contentCollector, text string) { + c.Handle(agentruntime.ToolUseEvent{ID: "toolu_01", Name: "Bash", Summary: text}) + })}, + {"listed argument", viaCollector(func(c *contentCollector, text string) { + c.Handle(agentruntime.ToolUseEvent{ID: "toolu_01", Name: "Bash", Arguments: `{"command":` + jsonStr(text) + `}`}) + })}, + {"unlisted argument", viaCollector(func(c *contentCollector, text string) { + c.Handle(agentruntime.ToolUseEvent{ID: "toolu_01", Name: "Write", Arguments: `{"file_path":"/x","content":` + jsonStr(text) + `}`}) + })}, + {"array under a listed member", viaCollector(func(c *contentCollector, text string) { + c.Handle(agentruntime.ToolUseEvent{ID: "toolu_01", Name: "Grep", Arguments: `{"path":[` + jsonStr(text) + `]}`}) + })}, + {"arguments that are not JSON", viaCollector(func(c *contentCollector, text string) { + c.Handle(agentruntime.ToolUseEvent{ID: "toolu_01", Name: "Bash", Arguments: text}) + })}, + {"trailing input after the object", viaCollector(func(c *contentCollector, text string) { + c.Handle(agentruntime.ToolUseEvent{ID: "toolu_01", Name: "Bash", Arguments: `{"command":"ls"} ` + text}) + })}, + {"tool result", viaCollector(func(c *contentCollector, text string) { + c.Handle(agentruntime.ToolUseEvent{ID: "toolu_01", Name: "Bash"}) + c.Handle(agentruntime.ToolResultEvent{ID: "toolu_01", Result: text}) + })}, + {"input message", func(t *testing.T, text string) []string { + // The real path: the feedback is redacted, then composed into the + // retry prompt (sanitized, cut, framed), then recorded. + fb := redactFeedback(text, runnerEnv) + prompt, sanitized := buildFeedbackPrompt(fb) + attrs := recordedInput(t, collector(), prompt, promptHidesAlterations(fb, sanitized)) + var out []string + for _, v := range attrs { + out = append(out, v.Emit()) + } + return out + }}, + {"span name", viaTracker(func(tr *toolSpanTracker, text string) { + tr.Handle(agentruntime.ToolUseEvent{ID: "toolu_01", Name: text}) + tr.Handle(agentruntime.ToolResultEvent{ID: "toolu_01"}) + })}, + {"span id", viaTracker(func(tr *toolSpanTracker, text string) { + tr.Handle(agentruntime.ToolUseEvent{ID: text, Name: "Bash"}) + tr.Handle(agentruntime.ToolResultEvent{ID: text}) + })}, + } + + for _, pair := range secrets { + a, b := pair[0], pair[1] + for _, w := range wrappings { + for _, s := range surfaces { + t.Run(fmt.Sprintf("%s/%s/%s", a.name, w.name, s.name), func(t *testing.T) { + if reason := w.excluded(a); reason != "" { + t.Skip("named exclusion: " + reason) + } + text := w.wrap(a, b) + require.NotEmpty(t, survives(text, a.body), "control: the detector must see the secret in the input") + for _, out := range s.run(t, text) { + if run := survives(out, a.body); run != "" { + t.Fatalf("%q of the first secret survives in %q", run, out) + } + if w.twoBodies { + if run := survives(out, b.body); run != "" { + t.Fatalf("%q of the second secret survives in %q", run, out) + } + } + } + }) + } + } + } +} diff --git a/internal/cli/run.go b/internal/cli/run.go index ec09538f1f..6978fc2b2d 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -2501,14 +2501,14 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep // append, the previous validation failure is injected so the agent // can self-correct instead of re-running blindly (#1050, #6494). agentPrompt := "" + promptSanitized := 0 if iteration > 1 && feedbackEnabled && validationFeedback != "" { - var sanitizedFindings int - agentPrompt, sanitizedFindings = buildFeedbackPrompt(validationFeedback) + agentPrompt, promptSanitized = buildFeedbackPrompt(validationFeedback) printer.StepInfo("Injecting validation feedback into agent prompt") - if sanitizedFindings > 0 { + if promptSanitized > 0 { printer.StepWarn(fmt.Sprintf( "Unicode sanitization altered validation feedback (%d finding(s) stripped)", - sanitizedFindings)) + promptSanitized)) } } @@ -2542,7 +2542,7 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep // tool-span tracker is per iteration for the same reason and is not // gated: execute_tool spans are metadata. collector := newContentCollectorIfEnabled(h.RunnerEnv) - collector.attachInput(agentSpan, agentPrompt) + collector.attachInput(agentSpan, agentPrompt, promptHidesAlterations(validationFeedback, promptSanitized)) toolSpans := newToolSpanTracker(tracer, agentCtx) toolSpans.runnerEnv = h.RunnerEnv var metrics agentruntime.RunMetrics @@ -3874,6 +3874,18 @@ func buildFeedbackPrompt(feedback string) (string, int) { sanitizedFindings } +// promptHidesAlterations reports whether the retry prompt's telemetry copy +// (contentCollector.attachInput) must be recorded masked whole: the +// sanitizer stripped something (sanitized > 0), which the collector's own +// normalizer can then no longer see — a stripped colour code can take a +// token's first letter with it, and what is left matches no pattern — or +// the feedback was cut at maxFeedbackBytes while it held compatibility +// characters, so the cut may have split a secret written in those forms, +// which only the collector's fold would have recognised whole. +func promptHidesAlterations(feedback string, sanitized int) bool { + return sanitized > 0 || (len(feedback) > maxFeedbackBytes && !security.NewUnicodeNormalizer().Scan(feedback).Safe) +} + // sanitizeFeedbackUnicode strips dangerous non-rendering Unicode characters // from validation feedback before it enters the agent prompt. It uses the // same UnicodeNormalizer that backs the PostToolUse hook chain's scan_text, diff --git a/internal/cli/telemetry_run_test.go b/internal/cli/telemetry_run_test.go index b1e95bf95c..30bcec0fb1 100644 --- a/internal/cli/telemetry_run_test.go +++ b/internal/cli/telemetry_run_test.go @@ -1660,7 +1660,7 @@ func TestContentCapture_EndToEndFileSink(t *testing.T) { prompt, _ := buildFeedbackPrompt(strings.Repeat("the validator said no. ", 500)) // cut at 10 KiB > 8192 _, span := tracer.Start(context.Background(), "agent") - c.attachInput(span, prompt) + c.attachInput(span, prompt, false) res := c.Result("stop") attachContent(span, res) span.End() @@ -1713,7 +1713,7 @@ func TestContentCapture_GateOffProducesNoContent(t *testing.T) { c.Handle(agentruntime.TextEvent{Text: "would-be content"}) // nil-safe no-op ctx, span := tracer.Start(context.Background(), "agent") - c.attachInput(span, "would-be prompt") // nil-safe no-op + c.attachInput(span, "would-be prompt", false) // nil-safe no-op handle := iterationEventHandler(func(agentruntime.AgentEvent) {}, c, newToolSpanTracker(tracer, ctx)) handle(agentruntime.ToolUseEvent{ID: "toolu_01", Name: "Bash", Arguments: `{"command":"would-be arguments"}`}) handle(agentruntime.ToolResultEvent{ID: "toolu_01", Result: "would-be result"}) diff --git a/internal/cli/tool_spans_test.go b/internal/cli/tool_spans_test.go index 562741d666..ca6d177570 100644 --- a/internal/cli/tool_spans_test.go +++ b/internal/cli/tool_spans_test.go @@ -434,6 +434,23 @@ func TestToolSpanTracker_ANameTheNormalizerStrippedIsMaskedWhole(t *testing.T) { } } +func TestToolSpanTracker_AnInvalidByteInsideASecretIsMaskedWhole(t *testing.T) { + // An invalid byte inside a token keeps every pattern from matching it, + // and ToValidUTF8 on the way to the span would then join the halves + // (an oversized result's id reaches the tracker as raw line bytes). + // redactText masks such a string whole; safeID drops the id. + token := "ghp_a" + "\xff" + strings.Repeat("r", 35) + tr, rec, _ := toolSpanFixture(t) + tr.Handle(agentruntime.ToolUseEvent{ID: token, Name: token}) + tr.Handle(agentruntime.ToolResultEvent{ID: token}) + spans := endedToolSpans(rec) + require.Len(t, spans, 1) + assert.NotContains(t, spans[0].Name, "rrrrrrrrrrrr") + attrs := toolSpanAttrs(spans[0]) + assert.Equal(t, "***", attrs["gen_ai.tool.name"].AsString()) + assert.NotContains(t, attrs, attribute.Key("gen_ai.tool.call.id")) +} + func TestRecordToolSpanOverflow_MarksAgentSpanOnlyWhenDropped(t *testing.T) { rec := tracetest.NewSpanRecorder() tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(rec)) From 574e44f4ba9264d70ecdeedff5f4922b7c207e02 Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Tue, 13 Oct 2026 14:57:51 -0400 Subject: [PATCH 32/40] fix(cli): treat the replacement character as a lost byte; a call not judged keeps no summary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 28055ba61 masked a string holding invalid UTF-8 whole, but our own decodes had already turned such a byte into U+FFFD on two paths before the collector saw it: the Claude parser decodes a member into a Go string to cut a summary from it, and the stream path decodes each partial_json delta before appending it to the arguments. Both arrive valid, with the token split by a character no pattern spans, and the record kept the halves. The replacement character is the mark a lossy decode leaves where a byte was lost, so redactText now masks a string holding it whole as it does the byte itself, and toolArguments drops such arguments before decoding. A call whose arguments were not judged keeps no summary: invalid_utf8 is a revealing finding, since the summary is cut from a member only that judgement sees whole — an invalid byte after a token, with the summary cut landing inside it, left a fragment otherwise. The invariant test gains the replacement character inside a body, an invalid byte after a body with the secret straddling the parser's summary cut, and a surface that derives the summary as the Claude parser does. Review findings on 28055ba61: 4262586421 (medium, logic error) and 4262586428 (medium, secret exposure). A pre-push review with the review agent's own lens found no further path; it brought the guides' summary-loss sentences, the wrapping list, three comments and the user guide (non-UTF-8 output is recorded "***" whole) up to date, and dropped the redundant half of the check, since ContainsRune reports an invalid byte as RuneError too. Signed-off-by: Dharit Shah --- docs/guides/dev/tracing.md | 21 +++++--- .../infrastructure/distributed-tracing.md | 21 ++++---- docs/guides/user/how-to-emit-traces.md | 2 +- internal/cli/content_collector.go | 52 +++++++++++-------- internal/cli/redaction_invariant_test.go | 14 +++++ 5 files changed, 73 insertions(+), 37 deletions(-) diff --git a/docs/guides/dev/tracing.md b/docs/guides/dev/tracing.md index a6151b70c6..becdd5d81c 100644 --- a/docs/guides/dev/tracing.md +++ b/docs/guides/dev/tracing.md @@ -237,7 +237,9 @@ ends at the next letter, so it can take a token's first letter, and a title code's payload is text no pattern sees, so nothing cut from such a string can be trusted, and no scan of it as written can stand in; so is a string holding invalid UTF-8, since the span and attribute writers would -join the halves of a value such a byte splits) — at assembly (redaction runs +join the halves of a value such a byte splits, or the replacement +character a lossy decode leaves in its place, which splits a value the +same way) — at assembly (redaction runs before the size budget — truncating first could split a secret past recognition), enforces a 256 KiB ordered-suffix budget (the ending survives — the final answer is what consumers judge) plus an 8 KiB per-tool-result @@ -270,10 +272,14 @@ string walked; a known secret is one a `SecretRedactor` pattern matches, or a runner env or provider-only env value the literal pass holds. `TestRedaction_NoSecretSurvivesAnyWrapping` (`internal/cli`) states it and checks it over every surface (text, names, summaries, arguments on each -path, results, the input message, span names and ids) and every wrapping a -review has constructed so far — terminal escapes before, around and inside -a secret, repeated and paired secrets, zero-width, bidi, tag, NUL and -variation-selector characters inside it, compatibility forms, and noise. +path, results, the input message, span names and ids, and a Claude call +with the summary the parser derives from it) and every wrapping a review +has constructed so far — terminal escapes before, around and inside a +secret, repeated and paired secrets, zero-width, bidi, tag, NUL and +variation-selector characters inside it, an invalid byte or the +replacement character inside it, an invalid byte after it with the +secret straddling the parser's summary cut, compatibility forms, and +noise, inside the record and across the feedback cut. Four limits are named there and here, and nowhere else: an env value written in compatibility (fullwidth) forms, since the normalizer rewrites it and the literal pass matches only what the env has; a secret split @@ -303,7 +309,10 @@ string is (above). Discarded content is redacted first, here as everywhere: a dropped member is scanned as text — the decoded string, or the compact encoding of an object or an array — so a secret in a file body still counts, and costs the summary as a secret -found anywhere in the arguments does. Arguments that are not one JSON +found anywhere in the arguments does; so do arguments that could not be +judged at all — stripped, or holding invalid UTF-8 or the replacement +character — since the summary was cut from a member no scan saw whole. +Arguments that are not one JSON object are scanned as text and dropped whole, charged; so are kept members whose encoding exceeds `maxToolArgumentsBytes`, charged as encoded. A call the stream reports without a name carries no arguments. diff --git a/docs/guides/infrastructure/distributed-tracing.md b/docs/guides/infrastructure/distributed-tracing.md index 36042d6213..5b5bac9b72 100644 --- a/docs/guides/infrastructure/distributed-tracing.md +++ b/docs/guides/infrastructure/distributed-tracing.md @@ -96,8 +96,9 @@ or spelled that way which a pattern also recognises — in one of those contexts, in a connection string, or by its own prefix — is masked by that pattern, and shows what the pattern's mask shows. A tool call loses its `summary` when redaction found a secret in its arguments, the dropped -members included, or stripped an escape sequence or tag characters from -them: the parser cut the summary out of them before anything scanned it, +members included, stripped an escape sequence or tag characters from +them, or could not judge them at all (invalid UTF-8 or the replacement +character): the parser cut the summary out of them before anything scanned it, so the secret could be there as a beginning. Where a runtime reports a summary and no arguments (pi, codex, OpenCode), its parser has already run the patterns over the summary and, for pi and codex, cut it: a value that straddles a @@ -136,9 +137,11 @@ dropped — scanned first, so a secret in them still counts — and the part is marked `fullsend.truncated`. The kept members are decoded, redacted one by one and encoded again, so key order, spacing and escapes are not the stream's; a string a terminal escape sequence or tag characters were -stripped from, or that holds invalid UTF-8, is masked whole, since the -stripping can leave most of a token and a later write would join the -halves of a value an invalid byte splits. +stripped from, or that holds invalid UTF-8 or the replacement character +a lossy decode leaves (U+FFFD), is masked whole, since the +stripping can leave most of a token, a later write would join the +halves of a value an invalid byte splits, and no pattern spans the +replacement character. On a retry iteration under `validation_loop.feedback_mode: append`, the prompt the runner composed — its fixed framing around the previous @@ -157,8 +160,8 @@ which sees the runtime's stream, not its API requests. **Redaction and size:** every part passes through security redaction (runner environment values, Unicode normalization, secret masking, then runner environment values again; a string the normalizer stripped an -escape sequence or tag characters from, or that holds invalid UTF-8, is -masked whole) before reaching the span. What +escape sequence or tag characters from, or that holds invalid UTF-8 or +the replacement character, is masked whole) before reaching the span. What that guarantees, and its four named limits, is stated in the [developer guide](../dev/tracing.md) beside the test that checks it. Content is bounded at 256 KiB per iteration — each tool result at 8 KiB, @@ -177,8 +180,8 @@ Truncation is marked via `fullsend.content.truncated` on the span and its tail; arguments over theirs, or not a complete JSON object, are dropped whole — a cut object is not JSON — and the marked `tool_call` part keeps its `id`, name and summary — not the summary when redaction -found a secret in the arguments or stripped an escape sequence or tag -characters from them. A member outside the recorded list (above) is +found a secret in the arguments, stripped an escape sequence or tag +characters from them, or could not judge them at all. A member outside the recorded list (above) is dropped and the part marked the same way, its listed members kept. Arguments also go, charged and marked, when the call's name redacts to nothing; that part survives only if it has a summary. So diff --git a/docs/guides/user/how-to-emit-traces.md b/docs/guides/user/how-to-emit-traces.md index cb8a6c6a86..8e3a4158a0 100644 --- a/docs/guides/user/how-to-emit-traces.md +++ b/docs/guides/user/how-to-emit-traces.md @@ -130,7 +130,7 @@ local file and at the endpoint. A retry that carries validation feedback also records the prompt the runner composed. Content is redacted for secrets and bounded per iteration; a string that held a terminal colour or title code or tag characters, as most -coloured CLI output does, is recorded as `***` whole, and so is a retry +coloured CLI output does, or that was not valid UTF-8, is recorded as `***` whole, and so is a retry prompt the runner sanitized or cut. Content may still contain proprietary code or PII — tool arguments carry the commands an agent ran and the paths and patterns it used, never a file body or an edit — so make sure your backend's diff --git a/internal/cli/content_collector.go b/internal/cli/content_collector.go index 97ac22bafe..faef098c1f 100644 --- a/internal/cli/content_collector.go +++ b/internal/cli/content_collector.go @@ -86,8 +86,9 @@ const maxToolResultBytes = 8 * 1024 // their redacted encoding. Arguments over it are dropped whole, not cut: // a cut object is not JSON, and the part keeps its id, name and summary // (not the summary when redaction found a secret in the arguments, -// stripped an escape sequence or tag characters from them, or masked a -// document there it could not judge; see Handle). +// stripped an escape sequence or tag characters from them, or could not +// judge them at all — invalid UTF-8 or the replacement character; see +// Handle). // Like maxToolResultBytes it exists because the total above is small; // what blocks raising it is the same unproven backend ceiling named on // maxContentBytes. @@ -426,10 +427,11 @@ func (c *contentCollector) Handle(evt agentruntime.AgentEvent) { if slices.ContainsFunc(c.findings[scanned:], revealing) { // The parser cut the summary out of these arguments // before anything scanned it, so a secret found in them, - // or stripped with an escape sequence or tag characters, - // can be in the summary as a beginning that neither the - // literal pass nor a pattern matches, nor the normalizer - // strips once its terminator is cut. + // stripped with an escape sequence or tag characters, or + // never judged (invalid UTF-8 or the replacement + // character), can be in the summary as a beginning that + // neither the literal pass nor a pattern matches, nor the + // normalizer strips once its terminator is cut. p.Summary = "" } } @@ -755,20 +757,23 @@ func (c *contentCollector) redact(text string, findings *[]security.Finding) str // mask shows; and a value the normalizer itself rewrites is matched only as // the env has it. An empty result with findings means fully redacted. A // string the normalizer stripped an escape sequence or tag characters -// from (stripped), or that holds invalid UTF-8, is masked *** whole. +// from (stripped), or that holds invalid UTF-8 or the replacement +// character a lossy decode leaves, is masked *** whole. func redactText(pipeline *security.Pipeline, runnerEnv map[string]string, text string, findings *[]security.Finding) string { if text == "" { return text } - if !utf8.ValidString(text) { + if strings.ContainsRune(text, utf8.RuneError) { // true for an invalid byte sequence too // An invalid byte splits a value as a zero-width character does, // but nothing here joins it: the normalizer leaves it, no pattern // spans it, the literal pass needs the value whole — and the span // and attribute writers' ToValidUTF8 would join the halves after - // this pass. Masked whole, as every string the normalizer stripped - // something from is. + // this pass. The replacement character is the mark a lossy decode + // leaves where such a byte was (the parser's summary, the stream + // path's partial_json), and splits a value the same way. Masked + // whole, as every string the normalizer stripped something from is. *findings = append(*findings, security.Finding{Scanner: "unicode_normalizer", Name: "invalid_utf8", Severity: "medium", - Detail: "invalid UTF-8 inside the string; masked whole", Position: -1}) + Detail: "invalid UTF-8, or the replacement character a lossy decode leaves, inside the string; masked whole", Position: -1}) return "***" } text = replaceEnv(runnerEnv, text, findings) @@ -834,9 +839,9 @@ var recordedArguments = map[string]bool{ // wire's. A kept number is scanned as its digits — a runner env value // can be sent as one — and stays a number when nothing was replaced. // Text that is not one JSON object (see ToolUseEvent.Arguments), or -// carries anything but white space after it, or holds invalid UTF-8, -// cannot be walked that way: it is scanned as text so its findings count, -// then dropped whole and charged. +// carries anything but white space after it, or holds invalid UTF-8 or +// the replacement character, cannot be walked that way: it is scanned as +// text so its findings count, then dropped whole and charged. // So are arguments whose encoding exceeds maxToolArgumentsBytes, charged // as encoded. A dropped member is charged as the redacted text it was // scanned as. @@ -844,10 +849,12 @@ func (c *contentCollector) toolArguments(args string) (json.RawMessage, bool) { if strings.TrimSpace(args) == "" { return nil, false } - if !utf8.ValidString(args) { - // The decoder would coerce an invalid byte to a visible U+FFFD, - // and a value it splits would reach the record in halves no pattern - // spans. Scanned as text — which masks it whole — and dropped. + if strings.ContainsRune(args, utf8.RuneError) { // true for an invalid byte sequence too + // The decoder would coerce an invalid byte to a visible U+FFFD — + // or the stream path already has — and a value it splits would + // reach the record in halves no pattern spans. Scanned as text, + // which masks it whole and clears the summary (revealing), and + // dropped. c.evicted += len(c.redact(args, &c.findings)) return nil, true } @@ -923,10 +930,13 @@ func decodeJSON(s string) (any, error) { return v, nil } -// revealing reports a finding that shows a secret was there: any but the -// normalizer's, and the normalizer's stripping of content. +// revealing reports a finding that costs a call its summary: any but the +// normalizer's, which shows a secret was there; the normalizer's stripping +// of content; and invalid UTF-8, after which the arguments were not judged +// at all — and a summary is cut from a member only that judgement sees +// whole. func revealing(f security.Finding) bool { - return f.Scanner != "unicode_normalizer" || stripped(f) + return f.Scanner != "unicode_normalizer" || stripped(f) || f.Name == "invalid_utf8" } // stripped reports the normalizer's removal of an escape sequence or of diff --git a/internal/cli/redaction_invariant_test.go b/internal/cli/redaction_invariant_test.go index 9c2245e2d4..603327dd8a 100644 --- a/internal/cli/redaction_invariant_test.go +++ b/internal/cli/redaction_invariant_test.go @@ -156,6 +156,8 @@ func TestRedaction_NoSecretSurvivesAnyWrapping(t *testing.T) { {"tag characters every four bytes", func(a, _ secret) string { return tags(a) }, none, false}, {"NUL inside", func(a, _ secret) string { return insert(a, 5, "\x00") }, none, false}, {"invalid byte inside the body", func(a, _ secret) string { return insert(a, 1, "\xff") }, none, false}, + {"replacement character inside the body", func(a, _ secret) string { return insert(a, 1, "\ufffd") }, none, false}, + {"invalid byte after the body, secret straddling the summary cut", func(a, _ secret) string { return noise(4, 100) + " " + a.text + " \xff" }, none, false}, {"variation selector inside", func(a, _ secret) string { return insert(a, 5, "\ufe0f") }, none, false}, {"fullwidth body", func(a, _ secret) string { return fullwidth(a) }, envExcluded, false}, {"inside 2 KiB of noise each side", func(a, _ secret) string { return noise(1, 2048) + " " + a.text + " " + noise(2, 2048) }, none, false}, @@ -280,6 +282,18 @@ func TestRedaction_NoSecretSurvivesAnyWrapping(t *testing.T) { {"array under a listed member", viaCollector(func(c *contentCollector, text string) { c.Handle(agentruntime.ToolUseEvent{ID: "toolu_01", Name: "Grep", Arguments: `{"path":[` + jsonStr(text) + `]}`}) })}, + {"Claude call with its derived summary", viaCollector(func(c *contentCollector, text string) { + // As the Claude parser derives a Bash summary: the member decoded + // into a Go string (an invalid byte becomes U+FFFD) and cut to + // maxCommandDisplay runes. + var cmd string + _ = json.Unmarshal([]byte(jsonStr(text)), &cmd) + summary := cmd + if r := []rune(cmd); len(r) > 120 { + summary = string(r[:120]) + "…" + } + c.Handle(agentruntime.ToolUseEvent{ID: "toolu_01", Name: "Bash", Summary: summary, Arguments: `{"command":` + jsonStr(text) + `}`}) + })}, {"arguments that are not JSON", viaCollector(func(c *contentCollector, text string) { c.Handle(agentruntime.ToolUseEvent{ID: "toolu_01", Name: "Bash", Arguments: text}) })}, From 3c529ce36d74b91bcaccbc90e0fb0a64f803b156 Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Tue, 13 Oct 2026 15:12:06 -0400 Subject: [PATCH 33/40] docs(guides): name the replacement character in the user guide The user guide said a string that was not valid UTF-8 is recorded `***` whole; 574e44f4b made the replacement character (U+FFFD) count the same, and the two other guides say so. This one now does too. Signed-off-by: Dharit Shah --- docs/guides/user/how-to-emit-traces.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guides/user/how-to-emit-traces.md b/docs/guides/user/how-to-emit-traces.md index 8e3a4158a0..a7e30549b1 100644 --- a/docs/guides/user/how-to-emit-traces.md +++ b/docs/guides/user/how-to-emit-traces.md @@ -130,7 +130,7 @@ local file and at the endpoint. A retry that carries validation feedback also records the prompt the runner composed. Content is redacted for secrets and bounded per iteration; a string that held a terminal colour or title code or tag characters, as most -coloured CLI output does, or that was not valid UTF-8, is recorded as `***` whole, and so is a retry +coloured CLI output does, or that was not valid UTF-8 or held the replacement character (U+FFFD), is recorded as `***` whole, and so is a retry prompt the runner sanitized or cut. Content may still contain proprietary code or PII — tool arguments carry the commands an agent ran and the paths and patterns it used, never a file body or an edit — so make sure your backend's From ed17071303f74e83a8086d19e8fba1c3bec4197a Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Tue, 13 Oct 2026 15:23:44 -0400 Subject: [PATCH 34/40] docs(guides): a prompt cut for size alone is still recorded The user guide said a retry prompt the runner sanitized or cut is recorded `***` whole. A cut hides a secret only when the feedback also held compatibility characters (promptHidesAlterations); feedback cut for size alone is recorded. Say so, as the dev guide does. Signed-off-by: Dharit Shah --- docs/guides/user/how-to-emit-traces.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/guides/user/how-to-emit-traces.md b/docs/guides/user/how-to-emit-traces.md index a7e30549b1..191299d514 100644 --- a/docs/guides/user/how-to-emit-traces.md +++ b/docs/guides/user/how-to-emit-traces.md @@ -131,7 +131,8 @@ at the endpoint. A retry that carries validation feedback also records the prompt the runner composed. Content is redacted for secrets and bounded per iteration; a string that held a terminal colour or title code or tag characters, as most coloured CLI output does, or that was not valid UTF-8 or held the replacement character (U+FFFD), is recorded as `***` whole, and so is a retry -prompt the runner sanitized or cut. Content may still contain proprietary code or PII — tool arguments carry the +prompt the runner sanitized, or cut at its size bound while it held compatibility +characters; feedback cut for size alone is still recorded. Content may still contain proprietary code or PII — tool arguments carry the commands an agent ran and the paths and patterns it used, never a file body or an edit — so make sure your backend's access controls fit before enabling it. From 967e57a551f2a7b170f05fcae75fefa2fe2cca49 Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Wed, 14 Oct 2026 09:41:41 -0400 Subject: [PATCH 35/40] fix(cli): arguments listed by tool, URLs masked, policy drops unmarked, sanitizations apart MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Maintainer review of ed1707130, five medium findings on the Level 3 argument record, and the pre-push review of the first fix for them: - recordedArguments is keyed by tool as well as member, for Claude Code's ten built-in tools. Any other name records nothing of its input, every mcp__ tool among them, `{}` included: on those, description, url or path carry bulk content as often as not. - redactText masks, last, what a URL carries that no pattern is written for: everything after its first '?' or '#' and everything from "://" to its last '@' (userinfo, or a URL nested in another's path), as ***, with one finding that costs a call its summary. It runs on every recorded string, after the normalizer folds a compatibility '？' or '＠' and the patterns mask a token in the path, and re-encodes nothing; a URL runs to white space, so shell quoting does not end it. The first fix parsed kept members with net/url before the normalizer: a fold rebuilt the query after it, u.String() percent-encoded a fullwidth token past the patterns, and a quote ended the run early. The invariant test gains seven URL shapes; all were red there. - A member the list leaves out is scanned for findings but neither charged to dropped_bytes nor marked fullsend.truncated: it is the documented policy, and the markers mean loss — the 8 KiB bound, text that is not one JSON object, invalid UTF-8, parser-side cuts. The Result branch for a name that redacts to nothing taking its accepted arguments is deleted: arguments are kept only under an exact built-in name, plain words no scan touches. - fullsend.content.redactions counts known secrets matched, the patterns' and the runner-env literal pass's; what was masked or rewritten for its shape (the normalizer's findings, a retry prompt altered upstream, a URL's userinfo, query or fragment) is counted in fullsend.content.sanitizations. Measured first, as the review asked: the three captured live streams replayed at ed1707130 with arguments on show 0 of 552 text and tool-result parts masked whole, none holding an escape sequence, invalid UTF-8 or U+FFFD. The coarse rule stays. - The user guide, the reference, the dev guide, the code comment and the ADR 0050 and 0108 annotations say what is recorded: a Bash command whole, up to 8 KiB, which can embed what it writes; not a Write, Edit or NotebookEdit body, an Agent or WebFetch prompt or an MCP tool's input as arguments, though a result can quote what was written and a summary keeps the input's first 120 characters. Replayed with this change: the five tool_call parts the three streams had marked for policy drops become one, the call whose arguments held U+FFFD; the URL mask changes 54 runs in tool results (about 2.5 KB, all fragments or queries, no credential) and none in arguments, text or summaries; redactions 0 / 10 / 0, sanitizations 88 / 33 / 16. Signed-off-by: Dharit Shah --- ...050-distributed-tracing-instrumentation.md | 6 +- docs/ADRs/0108-tool-call-span-topology.md | 6 +- docs/guides/dev/tracing.md | 52 ++-- .../infrastructure/distributed-tracing.md | 66 +++-- docs/guides/user/how-to-emit-traces.md | 11 +- internal/cli/content_capture_record_test.go | 7 +- internal/cli/content_collector.go | 185 ++++++++----- internal/cli/content_collector_test.go | 254 ++++++++++++------ internal/cli/redaction_invariant_test.go | 21 +- internal/cli/telemetry_run_test.go | 6 +- 10 files changed, 407 insertions(+), 207 deletions(-) diff --git a/docs/ADRs/0050-distributed-tracing-instrumentation.md b/docs/ADRs/0050-distributed-tracing-instrumentation.md index 55e3c63056..8731f1f696 100644 --- a/docs/ADRs/0050-distributed-tracing-instrumentation.md +++ b/docs/ADRs/0050-distributed-tracing-instrumentation.md @@ -183,8 +183,10 @@ remains deferred. under the content gate, a retry iteration that carries validation feedback records the prompt the runner composed as `gen_ai.input.messages`, and `tool_call` parts on the message record carry the members of the call's -arguments that name what was called (a path, a pattern, a command), not a -file body or a prompt — Claude Code today +arguments that name what was called (a path, a pattern, a `Bash` command +whole, which can embed what it writes), not a `Write`, `Edit` or +`NotebookEdit` body, an `Agent` or `WebFetch` prompt or the input of an +MCP tool — Claude Code today ([#7414](https://github.com/fullsend-ai/fullsend/issues/7414)). The rest of the model's input is not recorded ([reference](../guides/infrastructure/distributed-tracing.md#content-capture-level-3)). diff --git a/docs/ADRs/0108-tool-call-span-topology.md b/docs/ADRs/0108-tool-call-span-topology.md index 47a9762ee1..663325d634 100644 --- a/docs/ADRs/0108-tool-call-span-topology.md +++ b/docs/ADRs/0108-tool-call-span-topology.md @@ -128,7 +128,9 @@ convention bump. arguments next" is in place for the members that name what was called: `tool_call` parts on the `agent` span's `gen_ai.output.messages` record carry the call's path, pattern, command and the like, redacted, where the -runtime's stream provides them, Claude Code today; a file body, an edit or -a prompt is not recorded. The list, the bound and the drop rules are in the +runtime's stream provides them, Claude Code today, for its own tools (a +`Bash` command whole, which can embed what it writes); a `Write`, `Edit` or +`NotebookEdit` body, an `Agent` or `WebFetch` prompt and the input of an +MCP tool are not recorded. The list, the bound and the drop rules are in the [reference](../guides/infrastructure/distributed-tracing.md#content-capture-level-3). `execute_tool` spans still carry metadata only. diff --git a/docs/guides/dev/tracing.md b/docs/guides/dev/tracing.md index becdd5d81c..fbde5877e1 100644 --- a/docs/guides/dev/tracing.md +++ b/docs/guides/dev/tracing.md @@ -239,7 +239,9 @@ string can be trusted, and no scan of it as written can stand in; so is a string holding invalid UTF-8, since the span and attribute writers would join the halves of a value such a byte splits, or the replacement character a lossy decode leaves in its place, which splits a value the -same way) — at assembly (redaction runs +same way; last, every URL's userinfo, query and fragment are masked, +`maskURLs`, on text the normalizer has folded and the patterns masked) — +at assembly (redaction runs before the size budget — truncating first could split a secret past recognition), enforces a 256 KiB ordered-suffix budget (the ending survives — the final answer is what consumers judge) plus an 8 KiB per-tool-result @@ -248,9 +250,9 @@ bound (tail-kept, redacted before the cut, the part marked arguments are dropped whole and the part marked, since a cut object is not JSON), with exact dropped-byte accounting across content, tool names, summaries, responses, part -ids, and dropped arguments (charged as the kept encoding when the bound -drops them, otherwise as the redacted text they were scanned as, and as -written when the call's name redacts away), then holds the marshaled string to `maxEncodedContentBytes` +ids, and arguments dropped whole (charged as the kept encoding when the +bound drops them, otherwise as the redacted text they were scanned as; a +member the recorded list leaves out is not charged), then holds the marshaled string to `maxEncodedContentBytes` (255,000 — just under the one size the pilot backend is proven to accept) by trimming the oldest content again, measured on the encoding itself and still charged in raw bytes (a separate, earlier boundary — the parser's 1 MiB @@ -268,15 +270,18 @@ content. **Guarantee and limits.** For every string the record or an `execute_tool` span exports, no 12-byte run of a known secret's body appears, read as raw bytes, after the normalizer, or after decoding the record's JSON with every -string walked; a known secret is one a `SecretRedactor` pattern matches, or -a runner env or provider-only env value the literal pass holds. +string walked; a known secret is one a `SecretRedactor` pattern matches, +a runner env or provider-only env value the literal pass holds, or a +URL's userinfo, query or fragment. `TestRedaction_NoSecretSurvivesAnyWrapping` (`internal/cli`) states it and checks it over every surface (text, names, summaries, arguments on each path, results, the input message, span names and ids, and a Claude call with the summary the parser derives from it) and every wrapping a review has constructed so far — terminal escapes before, around and inside a secret, repeated and paired secrets, zero-width, bidi, tag, NUL and -variation-selector characters inside it, an invalid byte or the +variation-selector characters inside it, a URL's delimiters in +compatibility forms, a password quoted for the shell, a URL nested in +another's path, an invalid byte or the replacement character inside it, an invalid byte after it with the secret straddling the parser's summary cut, compatibility forms, and noise, inside the record and across the feedback cut. @@ -284,20 +289,24 @@ Four limits are named there and here, and nowhere else: an env value written in compatibility (fullwidth) forms, since the normalizer rewrites it and the literal pass matches only what the env has; a secret split across two parts or events; a secret that matches no pattern and is in no -env; a summary a pi or codex parser cut before the collector saw it, since +env, a token in a URL's path among them; a summary a pi or codex parser cut before the collector saw it, since that cut can split a value the parser's own scan did not see joined (a Claude summary is cut from a member the collector scans whole). A finding inside the guarantee is a defect; one on this list is a documented limit. `toolArguments` records the members of a call's input that -`recordedArguments` names — paths, patterns, commands, modes and bounds: -`file_path`, `pattern`, `path`, `command`, `description`, `url`, the -notebook and bound members, the Grep flags — and nothing else: a file -body, an edit, a prompt, a notebook source or a todo list, and any member -whose value is an object or an array, is dropped, charged as the redacted -text it was scanned as, and the part marked `fullsend.truncated`. The -guarantee is one sentence: the record holds these members of a call, each -redacted as text, and nothing else of its input. The input is decoded +`recordedArguments` names for the tool — paths, patterns, commands, modes +and bounds; the reference lists them tool by tool — and nothing else: a +file body, an edit, a prompt, a notebook source or a todo list, any member +whose value is an object or an array, and the whole input of a tool the +list does not name (every `mcp__` tool) is dropped: scanned as text so a +secret in it still counts, then left out, neither charged nor marked, +since the list is the documented policy and the markers mean loss. A +`Bash` `command` is kept whole, so a heredoc or an inline script records +what it writes. A URL's userinfo, query and fragment are masked in every +recorded string (`maskURLs`, above), with a finding that costs a call its +summary. The guarantee is one sentence: the record holds these members of these tools, +each redacted as text, and nothing else of any call's input. The input is decoded (one JSON object, white space at most after it) and each kept string, and each kept number as its digits, goes through the text pipeline on its own, since the redactor's patterns are written for plain @@ -317,9 +326,8 @@ object are scanned as text and dropped whole, charged; so are kept members whose encoding exceeds `maxToolArgumentsBytes`, charged as encoded. A call the stream reports without a name carries no arguments. This happens once, when the event is handled; eviction and `Result` do -not rescan it. A name that redacts to nothing at `Result` takes the -arguments with it: they are charged to the dropped bytes, and the part — -kept only when it has a summary — is marked. +not rescan it. Arguments are kept only under a tool's exact name, plain +words no scan touches, so a name that redacts to nothing never had any. The input message does not come from the stream. When `runAgent` composes a retry prompt (`buildFeedbackPrompt`, under `feedback_mode: append`), @@ -343,7 +351,11 @@ stay within the proven size. JSON; check `fullsend.content.truncated` / `fullsend.content.dropped_bytes` before treating content as complete; masked secrets appear as the redactor's mask tokens, or as `[REDACTED:]` for a runner env or -provider-only value, and are counted in `fullsend.content.redactions`. +provider-only value, and are counted in `fullsend.content.redactions`; +what was masked or rewritten for its shape — the normalizer's folds, +removals, strippings and invalid UTF-8, a retry prompt altered upstream, +a URL's userinfo, query or fragment — is counted apart in +`fullsend.content.sanitizations`. A `tool_call` part's `arguments`, when present, is a JSON value (an object for the tools seen so far); a `tool_call` part marked `fullsend.truncated` had arguments that were dropped whole — on that part type the marker diff --git a/docs/guides/infrastructure/distributed-tracing.md b/docs/guides/infrastructure/distributed-tracing.md index 5b5bac9b72..a87c7017ae 100644 --- a/docs/guides/infrastructure/distributed-tracing.md +++ b/docs/guides/infrastructure/distributed-tracing.md @@ -96,9 +96,9 @@ or spelled that way which a pattern also recognises — in one of those contexts, in a connection string, or by its own prefix — is masked by that pattern, and shows what the pattern's mask shows. A tool call loses its `summary` when redaction found a secret in its arguments, the dropped -members included, stripped an escape sequence or tag characters from -them, or could not judge them at all (invalid UTF-8 or the replacement -character): the parser cut the summary out of them before anything scanned it, +members included, masked a URL's userinfo, query or fragment in them, +stripped an escape sequence or tag characters from them, or could not +judge them at all (invalid UTF-8 or the replacement character): the parser cut the summary out of them before anything scanned it, so the secret could be there as a beginning. Where a runtime reports a summary and no arguments (pi, codex, OpenCode), its parser has already run the patterns over the summary and, for pi and codex, cut it: a value that straddles a @@ -126,17 +126,31 @@ only when the runtime's stream provides them: Claude runs do; the pi and codex parsers emit none of the three yet ([#7414](https://github.com/fullsend-ai/fullsend/issues/7414)). `arguments` holds the members of the call's input that name what was -called, each redacted as text: `file_path`, `notebook_path`, `path`, -`pattern`, `glob`, `command`, `description`, `url`, `output_mode`, -`model`, `subagent_type`, `skill`, `cell_id`, `cell_type`, `edit_mode`, -`replace_all`, `multiline`, `type`, `limit`, `offset`, `head_limit`, -`timeout`, `context`, `-A`, `-B`, `-C`, `-i`, `-n`. Nothing else of the -input is recorded: a file body, an edit, a prompt, a notebook source, a -todo list, and any member whose value is an object or an array are -dropped — scanned first, so a secret in them still counts — and the part -is marked `fullsend.truncated`. The kept members are decoded, redacted -one by one and encoded again, so key order, spacing and escapes are not -the stream's; a string a terminal escape sequence or tag characters were +called, listed by tool, each redacted as text: `Bash` — `command`, +`description`, `timeout`; `Read` — `file_path`, `limit`, `offset`; +`Write` — `file_path`; `Edit` — `file_path`, `replace_all`; +`NotebookEdit` — `notebook_path`, `cell_id`, `cell_type`, `edit_mode`; +`Grep` — `pattern`, `path`, `glob`, `type`, `output_mode`, `context`, +`head_limit`, `multiline`, `-A`, `-B`, `-C`, `-i`, `-n`; `Glob` — +`pattern`, `path`; `WebFetch` — `url`; `Agent` — `description`, `model`, +`subagent_type`; `Skill` — `skill`. Nothing else of the input is +recorded: a file body, an edit, a prompt, a notebook source, a todo list, +any member whose value is an object or an array, and the whole input of +a tool not listed here (every `mcp__` tool among them) are dropped — +scanned first, so a secret in them still counts, but neither charged nor +marked, since the list is the policy and the markers mean loss. A `Bash` +`command` is recorded whole, up to 8 KiB, so a heredoc or an inline +script records what it writes. A URL anywhere in the record — a +member, text, a result, a summary, the input message, a name — has +everything from `://` to its last `@` (userinfo, or a URL nested in +another's path) and everything after its first `?` or `#` masked `***`, +after the patterns ran, and counted in `fullsend.content.sanitizations`; +the URL runs to the next white space, so in compact JSON the mask can +take the rest of the line. A token carried in the path itself, as a +webhook URL's is, matches no pattern and is kept. The kept +members are decoded, redacted one by one and encoded again, so key +order, spacing and escapes are not the stream's; a string a terminal +escape sequence or tag characters were stripped from, or that holds invalid UTF-8 or the replacement character a lossy decode leaves (U+FFFD), is masked whole, since the stripping can leave most of a token, a later write would join the @@ -169,7 +183,7 @@ each call's arguments at 8 KiB — kept as an ordered suffix; overflow drops the oldest content first. Those bounds count raw bytes, except that arguments count as the re-encoded JSON described above, against their own bound and against the -total (as redacted text when a member was dropped by name or the arguments were not one JSON object). That encoding writes `<`, `>` and `&` as six bytes each, so +total (as redacted text when the arguments were not one JSON object). That encoding writes `<`, `>` and `&` as six bytes each, so arguments dense in those characters reach their bound at about a sixth of that size on the stream. The exported JSON string is bounded as well, at 255,000 bytes, because encoding adds 8–10% at these bounds on real runs @@ -180,14 +194,15 @@ Truncation is marked via `fullsend.content.truncated` on the span and its tail; arguments over theirs, or not a complete JSON object, are dropped whole — a cut object is not JSON — and the marked `tool_call` part keeps its `id`, name and summary — not the summary when redaction -found a secret in the arguments, stripped an escape sequence or tag -characters from them, or could not judge them at all. A member outside the recorded list (above) is -dropped and the part marked the same way, its listed members kept. -Arguments also go, charged and marked, when the call's name redacts to -nothing; that part survives only if it has a summary. So +found a secret in the arguments, masked a URL's userinfo, query or +fragment in them, stripped an escape sequence or tag characters from +them, or could not judge them at all. A member outside the recorded list (above) is +dropped without a mark, its listed members kept. A name that redacts to +nothing never had arguments: they are kept only under a tool's exact +name, plain words no scan touches. So `fullsend.truncated` means different things by part type: on a tool -result, part of the response is kept; on a tool call, some or all of the -input is not. Read it together with the part's `type`. The input message is not cut at this stage: +result, part of the response is kept; on a tool call, the arguments were +dropped whole. Read it together with the part's `type`. The input message is not cut at this stage: the validation output inside it is cut at 10 KiB when the prompt is composed (the prompt then says `[truncated]`), and the two markers above do not describe it. The recorded copy is the redacted one, so it can @@ -338,9 +353,10 @@ The `agent` span's provider identity reflects only the parent run's serving endp | `fullsend.transcript_error` | `agent` | Present (`true`) when the agent exited 0 but its transcript reported an error — the span's status is Error while `exit_code` keeps the raw process exit | | `gen_ai.input.messages` | `agent` | Level 3 only, on a retry iteration that carries validation feedback: the prompt the runner composed, as a JSON string (see Content capture) | | `gen_ai.output.messages` | `agent` | Level 3 only: the iteration's conversation content as a JSON string (see Content capture) | -| `fullsend.content.truncated` | `agent` | Level 3 only: present (`true`) when the size budget cut or dropped content, a tool call lost its arguments, or a kept tool result is a parser-side fragment or an oversized line's empty stand-in (`fullsend.truncated` on the part; no byte count for the last two) | -| `fullsend.content.dropped_bytes` | `agent` | Level 3 only: exact part bytes removed by the size budget — content, ids, dropped arguments (counted as re-encoded JSON when the bound drops them, or as redacted text when a member was dropped by name or the arguments were not one JSON object), and the fixed footprint of an errored-empty or oversized stand-in part — otherwise in raw bytes, whichever bound made the cut; the bytes of a skipped oversized line were never decoded and are not counted | -| `fullsend.content.redactions` | `agent` | Level 3 only: number of security findings raised while redacting content at assembly, the input message included (and findings from parts the size budget later dropped) | +| `fullsend.content.truncated` | `agent` | Level 3 only: present (`true`) when the size budget cut or dropped content, a tool call's arguments were dropped whole, or a kept tool result is a parser-side fragment or an oversized line's empty stand-in (`fullsend.truncated` on the part; no byte count for the last two) | +| `fullsend.content.dropped_bytes` | `agent` | Level 3 only: exact part bytes removed by the size budget — content, ids, arguments dropped whole (counted as re-encoded JSON when the bound drops them, or as redacted text when the arguments were not one JSON object; a member the recorded list leaves out is not counted), and the fixed footprint of an errored-empty or oversized stand-in part — otherwise in raw bytes, whichever bound made the cut; the bytes of a skipped oversized line were never decoded and are not counted | +| `fullsend.content.redactions` | `agent` | Level 3 only: number of known secrets matched while redacting content at assembly — the secret redactor's patterns and the runner-env literal pass — the input message included (and findings from parts the size budget later dropped) | +| `fullsend.content.sanitizations` | `agent` | Level 3 only: number of strings masked or rewritten for their shape rather than for a secret they matched — compatibility characters folded, invisible characters removed, an escape sequence or tag characters stripped or invalid UTF-8 or U+FFFD present (each such string is `***` whole, since it can hide a secret), a retry prompt altered upstream (`***` whole), a URL's userinfo, query or fragment masked | | `fullsend.tool.unmatched` | `execute_tool` | Present (`true`) when a result arrived for a call the stream never reported; the span has near-zero duration | | `fullsend.tool.result_oversized` | `execute_tool` | Present (`true`) when the result's stream line exceeded the parser's 1 MiB bound: the call was answered but nothing of the result was decoded, so the span has no status and no `error.type` | | `fullsend.tool_spans.dropped` | `agent` | Present when the iteration hit the 1,024-span cap and at least one id-bearing call was refused a span: the number of `tool_use` events with a usable id that arrived past the cap, each counted once whatever its result later does; a result with no open span past the cap is not counted | diff --git a/docs/guides/user/how-to-emit-traces.md b/docs/guides/user/how-to-emit-traces.md index 191299d514..8748292618 100644 --- a/docs/guides/user/how-to-emit-traces.md +++ b/docs/guides/user/how-to-emit-traces.md @@ -132,9 +132,14 @@ prompt the runner composed. Content is redacted for secrets and bounded per iter a string that held a terminal colour or title code or tag characters, as most coloured CLI output does, or that was not valid UTF-8 or held the replacement character (U+FFFD), is recorded as `***` whole, and so is a retry prompt the runner sanitized, or cut at its size bound while it held compatibility -characters; feedback cut for size alone is still recorded. Content may still contain proprietary code or PII — tool arguments carry the -commands an agent ran and the paths and patterns it used, never a file body -or an edit — so make sure your backend's +characters; feedback cut for size alone is still recorded. Content may still contain proprietary code or PII: +tool arguments carry the commands an agent ran and the paths and patterns it used. +A `Write`, `Edit` or `NotebookEdit` body, an `Agent` or `WebFetch` prompt and the +input of MCP tools are not recorded as arguments, though a tool's result can quote +what it wrote and a call's summary keeps the first 120 characters of its input; +a `Bash` command is recorded whole (up to 8 KiB) and can embed the file content +it writes. A URL anywhere in the record has its userinfo, query and fragment +masked. Make sure your backend's access controls fit before enabling it. ```bash diff --git a/internal/cli/content_capture_record_test.go b/internal/cli/content_capture_record_test.go index f80c85ad12..c47b500887 100644 --- a/internal/cli/content_capture_record_test.go +++ b/internal/cli/content_capture_record_test.go @@ -134,7 +134,7 @@ func TestContentCapture_SecretsAcrossArgumentsNeverReachTheRecord(t *testing.T) } require.Len(t, calls, 4, "every call is in the record, arguments or not") assert.Equal(t, map[string]any{"notebook_path": "notes.ipynb"}, calls["NotebookEdit"]["arguments"], "the cells are not recorded; the path is") - assert.Equal(t, true, calls["NotebookEdit"]["fullsend.truncated"]) + assert.NotContains(t, calls["NotebookEdit"], "fullsend.truncated", "a member the list leaves out is policy, not loss") assert.NotContains(t, calls["NotebookEdit"], "summary", "a secret in the dropped cells still costs the summary") assert.Equal(t, map[string]any{"file_path": "cfg.json"}, calls["Write"]["arguments"], "the file body is not recorded; the path is") assert.Equal(t, "cfg.json", calls["Write"]["summary"], "no secret was found in it") @@ -142,7 +142,6 @@ func TestContentCapture_SecretsAcrossArgumentsNeverReachTheRecord(t *testing.T) assert.Equal(t, map[string]any{"file_path": "README.md"}, calls["Read"]["arguments"], "the clean call's arguments are recorded") assert.Equal(t, "README.md", calls["Read"]["summary"]) - truncated, ok := recordAttribute(t, dir, "fullsend.content.truncated") - assert.True(t, ok, "the dropped arguments are marked on the span") - assert.Equal(t, "true", truncated) + _, marked := recordAttribute(t, dir, "fullsend.content.truncated") + assert.False(t, marked, "nothing was cut or dropped whole, so the span is not marked") } diff --git a/internal/cli/content_collector.go b/internal/cli/content_collector.go index faef098c1f..e368b36776 100644 --- a/internal/cli/content_collector.go +++ b/internal/cli/content_collector.go @@ -5,6 +5,7 @@ import ( "errors" "io" "maps" + "regexp" "slices" "sort" "strings" @@ -152,9 +153,21 @@ func attachContent(span trace.Span, res contentResult) { if res.DroppedBytes > 0 { attrs = append(attrs, attribute.Int("fullsend.content.dropped_bytes", res.DroppedBytes)) } - if n := len(res.Findings); n > 0 { + // What was masked or rewritten for its shape — the normalizer's folds, + // removals, strippings and invalid UTF-8, a URL's userinfo, query or + // fragment — is counted apart from what matched a known secret. + sanitized := 0 + for _, f := range res.Findings { + if f.Scanner == "unicode_normalizer" || f.Scanner == "url" { + sanitized++ + } + } + if n := len(res.Findings) - sanitized; n > 0 { attrs = append(attrs, attribute.Int("fullsend.content.redactions", n)) } + if sanitized > 0 { + attrs = append(attrs, attribute.Int("fullsend.content.sanitizations", sanitized)) + } if len(attrs) > 0 { span.SetAttributes(attrs...) } @@ -237,7 +250,7 @@ type contentPart struct { // content-bearing (an errored empty result is signal, not absence) // and accounts a fixed footprint. Truncated marks a part whose bulk // field was cut, so a consumer never reads a fragment as a whole - // result, and a tool_call whose arguments were dropped; it is set by + // result, and a tool_call whose arguments were dropped whole; it is set by // the collector's own cuts and by parser-side loss (Partial, // Oversized), and stays outside the accounting except for an oversized // stand-in's fixed footprint (see oversized). @@ -349,15 +362,17 @@ type contentResult struct { OutputMessages string // DroppedBytes counts raw part bytes removed by the size budget // (content, tool name, summary, tool response, and id bytes alike) and - // the bytes of dropped tool arguments: as re-encoded JSON (on a key + // the bytes of arguments dropped whole: as re-encoded JSON (on a key // collision, without the member the later key in sorted order - // replaced), or as - // redacted text when they were not one JSON value. + // replaced) when the bound dropped them, or + // as redacted text when they were not one JSON object. A member the + // list leaves out is policy, not loss, and is not counted. DroppedBytes int // Truncated reports whether the budget cut or dropped anything, a tool - // call's arguments were dropped (toolArguments, or Result when the name - // redacted away; a kept part is marked), or a kept tool result was a - // parser-side fragment (its part is marked). + // call's arguments were dropped whole (toolArguments; a kept part is + // marked), or a kept tool + // result was a parser-side fragment (its part is marked). A member + // the list leaves out does not set it. Truncated bool // Findings are the security findings raised during redaction. Findings []security.Finding @@ -383,7 +398,7 @@ type contentCollector struct { total int // evicted counts bytes discarded before Result's budget runs: old // parts and a lone oversized part's head (evictOverflow), the head a - // tool result loses to maxToolResultBytes, and dropped tool arguments + // tool result loses to maxToolResultBytes, and arguments dropped whole // (toolArguments). // Eviction keeps memory bounded on long sessions by approximating the // Result budget on sizes as accumulated — pre-redaction — so it can @@ -423,7 +438,7 @@ func (c *contentCollector) Handle(evt agentruntime.AgentEvent) { // alone must not keep a nameless call, nor charge for one that // appendPart then refuses. scanned := len(c.findings) - p.Arguments, p.Truncated = c.toolArguments(e.Arguments) + p.Arguments, p.Truncated = c.toolArguments(e.Name, e.Arguments) if slices.ContainsFunc(c.findings[scanned:], revealing) { // The parser cut the summary out of these arguments // before anything scanned it, so a secret found in them, @@ -593,15 +608,10 @@ func (c *contentCollector) Result(finishReason string) contentResult { if bulk != &p.Response { p.Response = c.redact(p.Response, &res.Findings) } + // A name that redacts to nothing never had arguments: they are + // kept only under a tool's exact name (recordedArguments), plain + // words no scan touches. p.Name = c.redact(p.Name, &res.Findings) - if p.Name == "" && p.Arguments != nil { - // As in Handle, for a name that redacts to nothing — but these - // arguments were accepted, so losing them is charged and - // marked. Ones Handle had already dropped stay as they were. - res.DroppedBytes += len(p.Arguments) - res.Truncated = true - p.Arguments, p.Truncated = nil, true - } p.Summary = c.redact(p.Summary, &res.Findings) p.ID = c.redactID(p.ID, &res.Findings) if contentBytes(p) == 0 { @@ -758,7 +768,8 @@ func (c *contentCollector) redact(text string, findings *[]security.Finding) str // the env has it. An empty result with findings means fully redacted. A // string the normalizer stripped an escape sequence or tag characters // from (stripped), or that holds invalid UTF-8 or the replacement -// character a lossy decode leaves, is masked *** whole. +// character a lossy decode leaves, is masked *** whole. Last, every URL +// loses its userinfo, query and fragment (maskURLs). func redactText(pipeline *security.Pipeline, runnerEnv map[string]string, text string, findings *[]security.Finding) string { if text == "" { return text @@ -791,45 +802,84 @@ func redactText(pipeline *security.Pipeline, runnerEnv map[string]string, text s return "***" } if scanned.Sanitized != "" { - return replaceEnv(runnerEnv, scanned.Sanitized, findings) - } - if len(scanned.Findings) > 0 { + text = replaceEnv(runnerEnv, scanned.Sanitized, findings) + } else if len(scanned.Findings) > 0 { return "" } + return maskURLs(text, findings) +} + +// urlRun matches a URL as a scheme, "://" and the rest of the run up to +// white space — the only end it has, so a quote inside it (a password +// quoted for the shell, a query quoted half-way) never ends it early. +var urlRun = regexp.MustCompile(`[A-Za-z][A-Za-z0-9+.-]*://\S+`) + +// maskURLs masks what a URL carries that no pattern is written for: +// everything after its first '?' or '#' (a presigned signature, an access +// token, a token in a fragment) and everything from "://" to its last +// '@' (userinfo, or a URL nested in another's path), as ***, with one +// finding for a string that had any. It runs last in redactText, on text +// the normalizer has folded and the patterns have masked, so a +// compatibility '？' or '＠' is a plain one here, and nothing is +// re-encoded: the mask only replaces bytes. A token carried in the path +// itself, as a webhook URL's is, matches no pattern and is kept. +func maskURLs(text string, findings *[]security.Finding) string { + masked := false + text = urlRun.ReplaceAllStringFunc(text, func(run string) string { + out := run + if i := strings.IndexAny(out, "?#"); i >= 0 { + out = out[:i+1] + "***" + } + if s, a := strings.Index(out, "://"), strings.LastIndex(out, "@"); a > s { + out = out[:s+3] + "***" + out[a:] + } + masked = masked || out != run + return out + }) + if masked { + *findings = append(*findings, security.Finding{Scanner: "url", Name: "url_masked", Severity: "medium", + Detail: "a URL's userinfo, query or fragment; masked", Position: -1}) + } return text } -// recordedArguments names the members of a tool call's arguments that -// the record keeps: paths, patterns, commands, modes and bounds — what a -// trace needs to say which call was made — and never a file body, an -// edit, a prompt, a notebook source or a todo list, the members that -// carry what the agent read or wrote. A member outside the list, or one -// whose value is an object or an array, is dropped: scanned as text so -// its findings count, charged and marked, like any other discarded -// content. Each kept string goes through the text pipeline on its own. -// The guarantee is then one sentence: the record holds these members of -// a call, each redacted as text, and nothing else of its input. Claude -// Code's tools today: Bash (command, description, timeout), Read -// (file_path, limit, offset), Write, Edit and NotebookEdit (file_path, -// notebook_path, cell_id, replace_all), Grep and Glob (pattern, path, -// glob, output_mode, the context flags), WebFetch (url), Agent -// (description, model, subagent_type), Skill (skill). The reference -// lists them; a tool that names its path or pattern otherwise records -// nothing of its input, which is the safe default. -var recordedArguments = map[string]bool{ - "-A": true, "-B": true, "-C": true, "-i": true, "-n": true, - "cell_id": true, "cell_type": true, "command": true, "context": true, - "description": true, "edit_mode": true, "file_path": true, "glob": true, - "head_limit": true, "limit": true, "model": true, "multiline": true, - "notebook_path": true, "offset": true, "output_mode": true, "path": true, - "pattern": true, "replace_all": true, "skill": true, "subagent_type": true, - "timeout": true, "type": true, "url": true, +// recordedArguments names, tool by tool, the members of a call's input +// that the record keeps: paths, patterns, commands, modes and bounds — +// what a trace needs to say which call was made. A Write, Edit or +// NotebookEdit body, an Agent or WebFetch prompt and a todo list are not +// listed; a member outside the list, or one whose value is an object or +// an array, is dropped: scanned as text so a secret in it still counts, +// then left out — the documented policy, so neither charged nor marked; +// the tool's name says what was not recorded. A tool the list does not +// name, every mcp__ tool among them, records nothing of its input: on +// those, description, url or path carry bulk content as often as not. +// Each kept string goes through the text pipeline on its own. A Bash command is +// kept whole, up to maxToolArgumentsBytes, so a heredoc or an inline +// script records what it writes. The guarantee is then one sentence: +// the record holds these members of these tools, each redacted as text, +// and nothing else of any call's input. The reference lists them. +var recordedArguments = map[string]map[string]bool{ + "Agent": {"description": true, "model": true, "subagent_type": true}, + "Bash": {"command": true, "description": true, "timeout": true}, + "Edit": {"file_path": true, "replace_all": true}, + "Glob": {"path": true, "pattern": true}, + "Grep": { + "-A": true, "-B": true, "-C": true, "-i": true, "-n": true, "context": true, "glob": true, + "head_limit": true, "multiline": true, "output_mode": true, "path": true, "pattern": true, "type": true, + }, + "NotebookEdit": {"cell_id": true, "cell_type": true, "edit_mode": true, "notebook_path": true}, + "Read": {"file_path": true, "limit": true, "offset": true}, + "Skill": {"skill": true}, + "WebFetch": {"url": true}, + "Write": {"file_path": true}, } // toolArguments returns the recorded members of a tool call's arguments -// (recordedArguments), redacted and encoded again, and whether any of -// the input was dropped instead. There are none to return when the event -// carries none, white space only, or JSON null. +// (recordedArguments, by the tool's name), redacted and encoded again, +// and whether the input was dropped whole instead. There are none to +// return when the event carries none, white space only, or JSON null — +// and none, unmarked, when the list keeps no member of this call or does +// not name its tool. // // The input is decoded and each kept string is redacted on its own: the // redactor's patterns are written for plain text, and run over serialised @@ -843,9 +893,8 @@ var recordedArguments = map[string]bool{ // the replacement character, cannot be walked that way: it is scanned as // text so its findings count, then dropped whole and charged. // So are arguments whose encoding exceeds maxToolArgumentsBytes, charged -// as encoded. A dropped member is charged as the redacted text it was -// scanned as. -func (c *contentCollector) toolArguments(args string) (json.RawMessage, bool) { +// as encoded. A member the list leaves out is scanned and not charged. +func (c *contentCollector) toolArguments(name, args string) (json.RawMessage, bool) { if strings.TrimSpace(args) == "" { return nil, false } @@ -867,20 +916,21 @@ func (c *contentCollector) toolArguments(args string) (json.RawMessage, bool) { c.evicted += len(c.redact(args, &c.findings)) return nil, true } - out, dropped := make(map[string]any, len(obj)), false + listed := recordedArguments[name] + out, filtered := make(map[string]any, len(obj)), false for _, k := range slices.Sorted(maps.Keys(obj)) { switch t := obj[k].(type) { case string: - if recordedArguments[k] { + if listed[k] { out[k] = c.redact(t, &c.findings) continue } - c.evicted += len(c.redact(t, &c.findings)) + c.redact(t, &c.findings) case map[string]any, []any: enc, _ := json.Marshal(t) // decoded JSON values marshal unconditionally - c.evicted += len(c.redact(string(enc), &c.findings)) + c.redact(string(enc), &c.findings) case json.Number: - if recordedArguments[k] { + if listed[k] { if r := c.redact(string(t), &c.findings); r != string(t) { out[k] = r // the mask, as the string it is } else { @@ -888,19 +938,17 @@ func (c *contentCollector) toolArguments(args string) (json.RawMessage, bool) { } continue } - c.evicted += len(c.redact(string(t), &c.findings)) - default: // a boolean, null - if recordedArguments[k] { + c.redact(string(t), &c.findings) + default: // a boolean, null: nothing to scan + if listed[k] { out[k] = t continue } - enc, _ := json.Marshal(t) - c.evicted += len(c.redact(string(enc), &c.findings)) } - dropped = true + filtered = true } - if len(out) == 0 && dropped { - return nil, true + if listed == nil || len(out) == 0 && filtered { + return nil, false } enc, err := json.Marshal(out) if err != nil { @@ -910,7 +958,7 @@ func (c *contentCollector) toolArguments(args string) (json.RawMessage, bool) { c.evicted += len(enc) return nil, true } - return enc, dropped + return enc, false } var errTrailingInput = errors.New("trailing input after the JSON value") @@ -931,7 +979,8 @@ func decodeJSON(s string) (any, error) { } // revealing reports a finding that costs a call its summary: any but the -// normalizer's, which shows a secret was there; the normalizer's stripping +// normalizer's, which shows a secret — or a URL's userinfo, query or +// fragment — was there; the normalizer's stripping // of content; and invalid UTF-8, after which the arguments were not judged // at all — and a summary is cut from a member only that judgement sees // whole. diff --git a/internal/cli/content_collector_test.go b/internal/cli/content_collector_test.go index 03b857bf83..655fa1e44c 100644 --- a/internal/cli/content_collector_test.go +++ b/internal/cli/content_collector_test.go @@ -14,6 +14,7 @@ import ( "go.opentelemetry.io/otel/sdk/trace/tracetest" agentruntime "github.com/fullsend-ai/fullsend/internal/runtime" + "github.com/fullsend-ai/fullsend/internal/security" "github.com/fullsend-ai/fullsend/internal/telemetry" ) @@ -149,77 +150,75 @@ func TestContentCollector_RedactsSecretsInArguments(t *testing.T) { } func TestContentCollector_RecordsOnlyListedArguments(t *testing.T) { - // The record keeps the members recordedArguments names — paths, - // patterns, commands, modes and bounds — and nothing else of a call's - // input: a file body, an edit, a prompt, a notebook source or a todo - // list is dropped, charged and marked, with no finding and the - // summary kept, since nothing found a secret. + // The record keeps the members recordedArguments names for the tool — + // paths, patterns, commands, modes and bounds — and nothing else of a + // call's input: a file body, an edit, a prompt, a notebook source or + // a todo list is dropped with no finding and the summary kept, since + // nothing found a secret, and neither charged nor marked, since the + // list is the documented policy, not a loss. A tool the list does not + // name — an MCP tool, a tool of another name — records nothing. const opaque = "opaque-credential-123" for name, tc := range map[string]struct { - args, want string - dropped bool + tool, args, want string }{ - "Write keeps the path": {`{"file_path":"/x/a.json","content":"{\"credentials\":{\"value\":\"` + opaque + `\"}}"}`, `{"file_path":"/x/a.json"}`, true}, - "Edit keeps the path": {`{"file_path":"/x/a.go","old_string":"` + opaque + `","new_string":"x","replace_all":true}`, `{"file_path":"/x/a.go","replace_all":true}`, true}, - "MultiEdit keeps the path": {`{"file_path":"/x/a.go","edits":[{"old_string":"` + opaque + `","new_string":"x"}]}`, `{"file_path":"/x/a.go"}`, true}, - "NotebookEdit keeps the path": {`{"notebook_path":"n.ipynb","cell_id":"c1","new_source":"print('` + opaque + `')"}`, `{"cell_id":"c1","notebook_path":"n.ipynb"}`, true}, - "Agent keeps its description": {`{"description":"scan","prompt":"the token is ` + opaque + `","model":"sonnet","subagent_type":"Explore"}`, `{"description":"scan","model":"sonnet","subagent_type":"Explore"}`, true}, - "Grep keeps everything": {`{"pattern":"TODO","path":"src","output_mode":"content","-n":true,"context":2}`, `{"-n":true,"context":2,"output_mode":"content","path":"src","pattern":"TODO"}`, false}, - "Read keeps everything": {`{"file_path":"/x","offset":10,"limit":50}`, `{"file_path":"/x","limit":50,"offset":10}`, false}, - "a listed member holding an object": {`{"path":{"dir":"` + opaque + `"},"pattern":"x"}`, `{"pattern":"x"}`, true}, - "a listed member holding an array": {`{"pattern":["` + opaque + `"],"path":"src"}`, `{"path":"src"}`, true}, - "no members": {`{}`, `{}`, false}, + "Write keeps the path": {"Write", `{"file_path":"/x/a.json","content":"{\"credentials\":{\"value\":\"` + opaque + `\"}}"}`, `{"file_path":"/x/a.json"}`}, + "Edit keeps the path": {"Edit", `{"file_path":"/x/a.go","old_string":"` + opaque + `","new_string":"x","replace_all":true}`, `{"file_path":"/x/a.go","replace_all":true}`}, + "NotebookEdit keeps the path": {"NotebookEdit", `{"notebook_path":"n.ipynb","cell_id":"c1","new_source":"print('` + opaque + `')"}`, `{"cell_id":"c1","notebook_path":"n.ipynb"}`}, + "Agent keeps its description": {"Agent", `{"description":"scan","prompt":"the token is ` + opaque + `","model":"sonnet","subagent_type":"Explore"}`, `{"description":"scan","model":"sonnet","subagent_type":"Explore"}`}, + "WebFetch keeps the url": {"WebFetch", `{"url":"https://example.com/doc","prompt":"find ` + opaque + `"}`, `{"url":"https://example.com/doc"}`}, + "Bash keeps the command": {"Bash", `{"command":"ls","description":"list","timeout":5000,"run_in_background":true}`, `{"command":"ls","description":"list","timeout":5000}`}, + "Grep keeps everything": {"Grep", `{"pattern":"TODO","path":"src","output_mode":"content","-n":true,"context":2}`, `{"-n":true,"context":2,"output_mode":"content","path":"src","pattern":"TODO"}`}, + "Read keeps everything": {"Read", `{"file_path":"/x","offset":10,"limit":50}`, `{"file_path":"/x","limit":50,"offset":10}`}, + "a listed member holding an object": {"Grep", `{"path":{"dir":"` + opaque + `"},"pattern":"x"}`, `{"pattern":"x"}`}, + "a listed member holding an array": {"Grep", `{"pattern":["` + opaque + `"],"path":"src"}`, `{"path":"src"}`}, + "no members": {"Bash", `{}`, `{}`}, + "an MCP tool records nothing": {"mcp__gh__create_issue", `{"title":"bug","description":"the token is ` + opaque + `"}`, ""}, + "a tool of another name records nothing": {"MultiEdit", `{"file_path":"/x/a.go","edits":[{"old_string":"` + opaque + `","new_string":"x"}]}`, ""}, + "TodoWrite records nothing": {"TodoWrite", `{"todos":[{"content":"` + opaque + `","status":"pending"}]}`, ""}, } { t.Run(name, func(t *testing.T) { c := newContentCollector(4096) - c.Handle(agentruntime.ToolUseEvent{Name: "Tool", Summary: "s", Arguments: tc.args}) + c.Handle(agentruntime.ToolUseEvent{Name: tc.tool, Summary: "s", Arguments: tc.args}) res := c.Result("stop") - assert.Contains(t, res.OutputMessages, `"arguments":`+tc.want) + part := partAt(t, decodeOutputMessages(t, res.OutputMessages), 0) + if tc.want == "" { + assert.NotContains(t, part, "arguments") + } else { + assert.Contains(t, res.OutputMessages, `"arguments":`+tc.want) + } assert.NotContains(t, res.OutputMessages, opaque) - assert.Contains(t, res.OutputMessages, `"summary":"s"`, "nothing found a secret") + assert.Equal(t, "s", part["summary"], "nothing found a secret") assert.Empty(t, res.Findings) - assert.Equal(t, tc.dropped, res.Truncated) - assert.Equal(t, tc.dropped, strings.Contains(res.OutputMessages, `"fullsend.truncated":true`)) - assert.Equal(t, tc.dropped, res.DroppedBytes > 0, "a dropped member is charged") + assert.False(t, res.Truncated, "a member the list leaves out is policy, not loss") + assert.NotContains(t, part, "fullsend.truncated") + assert.Zero(t, res.DroppedBytes) }) } - t.Run("nothing listed records no arguments", func(t *testing.T) { - c := newContentCollector(4096) - c.Handle(agentruntime.ToolUseEvent{Name: "TodoWrite", Summary: "s", Arguments: `{"todos":[{"content":"` + opaque + `","status":"pending"}]}`}) - - res := c.Result("stop") - part := partAt(t, decodeOutputMessages(t, res.OutputMessages), 0) - assert.NotContains(t, part, "arguments") - assert.Equal(t, "s", part["summary"]) - assert.Equal(t, true, part["fullsend.truncated"]) - assert.NotContains(t, res.OutputMessages, opaque) - }) } -func TestContentCollector_DroppedArgumentsAreScannedAndCharged(t *testing.T) { +func TestContentCollector_DroppedMembersAreScannedAndNotCharged(t *testing.T) { // Discarded content is redacted first, so a secret in a member that // is not recorded still counts — and costs the summary, as a secret - // found anywhere in the arguments does — and the member is charged as - // the redacted text it was scanned as. + // found anywhere in the arguments does — but the member is neither + // charged nor marked: leaving it out is the list's policy, and the + // markers mean loss. token := "ghp_" + strings.Repeat("r", 36) - for name, tc := range map[string]struct { - args string - charged int - }{ - "a file body": {`{"file_path":"/x","content":"echo ` + token + `"}`, len("echo ghp_...")}, - "a listed member holding an object": {`{"file_path":"/x","path":{"t":"` + token + `"}}`, len(`{"t":"ghp_..."}`)}, - "a number outside the list": {`{"file_path":"/x","pin":12345678}`, len("12345678")}, + for name, args := range map[string]string{ + "a file body": `{"file_path":"/x","content":"echo ` + token + `"}`, + "a listed member holding an object": `{"file_path":"/x","path":{"t":"` + token + `"}}`, + "a number outside the list": `{"file_path":"/x","pin":12345678}`, } { t.Run(name, func(t *testing.T) { c := newContentCollector(4096) - c.Handle(agentruntime.ToolUseEvent{Name: "Write", Summary: "/x", Arguments: tc.args}) + c.Handle(agentruntime.ToolUseEvent{Name: "Write", Summary: "/x", Arguments: args}) res := c.Result("stop") assert.NotContains(t, res.OutputMessages, token[4:]) assert.Contains(t, res.OutputMessages, `"arguments":{"file_path":"/x"}`) - assert.Equal(t, tc.charged, res.DroppedBytes) - assert.True(t, res.Truncated) + assert.Zero(t, res.DroppedBytes) + assert.False(t, res.Truncated) + assert.NotContains(t, res.OutputMessages, `"fullsend.truncated"`) if strings.Contains(name, "number") { assert.Empty(t, res.Findings) assert.Contains(t, res.OutputMessages, `"summary":"/x"`) @@ -231,6 +230,101 @@ func TestContentCollector_DroppedArgumentsAreScannedAndCharged(t *testing.T) { } } +func TestContentCollector_AURLsUserinfoQueryAndFragmentAreMasked(t *testing.T) { + // No pattern is written for a password before a host, a presigned + // signature or an access token in a query string, or a token behind a + // fragment, and the semantic conventions say url.full must not carry + // userinfo and should redact signature query values. redactText masks + // them in every recorded string, last — after the normalizer folds a + // compatibility '？' or '＠' and the patterns mask a token in the path — + // counts one finding, which costs a call its summary, and re-encodes + // nothing. + const pw = "hunter2hunter2" + fw := func(r rune) string { return string(r) } + ghp := "ghp_" + strings.Repeat("a", 36) + var ghpFull strings.Builder + for _, r := range ghp[4:] { + ghpFull.WriteRune(r + 0xfee0) + } + for name, tc := range map[string]struct { + tool, args, want string + }{ + "a password before the host": {"WebFetch", `{"url":"https://alice:` + pw + `@example.com/repo"}`, `{"url":"https://***@example.com/repo"}`}, + "a presigned query": {"WebFetch", `{"url":"https://b.s3.amazonaws.com/k?X-Amz-Credential=AKIA` + pw + `&X-Amz-Signature=` + pw + `"}`, `{"url":"https://b.s3.amazonaws.com/k?***"}`}, + "an access token in the query": {"Bash", `{"command":"curl 'https://api.example.com/v1/me?access_token=` + pw + `' -s"}`, `{"command":"curl 'https://api.example.com/v1/me?*** -s"}`}, + "a token in a clone URL": {"Bash", `{"command":"git clone https://x-access-token:` + pw + `@github.com/o/r.git /tmp/r"}`, `{"command":"git clone https://***@github.com/o/r.git /tmp/r"}`}, + "a token behind the fragment": {"Bash", `{"command":"open https://app.example.com/cb#access_token=` + pw + `"}`, `{"command":"open https://app.example.com/cb#***"}`}, + "a password quoted for the shell": {"Bash", `{"command":"git clone https://deploy:'` + pw + `'@git.example.com/o/r"}`, `{"command":"git clone https://***@git.example.com/o/r"}`}, + "a query quoted half-way": {"Bash", `{"command":"curl 'https://h.example.com/x'?token=` + pw + `"}`, `{"command":"curl 'https://h.example.com/x'?***"}`}, + "a URL nested in another's path": {"Bash", `{"command":"curl https://proxy.example.com/https://u:` + pw + `@h.example.com/x"}`, `{"command":"curl https://***@h.example.com/x"}`}, + "a fullwidth question mark": {"Bash", `{"command":"curl https://api.example.com/v1/me` + fw(0xff1f) + `access_token=` + pw + `"}`, `{"command":"curl https://api.example.com/v1/me?***"}`}, + "a fullwidth at sign": {"Bash", `{"command":"git clone https://x-access-token` + fw(0xff1a) + pw + fw(0xff20) + `github.com/o/r"}`, `{"command":"git clone https://***@github.com/o/r"}`}, + "a zero-width split scheme": {"Bash", `{"command":"curl https:` + fw(0x200b) + `//h.example.com/p?access_token=` + pw + `"}`, `{"command":"curl https://h.example.com/p?***"}`}, + "a fullwidth token in the path": {"Bash", `{"command":"curl https://h.example.com/ghp_` + ghpFull.String() + `?x=1"}`, `{"command":"curl https://h.example.com/ghp_...?***"}`}, + } { + t.Run(name, func(t *testing.T) { + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{Name: tc.tool, Summary: "s", Arguments: tc.args}) + + res := c.Result("stop") + assert.Contains(t, res.OutputMessages, `"arguments":`+tc.want) + assert.NotContains(t, res.OutputMessages, pw) + assert.NotContains(t, res.OutputMessages, ghp[4:]) + assert.NotContains(t, res.OutputMessages, "%EF", "nothing is re-encoded") + assert.NotContains(t, res.OutputMessages, `"summary"`, "the summary was cut from the same member") + urls := 0 + for _, f := range res.Findings { + if f.Scanner == "url" { + urls++ + } + } + assert.Equal(t, 1, urls, "one finding for the string") + }) + } + t.Run("an Agent prompt's URL costs the summary cut from it", func(t *testing.T) { + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{Name: "Agent", Summary: "Clone https://deploy:" + pw + "@git.example.com/o/r and", + Arguments: `{"description":"clone","prompt":"Clone https://deploy:` + pw + `@git.example.com/o/r and read it"}`}) + + res := c.Result("stop") + assert.Contains(t, res.OutputMessages, `"arguments":{"description":"clone"}`) + assert.NotContains(t, res.OutputMessages, pw) + assert.NotContains(t, res.OutputMessages, `"summary"`) + }) + t.Run("a URL in a tool result or in text", func(t *testing.T) { + c := newContentCollector(4096) + c.Handle(agentruntime.TextEvent{Text: "pushing to https://deploy:" + pw + "@git.example.com/o/r now"}) + c.Handle(agentruntime.ToolResultEvent{ID: "toolu_01", Result: "origin\thttps://deploy:" + pw + "@git.example.com/o/r (fetch)"}) + + res := c.Result("stop") + assert.Contains(t, res.OutputMessages, `"content":"pushing to https://***@git.example.com/o/r now"`) + assert.Contains(t, res.OutputMessages, `"response":"origin\thttps://***@git.example.com/o/r (fetch)"`) + assert.NotContains(t, res.OutputMessages, pw) + }) + t.Run("a URL with none of them is kept as written", func(t *testing.T) { + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{Name: "WebFetch", Summary: "s", Arguments: `{"url":"https://example.com/docs/a.html"}`}) + + res := c.Result("stop") + assert.Contains(t, res.OutputMessages, `"arguments":{"url":"https://example.com/docs/a.html"}`) + assert.Contains(t, res.OutputMessages, `"summary":"s"`) + assert.Empty(t, res.Findings) + }) +} + +func TestContentCollector_AToolTheListDoesNotNameRecordsNoEmptyObject(t *testing.T) { + // `{}` under a name the list does not name is still nothing of that + // tool's input, and a name that redacts away is never one the list + // names: neither part carries arguments. + c := newContentCollector(4096) + c.Handle(agentruntime.ToolUseEvent{Name: "mcp__x__y", Summary: "s", Arguments: `{}`}) + c.Handle(agentruntime.ToolUseEvent{ID: "toolu_01", Name: string(rune(0x200B)), Arguments: `{}`}) + + res := c.Result("stop") + assert.NotContains(t, res.OutputMessages, "arguments") + assert.NotContains(t, res.OutputMessages, "toolu_01", "a nameless part with no summary is not kept") +} + func TestContentCollector_AStringTheNormalizerStrippedIsMaskedWhole(t *testing.T) { // The normalizer strips a terminal escape sequence or tag characters // before the patterns run, and the stripping can take the first @@ -445,13 +539,13 @@ func TestContentCollector_ReplacesRunnerEnvValues(t *testing.T) { "API_KEY": "short", // under minRedactableSecretLen }) c.Handle(agentruntime.TextEvent{Text: "pushing " + secret + " to feature-branch-name, short"}) - c.Handle(agentruntime.ToolUseEvent{ID: "toolu_01", Name: "Bash", Arguments: `{"command":"git push https://x:` + secret + `@host/repo"}`}) + c.Handle(agentruntime.ToolUseEvent{ID: "toolu_01", Name: "Bash", Arguments: `{"command":"deploy --token ` + secret + ` origin"}`}) c.Handle(agentruntime.ToolResultEvent{ID: "toolu_" + secret, Result: "remote: " + secret}) res := c.Result("stop") assert.NotContains(t, res.OutputMessages, secret) assert.Contains(t, res.OutputMessages, `"content":"pushing [REDACTED:PUSH_TOKEN] to feature-branch-name, short"`) - assert.Contains(t, res.OutputMessages, `"arguments":{"command":"git push https://x:[REDACTED:PUSH_TOKEN]@host/repo"}`) + assert.Contains(t, res.OutputMessages, `"arguments":{"command":"deploy --token [REDACTED:PUSH_TOKEN] origin"}`) assert.Contains(t, res.OutputMessages, `{"type":"tool_call_response","response":"remote: [REDACTED:PUSH_TOKEN]"}`, "an id that holds a value is dropped") assert.Equal(t, 4, runnerEnvFindings(res), "one per replaced key per pass per scanned string") assert.Len(t, res.Findings, 4) @@ -550,14 +644,14 @@ func TestContentCollector_DropsTheSummaryWhenTheArgumentsHeldARunnerEnvValue(t * c := newContentCollectorIfEnabled(map[string]string{"PUSH_TOKEN": secret}) c.Handle(agentruntime.ToolUseEvent{ Name: "Bash", - Summary: "git push https://x:" + secret[:20] + "…", - Arguments: `{"command":"git push https://x:` + secret + `@host/repo"}`, + Summary: "deploy --token " + secret[:20] + "…", + Arguments: `{"command":"deploy --token ` + secret + ` origin"}`, }) c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Summary: "ls", Arguments: `{"command":"ls"}`}) res := c.Result("stop") assert.NotContains(t, res.OutputMessages, secret[:5]) - assert.Contains(t, res.OutputMessages, `{"type":"tool_call","name":"Bash","arguments":{"command":"git push https://x:[REDACTED:PUSH_TOKEN]@host/repo"}}`) + assert.Contains(t, res.OutputMessages, `{"type":"tool_call","name":"Bash","arguments":{"command":"deploy --token [REDACTED:PUSH_TOKEN] origin"}}`) assert.Contains(t, res.OutputMessages, `"summary":"ls"`, "a call whose arguments held no value keeps its summary") } @@ -637,31 +731,26 @@ func TestContentCollector_ReplacesARunnerEnvValueSentAsANumber(t *testing.T) { assert.Contains(t, res.OutputMessages, `"arguments":{"timeout":42}`) assert.NotContains(t, res.OutputMessages, digits) assert.Equal(t, 1, runnerEnvFindings(res)) - assert.Equal(t, len("[REDACTED:DEPLOY_PASSWORD]"), res.DroppedBytes) -} - -func TestContentCollector_ANameRedactedAwayTakesItsArgumentsAlong(t *testing.T) { - const args = `{"command":"ls"}` - c := newContentCollector(4096) - c.Handle(agentruntime.ToolUseEvent{ID: "toolu_01", Name: "\u200B", Arguments: args}) - - res := c.Result("stop") - assert.Empty(t, res.OutputMessages, "nothing is left of the part") - assert.Len(t, res.Findings, 1) - assert.True(t, res.Truncated) - assert.Equal(t, len(args), res.DroppedBytes) + assert.Zero(t, res.DroppedBytes, "a member the list leaves out is not charged") } -func TestContentCollector_ANameRedactedAwayMarksTheCallItsSummaryKeeps(t *testing.T) { - const args = `{"command":"ls"}` +func TestContentCollector_ANameRedactedAwayNeverHadArguments(t *testing.T) { + // Arguments are kept only under a tool's exact name, and those names + // are plain words no scan touches: a name that redacts to nothing was + // never one of them, so nothing is lost with it — the part survives + // with its summary, unmarked and uncharged. + name := string(rune(0x200B)) c := newContentCollector(4096) - c.Handle(agentruntime.ToolUseEvent{Name: "\u200B", Summary: "ls", Arguments: args}) + c.Handle(agentruntime.ToolUseEvent{Name: name, Summary: "ls", Arguments: `{"command":"ls"}`}) + c.Handle(agentruntime.ToolUseEvent{ID: "toolu_01", Name: name, Arguments: `{"command":"ls"}`}) res := c.Result("stop") - assert.Contains(t, res.OutputMessages, `"summary":"ls","fullsend.truncated":true`) + assert.Contains(t, res.OutputMessages, `{"type":"tool_call","summary":"ls"}`) assert.NotContains(t, res.OutputMessages, "arguments") - assert.True(t, res.Truncated) - assert.Equal(t, len(args), res.DroppedBytes) + assert.NotContains(t, res.OutputMessages, "toolu_01", "nothing is left of the part without a summary") + assert.Len(t, res.Findings, 2) + assert.False(t, res.Truncated) + assert.Zero(t, res.DroppedBytes) } func TestContentCollector_DroppedBytesCountArgumentsOnEveryDropPath(t *testing.T) { @@ -699,13 +788,13 @@ func TestContentCollector_ADroppedResultIsReportedWithNoPartsLeft(t *testing.T) } func TestContentCollector_ArgumentsAreScannedOnce(t *testing.T) { - // A masked connection string still matches its own pattern, so a second - // pass over redacted arguments would count the same secret twice. + // A connection string counts the pattern's finding and the URL mask's, + // once each, though the call is evicted before Result. c := newContentCollector(64) c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Arguments: `{"command":"psql postgres://u:hunter2hunter2@db/x"}`}) c.Handle(agentruntime.TextEvent{Text: strings.Repeat("x", 200)}) // evicts the call - assert.Len(t, c.Result("stop").Findings, 1) + assert.Equal(t, []string{"db_connection_password", "url_masked"}, findingNames(c.Result("stop").Findings)) } func TestContentCollector_FullwidthQuotesCannotAddAMember(t *testing.T) { @@ -1208,9 +1297,9 @@ func TestContentCollector_RedactionUnderCapDoesNotMarkTruncated(t *testing.T) { func TestContentCollector_CappedResultScannedOnce(t *testing.T) { // The cap path already scans the response; Result must not scan the - // same bytes again — a re-scan re-matches masked db-URL passwords - // (supe... still fits the 4+-char capture) and double-counts the - // finding. + // same bytes again. The URL mask leaves nothing a second pass would + // match, so this pins the count: the pattern's finding and the URL + // mask's, once each. c := newContentCollector(maxContentBytes) c.Handle(agentruntime.ToolResultEvent{ ID: "toolu_db", @@ -1219,8 +1308,7 @@ func TestContentCollector_CappedResultScannedOnce(t *testing.T) { res := c.Result("stop") assert.NotContains(t, res.OutputMessages, "supersecret") - assert.Len(t, res.Findings, 1, - "one secret must yield exactly one finding, not one per scan") + assert.Equal(t, []string{"db_connection_password", "url_masked"}, findingNames(res.Findings)) } func TestContentCollector_CoalescedAfterPreTrimStillScanned(t *testing.T) { @@ -1860,3 +1948,11 @@ func TestAttachInput_WorstCaseFeedbackSharesTheEncodedCeiling(t *testing.T) { assert.Greater(t, len(in)+len(out), maxEncodedContentBytes-100, "the output keeps everything the input left") } + +func findingNames(fs []security.Finding) []string { + names := make([]string, 0, len(fs)) + for _, f := range fs { + names = append(names, f.Name) + } + return names +} diff --git a/internal/cli/redaction_invariant_test.go b/internal/cli/redaction_invariant_test.go index 603327dd8a..9159b2a9d7 100644 --- a/internal/cli/redaction_invariant_test.go +++ b/internal/cli/redaction_invariant_test.go @@ -19,8 +19,9 @@ import ( // a span exports, no 12-byte run of a known secret's body appears, read // three ways — as raw bytes, after the normalizer, and after decoding the // record's JSON with every string walked. A known secret is one a -// SecretRedactor pattern matches, or a runner env / provider-only env -// value the literal pass holds. +// SecretRedactor pattern matches, a runner env / provider-only env +// value the literal pass holds, or what a URL carries before its host or +// after its path — userinfo, a query, a fragment. // // Named exclusions, skipped below with the reason beside them and listed // in docs/guides/dev/tracing.md ("Guarantee and limits"): an env value @@ -88,6 +89,20 @@ func TestRedaction_NoSecretSurvivesAnyWrapping(t *testing.T) { out[i] = secret{name, "AWS_SECRET_ACCESS_KEY=" + b, b, false} case "auth_header": out[i] = secret{name, "Authorization: Bearer " + b[:32], b[:32], false} + case "url_userinfo": + out[i] = secret{name, "https://deploy:" + b[:16] + "@git.example.com/o/r", b[:16], false} + case "url_userinfo_quoted": // shell quoting around the password + out[i] = secret{name, "https://deploy:'" + b[:16] + "'@git.example.com/o/r", b[:16], false} + case "url_userinfo_fullwidth": // a colon and an at sign the normalizer folds + out[i] = secret{name, "https://deploy" + string(rune(0xff1a)) + b[:16] + string(rune(0xff20)) + "git.example.com/o/r", b[:16], false} + case "url_query": + out[i] = secret{name, "https://h.example.com/x?access_token=" + b[:16], b[:16], false} + case "url_query_fullwidth": // a question mark the normalizer folds + out[i] = secret{name, "https://h.example.com/x" + string(rune(0xff1f)) + "access_token=" + b[:16], b[:16], false} + case "url_fragment": + out[i] = secret{name, "https://app.example.com/cb#access_token=" + b[:16], b[:16], false} + case "url_nested": // a URL in another's path + out[i] = secret{name, "https://proxy.example.com/https://u:" + b[:16] + "@h.example.com/x", b[:16], false} } } return out @@ -96,6 +111,8 @@ func TestRedaction_NoSecretSurvivesAnyWrapping(t *testing.T) { mk("github_pat", 100), mk("aws_access_key", 200), mk("runner_env", 300), mk("provider_env", 400), mk("private_key", 500), mk("db_connection_password", 600), mk("json_field", 700), mk("env_assignment", 800), mk("auth_header", 900), + mk("url_userinfo", 1000), mk("url_userinfo_quoted", 1100), mk("url_userinfo_fullwidth", 1200), + mk("url_query", 1300), mk("url_query_fullwidth", 1400), mk("url_fragment", 1500), mk("url_nested", 1600), } // Wrappings over the secret text; the ones that insert a character diff --git a/internal/cli/telemetry_run_test.go b/internal/cli/telemetry_run_test.go index 30bcec0fb1..1697faeaa6 100644 --- a/internal/cli/telemetry_run_test.go +++ b/internal/cli/telemetry_run_test.go @@ -1533,7 +1533,8 @@ func TestAttachContent_SetsContentAndMarkerAttrs(t *testing.T) { OutputMessages: `[{"role":"assistant","parts":[{"type":"text","content":"hi"}]}]`, DroppedBytes: 7, Truncated: true, - Findings: []security.Finding{{Severity: "high"}}, + Findings: []security.Finding{{Severity: "high"}, + {Scanner: "unicode_normalizer", Name: "fullwidth"}, {Scanner: "unicode_normalizer", Name: "ansi_escape"}, {Scanner: "url", Name: "url_masked"}}, }) span.End() @@ -1547,7 +1548,8 @@ func TestAttachContent_SetsContentAndMarkerAttrs(t *testing.T) { assert.Contains(t, attrs[attribute.Key("gen_ai.output.messages")].AsString(), `"type":"text"`) assert.Equal(t, int64(7), attrs[attribute.Key("fullsend.content.dropped_bytes")].AsInt64()) assert.True(t, attrs[attribute.Key("fullsend.content.truncated")].AsBool()) - assert.Equal(t, int64(1), attrs[attribute.Key("fullsend.content.redactions")].AsInt64()) + assert.Equal(t, int64(1), attrs[attribute.Key("fullsend.content.redactions")].AsInt64(), "only what matched a known secret") + assert.Equal(t, int64(3), attrs[attribute.Key("fullsend.content.sanitizations")].AsInt64(), "what was masked for its shape, apart") } func TestAttachContent_EmptyResultAddsNothing(t *testing.T) { From d3c1d7ee2489a62eda1ebd6e8434fd5e386b62a0 Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Wed, 14 Oct 2026 10:22:32 -0400 Subject: [PATCH 36/40] fix(cli): mask a retry prompt cut inside a URL; escaped separators Review of 967e57a5, one medium and two low: - A retry prompt's feedback is cut at maxFeedbackBytes before the collector records it, and a cut inside a URL password takes the '@' that marks it, leaving the password's first bytes where maskURLs no longer sees userinfo. promptHidesAlterations now also masks the recorded copy whole when the feedback was cut and holds a URL maskURLs would mask, beside the compatibility-character clause; cut feedback with neither is still recorded. The invariant test gains a wrapping that puts any secret across the cut with 12 bytes before it; the three ASCII userinfo shapes were red on the input message. - urlRun and the new urlSep accept a separator with its slashes escaped any number of times (`https:\/\/` from a shell or a JSON encoder), and maskURLs masks userinfo from the end of that separator. Two escaped URL shapes join the invariant table, red on every surface before. - The two scanned-once tests count, through a scanner placed ahead of the collector's pipeline, the scans of a word that survives redaction; each fails when its guard is removed (rescanning a capped result in Result, rescanning arguments at eviction). Signed-off-by: Dharit Shah --- docs/guides/dev/tracing.md | 11 ++- .../infrastructure/distributed-tracing.md | 2 +- docs/guides/user/how-to-emit-traces.md | 2 +- internal/cli/content_collector.go | 25 ++++--- internal/cli/content_collector_test.go | 74 +++++++++++++------ internal/cli/redaction_invariant_test.go | 8 ++ internal/cli/run.go | 18 ++++- 7 files changed, 98 insertions(+), 42 deletions(-) diff --git a/docs/guides/dev/tracing.md b/docs/guides/dev/tracing.md index fbde5877e1..2e37ae8112 100644 --- a/docs/guides/dev/tracing.md +++ b/docs/guides/dev/tracing.md @@ -280,11 +280,13 @@ with the summary the parser derives from it) and every wrapping a review has constructed so far — terminal escapes before, around and inside a secret, repeated and paired secrets, zero-width, bidi, tag, NUL and variation-selector characters inside it, a URL's delimiters in -compatibility forms, a password quoted for the shell, a URL nested in -another's path, an invalid byte or the +compatibility forms, separators escaped for the shell or JSON, a +password quoted for the shell, a URL nested in another's path, an +invalid byte or the replacement character inside it, an invalid byte after it with the secret straddling the parser's summary cut, compatibility forms, and -noise, inside the record and across the feedback cut. +noise, inside the record and across the feedback cut, in compatibility +forms or plain with 12 bytes before the cut. Four limits are named there and here, and nowhere else: an env value written in compatibility (fullwidth) forms, since the normalizer rewrites it and the literal pass matches only what the env has; a secret split @@ -336,7 +338,8 @@ ran before the feedback was sanitized and framed, and sanitizing can join a token that scan saw split, so the recorded copy is pattern-scanned again; a prompt the composition altered in a way that pass cannot see — sanitized, or cut at `maxFeedbackBytes` while holding compatibility -characters (`promptHidesAlterations`) — is recorded `***` whole; the +characters or a URL `maskURLs` would mask, since the cut can take the `@` +that marks a password (`promptHidesAlterations`) — is recorded `***` whole; the prompt the agent is sent is not changed, and the mask the first scan leaves for a connection-string password of ten or more bytes matches its own pattern and counts a finding — diff --git a/docs/guides/infrastructure/distributed-tracing.md b/docs/guides/infrastructure/distributed-tracing.md index a87c7017ae..4c3cbc3434 100644 --- a/docs/guides/infrastructure/distributed-tracing.md +++ b/docs/guides/infrastructure/distributed-tracing.md @@ -142,7 +142,7 @@ marked, since the list is the policy and the markers mean loss. A `Bash` `command` is recorded whole, up to 8 KiB, so a heredoc or an inline script records what it writes. A URL anywhere in the record — a member, text, a result, a summary, the input message, a name — has -everything from `://` to its last `@` (userinfo, or a URL nested in +everything from its `://` (or `:\/\/`, escaped for a shell or JSON) to its last `@` (userinfo, or a URL nested in another's path) and everything after its first `?` or `#` masked `***`, after the patterns ran, and counted in `fullsend.content.sanitizations`; the URL runs to the next white space, so in compact JSON the mask can diff --git a/docs/guides/user/how-to-emit-traces.md b/docs/guides/user/how-to-emit-traces.md index 8748292618..497c48509a 100644 --- a/docs/guides/user/how-to-emit-traces.md +++ b/docs/guides/user/how-to-emit-traces.md @@ -132,7 +132,7 @@ prompt the runner composed. Content is redacted for secrets and bounded per iter a string that held a terminal colour or title code or tag characters, as most coloured CLI output does, or that was not valid UTF-8 or held the replacement character (U+FFFD), is recorded as `***` whole, and so is a retry prompt the runner sanitized, or cut at its size bound while it held compatibility -characters; feedback cut for size alone is still recorded. Content may still contain proprietary code or PII: +characters or a URL with userinfo, a query or a fragment; feedback cut otherwise is still recorded. Content may still contain proprietary code or PII: tool arguments carry the commands an agent ran and the paths and patterns it used. A `Write`, `Edit` or `NotebookEdit` body, an `Agent` or `WebFetch` prompt and the input of MCP tools are not recorded as arguments, though a tool's result can quote diff --git a/internal/cli/content_collector.go b/internal/cli/content_collector.go index e368b36776..7f226c096b 100644 --- a/internal/cli/content_collector.go +++ b/internal/cli/content_collector.go @@ -200,8 +200,8 @@ func attachContent(span trace.Span, res contentResult) { // maxEncoded: the two content attributes of one span together stay // within the size maxEncodedContentBytes is proven for. A prompt the // composition altered in a way this pass cannot see — sanitized, or cut -// while holding compatibility characters (promptHidesAlterations) — is -// recorded "***" whole. +// while holding compatibility characters or a URL maskURLs would mask +// (promptHidesAlterations) — is recorded "***" whole. func (c *contentCollector) attachInput(span trace.Span, prompt string, alteredUpstream bool) { if c == nil || prompt == "" { return @@ -809,15 +809,20 @@ func redactText(pipeline *security.Pipeline, runnerEnv map[string]string, text s return maskURLs(text, findings) } -// urlRun matches a URL as a scheme, "://" and the rest of the run up to -// white space — the only end it has, so a quote inside it (a password -// quoted for the shell, a query quoted half-way) never ends it early. -var urlRun = regexp.MustCompile(`[A-Za-z][A-Za-z0-9+.-]*://\S+`) +// urlRun matches a URL as a scheme, its separator (urlSep) and the rest +// of the run up to white space — the only end it has, so a quote inside +// it (a password quoted for the shell, a query quoted half-way) never +// ends it early. +var urlRun = regexp.MustCompile(`[A-Za-z][A-Za-z0-9+.-]*:(?:\\*/){2}\S+`) + +// urlSep matches a URL's "://", or the same with its slashes escaped, as +// a shell command or a JSON encoder writes them, once or more. +var urlSep = regexp.MustCompile(`:(?:\\*/){2}`) // maskURLs masks what a URL carries that no pattern is written for: // everything after its first '?' or '#' (a presigned signature, an access -// token, a token in a fragment) and everything from "://" to its last -// '@' (userinfo, or a URL nested in another's path), as ***, with one +// token, a token in a fragment) and everything from its separator to its +// last '@' (userinfo, or a URL nested in another's path), as ***, with one // finding for a string that had any. It runs last in redactText, on text // the normalizer has folded and the patterns have masked, so a // compatibility '？' or '＠' is a plain one here, and nothing is @@ -830,8 +835,8 @@ func maskURLs(text string, findings *[]security.Finding) string { if i := strings.IndexAny(out, "?#"); i >= 0 { out = out[:i+1] + "***" } - if s, a := strings.Index(out, "://"), strings.LastIndex(out, "@"); a > s { - out = out[:s+3] + "***" + out[a:] + if s, a := urlSep.FindStringIndex(out)[1], strings.LastIndex(out, "@"); a > s { + out = out[:s] + "***" + out[a:] } masked = masked || out != run return out diff --git a/internal/cli/content_collector_test.go b/internal/cli/content_collector_test.go index 655fa1e44c..b364fcc305 100644 --- a/internal/cli/content_collector_test.go +++ b/internal/cli/content_collector_test.go @@ -249,18 +249,20 @@ func TestContentCollector_AURLsUserinfoQueryAndFragmentAreMasked(t *testing.T) { for name, tc := range map[string]struct { tool, args, want string }{ - "a password before the host": {"WebFetch", `{"url":"https://alice:` + pw + `@example.com/repo"}`, `{"url":"https://***@example.com/repo"}`}, - "a presigned query": {"WebFetch", `{"url":"https://b.s3.amazonaws.com/k?X-Amz-Credential=AKIA` + pw + `&X-Amz-Signature=` + pw + `"}`, `{"url":"https://b.s3.amazonaws.com/k?***"}`}, - "an access token in the query": {"Bash", `{"command":"curl 'https://api.example.com/v1/me?access_token=` + pw + `' -s"}`, `{"command":"curl 'https://api.example.com/v1/me?*** -s"}`}, - "a token in a clone URL": {"Bash", `{"command":"git clone https://x-access-token:` + pw + `@github.com/o/r.git /tmp/r"}`, `{"command":"git clone https://***@github.com/o/r.git /tmp/r"}`}, - "a token behind the fragment": {"Bash", `{"command":"open https://app.example.com/cb#access_token=` + pw + `"}`, `{"command":"open https://app.example.com/cb#***"}`}, - "a password quoted for the shell": {"Bash", `{"command":"git clone https://deploy:'` + pw + `'@git.example.com/o/r"}`, `{"command":"git clone https://***@git.example.com/o/r"}`}, - "a query quoted half-way": {"Bash", `{"command":"curl 'https://h.example.com/x'?token=` + pw + `"}`, `{"command":"curl 'https://h.example.com/x'?***"}`}, - "a URL nested in another's path": {"Bash", `{"command":"curl https://proxy.example.com/https://u:` + pw + `@h.example.com/x"}`, `{"command":"curl https://***@h.example.com/x"}`}, - "a fullwidth question mark": {"Bash", `{"command":"curl https://api.example.com/v1/me` + fw(0xff1f) + `access_token=` + pw + `"}`, `{"command":"curl https://api.example.com/v1/me?***"}`}, - "a fullwidth at sign": {"Bash", `{"command":"git clone https://x-access-token` + fw(0xff1a) + pw + fw(0xff20) + `github.com/o/r"}`, `{"command":"git clone https://***@github.com/o/r"}`}, - "a zero-width split scheme": {"Bash", `{"command":"curl https:` + fw(0x200b) + `//h.example.com/p?access_token=` + pw + `"}`, `{"command":"curl https://h.example.com/p?***"}`}, - "a fullwidth token in the path": {"Bash", `{"command":"curl https://h.example.com/ghp_` + ghpFull.String() + `?x=1"}`, `{"command":"curl https://h.example.com/ghp_...?***"}`}, + "a password before the host": {"WebFetch", `{"url":"https://alice:` + pw + `@example.com/repo"}`, `{"url":"https://***@example.com/repo"}`}, + "a presigned query": {"WebFetch", `{"url":"https://b.s3.amazonaws.com/k?X-Amz-Credential=AKIA` + pw + `&X-Amz-Signature=` + pw + `"}`, `{"url":"https://b.s3.amazonaws.com/k?***"}`}, + "an access token in the query": {"Bash", `{"command":"curl 'https://api.example.com/v1/me?access_token=` + pw + `' -s"}`, `{"command":"curl 'https://api.example.com/v1/me?*** -s"}`}, + "a token in a clone URL": {"Bash", `{"command":"git clone https://x-access-token:` + pw + `@github.com/o/r.git /tmp/r"}`, `{"command":"git clone https://***@github.com/o/r.git /tmp/r"}`}, + "a token behind the fragment": {"Bash", `{"command":"open https://app.example.com/cb#access_token=` + pw + `"}`, `{"command":"open https://app.example.com/cb#***"}`}, + "a password quoted for the shell": {"Bash", `{"command":"git clone https://deploy:'` + pw + `'@git.example.com/o/r"}`, `{"command":"git clone https://***@git.example.com/o/r"}`}, + "a query quoted half-way": {"Bash", `{"command":"curl 'https://h.example.com/x'?token=` + pw + `"}`, `{"command":"curl 'https://h.example.com/x'?***"}`}, + "a URL nested in another's path": {"Bash", `{"command":"curl https://proxy.example.com/https://u:` + pw + `@h.example.com/x"}`, `{"command":"curl https://***@h.example.com/x"}`}, + "a fullwidth question mark": {"Bash", `{"command":"curl https://api.example.com/v1/me` + fw(0xff1f) + `access_token=` + pw + `"}`, `{"command":"curl https://api.example.com/v1/me?***"}`}, + "a fullwidth at sign": {"Bash", `{"command":"git clone https://x-access-token` + fw(0xff1a) + pw + fw(0xff20) + `github.com/o/r"}`, `{"command":"git clone https://***@github.com/o/r"}`}, + "a zero-width split scheme": {"Bash", `{"command":"curl https:` + fw(0x200b) + `//h.example.com/p?access_token=` + pw + `"}`, `{"command":"curl https://h.example.com/p?***"}`}, + "a fullwidth token in the path": {"Bash", `{"command":"curl https://h.example.com/ghp_` + ghpFull.String() + `?x=1"}`, `{"command":"curl https://h.example.com/ghp_...?***"}`}, + "separators escaped for the shell": {"Bash", `{"command":"curl https:\\\\/\\\\/h.example.com/p?access_token=` + pw + `"}`, `{"command":"curl https:\\\\/\\\\/h.example.com/p?***"}`}, + "a password behind escaped separators": {"Bash", `{"command":"git clone https:\\\\/\\\\/deploy:` + pw + `@git.example.com/o/r"}`, `{"command":"git clone https:\\\\/\\\\/***@git.example.com/o/r"}`}, } { t.Run(name, func(t *testing.T) { c := newContentCollector(4096) @@ -295,10 +297,12 @@ func TestContentCollector_AURLsUserinfoQueryAndFragmentAreMasked(t *testing.T) { c := newContentCollector(4096) c.Handle(agentruntime.TextEvent{Text: "pushing to https://deploy:" + pw + "@git.example.com/o/r now"}) c.Handle(agentruntime.ToolResultEvent{ID: "toolu_01", Result: "origin\thttps://deploy:" + pw + "@git.example.com/o/r (fetch)"}) + c.Handle(agentruntime.ToolResultEvent{ID: "toolu_02", Result: `{"next":"https:\/\/api.example.com\/v1?access_token=` + pw + `"}`}) res := c.Result("stop") assert.Contains(t, res.OutputMessages, `"content":"pushing to https://***@git.example.com/o/r now"`) assert.Contains(t, res.OutputMessages, `"response":"origin\thttps://***@git.example.com/o/r (fetch)"`) + assert.Contains(t, res.OutputMessages, `/v1?***"`, "a JSON encoder's escaped slashes") assert.NotContains(t, res.OutputMessages, pw) }) t.Run("a URL with none of them is kept as written", func(t *testing.T) { @@ -457,9 +461,10 @@ func TestAttachInput_APromptAlteredUpstreamIsMaskedWhole(t *testing.T) { // the sanitizer stripped: a colour code that ate a token's first // letter plus a zero-width character that kept the token from the // pattern scan leave "hp_" and the body; and a token in compatibility - // forms straddling the cut is split before anything can fold it. Both - // are recorded masked whole (promptHidesAlterations). Big plain - // feedback is not. + // forms straddling the cut is split before anything can fold it; and a + // URL password straddling the cut loses the '@' that marks it. All are + // recorded masked whole (promptHidesAlterations). Big plain feedback, + // a URL with nothing to mask in it included, is not. token := "ghp_" + strings.Repeat("r", 36) var fw strings.Builder for _, r := range token { @@ -472,7 +477,8 @@ func TestAttachInput_APromptAlteredUpstreamIsMaskedWhole(t *testing.T) { }{ "a colour code ate the first letter of a zero-width-split token": {"\x1b[" + token[:20] + "\u200b" + token[20:], true}, "a fullwidth token straddling the cut": {strings.Repeat("x", maxFeedbackBytes-48) + " " + fw.String(), true}, - "plain feedback over the cut": {plain, false}, + "a URL password straddling the cut, 12 bytes before it": {strings.Repeat("x", maxFeedbackBytes-28) + " https://deploy:" + token[4:20] + "@git.example.com/o/r", true}, + "plain feedback over the cut": {plain + " see https://example.com/docs", false}, } { t.Run(name, func(t *testing.T) { fb := redactFeedback(tc.feedback, nil) @@ -788,13 +794,15 @@ func TestContentCollector_ADroppedResultIsReportedWithNoPartsLeft(t *testing.T) } func TestContentCollector_ArgumentsAreScannedOnce(t *testing.T) { - // A connection string counts the pattern's finding and the URL mask's, - // once each, though the call is evicted before Result. + // Arguments are scanned when the event is handled, and neither + // eviction nor Result scans them again. c := newContentCollector(64) + scans := countScans(c, "psql") // survives redaction, so a second scan would see it c.Handle(agentruntime.ToolUseEvent{Name: "Bash", Arguments: `{"command":"psql postgres://u:hunter2hunter2@db/x"}`}) c.Handle(agentruntime.TextEvent{Text: strings.Repeat("x", 200)}) // evicts the call assert.Equal(t, []string{"db_connection_password", "url_masked"}, findingNames(c.Result("stop").Findings)) + assert.Equal(t, 1, *scans) } func TestContentCollector_FullwidthQuotesCannotAddAMember(t *testing.T) { @@ -1297,18 +1305,18 @@ func TestContentCollector_RedactionUnderCapDoesNotMarkTruncated(t *testing.T) { func TestContentCollector_CappedResultScannedOnce(t *testing.T) { // The cap path already scans the response; Result must not scan the - // same bytes again. The URL mask leaves nothing a second pass would - // match, so this pins the count: the pattern's finding and the URL - // mask's, once each. + // same bytes again (bulkScanned). c := newContentCollector(maxContentBytes) + scans := countScans(c, "scanned-once") // survives redaction and the cut c.Handle(agentruntime.ToolResultEvent{ ID: "toolu_db", - Result: strings.Repeat("x", maxToolResultBytes) + " postgres://user:supersecret@host", + Result: strings.Repeat("x", maxToolResultBytes) + " scanned-once postgres://user:supersecret@host", }) res := c.Result("stop") assert.NotContains(t, res.OutputMessages, "supersecret") assert.Equal(t, []string{"db_connection_password", "url_masked"}, findingNames(res.Findings)) + assert.Equal(t, 1, *scans) } func TestContentCollector_CoalescedAfterPreTrimStillScanned(t *testing.T) { @@ -1949,6 +1957,28 @@ func TestAttachInput_WorstCaseFeedbackSharesTheEncodedCeiling(t *testing.T) { "the output keeps everything the input left") } +// scanCounter counts the strings holding marker that reach a pipeline. +type scanCounter struct { + marker string + n *int +} + +func (s scanCounter) Name() string { return "scan_counter" } + +func (s scanCounter) Scan(text string) security.ScanResult { + if strings.Contains(text, s.marker) { + *s.n++ + } + return security.ScanResult{Safe: true} +} + +// countScans puts a scanCounter ahead of c's pipeline and returns its count. +func countScans(c *contentCollector, marker string) *int { + n := new(int) + c.pipeline = security.NewPipeline(scanCounter{marker, n}, security.NewUnicodeNormalizer(), security.NewSecretRedactor()) + return n +} + func findingNames(fs []security.Finding) []string { names := make([]string, 0, len(fs)) for _, f := range fs { diff --git a/internal/cli/redaction_invariant_test.go b/internal/cli/redaction_invariant_test.go index 9159b2a9d7..4f5d19fa52 100644 --- a/internal/cli/redaction_invariant_test.go +++ b/internal/cli/redaction_invariant_test.go @@ -101,6 +101,10 @@ func TestRedaction_NoSecretSurvivesAnyWrapping(t *testing.T) { out[i] = secret{name, "https://h.example.com/x" + string(rune(0xff1f)) + "access_token=" + b[:16], b[:16], false} case "url_fragment": out[i] = secret{name, "https://app.example.com/cb#access_token=" + b[:16], b[:16], false} + case "url_userinfo_escaped": // separators escaped for the shell or JSON + out[i] = secret{name, `https:\/\/deploy:` + b[:16] + `@git.example.com\/o\/r`, b[:16], false} + case "url_query_escaped": + out[i] = secret{name, `https:\/\/h.example.com\/x?access_token=` + b[:16], b[:16], false} case "url_nested": // a URL in another's path out[i] = secret{name, "https://proxy.example.com/https://u:" + b[:16] + "@h.example.com/x", b[:16], false} } @@ -113,6 +117,7 @@ func TestRedaction_NoSecretSurvivesAnyWrapping(t *testing.T) { mk("env_assignment", 800), mk("auth_header", 900), mk("url_userinfo", 1000), mk("url_userinfo_quoted", 1100), mk("url_userinfo_fullwidth", 1200), mk("url_query", 1300), mk("url_query_fullwidth", 1400), mk("url_fragment", 1500), mk("url_nested", 1600), + mk("url_userinfo_escaped", 1700), mk("url_query_escaped", 1800), } // Wrappings over the secret text; the ones that insert a character @@ -179,6 +184,9 @@ func TestRedaction_NoSecretSurvivesAnyWrapping(t *testing.T) { {"fullwidth body", func(a, _ secret) string { return fullwidth(a) }, envExcluded, false}, {"inside 2 KiB of noise each side", func(a, _ secret) string { return noise(1, 2048) + " " + a.text + " " + noise(2, 2048) }, none, false}, {"fullwidth body straddling the feedback cut", func(a, _ secret) string { return noise(3, maxFeedbackBytes-48) + " " + fullwidth(a) }, envExcluded, false}, + {"body straddling the feedback cut, 12 bytes before it", func(a, _ secret) string { + return noise(5, maxFeedbackBytes-1-window-strings.Index(a.text, a.body)) + " " + a.text + }, none, false}, } // Views of one exported string. diff --git a/internal/cli/run.go b/internal/cli/run.go index 6978fc2b2d..56bf4e392d 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -3879,11 +3879,21 @@ func buildFeedbackPrompt(feedback string) (string, int) { // sanitizer stripped something (sanitized > 0), which the collector's own // normalizer can then no longer see — a stripped colour code can take a // token's first letter with it, and what is left matches no pattern — or -// the feedback was cut at maxFeedbackBytes while it held compatibility -// characters, so the cut may have split a secret written in those forms, -// which only the collector's fold would have recognised whole. +// the feedback was cut at maxFeedbackBytes while it held what the +// collector recognises only whole: compatibility characters, which only +// its fold joins, or a URL whose userinfo, query or fragment maskURLs +// masks — a cut can take the '@' that marks userinfo and leave the +// password before it. Feedback cut with neither is recorded. func promptHidesAlterations(feedback string, sanitized int) bool { - return sanitized > 0 || (len(feedback) > maxFeedbackBytes && !security.NewUnicodeNormalizer().Scan(feedback).Safe) + if sanitized > 0 { + return true + } + if len(feedback) <= maxFeedbackBytes { + return false + } + var urls []security.Finding + maskURLs(feedback, &urls) + return len(urls) > 0 || !security.NewUnicodeNormalizer().Scan(feedback).Safe } // sanitizeFeedbackUnicode strips dangerous non-rendering Unicode characters From 69d849e1762abb260d1b638361efc4d27cbfd605 Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Wed, 14 Oct 2026 19:48:27 -0400 Subject: [PATCH 37/40] fix(cli): end a URL at a quote that closes a JSON string Maintainer review of d3c1d7ee: a URL ran to the next white space, so in minified JSON (raw gh api output) the query mask took the fields after the URL, and an email in a later field read as userinfo. urlRun now ends a URL at white space or at a double quote that could close a JSON string: one followed by white space, ',', ':', '}', ']' or the end of the text. A URL inside JSON ends with its string and the fields after it are kept. A quote followed by anything else, as in a password or a query quoted for the shell, and a backslash-escaped character, as in JSON escaped inside another string, do not end it. Userinfo still runs from the separator to the last '@' in the URL. The pre-push review found that bounding it at the first '/' keeps a password holding a '/' (a base64 secret), and that ending at any quote keeps a password holding one; the invariant table gains both shapes, red on the bounded rule, and the quote rule is the one the review probed. An '@' in a path is masked with the userinfo, the safe direction. On the three captured streams the URL mask hides 1,424 bytes instead of 2,492; the minified gh api stream goes from 1,248 bytes to 180, with the same 54 runs masked. The reference, the dev guide and the user guide say where a URL ends, and the named limit for a secret no pattern matches lists what the mask does not read: a token in a URL's path, what follows a quote that ends a URL, and a URL written percent-encoded or with an escaped colon. Signed-off-by: Dharit Shah --- docs/guides/dev/tracing.md | 5 +++- .../infrastructure/distributed-tracing.md | 18 ++++++++----- docs/guides/user/how-to-emit-traces.md | 5 ++-- internal/cli/content_collector.go | 26 ++++++++++++------- internal/cli/content_collector_test.go | 19 +++++++++++++- internal/cli/redaction_invariant_test.go | 9 ++++++- 6 files changed, 61 insertions(+), 21 deletions(-) diff --git a/docs/guides/dev/tracing.md b/docs/guides/dev/tracing.md index 2e37ae8112..bed212d88e 100644 --- a/docs/guides/dev/tracing.md +++ b/docs/guides/dev/tracing.md @@ -291,7 +291,10 @@ Four limits are named there and here, and nowhere else: an env value written in compatibility (fullwidth) forms, since the normalizer rewrites it and the literal pass matches only what the env has; a secret split across two parts or events; a secret that matches no pattern and is in no -env, a token in a URL's path among them; a summary a pi or codex parser cut before the collector saw it, since +env, among them a token in a URL's path, what follows a double quote +that ends a URL (one that could close a JSON string, so a URL inside +JSON ends with its string), and a URL written percent-encoded or with +an escaped colon, which is not read as one; a summary a pi or codex parser cut before the collector saw it, since that cut can split a value the parser's own scan did not see joined (a Claude summary is cut from a member the collector scans whole). A finding inside the guarantee is a defect; one on this list is a documented limit. diff --git a/docs/guides/infrastructure/distributed-tracing.md b/docs/guides/infrastructure/distributed-tracing.md index 4c3cbc3434..0e7a973151 100644 --- a/docs/guides/infrastructure/distributed-tracing.md +++ b/docs/guides/infrastructure/distributed-tracing.md @@ -142,12 +142,18 @@ marked, since the list is the policy and the markers mean loss. A `Bash` `command` is recorded whole, up to 8 KiB, so a heredoc or an inline script records what it writes. A URL anywhere in the record — a member, text, a result, a summary, the input message, a name — has -everything from its `://` (or `:\/\/`, escaped for a shell or JSON) to its last `@` (userinfo, or a URL nested in -another's path) and everything after its first `?` or `#` masked `***`, -after the patterns ran, and counted in `fullsend.content.sanitizations`; -the URL runs to the next white space, so in compact JSON the mask can -take the rest of the line. A token carried in the path itself, as a -webhook URL's is, matches no pattern and is kept. The kept +everything from its `://` (or `:\/\/`, escaped for a shell or JSON) to its last `@` +(userinfo, a password holding a `/` or a URL nested in another's path +included; an `@` in a path is masked with it) and everything after its +first `?` or `#` masked `***`, after the patterns ran, and counted in +`fullsend.content.sanitizations`. The URL runs to white space or to a +double quote that could close a JSON string — one followed by white +space, `,`, `:`, `}`, `]` or the end — so one inside JSON ends with its +string and the fields after it are kept; inside JSON escaped in another +string (`\"`) it runs on. Kept as well, under the named limit for a +secret no pattern matches: a token carried in the path itself, as a +webhook URL's is; what follows a quote that ends a URL; a URL written +percent-encoded or with an escaped colon. The kept members are decoded, redacted one by one and encoded again, so key order, spacing and escapes are not the stream's; a string a terminal escape sequence or tag characters were diff --git a/docs/guides/user/how-to-emit-traces.md b/docs/guides/user/how-to-emit-traces.md index 497c48509a..fe6b156060 100644 --- a/docs/guides/user/how-to-emit-traces.md +++ b/docs/guides/user/how-to-emit-traces.md @@ -138,8 +138,9 @@ A `Write`, `Edit` or `NotebookEdit` body, an `Agent` or `WebFetch` prompt and th input of MCP tools are not recorded as arguments, though a tool's result can quote what it wrote and a call's summary keeps the first 120 characters of its input; a `Bash` command is recorded whole (up to 8 KiB) and can embed the file content -it writes. A URL anywhere in the record has its userinfo, query and fragment -masked. Make sure your backend's +it writes. A URL in the record has its userinfo, query and fragment masked, as +written in a command, text or JSON; the developer guide's tracing page lists the +forms it does not read. Make sure your backend's access controls fit before enabling it. ```bash diff --git a/internal/cli/content_collector.go b/internal/cli/content_collector.go index 7f226c096b..f58b988a7a 100644 --- a/internal/cli/content_collector.go +++ b/internal/cli/content_collector.go @@ -810,10 +810,13 @@ func redactText(pipeline *security.Pipeline, runnerEnv map[string]string, text s } // urlRun matches a URL as a scheme, its separator (urlSep) and the rest -// of the run up to white space — the only end it has, so a quote inside -// it (a password quoted for the shell, a query quoted half-way) never -// ends it early. -var urlRun = regexp.MustCompile(`[A-Za-z][A-Za-z0-9+.-]*:(?:\\*/){2}\S+`) +// of the run, up to white space or a double quote that could close a +// JSON string — one followed by white space, ',', ':', '}', ']' or the +// end of the text — so a URL inside JSON ends with its string and the +// fields after it are not masked with it. Any other quote (a password or +// a query quoted half-way for the shell) and a backslash-escaped +// character do not end it. +var urlRun = regexp.MustCompile(`[A-Za-z][A-Za-z0-9+.-]*:(?:\\*/){2}(?:[^\s"\\]|\\.|"[^\s"\\,:}\]])+`) // urlSep matches a URL's "://", or the same with its slashes escaped, as // a shell command or a JSON encoder writes them, once or more. @@ -822,12 +825,15 @@ var urlSep = regexp.MustCompile(`:(?:\\*/){2}`) // maskURLs masks what a URL carries that no pattern is written for: // everything after its first '?' or '#' (a presigned signature, an access // token, a token in a fragment) and everything from its separator to its -// last '@' (userinfo, or a URL nested in another's path), as ***, with one -// finding for a string that had any. It runs last in redactText, on text -// the normalizer has folded and the patterns have masked, so a -// compatibility '？' or '＠' is a plain one here, and nothing is -// re-encoded: the mask only replaces bytes. A token carried in the path -// itself, as a webhook URL's is, matches no pattern and is kept. +// last '@' (userinfo, a password holding a '/' or a URL nested in +// another's path included; an '@' in a path is masked with it), as ***, +// with one finding for a string that had any. It runs last in +// redactText, on text the normalizer has folded and the patterns have +// masked, so a compatibility '？' or '＠' is a plain one here, and nothing +// is re-encoded: the mask only replaces bytes. Kept: a token carried in +// the path itself, as a webhook URL's is; a URL holding a quote that +// ends it (see urlRun), past that quote; a URL written percent-encoded +// or with an escaped colon, which is not read as one. func maskURLs(text string, findings *[]security.Finding) string { masked := false text = urlRun.ReplaceAllStringFunc(text, func(run string) string { diff --git a/internal/cli/content_collector_test.go b/internal/cli/content_collector_test.go index b364fcc305..7c628fbf35 100644 --- a/internal/cli/content_collector_test.go +++ b/internal/cli/content_collector_test.go @@ -257,6 +257,9 @@ func TestContentCollector_AURLsUserinfoQueryAndFragmentAreMasked(t *testing.T) { "a password quoted for the shell": {"Bash", `{"command":"git clone https://deploy:'` + pw + `'@git.example.com/o/r"}`, `{"command":"git clone https://***@git.example.com/o/r"}`}, "a query quoted half-way": {"Bash", `{"command":"curl 'https://h.example.com/x'?token=` + pw + `"}`, `{"command":"curl 'https://h.example.com/x'?***"}`}, "a URL nested in another's path": {"Bash", `{"command":"curl https://proxy.example.com/https://u:` + pw + `@h.example.com/x"}`, `{"command":"curl https://***@h.example.com/x"}`}, + "a password holding a '/'": {"Bash", `{"command":"git clone https://x-access-token:ab+c/` + pw + `=@github.com/o/r"}`, `{"command":"git clone https://***@github.com/o/r"}`}, + "a query holding a quote": {"Bash", `{"command":"curl 'https://h.example.com/?q=\"a\"&token=` + pw + `'"}`, `{"command":"curl 'https://h.example.com/?***"}`}, + "a query after a closing quote": {"Bash", `{"command":"curl \"https://h.example.com/x\"?token=` + pw + `"}`, `{"command":"curl \"https://h.example.com/x\"?***"}`}, "a fullwidth question mark": {"Bash", `{"command":"curl https://api.example.com/v1/me` + fw(0xff1f) + `access_token=` + pw + `"}`, `{"command":"curl https://api.example.com/v1/me?***"}`}, "a fullwidth at sign": {"Bash", `{"command":"git clone https://x-access-token` + fw(0xff1a) + pw + fw(0xff20) + `github.com/o/r"}`, `{"command":"git clone https://***@github.com/o/r"}`}, "a zero-width split scheme": {"Bash", `{"command":"curl https:` + fw(0x200b) + `//h.example.com/p?access_token=` + pw + `"}`, `{"command":"curl https://h.example.com/p?***"}`}, @@ -302,9 +305,23 @@ func TestContentCollector_AURLsUserinfoQueryAndFragmentAreMasked(t *testing.T) { res := c.Result("stop") assert.Contains(t, res.OutputMessages, `"content":"pushing to https://***@git.example.com/o/r now"`) assert.Contains(t, res.OutputMessages, `"response":"origin\thttps://***@git.example.com/o/r (fetch)"`) - assert.Contains(t, res.OutputMessages, `/v1?***"`, "a JSON encoder's escaped slashes") + assert.Contains(t, res.OutputMessages, `/v1?***\"}`, "a JSON encoder's escaped slashes") assert.NotContains(t, res.OutputMessages, pw) }) + t.Run("a URL in minified JSON masks nothing past its closing quote", func(t *testing.T) { + c := newContentCollector(4096) + c.Handle(agentruntime.ToolResultEvent{ID: "toolu_01", Result: `{"html_url":"https://github.com/o/r/pull/1?x=1","title":"Fix the parser"}`}) + c.Handle(agentruntime.ToolResultEvent{ID: "toolu_02", Result: `{"url":"https://github.com/o/r","author":{"email":"dev@example.com"}}`}) + c.Handle(agentruntime.ToolResultEvent{ID: "toolu_03", Result: `{"url":"https://github.com","email":"dev@example.com"}`}) + + res := c.Result("stop") + parts := decodeOutputMessages(t, res.OutputMessages)[0]["parts"].([]any) + require.Len(t, parts, 3) + assert.Equal(t, `{"html_url":"https://github.com/o/r/pull/1?***","title":"Fix the parser"}`, parts[0].(map[string]any)["response"]) + assert.Equal(t, `{"url":"https://github.com/o/r","author":{"email":"dev@example.com"}}`, parts[1].(map[string]any)["response"], "the quote ends the URL before the email") + assert.Equal(t, `{"url":"https://github.com","email":"dev@example.com"}`, parts[2].(map[string]any)["response"], "the quote ends the URL") + assert.Len(t, res.Findings, 1, "only the query was masked") + }) t.Run("a URL with none of them is kept as written", func(t *testing.T) { c := newContentCollector(4096) c.Handle(agentruntime.ToolUseEvent{Name: "WebFetch", Summary: "s", Arguments: `{"url":"https://example.com/docs/a.html"}`}) diff --git a/internal/cli/redaction_invariant_test.go b/internal/cli/redaction_invariant_test.go index 4f5d19fa52..9ca82f2ce8 100644 --- a/internal/cli/redaction_invariant_test.go +++ b/internal/cli/redaction_invariant_test.go @@ -28,7 +28,9 @@ import ( // written in compatibility (fullwidth) forms, since the normalizer // rewrites it and the literal pass matches only what the env has; a // secret split across two parts or events; a secret matching no pattern -// and in no env; a summary a pi or codex parser cut before the collector +// and in no env — a token in a URL's path, what follows a double quote +// that ends a URL (urlRun), a URL written percent-encoded or with an +// escaped colon among them; a summary a pi or codex parser cut before the collector // saw it, since that cut can split a value the parser's own scan did not // see joined (a Claude summary is cut from a member the collector scans // whole). Nothing else is excluded. @@ -105,6 +107,10 @@ func TestRedaction_NoSecretSurvivesAnyWrapping(t *testing.T) { out[i] = secret{name, `https:\/\/deploy:` + b[:16] + `@git.example.com\/o\/r`, b[:16], false} case "url_query_escaped": out[i] = secret{name, `https:\/\/h.example.com\/x?access_token=` + b[:16], b[:16], false} + case "url_userinfo_slash": // a base64 password holds a '/' + out[i] = secret{name, "https://deploy:ab+c/" + b[:16] + "=@git.example.com/o/r", b[:16], false} + case "url_userinfo_double_quote": // a password holding a quote + out[i] = secret{name, "https://deploy:" + b[:16] + `"x@git.example.com/o/r`, b[:16], false} case "url_nested": // a URL in another's path out[i] = secret{name, "https://proxy.example.com/https://u:" + b[:16] + "@h.example.com/x", b[:16], false} } @@ -118,6 +124,7 @@ func TestRedaction_NoSecretSurvivesAnyWrapping(t *testing.T) { mk("url_userinfo", 1000), mk("url_userinfo_quoted", 1100), mk("url_userinfo_fullwidth", 1200), mk("url_query", 1300), mk("url_query_fullwidth", 1400), mk("url_fragment", 1500), mk("url_nested", 1600), mk("url_userinfo_escaped", 1700), mk("url_query_escaped", 1800), + mk("url_userinfo_slash", 1900), mk("url_userinfo_double_quote", 2000), } // Wrappings over the secret text; the ones that insert a character From 845c0ee7ae4f685be0f81ecea5beebf32e011548 Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Thu, 15 Oct 2026 08:26:34 -0400 Subject: [PATCH 38/40] fix(cli): mask a URL whole past its separator when a ? or # precedes its last @ Maintainer review of 69d849e1: maskURLs cut a URL at its first '?' or '#' before it looked for the last '@', so a password holding one lost the '@' that marks it and its first bytes were recorded. The suggested order, userinfo first and the query cut after, opens the mirror case: an '@' in the query takes the '?' into the userinfo mask and the rest of the query survives. When the first '?' or '#' comes before the last '@' the two cannot be told apart, so everything after the separator is masked; otherwise userinfo is masked and the first '?' or '#' of what follows cuts the rest, as before. Two invariant rows, a secret before a '?' and before a '#' in the password, were red and are green; unit rows for both and for an '@' in the query. On the three captured streams no URL run changes. The named limit for a quote that ends a URL now reads: what follows the quote, and userinfo whose '@' follows it (promised in the review bot's thread on 69d849e1). Signed-off-by: Dharit Shah --- docs/guides/dev/tracing.md | 9 +++--- .../infrastructure/distributed-tracing.md | 15 +++++---- internal/cli/content_collector.go | 32 ++++++++++++------- internal/cli/content_collector_test.go | 3 ++ internal/cli/redaction_invariant_test.go | 13 +++++--- 5 files changed, 46 insertions(+), 26 deletions(-) diff --git a/docs/guides/dev/tracing.md b/docs/guides/dev/tracing.md index bed212d88e..b6a71fff20 100644 --- a/docs/guides/dev/tracing.md +++ b/docs/guides/dev/tracing.md @@ -291,10 +291,11 @@ Four limits are named there and here, and nowhere else: an env value written in compatibility (fullwidth) forms, since the normalizer rewrites it and the literal pass matches only what the env has; a secret split across two parts or events; a secret that matches no pattern and is in no -env, among them a token in a URL's path, what follows a double quote -that ends a URL (one that could close a JSON string, so a URL inside -JSON ends with its string), and a URL written percent-encoded or with -an escaped colon, which is not read as one; a summary a pi or codex parser cut before the collector saw it, since +env, among them a token in a URL's path, in a URL holding a double quote +that ends it (one that could close a JSON string, so a URL inside JSON +ends with its string), what follows the quote and userinfo whose `@` +follows it, and a URL written percent-encoded or with an escaped colon, +which is not read as one; a summary a pi or codex parser cut before the collector saw it, since that cut can split a value the parser's own scan did not see joined (a Claude summary is cut from a member the collector scans whole). A finding inside the guarantee is a defect; one on this list is a documented limit. diff --git a/docs/guides/infrastructure/distributed-tracing.md b/docs/guides/infrastructure/distributed-tracing.md index 0e7a973151..7e5d394404 100644 --- a/docs/guides/infrastructure/distributed-tracing.md +++ b/docs/guides/infrastructure/distributed-tracing.md @@ -143,17 +143,20 @@ marked, since the list is the policy and the markers mean loss. A `Bash` script records what it writes. A URL anywhere in the record — a member, text, a result, a summary, the input message, a name — has everything from its `://` (or `:\/\/`, escaped for a shell or JSON) to its last `@` -(userinfo, a password holding a `/` or a URL nested in another's path -included; an `@` in a path is masked with it) and everything after its -first `?` or `#` masked `***`, after the patterns ran, and counted in -`fullsend.content.sanitizations`. The URL runs to white space or to a +(userinfo — a password holding a `/` or a URL nested in another's path +included; an `@` in a path is masked with it, the host too) and +everything after its first `?` or `#` masked `***`, after the patterns +ran, and counted in `fullsend.content.sanitizations`; when the first `?` +or `#` comes before the last `@` — a password holding one, an `@` in the +query — everything after the separator is masked. The URL runs to white space or to a double quote that could close a JSON string — one followed by white space, `,`, `:`, `}`, `]` or the end — so one inside JSON ends with its string and the fields after it are kept; inside JSON escaped in another string (`\"`) it runs on. Kept as well, under the named limit for a secret no pattern matches: a token carried in the path itself, as a -webhook URL's is; what follows a quote that ends a URL; a URL written -percent-encoded or with an escaped colon. The kept +webhook URL's is; in a URL holding a quote that ends it, what follows the +quote and userinfo whose `@` follows it; a URL written percent-encoded or +with an escaped colon. The kept members are decoded, redacted one by one and encoded again, so key order, spacing and escapes are not the stream's; a string a terminal escape sequence or tag characters were diff --git a/internal/cli/content_collector.go b/internal/cli/content_collector.go index f58b988a7a..fa99ecd004 100644 --- a/internal/cli/content_collector.go +++ b/internal/cli/content_collector.go @@ -823,27 +823,35 @@ var urlRun = regexp.MustCompile(`[A-Za-z][A-Za-z0-9+.-]*:(?:\\*/){2}(?:[^\s"\\]| var urlSep = regexp.MustCompile(`:(?:\\*/){2}`) // maskURLs masks what a URL carries that no pattern is written for: -// everything after its first '?' or '#' (a presigned signature, an access -// token, a token in a fragment) and everything from its separator to its -// last '@' (userinfo, a password holding a '/' or a URL nested in -// another's path included; an '@' in a path is masked with it), as ***, -// with one finding for a string that had any. It runs last in -// redactText, on text the normalizer has folded and the patterns have +// everything from its separator to its last '@' (userinfo — a password +// holding a '/' or a URL nested in another's path included; an '@' in a +// path is masked with it, the host too) and everything after its first +// '?' or '#' (a presigned signature, an access token, a token in a +// fragment), as ***, with one finding for a string that had any. When +// the first '?' or '#' comes before the last '@' — a password holding +// one, or an '@' in the query — the two cannot be told apart and +// everything after the separator is masked. It runs last +// in redactText, on text the normalizer has folded and the patterns have // masked, so a compatibility '？' or '＠' is a plain one here, and nothing // is re-encoded: the mask only replaces bytes. Kept: a token carried in -// the path itself, as a webhook URL's is; a URL holding a quote that -// ends it (see urlRun), past that quote; a URL written percent-encoded -// or with an escaped colon, which is not read as one. +// the path itself, as a webhook URL's is; in a URL holding a quote that +// ends it (see urlRun), what follows the quote and userinfo whose '@' +// follows it; a URL written percent-encoded or with an escaped colon, +// which is not read as one. func maskURLs(text string, findings *[]security.Finding) string { masked := false text = urlRun.ReplaceAllStringFunc(text, func(run string) string { out := run + s, a, q := urlSep.FindStringIndex(run)[1], strings.LastIndex(run, "@"), strings.IndexAny(run, "?#") + switch { + case a > s && q >= 0 && q < a: + out = run[:s] + "***" + case a > s: + out = run[:s] + "***" + run[a:] + } if i := strings.IndexAny(out, "?#"); i >= 0 { out = out[:i+1] + "***" } - if s, a := urlSep.FindStringIndex(out)[1], strings.LastIndex(out, "@"); a > s { - out = out[:s] + "***" + out[a:] - } masked = masked || out != run return out }) diff --git a/internal/cli/content_collector_test.go b/internal/cli/content_collector_test.go index 7c628fbf35..f0c9984450 100644 --- a/internal/cli/content_collector_test.go +++ b/internal/cli/content_collector_test.go @@ -258,6 +258,9 @@ func TestContentCollector_AURLsUserinfoQueryAndFragmentAreMasked(t *testing.T) { "a query quoted half-way": {"Bash", `{"command":"curl 'https://h.example.com/x'?token=` + pw + `"}`, `{"command":"curl 'https://h.example.com/x'?***"}`}, "a URL nested in another's path": {"Bash", `{"command":"curl https://proxy.example.com/https://u:` + pw + `@h.example.com/x"}`, `{"command":"curl https://***@h.example.com/x"}`}, "a password holding a '/'": {"Bash", `{"command":"git clone https://x-access-token:ab+c/` + pw + `=@github.com/o/r"}`, `{"command":"git clone https://***@github.com/o/r"}`}, + "a password holding a '?'": {"Bash", `{"command":"git clone https://deploy:` + pw + `?x@github.com/o/r.git"}`, `{"command":"git clone https://***"}`}, + "a password holding a '#'": {"Bash", `{"command":"git clone https://deploy:` + pw + `#x@github.com/o/r.git"}`, `{"command":"git clone https://***"}`}, + "an '@' in the query": {"Bash", `{"command":"curl https://h.example.com/x?user=dev@example.com&token=` + pw + `"}`, `{"command":"curl https://***"}`}, "a query holding a quote": {"Bash", `{"command":"curl 'https://h.example.com/?q=\"a\"&token=` + pw + `'"}`, `{"command":"curl 'https://h.example.com/?***"}`}, "a query after a closing quote": {"Bash", `{"command":"curl \"https://h.example.com/x\"?token=` + pw + `"}`, `{"command":"curl \"https://h.example.com/x\"?***"}`}, "a fullwidth question mark": {"Bash", `{"command":"curl https://api.example.com/v1/me` + fw(0xff1f) + `access_token=` + pw + `"}`, `{"command":"curl https://api.example.com/v1/me?***"}`}, diff --git a/internal/cli/redaction_invariant_test.go b/internal/cli/redaction_invariant_test.go index 9ca82f2ce8..825ad80fa8 100644 --- a/internal/cli/redaction_invariant_test.go +++ b/internal/cli/redaction_invariant_test.go @@ -28,9 +28,10 @@ import ( // written in compatibility (fullwidth) forms, since the normalizer // rewrites it and the literal pass matches only what the env has; a // secret split across two parts or events; a secret matching no pattern -// and in no env — a token in a URL's path, what follows a double quote -// that ends a URL (urlRun), a URL written percent-encoded or with an -// escaped colon among them; a summary a pi or codex parser cut before the collector +// and in no env — a token in a URL's path, in a URL holding a double +// quote that ends it (urlRun), what follows the quote and userinfo whose +// '@' follows it, a URL written percent-encoded or with an escaped colon +// among them; a summary a pi or codex parser cut before the collector // saw it, since that cut can split a value the parser's own scan did not // see joined (a Claude summary is cut from a member the collector scans // whole). Nothing else is excluded. @@ -111,6 +112,10 @@ func TestRedaction_NoSecretSurvivesAnyWrapping(t *testing.T) { out[i] = secret{name, "https://deploy:ab+c/" + b[:16] + "=@git.example.com/o/r", b[:16], false} case "url_userinfo_double_quote": // a password holding a quote out[i] = secret{name, "https://deploy:" + b[:16] + `"x@git.example.com/o/r`, b[:16], false} + case "url_userinfo_question": // a password holding a '?' + out[i] = secret{name, "https://deploy:" + b[:16] + "?rest@git.example.com/o/r", b[:16], false} + case "url_userinfo_hash": // a password holding a '#' + out[i] = secret{name, "https://deploy:" + b[:16] + "#rest@git.example.com/o/r", b[:16], false} case "url_nested": // a URL in another's path out[i] = secret{name, "https://proxy.example.com/https://u:" + b[:16] + "@h.example.com/x", b[:16], false} } @@ -124,7 +129,7 @@ func TestRedaction_NoSecretSurvivesAnyWrapping(t *testing.T) { mk("url_userinfo", 1000), mk("url_userinfo_quoted", 1100), mk("url_userinfo_fullwidth", 1200), mk("url_query", 1300), mk("url_query_fullwidth", 1400), mk("url_fragment", 1500), mk("url_nested", 1600), mk("url_userinfo_escaped", 1700), mk("url_query_escaped", 1800), - mk("url_userinfo_slash", 1900), mk("url_userinfo_double_quote", 2000), + mk("url_userinfo_slash", 1900), mk("url_userinfo_double_quote", 2000), mk("url_userinfo_question", 2100), mk("url_userinfo_hash", 2200), } // Wrappings over the secret text; the ones that insert a character From 36a8978a0133b9b30e698e4c8c7b9bf49c3d1a3e Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Thu, 15 Oct 2026 09:23:25 -0400 Subject: [PATCH 39/40] fix(cli): read a URL with an escaped colon or a percent-encoded separator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The review bot has carried one low for three rounds: a URL whose colon is escaped for the shell (`https\:\/\/…`) or that is percent-encoded whole (`https%3A%2F%2F…`, as a URL carried in another's query is) was not read as one, so its userinfo, query and fragment were kept. urlRun and urlSep now accept any number of backslashes before the colon as before each slash, and the percent-encoded separator; maskURLs masks everything after a percent-encoded separator whole, since its '@', '?' and '#' are encoded too and nothing here decodes — anywhere in the run, not only as its first separator, so one nested in a plain URL's path is masked too, and encoded once or more, so one carried in a query that was itself encoded is read (both from the pre-push review). Four invariant rows were red and are green; unit rows for the escaped colon, a URL percent-encoded whole, one in another's query, one in another's path and one encoded twice. No URL run on the three captured streams has any of these forms. The named limit for these forms leaves the dev guide, the reference, the invariant header and the code comment. The two lows the bot still carries are documented limits that no coarse change closes: a quote then a colon before a URL's '@' (a fix would run past the quote that ends a URL inside JSON), and a token carried in a URL's path. The bot closes an open id only on a change it judges, or when a write-role human other than the author resolves the stamped thread; the PR body says so beside the two ids. Signed-off-by: Dharit Shah --- docs/guides/dev/tracing.md | 3 +-- .../infrastructure/distributed-tracing.md | 9 ++++---- internal/cli/content_collector.go | 23 +++++++++++++------ internal/cli/content_collector_test.go | 5 ++++ internal/cli/redaction_invariant_test.go | 13 ++++++++--- 5 files changed, 37 insertions(+), 16 deletions(-) diff --git a/docs/guides/dev/tracing.md b/docs/guides/dev/tracing.md index b6a71fff20..6e7262f675 100644 --- a/docs/guides/dev/tracing.md +++ b/docs/guides/dev/tracing.md @@ -294,8 +294,7 @@ across two parts or events; a secret that matches no pattern and is in no env, among them a token in a URL's path, in a URL holding a double quote that ends it (one that could close a JSON string, so a URL inside JSON ends with its string), what follows the quote and userinfo whose `@` -follows it, and a URL written percent-encoded or with an escaped colon, -which is not read as one; a summary a pi or codex parser cut before the collector saw it, since +follows it; a summary a pi or codex parser cut before the collector saw it, since that cut can split a value the parser's own scan did not see joined (a Claude summary is cut from a member the collector scans whole). A finding inside the guarantee is a defect; one on this list is a documented limit. diff --git a/docs/guides/infrastructure/distributed-tracing.md b/docs/guides/infrastructure/distributed-tracing.md index 7e5d394404..b05fc3be24 100644 --- a/docs/guides/infrastructure/distributed-tracing.md +++ b/docs/guides/infrastructure/distributed-tracing.md @@ -142,21 +142,22 @@ marked, since the list is the policy and the markers mean loss. A `Bash` `command` is recorded whole, up to 8 KiB, so a heredoc or an inline script records what it writes. A URL anywhere in the record — a member, text, a result, a summary, the input message, a name — has -everything from its `://` (or `:\/\/`, escaped for a shell or JSON) to its last `@` +everything from its `://` (or the same with its colon or slashes escaped for a shell or JSON) to its last `@` (userinfo — a password holding a `/` or a URL nested in another's path included; an `@` in a path is masked with it, the host too) and everything after its first `?` or `#` masked `***`, after the patterns ran, and counted in `fullsend.content.sanitizations`; when the first `?` or `#` comes before the last `@` — a password holding one, an `@` in the -query — everything after the separator is masked. The URL runs to white space or to a +query — everything after the separator is masked, and so it is after a +percent-encoded separator anywhere in the URL (`https%3A%2F%2F…`, +encoded once or more), since nothing decodes it. The URL runs to white space or to a double quote that could close a JSON string — one followed by white space, `,`, `:`, `}`, `]` or the end — so one inside JSON ends with its string and the fields after it are kept; inside JSON escaped in another string (`\"`) it runs on. Kept as well, under the named limit for a secret no pattern matches: a token carried in the path itself, as a webhook URL's is; in a URL holding a quote that ends it, what follows the -quote and userinfo whose `@` follows it; a URL written percent-encoded or -with an escaped colon. The kept +quote and userinfo whose `@` follows it. The kept members are decoded, redacted one by one and encoded again, so key order, spacing and escapes are not the stream's; a string a terminal escape sequence or tag characters were diff --git a/internal/cli/content_collector.go b/internal/cli/content_collector.go index fa99ecd004..d8f7ce77c6 100644 --- a/internal/cli/content_collector.go +++ b/internal/cli/content_collector.go @@ -816,11 +816,16 @@ func redactText(pipeline *security.Pipeline, runnerEnv map[string]string, text s // fields after it are not masked with it. Any other quote (a password or // a query quoted half-way for the shell) and a backslash-escaped // character do not end it. -var urlRun = regexp.MustCompile(`[A-Za-z][A-Za-z0-9+.-]*:(?:\\*/){2}(?:[^\s"\\]|\\.|"[^\s"\\,:}\]])+`) +var urlRun = regexp.MustCompile(`[A-Za-z][A-Za-z0-9+.-]*(?:\\*:(?:\\*/){2}|%(?:25)*3[Aa]%(?:25)*2[Ff]%(?:25)*2[Ff])(?:[^\s"\\]|\\.|"[^\s"\\,:}\]])+`) -// urlSep matches a URL's "://", or the same with its slashes escaped, as -// a shell command or a JSON encoder writes them, once or more. -var urlSep = regexp.MustCompile(`:(?:\\*/){2}`) +// urlSep matches a URL's "://", the same with its colon or slashes +// escaped as a shell command or a JSON encoder writes them, once or +// more, or percent-encoded (urlSepEncoded). +var urlSep = regexp.MustCompile(`\\*:(?:\\*/){2}|%(?:25)*3[Aa]%(?:25)*2[Ff]%(?:25)*2[Ff]`) + +// urlSepEncoded matches a percent-encoded "://" (%3A%2F%2F), as a URL +// carried in another's query is written, encoded once or more. +var urlSepEncoded = regexp.MustCompile(`%(?:25)*3[Aa]%(?:25)*2[Ff]%(?:25)*2[Ff]`) // maskURLs masks what a URL carries that no pattern is written for: // everything from its separator to its last '@' (userinfo — a password @@ -830,14 +835,15 @@ var urlSep = regexp.MustCompile(`:(?:\\*/){2}`) // fragment), as ***, with one finding for a string that had any. When // the first '?' or '#' comes before the last '@' — a password holding // one, or an '@' in the query — the two cannot be told apart and -// everything after the separator is masked. It runs last +// everything after the separator is masked; so it is after a +// percent-encoded separator anywhere in the run, whose '@', '?' and '#' +// are encoded too and nothing here decodes. It runs last // in redactText, on text the normalizer has folded and the patterns have // masked, so a compatibility '？' or '＠' is a plain one here, and nothing // is re-encoded: the mask only replaces bytes. Kept: a token carried in // the path itself, as a webhook URL's is; in a URL holding a quote that // ends it (see urlRun), what follows the quote and userinfo whose '@' -// follows it; a URL written percent-encoded or with an escaped colon, -// which is not read as one. +// follows it. func maskURLs(text string, findings *[]security.Finding) string { masked := false text = urlRun.ReplaceAllStringFunc(text, func(run string) string { @@ -849,6 +855,9 @@ func maskURLs(text string, findings *[]security.Finding) string { case a > s: out = run[:s] + "***" + run[a:] } + if e := urlSepEncoded.FindStringIndex(out); e != nil { + out = out[:e[1]] + "***" + } if i := strings.IndexAny(out, "?#"); i >= 0 { out = out[:i+1] + "***" } diff --git a/internal/cli/content_collector_test.go b/internal/cli/content_collector_test.go index f0c9984450..03ade9914c 100644 --- a/internal/cli/content_collector_test.go +++ b/internal/cli/content_collector_test.go @@ -267,6 +267,11 @@ func TestContentCollector_AURLsUserinfoQueryAndFragmentAreMasked(t *testing.T) { "a fullwidth at sign": {"Bash", `{"command":"git clone https://x-access-token` + fw(0xff1a) + pw + fw(0xff20) + `github.com/o/r"}`, `{"command":"git clone https://***@github.com/o/r"}`}, "a zero-width split scheme": {"Bash", `{"command":"curl https:` + fw(0x200b) + `//h.example.com/p?access_token=` + pw + `"}`, `{"command":"curl https://h.example.com/p?***"}`}, "a fullwidth token in the path": {"Bash", `{"command":"curl https://h.example.com/ghp_` + ghpFull.String() + `?x=1"}`, `{"command":"curl https://h.example.com/ghp_...?***"}`}, + "the colon escaped for the shell too": {"Bash", `{"command":"git clone https\\\\:\\\\/\\\\/deploy:` + pw + `@git.example.com/o/r"}`, `{"command":"git clone https\\\\:\\\\/\\\\/***@git.example.com/o/r"}`}, + "a URL percent-encoded whole": {"Bash", `{"command":"curl https%3A%2F%2Fdeploy%3A` + pw + `%40git.example.com%2Fo%2Fr"}`, `{"command":"curl https%3A%2F%2F***"}`}, + "a URL percent-encoded in a query": {"Bash", `{"command":"curl https://h.example.com/x?next=https%3A%2F%2Fdeploy%3A` + pw + `%40git.example.com"}`, `{"command":"curl https://h.example.com/x?***"}`}, + "a URL percent-encoded in a path": {"Bash", `{"command":"curl https://proxy.example.com/fetch/https%3A%2F%2Fdeploy%3A` + pw + `%40git.example.com"}`, `{"command":"curl https://proxy.example.com/fetch/https%3A%2F%2F***"}`}, + "a URL percent-encoded twice": {"Bash", `{"command":"curl https%253A%252F%252Fdeploy%253A` + pw + `%2540git.example.com"}`, `{"command":"curl https%253A%252F%252F***"}`}, "separators escaped for the shell": {"Bash", `{"command":"curl https:\\\\/\\\\/h.example.com/p?access_token=` + pw + `"}`, `{"command":"curl https:\\\\/\\\\/h.example.com/p?***"}`}, "a password behind escaped separators": {"Bash", `{"command":"git clone https:\\\\/\\\\/deploy:` + pw + `@git.example.com/o/r"}`, `{"command":"git clone https:\\\\/\\\\/***@git.example.com/o/r"}`}, } { diff --git a/internal/cli/redaction_invariant_test.go b/internal/cli/redaction_invariant_test.go index 825ad80fa8..f0d102b918 100644 --- a/internal/cli/redaction_invariant_test.go +++ b/internal/cli/redaction_invariant_test.go @@ -30,8 +30,7 @@ import ( // secret split across two parts or events; a secret matching no pattern // and in no env — a token in a URL's path, in a URL holding a double // quote that ends it (urlRun), what follows the quote and userinfo whose -// '@' follows it, a URL written percent-encoded or with an escaped colon -// among them; a summary a pi or codex parser cut before the collector +// '@' follows it among them; a summary a pi or codex parser cut before the collector // saw it, since that cut can split a value the parser's own scan did not // see joined (a Claude summary is cut from a member the collector scans // whole). Nothing else is excluded. @@ -116,6 +115,14 @@ func TestRedaction_NoSecretSurvivesAnyWrapping(t *testing.T) { out[i] = secret{name, "https://deploy:" + b[:16] + "?rest@git.example.com/o/r", b[:16], false} case "url_userinfo_hash": // a password holding a '#' out[i] = secret{name, "https://deploy:" + b[:16] + "#rest@git.example.com/o/r", b[:16], false} + case "url_userinfo_escaped_colon": // the colon escaped for the shell too + out[i] = secret{name, `https\:\/\/deploy:` + b[:16] + `@git.example.com/o/r`, b[:16], false} + case "url_percent_encoded": // the whole URL percent-encoded, as a query value is + out[i] = secret{name, "https%3A%2F%2Fdeploy%3A" + b[:16] + "%40git.example.com%2Fo%2Fr", b[:16], false} + case "url_percent_encoded_nested": // percent-encoded, in a plain URL's path + out[i] = secret{name, "https://proxy.example.com/fetch/https%3A%2F%2Fdeploy%3A" + b[:16] + "%40git.example.com", b[:16], false} + case "url_percent_encoded_twice": // encoded in a query that was itself encoded + out[i] = secret{name, "https%253A%252F%252Fdeploy%253A" + b[:16] + "%2540git.example.com", b[:16], false} case "url_nested": // a URL in another's path out[i] = secret{name, "https://proxy.example.com/https://u:" + b[:16] + "@h.example.com/x", b[:16], false} } @@ -129,7 +136,7 @@ func TestRedaction_NoSecretSurvivesAnyWrapping(t *testing.T) { mk("url_userinfo", 1000), mk("url_userinfo_quoted", 1100), mk("url_userinfo_fullwidth", 1200), mk("url_query", 1300), mk("url_query_fullwidth", 1400), mk("url_fragment", 1500), mk("url_nested", 1600), mk("url_userinfo_escaped", 1700), mk("url_query_escaped", 1800), - mk("url_userinfo_slash", 1900), mk("url_userinfo_double_quote", 2000), mk("url_userinfo_question", 2100), mk("url_userinfo_hash", 2200), + mk("url_userinfo_slash", 1900), mk("url_userinfo_double_quote", 2000), mk("url_userinfo_question", 2100), mk("url_userinfo_hash", 2200), mk("url_userinfo_escaped_colon", 2300), mk("url_percent_encoded", 2400), mk("url_percent_encoded_nested", 2500), mk("url_percent_encoded_twice", 2600), } // Wrappings over the secret text; the ones that insert a character From 31842099d3453ecfcd34f4059e79e342b680cf6d Mon Sep 17 00:00:00 2001 From: Dharit Shah Date: Thu, 15 Oct 2026 16:04:15 -0400 Subject: [PATCH 40/40] fix(cli): read a URL separator with any part percent-encoded MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Maintainer review of 36a8978a0: the URL mask read only a wholly percent-encoded separator (%3A%2F%2F), so the mixed forms two standard encoders write — Python's quote (https%3A//) and Go's PathEscape (https:%2F%2F) — and userinfo encoded behind a plain :// (deploy%3A…%40host) reached the record unmasked, with no finding, standalone and nested in another URL's path. urlSepPattern lets the colon and each slash be plain, escaped or percent-encoded, once or more; the run is clipped after the first separator holding a '%' before the userinfo rule; urlAt and urlCut read %40, %3F and %23 as the character they encode. Seven unit rows and seven invariant secrets cover the forms, standalone and nested, red first. The two guides state the rule. None of the three captured review streams holds any of the forms. Signed-off-by: Dharit Shah --- docs/guides/dev/tracing.md | 4 +- .../infrastructure/distributed-tracing.md | 8 ++- internal/cli/content_collector.go | 66 ++++++++++++++----- internal/cli/content_collector_test.go | 58 +++++++++------- internal/cli/redaction_invariant_test.go | 17 +++++ 5 files changed, 108 insertions(+), 45 deletions(-) diff --git a/docs/guides/dev/tracing.md b/docs/guides/dev/tracing.md index 6e7262f675..a608aa76a6 100644 --- a/docs/guides/dev/tracing.md +++ b/docs/guides/dev/tracing.md @@ -280,7 +280,9 @@ with the summary the parser derives from it) and every wrapping a review has constructed so far — terminal escapes before, around and inside a secret, repeated and paired secrets, zero-width, bidi, tag, NUL and variation-selector characters inside it, a URL's delimiters in -compatibility forms, separators escaped for the shell or JSON, a +compatibility forms, separators escaped for the shell or JSON, a URL +percent-encoded whole or in part (its separator's colon or slashes, its +`@`, `?` or `#`), a password quoted for the shell, a URL nested in another's path, an invalid byte or the replacement character inside it, an invalid byte after it with the diff --git a/docs/guides/infrastructure/distributed-tracing.md b/docs/guides/infrastructure/distributed-tracing.md index b05fc3be24..34c298b369 100644 --- a/docs/guides/infrastructure/distributed-tracing.md +++ b/docs/guides/infrastructure/distributed-tracing.md @@ -149,8 +149,12 @@ everything after its first `?` or `#` masked `***`, after the patterns ran, and counted in `fullsend.content.sanitizations`; when the first `?` or `#` comes before the last `@` — a password holding one, an `@` in the query — everything after the separator is masked, and so it is after a -percent-encoded separator anywhere in the URL (`https%3A%2F%2F…`, -encoded once or more), since nothing decodes it. The URL runs to white space or to a +separator with any part percent-encoded, anywhere in the URL +(`https%3A%2F%2F…`, `https%3A//…`, `https:%2F%2F…`, encoded once or more), +since nothing decodes it, and when the last `@` follows one — a URL with +userinfo nested in another's path; behind a plain `://`, a percent-encoded +`@`, `?` or `#` (`%40`, `%3F`, `%23`, encoded once or more) reads as the +character it encodes. The URL runs to white space or to a double quote that could close a JSON string — one followed by white space, `,`, `:`, `}`, `]` or the end — so one inside JSON ends with its string and the fields after it are kept; inside JSON escaped in another diff --git a/internal/cli/content_collector.go b/internal/cli/content_collector.go index d8f7ce77c6..d55222d6dc 100644 --- a/internal/cli/content_collector.go +++ b/internal/cli/content_collector.go @@ -809,6 +809,14 @@ func redactText(pipeline *security.Pipeline, runnerEnv map[string]string, text s return maskURLs(text, findings) } +// urlSepPattern matches a URL's "://" with its colon and slashes each +// written plain, escaped as a shell command or a JSON encoder writes +// them (once or more), or percent-encoded (%3A, %2F; encoded once or +// more): the whole separator encoded, as a query value carries a URL, +// and a part of it, as Python's quote (https%3A//) and Go's PathEscape +// (https:%2F%2F) write it. +const urlSepPattern = `\\*(?::|%(?:25)*3[Aa])(?:\\*(?:/|%(?:25)*2[Ff])){2}` + // urlRun matches a URL as a scheme, its separator (urlSep) and the rest // of the run, up to white space or a double quote that could close a // JSON string — one followed by white space, ',', ':', '}', ']' or the @@ -816,16 +824,19 @@ func redactText(pipeline *security.Pipeline, runnerEnv map[string]string, text s // fields after it are not masked with it. Any other quote (a password or // a query quoted half-way for the shell) and a backslash-escaped // character do not end it. -var urlRun = regexp.MustCompile(`[A-Za-z][A-Za-z0-9+.-]*(?:\\*:(?:\\*/){2}|%(?:25)*3[Aa]%(?:25)*2[Ff]%(?:25)*2[Ff])(?:[^\s"\\]|\\.|"[^\s"\\,:}\]])+`) +var urlRun = regexp.MustCompile(`[A-Za-z][A-Za-z0-9+.-]*(?:` + urlSepPattern + `)(?:[^\s"\\]|\\.|"[^\s"\\,:}\]])+`) -// urlSep matches a URL's "://", the same with its colon or slashes -// escaped as a shell command or a JSON encoder writes them, once or -// more, or percent-encoded (urlSepEncoded). -var urlSep = regexp.MustCompile(`\\*:(?:\\*/){2}|%(?:25)*3[Aa]%(?:25)*2[Ff]%(?:25)*2[Ff]`) +// urlSep matches a URL's separator (urlSepPattern); one holding a '%' +// has a part percent-encoded. +var urlSep = regexp.MustCompile(urlSepPattern) -// urlSepEncoded matches a percent-encoded "://" (%3A%2F%2F), as a URL -// carried in another's query is written, encoded once or more. -var urlSepEncoded = regexp.MustCompile(`%(?:25)*3[Aa]%(?:25)*2[Ff]%(?:25)*2[Ff]`) +// urlAt matches a URL's '@' and urlCut its '?' or '#', plain or +// percent-encoded once or more (%40, %3F, %23), so one encoded behind a +// plain separator reads as the character it encodes. +var ( + urlAt = regexp.MustCompile(`@|%(?:25)*40`) + urlCut = regexp.MustCompile(`[?#]|%(?:25)*(?:3[Ff]|23)`) +) // maskURLs masks what a URL carries that no pattern is written for: // everything from its separator to its last '@' (userinfo — a password @@ -835,9 +846,13 @@ var urlSepEncoded = regexp.MustCompile(`%(?:25)*3[Aa]%(?:25)*2[Ff]%(?:25)*2[Ff]` // fragment), as ***, with one finding for a string that had any. When // the first '?' or '#' comes before the last '@' — a password holding // one, or an '@' in the query — the two cannot be told apart and -// everything after the separator is masked; so it is after a -// percent-encoded separator anywhere in the run, whose '@', '?' and '#' -// are encoded too and nothing here decodes. It runs last +// everything after the separator is masked; so it is after a separator +// with any part percent-encoded (urlSep), anywhere in the run, whose '@', +// '?' and '#' may be encoded too and nothing here decodes, and when the +// last '@' follows one — a URL with userinfo nested in another's path. +// Behind a plain separator a percent-encoded '@', '?' or '#', encoded +// once or more, reads as the character it encodes (urlAt, urlCut). Each +// rule masks at least what the one before it did. It runs last // in redactText, on text the normalizer has folded and the patterns have // masked, so a compatibility '？' or '＠' is a plain one here, and nothing // is re-encoded: the mask only replaces bytes. Kept: a token carried in @@ -848,18 +863,24 @@ func maskURLs(text string, findings *[]security.Finding) string { masked := false text = urlRun.ReplaceAllStringFunc(text, func(run string) string { out := run - s, a, q := urlSep.FindStringIndex(run)[1], strings.LastIndex(run, "@"), strings.IndexAny(run, "?#") + s, a, q, c := urlSep.FindStringIndex(run)[1], -1, -1, encodedSepEnd(run) + if m := urlAt.FindAllStringIndex(run, -1); m != nil { + a = m[len(m)-1][0] + } + if m := urlCut.FindStringIndex(run); m != nil { + q = m[0] + } switch { - case a > s && q >= 0 && q < a: + case a > s && (q >= 0 && q < a || c >= 0 && c < a): out = run[:s] + "***" case a > s: out = run[:s] + "***" + run[a:] } - if e := urlSepEncoded.FindStringIndex(out); e != nil { - out = out[:e[1]] + "***" + if e := encodedSepEnd(out); e >= 0 { + out = out[:e] + "***" } - if i := strings.IndexAny(out, "?#"); i >= 0 { - out = out[:i+1] + "***" + if m := urlCut.FindStringIndex(out); m != nil { + out = out[:m[1]] + "***" } masked = masked || out != run return out @@ -871,6 +892,17 @@ func maskURLs(text string, findings *[]security.Finding) string { return text } +// encodedSepEnd returns where the first separator in s with a +// percent-encoded part (urlSep) ends, or -1. +func encodedSepEnd(s string) int { + for _, m := range urlSep.FindAllStringIndex(s, -1) { + if strings.Contains(s[m[0]:m[1]], "%") { + return m[1] + } + } + return -1 +} + // recordedArguments names, tool by tool, the members of a call's input // that the record keeps: paths, patterns, commands, modes and bounds — // what a trace needs to say which call was made. A Write, Edit or diff --git a/internal/cli/content_collector_test.go b/internal/cli/content_collector_test.go index 03ade9914c..75b4e62fc5 100644 --- a/internal/cli/content_collector_test.go +++ b/internal/cli/content_collector_test.go @@ -249,31 +249,39 @@ func TestContentCollector_AURLsUserinfoQueryAndFragmentAreMasked(t *testing.T) { for name, tc := range map[string]struct { tool, args, want string }{ - "a password before the host": {"WebFetch", `{"url":"https://alice:` + pw + `@example.com/repo"}`, `{"url":"https://***@example.com/repo"}`}, - "a presigned query": {"WebFetch", `{"url":"https://b.s3.amazonaws.com/k?X-Amz-Credential=AKIA` + pw + `&X-Amz-Signature=` + pw + `"}`, `{"url":"https://b.s3.amazonaws.com/k?***"}`}, - "an access token in the query": {"Bash", `{"command":"curl 'https://api.example.com/v1/me?access_token=` + pw + `' -s"}`, `{"command":"curl 'https://api.example.com/v1/me?*** -s"}`}, - "a token in a clone URL": {"Bash", `{"command":"git clone https://x-access-token:` + pw + `@github.com/o/r.git /tmp/r"}`, `{"command":"git clone https://***@github.com/o/r.git /tmp/r"}`}, - "a token behind the fragment": {"Bash", `{"command":"open https://app.example.com/cb#access_token=` + pw + `"}`, `{"command":"open https://app.example.com/cb#***"}`}, - "a password quoted for the shell": {"Bash", `{"command":"git clone https://deploy:'` + pw + `'@git.example.com/o/r"}`, `{"command":"git clone https://***@git.example.com/o/r"}`}, - "a query quoted half-way": {"Bash", `{"command":"curl 'https://h.example.com/x'?token=` + pw + `"}`, `{"command":"curl 'https://h.example.com/x'?***"}`}, - "a URL nested in another's path": {"Bash", `{"command":"curl https://proxy.example.com/https://u:` + pw + `@h.example.com/x"}`, `{"command":"curl https://***@h.example.com/x"}`}, - "a password holding a '/'": {"Bash", `{"command":"git clone https://x-access-token:ab+c/` + pw + `=@github.com/o/r"}`, `{"command":"git clone https://***@github.com/o/r"}`}, - "a password holding a '?'": {"Bash", `{"command":"git clone https://deploy:` + pw + `?x@github.com/o/r.git"}`, `{"command":"git clone https://***"}`}, - "a password holding a '#'": {"Bash", `{"command":"git clone https://deploy:` + pw + `#x@github.com/o/r.git"}`, `{"command":"git clone https://***"}`}, - "an '@' in the query": {"Bash", `{"command":"curl https://h.example.com/x?user=dev@example.com&token=` + pw + `"}`, `{"command":"curl https://***"}`}, - "a query holding a quote": {"Bash", `{"command":"curl 'https://h.example.com/?q=\"a\"&token=` + pw + `'"}`, `{"command":"curl 'https://h.example.com/?***"}`}, - "a query after a closing quote": {"Bash", `{"command":"curl \"https://h.example.com/x\"?token=` + pw + `"}`, `{"command":"curl \"https://h.example.com/x\"?***"}`}, - "a fullwidth question mark": {"Bash", `{"command":"curl https://api.example.com/v1/me` + fw(0xff1f) + `access_token=` + pw + `"}`, `{"command":"curl https://api.example.com/v1/me?***"}`}, - "a fullwidth at sign": {"Bash", `{"command":"git clone https://x-access-token` + fw(0xff1a) + pw + fw(0xff20) + `github.com/o/r"}`, `{"command":"git clone https://***@github.com/o/r"}`}, - "a zero-width split scheme": {"Bash", `{"command":"curl https:` + fw(0x200b) + `//h.example.com/p?access_token=` + pw + `"}`, `{"command":"curl https://h.example.com/p?***"}`}, - "a fullwidth token in the path": {"Bash", `{"command":"curl https://h.example.com/ghp_` + ghpFull.String() + `?x=1"}`, `{"command":"curl https://h.example.com/ghp_...?***"}`}, - "the colon escaped for the shell too": {"Bash", `{"command":"git clone https\\\\:\\\\/\\\\/deploy:` + pw + `@git.example.com/o/r"}`, `{"command":"git clone https\\\\:\\\\/\\\\/***@git.example.com/o/r"}`}, - "a URL percent-encoded whole": {"Bash", `{"command":"curl https%3A%2F%2Fdeploy%3A` + pw + `%40git.example.com%2Fo%2Fr"}`, `{"command":"curl https%3A%2F%2F***"}`}, - "a URL percent-encoded in a query": {"Bash", `{"command":"curl https://h.example.com/x?next=https%3A%2F%2Fdeploy%3A` + pw + `%40git.example.com"}`, `{"command":"curl https://h.example.com/x?***"}`}, - "a URL percent-encoded in a path": {"Bash", `{"command":"curl https://proxy.example.com/fetch/https%3A%2F%2Fdeploy%3A` + pw + `%40git.example.com"}`, `{"command":"curl https://proxy.example.com/fetch/https%3A%2F%2F***"}`}, - "a URL percent-encoded twice": {"Bash", `{"command":"curl https%253A%252F%252Fdeploy%253A` + pw + `%2540git.example.com"}`, `{"command":"curl https%253A%252F%252F***"}`}, - "separators escaped for the shell": {"Bash", `{"command":"curl https:\\\\/\\\\/h.example.com/p?access_token=` + pw + `"}`, `{"command":"curl https:\\\\/\\\\/h.example.com/p?***"}`}, - "a password behind escaped separators": {"Bash", `{"command":"git clone https:\\\\/\\\\/deploy:` + pw + `@git.example.com/o/r"}`, `{"command":"git clone https:\\\\/\\\\/***@git.example.com/o/r"}`}, + "a password before the host": {"WebFetch", `{"url":"https://alice:` + pw + `@example.com/repo"}`, `{"url":"https://***@example.com/repo"}`}, + "a presigned query": {"WebFetch", `{"url":"https://b.s3.amazonaws.com/k?X-Amz-Credential=AKIA` + pw + `&X-Amz-Signature=` + pw + `"}`, `{"url":"https://b.s3.amazonaws.com/k?***"}`}, + "an access token in the query": {"Bash", `{"command":"curl 'https://api.example.com/v1/me?access_token=` + pw + `' -s"}`, `{"command":"curl 'https://api.example.com/v1/me?*** -s"}`}, + "a token in a clone URL": {"Bash", `{"command":"git clone https://x-access-token:` + pw + `@github.com/o/r.git /tmp/r"}`, `{"command":"git clone https://***@github.com/o/r.git /tmp/r"}`}, + "a token behind the fragment": {"Bash", `{"command":"open https://app.example.com/cb#access_token=` + pw + `"}`, `{"command":"open https://app.example.com/cb#***"}`}, + "a password quoted for the shell": {"Bash", `{"command":"git clone https://deploy:'` + pw + `'@git.example.com/o/r"}`, `{"command":"git clone https://***@git.example.com/o/r"}`}, + "a query quoted half-way": {"Bash", `{"command":"curl 'https://h.example.com/x'?token=` + pw + `"}`, `{"command":"curl 'https://h.example.com/x'?***"}`}, + "a URL nested in another's path": {"Bash", `{"command":"curl https://proxy.example.com/https://u:` + pw + `@h.example.com/x"}`, `{"command":"curl https://***@h.example.com/x"}`}, + "a password holding a '/'": {"Bash", `{"command":"git clone https://x-access-token:ab+c/` + pw + `=@github.com/o/r"}`, `{"command":"git clone https://***@github.com/o/r"}`}, + "a password holding a '?'": {"Bash", `{"command":"git clone https://deploy:` + pw + `?x@github.com/o/r.git"}`, `{"command":"git clone https://***"}`}, + "a password holding a '#'": {"Bash", `{"command":"git clone https://deploy:` + pw + `#x@github.com/o/r.git"}`, `{"command":"git clone https://***"}`}, + "an '@' in the query": {"Bash", `{"command":"curl https://h.example.com/x?user=dev@example.com&token=` + pw + `"}`, `{"command":"curl https://***"}`}, + "a query holding a quote": {"Bash", `{"command":"curl 'https://h.example.com/?q=\"a\"&token=` + pw + `'"}`, `{"command":"curl 'https://h.example.com/?***"}`}, + "a query after a closing quote": {"Bash", `{"command":"curl \"https://h.example.com/x\"?token=` + pw + `"}`, `{"command":"curl \"https://h.example.com/x\"?***"}`}, + "a fullwidth question mark": {"Bash", `{"command":"curl https://api.example.com/v1/me` + fw(0xff1f) + `access_token=` + pw + `"}`, `{"command":"curl https://api.example.com/v1/me?***"}`}, + "a fullwidth at sign": {"Bash", `{"command":"git clone https://x-access-token` + fw(0xff1a) + pw + fw(0xff20) + `github.com/o/r"}`, `{"command":"git clone https://***@github.com/o/r"}`}, + "a zero-width split scheme": {"Bash", `{"command":"curl https:` + fw(0x200b) + `//h.example.com/p?access_token=` + pw + `"}`, `{"command":"curl https://h.example.com/p?***"}`}, + "a fullwidth token in the path": {"Bash", `{"command":"curl https://h.example.com/ghp_` + ghpFull.String() + `?x=1"}`, `{"command":"curl https://h.example.com/ghp_...?***"}`}, + "the colon escaped for the shell too": {"Bash", `{"command":"git clone https\\\\:\\\\/\\\\/deploy:` + pw + `@git.example.com/o/r"}`, `{"command":"git clone https\\\\:\\\\/\\\\/***@git.example.com/o/r"}`}, + "a URL percent-encoded whole": {"Bash", `{"command":"curl https%3A%2F%2Fdeploy%3A` + pw + `%40git.example.com%2Fo%2Fr"}`, `{"command":"curl https%3A%2F%2F***"}`}, + "a URL percent-encoded in a query": {"Bash", `{"command":"curl https://h.example.com/x?next=https%3A%2F%2Fdeploy%3A` + pw + `%40git.example.com"}`, `{"command":"curl https://***"}`}, + "a URL percent-encoded in a path": {"Bash", `{"command":"curl https://proxy.example.com/fetch/https%3A%2F%2Fdeploy%3A` + pw + `%40git.example.com"}`, `{"command":"curl https://***"}`}, + "a URL percent-encoded twice": {"Bash", `{"command":"curl https%253A%252F%252Fdeploy%253A` + pw + `%2540git.example.com"}`, `{"command":"curl https%253A%252F%252F***"}`}, + "separators escaped for the shell": {"Bash", `{"command":"curl https:\\\\/\\\\/h.example.com/p?access_token=` + pw + `"}`, `{"command":"curl https:\\\\/\\\\/h.example.com/p?***"}`}, + "a password behind escaped separators": {"Bash", `{"command":"git clone https:\\\\/\\\\/deploy:` + pw + `@git.example.com/o/r"}`, `{"command":"git clone https:\\\\/\\\\/***@git.example.com/o/r"}`}, + "the colon encoded, the slashes plain": {"Bash", `{"command":"curl https%3A//deploy%3A` + pw + `%40git.example.com/o/r"}`, `{"command":"curl https%3A//***"}`}, + "that, in a plain URL's path": {"Bash", `{"command":"curl https://proxy.example.com/fetch/https%3A//deploy%3A` + pw + `%40git.example.com"}`, `{"command":"curl https://***"}`}, + "the slashes encoded, the colon plain": {"Bash", `{"command":"curl https:%2F%2Fdeploy:` + pw + `@git.example.com%2Fo%2Fr"}`, `{"command":"curl https:%2F%2F***"}`}, + "that, in a plain URL's path too": {"Bash", `{"command":"curl https://proxy.example.com/fetch/https:%2F%2Fdeploy:` + pw + `@git.example.com%2Fo%2Fr"}`, `{"command":"curl https://***"}`}, + "userinfo encoded behind a plain '://'": {"Bash", `{"command":"git clone https://deploy%3A` + pw + `%40git.example.com/o/r"}`, `{"command":"git clone https://***%40git.example.com/o/r"}`}, + "that, in a plain URL's path as well": {"Bash", `{"command":"curl https://proxy.example.com/fetch/https://deploy%3A` + pw + `%40git.example.com"}`, `{"command":"curl https://***%40git.example.com"}`}, + "a fragment encoded behind a plain '://'": {"Bash", `{"command":"open https://app.example.com/cb%23access_token=` + pw + `"}`, `{"command":"open https://app.example.com/cb%23***"}`}, + "a query encoded behind a plain '://'": {"Bash", `{"command":"curl https://h.example.com/x%3Faccess_token=` + pw + `"}`, `{"command":"curl https://h.example.com/x%3F***"}`}, } { t.Run(name, func(t *testing.T) { c := newContentCollector(4096) diff --git a/internal/cli/redaction_invariant_test.go b/internal/cli/redaction_invariant_test.go index f0d102b918..c152e29381 100644 --- a/internal/cli/redaction_invariant_test.go +++ b/internal/cli/redaction_invariant_test.go @@ -123,6 +123,22 @@ func TestRedaction_NoSecretSurvivesAnyWrapping(t *testing.T) { out[i] = secret{name, "https://proxy.example.com/fetch/https%3A%2F%2Fdeploy%3A" + b[:16] + "%40git.example.com", b[:16], false} case "url_percent_encoded_twice": // encoded in a query that was itself encoded out[i] = secret{name, "https%253A%252F%252Fdeploy%253A" + b[:16] + "%2540git.example.com", b[:16], false} + case "url_mixed_colon_encoded": // the colon encoded, the slashes plain (Python's quote) + out[i] = secret{name, "https%3A//deploy%3A" + b[:16] + "%40git.example.com/o/r", b[:16], false} + case "url_mixed_colon_encoded_nested": // the same in a plain URL's path + out[i] = secret{name, "https://proxy.example.com/fetch/https%3A//deploy%3A" + b[:16] + "%40git.example.com", b[:16], false} + case "url_mixed_slashes_encoded": // the slashes encoded, the colon plain (Go's PathEscape) + out[i] = secret{name, "https:%2F%2Fdeploy:" + b[:16] + "@git.example.com%2Fo%2Fr", b[:16], false} + case "url_mixed_slashes_encoded_nested": // the same in a plain URL's path + out[i] = secret{name, "https://proxy.example.com/fetch/https:%2F%2Fdeploy:" + b[:16] + "@git.example.com%2Fo%2Fr", b[:16], false} + case "url_userinfo_percent_encoded": // userinfo encoded behind a plain separator + out[i] = secret{name, "https://deploy%3A" + b[:16] + "%40git.example.com/o/r", b[:16], false} + case "url_userinfo_percent_encoded_nested": // the same in a plain URL's path + out[i] = secret{name, "https://proxy.example.com/fetch/https://deploy%3A" + b[:16] + "%40git.example.com", b[:16], false} + case "url_query_percent_encoded": // the query's '?' encoded behind a plain separator + out[i] = secret{name, "https://h.example.com/x%3Faccess_token=" + b[:16], b[:16], false} + case "url_fragment_percent_encoded": // the fragment's '#' encoded behind a plain separator + out[i] = secret{name, "https://app.example.com/cb%23access_token=" + b[:16], b[:16], false} case "url_nested": // a URL in another's path out[i] = secret{name, "https://proxy.example.com/https://u:" + b[:16] + "@h.example.com/x", b[:16], false} } @@ -137,6 +153,7 @@ func TestRedaction_NoSecretSurvivesAnyWrapping(t *testing.T) { mk("url_query", 1300), mk("url_query_fullwidth", 1400), mk("url_fragment", 1500), mk("url_nested", 1600), mk("url_userinfo_escaped", 1700), mk("url_query_escaped", 1800), mk("url_userinfo_slash", 1900), mk("url_userinfo_double_quote", 2000), mk("url_userinfo_question", 2100), mk("url_userinfo_hash", 2200), mk("url_userinfo_escaped_colon", 2300), mk("url_percent_encoded", 2400), mk("url_percent_encoded_nested", 2500), mk("url_percent_encoded_twice", 2600), + mk("url_mixed_colon_encoded", 2700), mk("url_mixed_colon_encoded_nested", 2800), mk("url_mixed_slashes_encoded", 2900), mk("url_mixed_slashes_encoded_nested", 3000), mk("url_userinfo_percent_encoded", 3100), mk("url_userinfo_percent_encoded_nested", 3200), mk("url_query_percent_encoded", 3300), mk("url_fragment_percent_encoded", 3400), } // Wrappings over the secret text; the ones that insert a character