feat(tripwire): screen outside tool results for prompt injection

api/config/custom-environment-variables.js

@@ -58,6 +58,9 @@ export default {
     __name: 'CODE_MODE_MAX_WORKERS',
     __format: 'json'
   },
+  tripwire: {
+    enabled: { __name: 'TRIPWIRE_ENABLED', __format: 'json' }
+  },
   files: {
     storage: 'FILES_STORAGE',
     dataDir: 'FILES_DATA_DIR',

api/config/default.js

@@ -72,5 +72,8 @@ export default {
   taskTimeoutSeconds: 1800,
   // code_mode scripts running at once in this process: one QuickJS worker each (64 MiB heap). The pool refuses
   // rather than queues when full (api/src/code-mode).
-  codeModeMaxWorkers: 4
+  codeModeMaxWorkers: 4,
+  // The tool-output injection tripwire (api/src/tripwire): outside tool results are classified by the moderator
+  // role model before the agent model reads them. A deployment switch, not a per-account setting.
+  tripwire: { enabled: true }
 }

api/config/type/.type/index.d.ts

@@ -175,6 +175,12 @@ export type ApiConfig = {
    * Wall-clock ceiling of one planned task's run. Longer than a chat turn's: nobody waits on it, and code mode and files make long runs the normal case.
    */
   taskTimeoutSeconds: number;
+  tripwire?: {
+    /**
+     * Classify outside tool results for prompt injection before the model sees them (docs/architecture/moderation.md).
+     */
+    enabled?: boolean;
+  };
   /**
    * How many code_mode scripts this process runs at once (one QuickJS worker each, up to 64 MiB). A script started while all are busy is refused, not queued.
    */

api/config/type/schema.json

@@ -265,6 +265,17 @@
       "default": 1800,
       "description": "Wall-clock ceiling of one planned task's run. Longer than a chat turn's: nobody waits on it, and code mode and files make long runs the normal case."
     },
+    "tripwire": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "enabled": {
+          "type": "boolean",
+          "default": true,
+          "description": "Classify outside tool results for prompt injection before the model sees them (docs/architecture/moderation.md)."
+        }
+      }
+    },
     "codeModeMaxWorkers": {
       "type": "integer",
       "minimum": 1,

api/src/agent-session/browser-tools.ts

@@ -13,7 +13,7 @@
  */
 
 import { tool, jsonSchema, type Tool } from 'ai'
-import { withProvenance } from '../conversations/operations.ts'
+import { withProvenance, type ToolResultScreen } from '../conversations/operations.ts'
 import type { BrowserToolDescriptor } from '@agents/shared/agent-session-protocol'
 import type { AgentSession } from './session.ts'
 import { appendHostEvents } from './host-events.ts'
@@ -82,8 +82,9 @@ export function rawBrowserToolSet (session: AgentSession, descriptors?: BrowserT
 /**
  * The page's tools as the model is offered them: the declared tools, in the same envelope an MCP tool's result
  * gets, from the same implementation. A page's result is untrusted input in exactly the same way: it is content
- * the model will read, arriving from somewhere that is not this service.
+ * the model will read, arriving from somewhere that is not this service. Every page tool is screened by the
+ * tripwire when one is given: a page declares no annotations the service could trust to skip it.
  */
-export function browserToolSet (session: AgentSession, descriptors?: BrowserToolDescriptor[]): Record<string, Tool> {
-  return withProvenance(rawBrowserToolSet(session, descriptors), () => BROWSER_TOOL_SERVER)
+export function browserToolSet (session: AgentSession, descriptors?: BrowserToolDescriptor[], screen?: ToolResultScreen): Record<string, Tool> {
+  return withProvenance(rawBrowserToolSet(session, descriptors), () => BROWSER_TOOL_SERVER, screen)
 }

api/src/conversations/executor.ts

@@ -347,6 +347,10 @@ export const driveTurn = async (
   } finally {
     if (timeout) clearTimeout(timeout)
     stopWatching()
+    // The record is closed, by close or fail alike: a tool-output tripwire verdict coming late is applied to the
+    // stored part from now on (api/src/tripwire). On the fail path too, or a failed turn's stored result would be
+    // replayed by the next one.
+    live.tripwire?.closed(run.conversationId ? { conversationId: run.conversationId } : { runId: run.id })
     liveRuns.delete(run.id)
   }
 }

api/src/conversations/live-runs.ts

@@ -19,6 +19,7 @@ import mongo from '#mongo'
 import Debug from 'debug'
 import type { MessagePart } from './operations.ts'
 import { TurnStopped } from './interrupted-turn.ts'
+import type { TurnTripwire } from '../tripwire/service.ts'
 
 const debug = Debug('df-agents:autonomous-agent-executor')
 
@@ -30,6 +31,11 @@ export interface LiveRun {
   messageSeq?: number
   /** The turn's parts as the model loop produces them, once it has started. */
   parts?: () => MessagePart[]
+  /**
+   * The turn's tool-output tripwire, once the loop created it: the driver closes it when the record is stored,
+   * whether the turn returned or threw. The first pass's, for a task run reminded in a second pass.
+   */
+  tripwire?: TurnTripwire
 }
 
 export const liveRuns = new Map<string, LiveRun>()
@@ -91,15 +97,30 @@ export const abortRun = async (runId: string): Promise<boolean> => {
  * live run of any trigger is still aborted here.
  */
 export const abortRunsOfConversation = async (conversationId: string, why: 'person-spoke' | 'stop' = 'person-spoke'): Promise<void> => {
-  for (const live of liveRuns.values()) {
-    if (live.conversationId === conversationId) live.controller.abort(new TurnStopped(why))
+  const abortLive = () => {
+    for (const live of liveRuns.values()) {
+      if (live.conversationId === conversationId) live.controller.abort(new TurnStopped(why))
+    }
   }
+  abortLive()
   await mongo.runs.updateMany(
     { conversationId, status: 'running', ...(why === 'person-spoke' ? { trigger: 'user' } : {}) },
     { $set: { abortRequestedAt: new Date().toISOString(), abortReason: why } }
   )
+  // Again, once the write is acknowledged (STOP_ORDERING below).
+  abortLive()
 }
 
+/*
+ * STOP_ORDERING. A turn goes live (liveRuns.set) and then reads its run for a stop (watchStopRequests' initial
+ * read), while the stop above aborts the live turns and then writes the stop on the running runs. Nothing
+ * orders the turn's read and the stop's write: a turn going live after the first abort loop, whose read reaches
+ * mongo before the write, missed both, until its next poll STOP_POLL_MS later, by which time a short turn has
+ * answered (a hello and a prompt sent back to back got two answers). The loop repeated after the write is
+ * acknowledged closes it: a turn live by then is aborted in-process, and one going live after it reads the
+ * stop it writes.
+ */
+
 /**
  * Stop every live turn of one autonomous agent. Returns how many this process actually stopped.
  *
@@ -112,17 +133,23 @@ export const abortRunsOfConversation = async (conversationId: string, why: 'pers
  * ignore — the run's own deadline is the hard backstop.
  */
 export const abortRunsOfAgent = async (agentId: string): Promise<number> => {
-  let stopped = 0
-  for (const live of liveRuns.values()) {
-    if (live.agentId !== agentId) continue
-    live.controller.abort(new TurnStopped('agent-stopped'))
-    stopped++
+  const stopped = new Set<LiveRun>()
+  const abortLive = () => {
+    for (const live of liveRuns.values()) {
+      if (live.agentId !== agentId || stopped.has(live)) continue
+      live.controller.abort(new TurnStopped('agent-stopped'))
+      stopped.add(live)
+    }
   }
+  abortLive()
   await mongo.runs.updateMany(
     { agentId, status: 'running' },
     { $set: { abortRequestedAt: new Date().toISOString(), abortReason: 'agent-stopped' } }
   )
-  return stopped
+  // Again, once the write is acknowledged: a turn that went live meanwhile may have read its run before it
+  // (STOP_ORDERING, above).
+  abortLive()
+  return stopped.size
 }
 
 /**