fix(tests): reserve run slots, await pending assertions, keep dynamic tool args

- Each test workflow run reserves and releases an execution slot.
- A case waits for assertions it did not await and fails if one fails.
- Mocked MCP and custom tools keep the arguments their schema declares.
- A closed session refuses starts still awaiting their lookups.
- Stable refresh callback; scroll fade on the results pane.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EHGBgHrtePpi7KMrEfy41R

Author: TheodoreSpeaks

apps/sim/app/workspace/[workspaceId]/tests/components/test-dashboard.tsx

@@ -1,16 +1,19 @@
 'use client'
 
-import { lazy, type ReactNode, Suspense, useEffect, useState } from 'react'
+import { lazy, type ReactNode, Suspense, useEffect, useRef, useState } from 'react'
 import {
   Badge,
   cn,
   DashboardMetric,
+  scrollFadeAttributes,
+  scrollFadeClass,
   Table,
   TableBody,
   TableCell,
   TableHead,
   TableHeader,
   TableRow,
+  useScrollEdges,
 } from '@sim/emcn'
 import { Loader } from '@sim/emcn/icons'
 import { formatDuration, formatRelativeTime } from '@sim/utils/formatting'
@@ -59,6 +62,8 @@ interface TestDashboardProps {
  * freshness, every case, and what it ran against. A latest run still going shows live.
  */
 export function TestDashboard({ workspaceId, name, detail }: TestDashboardProps) {
+  const scrollRef = useRef<HTMLDivElement>(null)
+  const edges = useScrollEdges(scrollRef)
   const selectedRunId =
     useTestRunSelectionStore(
       (state) => state.selectedRunIds[testRunSelectionKey(workspaceId, name)]
@@ -89,7 +94,11 @@ export function TestDashboard({ workspaceId, name, detail }: TestDashboardProps)
   const durationMs = run?.report?.tests.reduce((total, result) => total + result.durationMs, 0)
 
   return (
-    <div className='@container/dashboard min-h-0 flex-1 overflow-y-auto'>
+    <div
+      ref={scrollRef}
+      className={cn('@container/dashboard min-h-0 flex-1 overflow-y-auto', scrollFadeClass)}
+      {...scrollFadeAttributes(edges)}
+    >
       <div className='flex flex-col gap-8 @min-[640px]/dashboard:px-8 px-4 py-6'>
         <div className='flex flex-col gap-4'>
           <div className='-ml-2'>

apps/sim/hooks/queries/workflow-tests.ts

@@ -1,3 +1,4 @@
+import { useCallback } from 'react'
 import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
 import { requestJson } from '@/lib/api/client/request'
 import {
@@ -83,10 +84,10 @@ export function useWorkflowTestRun(workspaceId: string, name: string, runId: str
 /** Refetches one test after its file changes, so its cases follow the edit. */
 export function useRefreshWorkflowTest(workspaceId: string, name: string) {
   const queryClient = useQueryClient()
-  return () => {
+  return useCallback(() => {
     queryClient.invalidateQueries({ queryKey: workflowTestKeys.lists() })
     queryClient.invalidateQueries({ queryKey: workflowTestKeys.test(workspaceId, name) })
-  }
+  }, [queryClient, workspaceId, name])
 }
 
 export function useRunWorkflowTests(workspaceId: string) {

apps/sim/lib/workflow-tests/session.ts

@@ -468,6 +468,8 @@ export class WorkflowTestSession {
   private readonly runs = new Map<string, TestWorkflowRun>()
   /** Every workflow this file's runs executed, children included. */
   readonly enteredWorkflowIds = new Set<string>()
+  /** Set when the sandbox run ends; a start still awaiting its lookups then starts nothing. */
+  private closed = false
 
   constructor(readonly config: WorkflowTestSessionConfig) {}
 
@@ -481,6 +483,7 @@ export class WorkflowTestSession {
     const workflowId = await resolveWorkflowId(workspaceId, args.workflow)
     const blocks = await loadWorkflowBlocks(workflowId, workspaceId, version)
     const triggerBlockId = chooseTrigger(args.workflow, blocks, args.trigger)
+    if (this.closed) throw new Error('This workflow test run has ended')
     const matcher = new RunMatcher(args.targets, loadChildBlockNames, (id) =>
       this.enteredWorkflowIds.add(id)
     )
@@ -527,6 +530,7 @@ export class WorkflowTestSession {
   }
 
   close(): void {
+    this.closed = true
     this.cancel([...this.runs.keys()])
   }
 

apps/sim/lib/workflows/application/run-workflow-for-test.ts

@@ -5,8 +5,13 @@ import {
 } from '@sim/auth/principal'
 import { generateId } from '@sim/utils/id'
 import { getActivelyBannedUserIds } from '@/lib/auth/ban'
+import {
+  releaseExecutionSlot,
+  reserveExecutionSlot,
+} from '@/lib/billing/calculations/usage-reservation'
 import { resolveBillingAttribution } from '@/lib/billing/core/billing-attribution'
 import { checkExecutionUsageLimits } from '@/lib/billing/core/usage-gate-cache'
+import { getExecutionTimeout, RESERVATION_TTL_BUFFER_MS } from '@/lib/core/execution-limits/types'
 import { OrchestrationError } from '@/lib/core/orchestration/types'
 import { generateRequestId } from '@/lib/core/utils/request'
 import { resolveActiveWorkflowApplicationContext } from '@/lib/workflows/application/context'
@@ -80,6 +85,36 @@ export async function runWorkflowForTest(
   if (usage.isExceeded) {
     throw new OrchestrationError('forbidden', usage.message ?? 'Usage limit exceeded')
   }
+  if (usage.payerUsage) {
+    const plan = billingAttribution.payerSubscription?.plan
+    const reservation = await reserveExecutionSlot({
+      billingEntity: billingAttribution.billingEntity,
+      reservationId: executionId,
+      plan,
+      enterpriseConcurrencyLimit: billingAttribution.payerSubscription?.enterpriseConcurrencyLimit,
+      currentUsage: usage.payerUsage.currentUsage,
+      limit: usage.payerUsage.limit,
+      expiresAt: Date.now() + getExecutionTimeout(plan, 'sync') + RESERVATION_TTL_BUFFER_MS,
+      ...(billingAttribution.organizationId &&
+      usage.memberUsage?.limit !== null &&
+      usage.memberUsage?.limit !== undefined
+        ? {
+            member: {
+              organizationId: billingAttribution.organizationId,
+              actorUserId: billingAttribution.actorUserId,
+              currentUsage: usage.memberUsage.currentUsage,
+              limit: usage.memberUsage.limit,
+            },
+          }
+        : {}),
+    })
+    if (!reservation.reserved) {
+      throw new OrchestrationError(
+        'forbidden',
+        `The workflow cannot start now (${reservation.reason}); wait for running executions to finish`
+      )
+    }
+  }
   try {
     const result = await executeWorkflow(
       {
@@ -108,6 +143,8 @@ export async function runWorkflowForTest(
   } catch (error) {
     attachAttemptedExecutionId(error, executionId)
     throw error
+  } finally {
+    await releaseExecutionSlot(executionId)
   }
 }
 

apps/sim/providers/runtime-context.ts

@@ -1,7 +1,7 @@
 import { AsyncLocalStorage } from 'node:async_hooks'
 import { createLogger } from '@sim/logger'
 import { getErrorMessage } from '@sim/utils/errors'
-import { isRecordLike, omit, toRecord } from '@sim/utils/object'
+import { isRecordLike, omit, toRecord, toRecordOrNull } from '@sim/utils/object'
 import {
   durableSecretProvenanceFromRegistry,
   importDurableSecretProvenance,
@@ -133,13 +133,18 @@ function modelSuppliedToolInput(
   toolId: string,
   params: Record<string, unknown>
 ): Record<string, unknown> {
-  const declared = getToolParams(toolId) ?? {}
+  const declared = getToolParams(toolId)
+  // MCP and custom tools declare their arguments only in the schema the model was given.
+  const dynamicArgs = declared
+    ? undefined
+    : toRecord(toRecordOrNull(params._toolSchema)?.properties ?? {})
   const input: Record<string, unknown> = {}
   for (const [key, value] of Object.entries(params)) {
-    const visibility = declared[key]?.visibility
-    if (Object.hasOwn(declared, key) && visibility !== 'user-only' && visibility !== 'hidden') {
-      input[key] = value
-    }
+    const visibility = declared?.[key]?.visibility
+    const isArgument = declared
+      ? Object.hasOwn(declared, key)
+      : Object.hasOwn(dynamicArgs ?? {}, key)
+    if (isArgument && visibility !== 'user-only' && visibility !== 'hidden') input[key] = value
   }
   return input
 }

apps/sim/sandbox-tasks/workflow-test.ts

@@ -115,7 +115,7 @@ const HARNESS = `
     check.status = status;
     if (error) Object.assign(check, describeError(error));
   }
-  function track(assertion, check) {
+  function track(assertion, check, test) {
     return new Proxy(assertion, {
       get(target, key) {
         let value;
@@ -135,16 +135,18 @@ const HARNESS = `
               throw error;
             }
             if (result && typeof result.then === 'function') {
-              return result.then(
+              const settled = result.then(
                 (resolved) => { settle(check, 'pass'); return resolved; },
                 (error) => { settle(check, 'fail', error); throw error; }
               );
+              test.pendingAssertions.push(settled.catch(() => {}));
+              return settled;
             }
             settle(check, 'pass');
-            return result && typeof result === 'object' && result.__flags ? track(result, check) : result;
+            return result && typeof result === 'object' && result.__flags ? track(result, check, test) : result;
           };
         }
-        if (value && typeof value === 'object' && value.__flags) return track(value, check);
+        if (value && typeof value === 'object' && value.__flags) return track(value, check, test);
         return value;
       },
     });
@@ -155,7 +157,7 @@ const HARNESS = `
     const check = { status: 'pending', ...(callSite(new Error().stack) || {}) };
     currentTest.checks.push(check);
     currentTest.activeCheck = check;
-    return track(assertion, check);
+    return track(assertion, check, currentTest);
   };
   Object.setPrototypeOf(expect, baseExpect);
   expect.assertions = (count) => {
@@ -385,7 +387,7 @@ const HARNESS = `
 
   async function runOne(suite, test, onlyNames) {
     const path = [...suitePath(suite), test.name];
-    const result = { path, status: 'pass', durationMs: 0, checks: [], executions: [], logs: [], runIds: [], matchedKeys: new Set(), activeCheck: null, expectedAssertions: null, expectsAssertions: false };
+    const result = { path, status: 'pass', durationMs: 0, checks: [], executions: [], logs: [], runIds: [], matchedKeys: new Set(), activeCheck: null, expectedAssertions: null, expectsAssertions: false, pendingAssertions: [] };
     if (!willRun(suite, test, onlyNames)) {
       result.status = 'skip';
       return result;
@@ -407,6 +409,10 @@ const HARNESS = `
         } finally {
           for (const s of suiteChain(suite).reverse()) await runHooks(s.hooks.afterEach);
         }
+        await Promise.all(result.pendingAssertions);
+        if (result.checks.some((check) => check.status === 'fail')) {
+          throw new Error('An assertion failed after its test returned; await expect(...).resolves, .rejects, and toMatchRubric');
+        }
         checkAssertionCount(result);
         const unmatched = [...registrations.values()].filter((r) => r.test === result && result.runIds.length > 0 && !result.matchedKeys.has(r.key));
         if (unmatched.length > 0) {