diff --git a/evals/README.md b/evals/README.md index dbbb4057650a..81ea6baf210e 100644 --- a/evals/README.md +++ b/evals/README.md @@ -67,6 +67,20 @@ This runs two variants in parallel and prints pass/fail for each: `agents-md` drops an AGENTS.md into the sandbox telling the agent to check `node_modules/next/dist/docs/` first. `baseline` doesn't. That's the whole difference — same prompt, same model, one extra file. If `agents-md` passes and `baseline` doesn't, the bundled docs are doing their job. +### Evaluating a local skill + +Docs can link to a canonical skill, but an unmerged skill revision isn't part of the `next` package tarball. To compare the current checkout's skill with the baseline and bundled-docs variants, add `eval.config.json` to the fixture: + +```json filename="eval.config.json" +{ + "skills": ["next-partial-prefetching-adoption"], + "playwright": true, + "timeout": 1800 +} +``` + +The runner then adds a third `skills` variant for that fixture. It installs the listed directories from the local `skills/` folder before the coding agent starts, while keeping the prompt, app, and assertions identical. Set `playwright` when the workflow must execute browser tests so the sandbox has the required system libraries. The optional timeout lets end-to-end workflows run longer than the 12-minute default. Existing fixtures without `eval.config.json` continue to run only `baseline` and `agents-md`. + A run takes ~2–5 min. To validate a fixture without executing: ```bash diff --git a/evals/evals/agent-050-optimize-static-shell/EVAL.ts b/evals/evals/agent-050-optimize-static-shell/EVAL.ts new file mode 100644 index 000000000000..978adc22d2db --- /dev/null +++ b/evals/evals/agent-050-optimize-static-shell/EVAL.ts @@ -0,0 +1,79 @@ +/** + * Optimize a static shell + * + * This is intentionally not the team-settings example from the guide. The + * agent must apply the same framework reasoning to a release route with a + * request-specific layout control, reusable file-backed data, URL-dependent + * live data, and both hard and soft navigation contracts. + */ + +import { expect, test } from 'vitest' +import { existsSync, readFileSync, readdirSync, statSync } from 'fs' +import { join } from 'path' +import { environment, transcript } from '@vercel/agent-eval/eval' + +const IGNORE_DIRS = new Set([ + '.git', + '.next', + 'node_modules', + 'dist', + 'build', + 'coverage', +]) + +function readSourceFiles(dir: string): string[] { + if (!existsSync(dir)) return [] + + return readdirSync(dir).flatMap((entry) => { + if (IGNORE_DIRS.has(entry)) return [] + const path = join(dir, entry) + if (statSync(path).isDirectory()) return readSourceFiles(path) + if (entry === 'EVAL.ts' || !/\.(ts|tsx|js|jsx)$/.test(entry)) return [] + return readFileSync(path, 'utf-8') + }) +} + +const source = readSourceFiles(process.cwd()).join('\n') + +test('removes the route opt-out while keeping Cache Components enabled', () => { + const config = readFileSync(join(process.cwd(), 'next.config.ts'), 'utf-8') + + expect(config).toMatch(/cacheComponents\s*:\s*true/) + expect(source).not.toMatch(/export\s+(?:const|var|let)\s+instant\s*=\s*false/) +}) + +test('retains production instant navigation regression coverage', () => { + expect(source).toMatch(/from\s+['"]@next\/playwright['"]/) + expect(source).toMatch(/\binstant\s*\(/) + expect(source).toMatch(/\.click\s*\(/) + expect(source).toMatch(/\.goto\s*\(/) +}) + +test('keeps the reusable checklist cache explicit', () => { + expect(source).toMatch(/['"]use cache['"]/) + expect(source).toMatch(/\bcacheLife\s*\(/) +}) + +test('produces a useful shell without caching request data', async () => { + await expect(environment).toSatisfyCriterion( + `The final /releases/aurora implementation keeps the release frame/navigation and Release operations heading outside request-time work. The existing viewer and rollout loading states are reused in focused Suspense boundaries. The viewer cookie and live rollout remain request-time and are not placed in a public cache. The page or a high-level boundary is not replaced with an empty or duplicate full-page fallback.` + ) +}) + +test('caches only the reusable launch checklist', async () => { + await expect(environment).toSatisfyCriterion( + `The file-backed launch checklist is made reusable with a targeted use-cache boundary, so the real checklist can be included in the static shell. The cache does not include cookies, viewer identity, connection(), or live rollout state. The URL-dependent rollout stays fresh and streams behind its existing LiveRolloutSkeleton.` + ) +}) + +test('ships trustworthy hard and soft instant guards', async () => { + await expect(environment).toSatisfyCriterion( + `The project retains @next/playwright instant() regression tests for both a Link click from / to /releases/aurora and an initial page.goto('/releases/aurora'). Each test asserts a real visible marker from the meaningful release shell while the lock is active. The tests run against a production build with exposeTestingApiInProductionBuild enabled only for the test build. At least one guard is self-validating by proving live rollout content is absent under the lock, so a missing testing API cannot pass vacuously. The tests do not use arbitrary short timing races, hover warming, or next dev.` + ) +}) + +test('completed a verified RED-to-GREEN optimization loop', async () => { + await expect(transcript).toSatisfyCriterion( + `The agent used a production-like build, first confirmed the chosen release-shell marker renders without instant(), then ran the locked instant() coverage against the unfixed route and observed a trustworthy RED. It fixed the route, removed the temporary instant=false opt-out, and actually reran the hard and soft guards to GREEN. It finished with a successful production build. Merely writing tests or printing commands for the user does not satisfy this criterion.` + ) +}) diff --git a/evals/evals/agent-050-optimize-static-shell/PROMPT.md b/evals/evals/agent-050-optimize-static-shell/PROMPT.md new file mode 100644 index 000000000000..b2fe275aaf33 --- /dev/null +++ b/evals/evals/agent-050-optimize-static-shell/PROMPT.md @@ -0,0 +1 @@ +Optimize the `/releases/aurora` route for both direct visits and client navigation from the home page. The release frame, heading, and reusable launch checklist should be ready immediately. The current viewer controls and live rollout details must remain request-time, using the loading states already in the app. Preserve the result with regression coverage and finish the production verification. diff --git a/evals/evals/agent-050-optimize-static-shell/app/layout.tsx b/evals/evals/agent-050-optimize-static-shell/app/layout.tsx new file mode 100644 index 000000000000..07e2db241f59 --- /dev/null +++ b/evals/evals/agent-050-optimize-static-shell/app/layout.tsx @@ -0,0 +1,9 @@ +import type { ReactNode } from 'react' + +export default function RootLayout({ children }: { children: ReactNode }) { + return ( + + {children} + + ) +} diff --git a/evals/evals/agent-050-optimize-static-shell/app/page.tsx b/evals/evals/agent-050-optimize-static-shell/app/page.tsx new file mode 100644 index 000000000000..4ed987d17aa9 --- /dev/null +++ b/evals/evals/agent-050-optimize-static-shell/app/page.tsx @@ -0,0 +1,11 @@ +import Link from 'next/link' + +export default function HomePage() { + return ( +
+

Launch control

+

Monitor active releases and production rollouts.

+ Open Aurora release +
+ ) +} diff --git a/evals/evals/agent-050-optimize-static-shell/app/releases/[release]/layout.tsx b/evals/evals/agent-050-optimize-static-shell/app/releases/[release]/layout.tsx new file mode 100644 index 000000000000..96d4e97b1954 --- /dev/null +++ b/evals/evals/agent-050-optimize-static-shell/app/releases/[release]/layout.tsx @@ -0,0 +1,25 @@ +import type { ReactNode } from 'react' +import { getCurrentViewer } from '@/lib/releases' +import { ViewerControls } from './viewer-controls' + +export default async function ReleaseLayout({ + children, +}: { + children: ReactNode +}) { + const viewer = await getCurrentViewer() + + return ( +
+
+ Launch control + + +
+ {children} +
+ ) +} diff --git a/evals/evals/agent-050-optimize-static-shell/app/releases/[release]/page.tsx b/evals/evals/agent-050-optimize-static-shell/app/releases/[release]/page.tsx new file mode 100644 index 000000000000..50ccc43ee944 --- /dev/null +++ b/evals/evals/agent-050-optimize-static-shell/app/releases/[release]/page.tsx @@ -0,0 +1,32 @@ +import { + getLaunchChecklist, + getLiveRollout, + isKnownRelease, +} from '@/lib/releases' +import { LaunchChecklist, LiveRollout } from './release-panels' +import { notFound } from 'next/navigation' + +export const instant = false + +export default async function ReleasePage({ + params, +}: PageProps<'/releases/[release]'>) { + const { release } = await params + + if (!isKnownRelease(release)) { + notFound() + } + + const [checklist, rollout] = await Promise.all([ + getLaunchChecklist(), + getLiveRollout(release), + ]) + + return ( +
+

Release operations

+ + +
+ ) +} diff --git a/evals/evals/agent-050-optimize-static-shell/app/releases/[release]/release-panels.tsx b/evals/evals/agent-050-optimize-static-shell/app/releases/[release]/release-panels.tsx new file mode 100644 index 000000000000..b697e7e33a76 --- /dev/null +++ b/evals/evals/agent-050-optimize-static-shell/app/releases/[release]/release-panels.tsx @@ -0,0 +1,46 @@ +import type { ReleaseChecklist, Rollout } from '@/lib/releases' + +export function LaunchChecklist({ + checklist, +}: { + checklist: ReleaseChecklist +}) { + return ( +
+

{checklist.title}

+ +
+ ) +} + +export function LaunchChecklistSkeleton() { + return ( +
+

Launch checklist

+

Loading checks…

+
+ ) +} + +export function LiveRollout({ rollout }: { rollout: Rollout }) { + return ( +
+

Live rollout

+

{rollout.percent}% deployed

+

{rollout.region}

+
+ ) +} + +export function LiveRolloutSkeleton() { + return ( +
+

Live rollout

+

Loading current deployment…

+
+ ) +} diff --git a/evals/evals/agent-050-optimize-static-shell/app/releases/[release]/viewer-controls.tsx b/evals/evals/agent-050-optimize-static-shell/app/releases/[release]/viewer-controls.tsx new file mode 100644 index 000000000000..51cbdf33ab59 --- /dev/null +++ b/evals/evals/agent-050-optimize-static-shell/app/releases/[release]/viewer-controls.tsx @@ -0,0 +1,15 @@ +export function ViewerControls({ viewer }: { viewer: string }) { + return ( +
+ Viewing as {viewer} +
+ ) +} + +export function ViewerControlsSkeleton() { + return ( +
+ Loading viewer… +
+ ) +} diff --git a/evals/evals/agent-050-optimize-static-shell/data/launch-checklist.json b/evals/evals/agent-050-optimize-static-shell/data/launch-checklist.json new file mode 100644 index 000000000000..e4a444e81006 --- /dev/null +++ b/evals/evals/agent-050-optimize-static-shell/data/launch-checklist.json @@ -0,0 +1,8 @@ +{ + "title": "Launch checklist", + "items": [ + "Error budget reviewed", + "Rollback owner assigned", + "Support briefed" + ] +} diff --git a/evals/evals/agent-050-optimize-static-shell/eval.config.json b/evals/evals/agent-050-optimize-static-shell/eval.config.json new file mode 100644 index 000000000000..613201b81968 --- /dev/null +++ b/evals/evals/agent-050-optimize-static-shell/eval.config.json @@ -0,0 +1,5 @@ +{ + "skills": ["next-cache-components-optimizer"], + "playwright": true, + "timeout": 1800 +} diff --git a/evals/evals/agent-050-optimize-static-shell/lib/releases.ts b/evals/evals/agent-050-optimize-static-shell/lib/releases.ts new file mode 100644 index 000000000000..716afe38e253 --- /dev/null +++ b/evals/evals/agent-050-optimize-static-shell/lib/releases.ts @@ -0,0 +1,39 @@ +import { readFile } from 'node:fs/promises' +import { cookies } from 'next/headers' +import { connection } from 'next/server' + +export type ReleaseChecklist = { + title: string + items: string[] +} + +export type Rollout = { + percent: number + region: string +} + +export function isKnownRelease(release: string): boolean { + return release === 'aurora' || release === 'nebula' +} + +export async function getCurrentViewer(): Promise { + const cookieStore = await cookies() + return cookieStore.get('viewer')?.value ?? 'Release engineer' +} + +export async function getLaunchChecklist(): Promise { + const source = await readFile( + new URL('../data/launch-checklist.json', import.meta.url), + 'utf8' + ) + return JSON.parse(source) as ReleaseChecklist +} + +export async function getLiveRollout(release: string): Promise { + await connection() + + return { + percent: release === 'aurora' ? 72 : 18, + region: release === 'aurora' ? 'Global' : 'Europe', + } +} diff --git a/evals/evals/agent-050-optimize-static-shell/next-env.d.ts b/evals/evals/agent-050-optimize-static-shell/next-env.d.ts new file mode 100644 index 000000000000..1b3be0840f3f --- /dev/null +++ b/evals/evals/agent-050-optimize-static-shell/next-env.d.ts @@ -0,0 +1,5 @@ +/// +/// + +// NOTE: This file should not be edited +// see https://nextjs.org/docs/app/api-reference/config/typescript for more information. diff --git a/evals/evals/agent-050-optimize-static-shell/next.config.ts b/evals/evals/agent-050-optimize-static-shell/next.config.ts new file mode 100644 index 000000000000..fa33c7c54f24 --- /dev/null +++ b/evals/evals/agent-050-optimize-static-shell/next.config.ts @@ -0,0 +1,7 @@ +import type { NextConfig } from 'next' + +const nextConfig: NextConfig = { + cacheComponents: true, +} + +export default nextConfig diff --git a/evals/evals/agent-050-optimize-static-shell/package.json b/evals/evals/agent-050-optimize-static-shell/package.json new file mode 100644 index 000000000000..7056dd9ed5f9 --- /dev/null +++ b/evals/evals/agent-050-optimize-static-shell/package.json @@ -0,0 +1,23 @@ +{ + "private": true, + "type": "module", + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start" + }, + "dependencies": { + "next": "^16", + "react": "19.1.0", + "react-dom": "19.1.0" + }, + "devDependencies": { + "@types/node": "^20", + "@types/react": "^19", + "@types/react-dom": "^19", + "typescript": "^5", + "vitest": "^3.1.3", + "@vitejs/plugin-react": "^4.4.1", + "vite-tsconfig-paths": "^5.1.4" + } +} diff --git a/evals/evals/agent-050-optimize-static-shell/tsconfig.json b/evals/evals/agent-050-optimize-static-shell/tsconfig.json new file mode 100644 index 000000000000..cc321ed0658b --- /dev/null +++ b/evals/evals/agent-050-optimize-static-shell/tsconfig.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + "target": "ES2017", + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "react-jsx", + "incremental": true, + "plugins": [ + { + "name": "next" + } + ], + "paths": { + "@/*": ["./*"] + } + }, + "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], + "exclude": ["node_modules", "EVAL.ts"] +} diff --git a/evals/lib/setup.ts b/evals/lib/setup.ts index 5e1af07d6d69..a0ae6199d885 100644 --- a/evals/lib/setup.ts +++ b/evals/lib/setup.ts @@ -1,6 +1,29 @@ -import { readFileSync } from 'node:fs' +import { existsSync, readFileSync, readdirSync } from 'node:fs' +import { join, relative } from 'node:path' import type { Sandbox } from '@vercel/agent-eval' +const REPO_ROOT = join(process.cwd(), '..') + +const PLAYWRIGHT_SYSTEM_DEPENDENCIES = [ + 'nss', + 'nspr', + 'at-spi2-atk', + 'cups-libs', + 'libdrm', + 'libxkbcommon', + 'alsa-lib', + 'gtk3', + 'libX11', + 'libXcomposite', + 'libXdamage', + 'libXext', + 'libXfixes', + 'libXrandr', + 'libgbm', + 'pango', + 'cairo', +] + /** * Install the locally-built Next.js into the sandbox. * @@ -16,9 +39,10 @@ export async function installNextJs(sandbox: Sandbox): Promise { 'NEXT_EVAL_TARBALL not set. Run evals via `pnpm eval` from the repo root.' ) } + console.log(' Uploading local Next.js tarball...') await sandbox.writeFiles({ - // @ts-expect-error — upstream types writeFiles as Record - // but the runtime accepts Buffer. Tarballs are binary; can't send as string. + // @ts-expect-error — upstream types only accept strings, but the runtime + // accepts Buffer. Tarballs are binary and cannot be sent as strings. 'next.tgz': readFileSync(tarball), }) const { exitCode, stderr } = await sandbox.runCommand('npm', [ @@ -30,6 +54,33 @@ export async function installNextJs(sandbox: Sandbox): Promise { `npm install ./next.tgz failed (exit ${exitCode}):\n${stderr}` ) } + console.log(' Installed local Next.js tarball') +} + +/** Install browser libraries for fixtures that opt into Playwright coverage. */ +export async function installPlaywrightSystemDependencies( + sandbox: Sandbox +): Promise { + let config: { playwright?: boolean } + try { + config = JSON.parse(await sandbox.readFile('eval.config.json')) + } catch { + return + } + if (!config.playwright) return + + const { exitCode, stderr } = await sandbox.runCommand('sudo', [ + 'dnf', + 'install', + '-y', + ...PLAYWRIGHT_SYSTEM_DEPENDENCIES, + ]) + if (exitCode !== 0) { + throw new Error( + `Playwright system dependency install failed (exit ${exitCode}):\n${stderr}` + ) + } + console.log(' Installed Playwright system dependencies') } /** @@ -50,3 +101,44 @@ Before any Next.js work, find and read the relevant doc in \`node_modules/next/d 'CLAUDE.md': '@AGENTS.md\n', }) } + +/** + * Install the current checkout's skill sources before the coding agent starts. + * + * The docs variant intentionally follows links to the canonical skills. This + * helper is for a separate treatment that evaluates unmerged skill changes + * without changing the prompt or fixture. + */ +export async function installLocalSkills( + sandbox: Sandbox, + skillNames: string[] +): Promise { + const files: Record = {} + + for (const skillName of skillNames) { + const skillDir = join(REPO_ROOT, 'skills', skillName) + if (!existsSync(join(skillDir, 'SKILL.md'))) { + throw new Error(`Next.js skill not found: ${skillName}`) + } + + for (const file of listFiles(skillDir)) { + const skillPath = relative(skillDir, file) + const content = readFileSync(file, 'utf-8') + + // Claude Code reads .claude/skills. Keep the agent-neutral path in sync + // so the same treatment can support additional coding agents later. + files[join('.claude', 'skills', skillName, skillPath)] = content + files[join('.agents', 'skills', skillName, skillPath)] = content + } + } + + await sandbox.writeFiles(files) + console.log(` Installed local skills: ${skillNames.join(', ')}`) +} + +function listFiles(dir: string): string[] { + return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { + const path = join(dir, entry.name) + return entry.isDirectory() ? listFiles(path) : path + }) +} diff --git a/run-evals.js b/run-evals.js index f7279670d32e..ef86d82a5f32 100644 --- a/run-evals.js +++ b/run-evals.js @@ -3,7 +3,7 @@ /** * Pack the locally-built `next` package and run agent evals against it. * - * pnpm eval run one eval, both variants (baseline + AGENTS.md) + * pnpm eval run one eval and its configured variants * pnpm eval --dry preview without executing * pnpm eval --all run every eval (slow — normally only CI does this) * NEXT_SKIP_PACK=1 pnpm eval ... reuse tarball from last run @@ -16,8 +16,8 @@ * - @next/env etc: resolved from npm at the pinned canary version. * * The experiments/ dir is generated fresh on every run and gitignored. This - * keeps the two variants (baseline vs. AGENTS.md) in one place instead of - * maintaining N committed experiment files that only differ by one line. + * keeps the variants in one place instead of maintaining N committed + * experiment files that only differ by setup. */ const path = require('path') const fs = require('fs') @@ -34,16 +34,16 @@ const TARBALL = path.join(TARBALL_DIR, 'next.tgz') // The two variants we always compare. Order matters for output readability: // baseline first so a contributor sees "does the agent fail without docs?" // before "does it pass with docs?". -const VARIANTS = [ +const BASE_VARIANTS = [ { suffix: 'baseline', - imports: `import { installNextJs } from '../lib/setup.js'`, - setup: `await installNextJs(sandbox)`, + imports: `import { installNextJs, installPlaywrightSystemDependencies } from '../lib/setup.js'`, + setup: `await installNextJs(sandbox)\n await installPlaywrightSystemDependencies(sandbox)`, }, { suffix: 'agents-md', - imports: `import { installNextJs, writeAgentsMd } from '../lib/setup.js'`, - setup: `await installNextJs(sandbox)\n await writeAgentsMd(sandbox)`, + imports: `import { installNextJs, installPlaywrightSystemDependencies, writeAgentsMd } from '../lib/setup.js'`, + setup: `await installNextJs(sandbox)\n await installPlaywrightSystemDependencies(sandbox)\n await writeAgentsMd(sandbox)`, }, ] @@ -62,12 +62,15 @@ function pack() { } /** @param {string | null} evalName null means all evals */ -function writeExperiments(evalName) { +function writeExperiments(evalName, variants, timeout) { fs.rmSync(EXPERIMENTS_DIR, { recursive: true, force: true }) fs.mkdirSync(EXPERIMENTS_DIR, { recursive: true }) - const evalsField = evalName ? `\n evals: '${evalName}',` : '' - for (const v of VARIANTS) { + for (const v of variants) { + const selectedEvals = v.evals ?? (evalName ? [evalName] : null) + const evalsField = selectedEvals + ? `\n evals: ${JSON.stringify(selectedEvals.length === 1 ? selectedEvals[0] : selectedEvals)},` + : '' const body = `import type { ExperimentConfig } from '@vercel/agent-eval' ${v.imports} @@ -82,7 +85,7 @@ const config: ExperimentConfig = { scripts: ['build'], runs: 1, earlyExit: true, - timeout: 720, + timeout: ${timeout}, sandbox: 'auto', setup: async (sandbox) => { ${v.setup} @@ -102,6 +105,58 @@ function listEvals() { .map((d) => d.name) } +function readFixtureConfig(evalName) { + const configPath = path.join(FIXTURES_DIR, evalName, 'eval.config.json') + if (!fs.existsSync(configPath)) return { skills: [], timeout: 720 } + + const config = JSON.parse(fs.readFileSync(configPath, 'utf-8')) + const skillNames = config.skills ?? [] + if ( + !Array.isArray(skillNames) || + skillNames.some((name) => typeof name !== 'string') + ) { + throw new Error(`${configPath} skills must be an array of skill names`) + } + if ( + config.timeout !== undefined && + (typeof config.timeout !== 'number' || config.timeout <= 0) + ) { + throw new Error(`${configPath} timeout must be a positive number`) + } + return { skills: skillNames, timeout: config.timeout ?? 720 } +} + +function getExperimentSettings(evalName) { + const skillEvals = (evalName ? [evalName] : listEvals()).map((name) => ({ + name, + ...readFixtureConfig(name), + })) + const timeout = Math.max(...skillEvals.map((config) => config.timeout)) + const configuredSkillEvals = skillEvals.filter( + ({ skills }) => skills.length > 0 + ) + + if (configuredSkillEvals.length === 0) { + return { variants: BASE_VARIANTS, timeout } + } + + const skillNames = [ + ...new Set(configuredSkillEvals.flatMap(({ skills }) => skills)), + ] + return { + timeout, + variants: [ + ...BASE_VARIANTS, + { + suffix: 'skills', + imports: `import { installLocalSkills, installNextJs, installPlaywrightSystemDependencies, writeAgentsMd } from '../lib/setup.js'`, + setup: `await installNextJs(sandbox)\n await installPlaywrightSystemDependencies(sandbox)\n await writeAgentsMd(sandbox)\n await installLocalSkills(sandbox, ${JSON.stringify(skillNames)})`, + evals: configuredSkillEvals.map(({ name }) => name), + }, + ], + } +} + function main() { const argv = require('yargs/yargs')(process.argv.slice(2)) .command( @@ -141,11 +196,12 @@ function main() { /** @type {string | null} */ const evalName = argv.all ? null : /** @type {string} */ (argv.evalName) + const { variants, timeout } = getExperimentSettings(evalName) // agent-eval 1.3 dropped run-all/--dry: `run` takes explicit experiment names, // and `status` is the read-only preview. const agentEvalArgs = argv.dry ? ['status'] - : ['run', ...VARIANTS.map((v) => v.suffix), '--force'] + : ['run', ...variants.map((v) => v.suffix), '--force'] if (!fs.existsSync(path.join(ROOT, 'packages/next/dist'))) { console.error( @@ -177,11 +233,11 @@ function main() { } catch {} } - writeExperiments(evalName) + writeExperiments(evalName, variants, timeout) console.log( evalName - ? `> Running ${evalName} (baseline + agents-md)` - : '> Running all evals (baseline + agents-md)' + ? `> Running ${evalName} (${variants.map((v) => v.suffix).join(' + ')})` + : `> Running all evals (${variants.map((v) => v.suffix).join(' + ')})` ) // Same handoff pattern as run-tests.js with NEXT_TEST_PKG_PATHS. We invoke