test: evaluate static shell optimization

evals/README.md

@@ -67,6 +67,20 @@ This runs two variants in parallel and prints pass/fail for each:
 
 `agents-md` drops an AGENTS.md into the sandbox telling the agent to check `node_modules/next/dist/docs/` first. `baseline` doesn't. That's the whole difference — same prompt, same model, one extra file. If `agents-md` passes and `baseline` doesn't, the bundled docs are doing their job.
 
+### Evaluating a local skill
+
+Docs can link to a canonical skill, but an unmerged skill revision isn't part of the `next` package tarball. To compare the current checkout's skill with the baseline and bundled-docs variants, add `eval.config.json` to the fixture:
+
+```json filename="eval.config.json"
+{
+  "skills": ["next-partial-prefetching-adoption"],
+  "playwright": true,
+  "timeout": 1800
+}
+```
+
+The runner then adds a third `skills` variant for that fixture. It installs the listed directories from the local `skills/` folder before the coding agent starts, while keeping the prompt, app, and assertions identical. Set `playwright` when the workflow must execute browser tests so the sandbox has the required system libraries. The optional timeout lets end-to-end workflows run longer than the 12-minute default. Existing fixtures without `eval.config.json` continue to run only `baseline` and `agents-md`.
+
 A run takes ~2–5 min. To validate a fixture without executing:
 
 ```bash

evals/evals/agent-050-optimize-static-shell/EVAL.ts

@@ -0,0 +1,79 @@
+/**
+ * Optimize a static shell
+ *
+ * This is intentionally not the team-settings example from the guide. The
+ * agent must apply the same framework reasoning to a release route with a
+ * request-specific layout control, reusable file-backed data, URL-dependent
+ * live data, and both hard and soft navigation contracts.
+ */
+
+import { expect, test } from 'vitest'
+import { existsSync, readFileSync, readdirSync, statSync } from 'fs'
+import { join } from 'path'
+import { environment, transcript } from '@vercel/agent-eval/eval'
+
+const IGNORE_DIRS = new Set([
+  '.git',
+  '.next',
+  'node_modules',
+  'dist',
+  'build',
+  'coverage',
+])
+
+function readSourceFiles(dir: string): string[] {
+  if (!existsSync(dir)) return []
+
+  return readdirSync(dir).flatMap((entry) => {
+    if (IGNORE_DIRS.has(entry)) return []
+    const path = join(dir, entry)
+    if (statSync(path).isDirectory()) return readSourceFiles(path)
+    if (entry === 'EVAL.ts' || !/\.(ts|tsx|js|jsx)$/.test(entry)) return []
+    return readFileSync(path, 'utf-8')
+  })
+}
+
+const source = readSourceFiles(process.cwd()).join('\n')
+
+test('removes the route opt-out while keeping Cache Components enabled', () => {
+  const config = readFileSync(join(process.cwd(), 'next.config.ts'), 'utf-8')
+
+  expect(config).toMatch(/cacheComponents\s*:\s*true/)
+  expect(source).not.toMatch(/export\s+(?:const|var|let)\s+instant\s*=\s*false/)
+})
+
+test('retains production instant navigation regression coverage', () => {
+  expect(source).toMatch(/from\s+['"]@next\/playwright['"]/)
+  expect(source).toMatch(/\binstant\s*\(/)
+  expect(source).toMatch(/\.click\s*\(/)
+  expect(source).toMatch(/\.goto\s*\(/)
+})
+
+test('keeps the reusable checklist cache explicit', () => {
+  expect(source).toMatch(/['"]use cache['"]/)
+  expect(source).toMatch(/\bcacheLife\s*\(/)
+})
+
+test('produces a useful shell without caching request data', async () => {
+  await expect(environment).toSatisfyCriterion(
+    `The final /releases/aurora implementation keeps the release frame/navigation and Release operations heading outside request-time work. The existing viewer and rollout loading states are reused in focused Suspense boundaries. The viewer cookie and live rollout remain request-time and are not placed in a public cache. The page or a high-level boundary is not replaced with an empty or duplicate full-page fallback.`
+  )
+})
+
+test('caches only the reusable launch checklist', async () => {
+  await expect(environment).toSatisfyCriterion(
+    `The file-backed launch checklist is made reusable with a targeted use-cache boundary, so the real checklist can be included in the static shell. The cache does not include cookies, viewer identity, connection(), or live rollout state. The URL-dependent rollout stays fresh and streams behind its existing LiveRolloutSkeleton.`
+  )
+})
+
+test('ships trustworthy hard and soft instant guards', async () => {
+  await expect(environment).toSatisfyCriterion(
+    `The project retains @next/playwright instant() regression tests for both a Link click from / to /releases/aurora and an initial page.goto('/releases/aurora'). Each test asserts a real visible marker from the meaningful release shell while the lock is active. The tests run against a production build with exposeTestingApiInProductionBuild enabled only for the test build. At least one guard is self-validating by proving live rollout content is absent under the lock, so a missing testing API cannot pass vacuously. The tests do not use arbitrary short timing races, hover warming, or next dev.`
+  )
+})
+
+test('completed a verified RED-to-GREEN optimization loop', async () => {
+  await expect(transcript).toSatisfyCriterion(
+    `The agent used a production-like build, first confirmed the chosen release-shell marker renders without instant(), then ran the locked instant() coverage against the unfixed route and observed a trustworthy RED. It fixed the route, removed the temporary instant=false opt-out, and actually reran the hard and soft guards to GREEN. It finished with a successful production build. Merely writing tests or printing commands for the user does not satisfy this criterion.`
+  )
+})

evals/evals/agent-050-optimize-static-shell/PROMPT.md

@@ -0,0 +1 @@
+Optimize the `/releases/aurora` route for both direct visits and client navigation from the home page. The release frame, heading, and reusable launch checklist should be ready immediately. The current viewer controls and live rollout details must remain request-time, using the loading states already in the app. Preserve the result with regression coverage and finish the production verification.

evals/evals/agent-050-optimize-static-shell/app/layout.tsx

@@ -0,0 +1,9 @@
+import type { ReactNode } from 'react'
+
+export default function RootLayout({ children }: { children: ReactNode }) {
+  return (
+    <html lang="en">
+      <body>{children}</body>
+    </html>
+  )
+}

evals/evals/agent-050-optimize-static-shell/app/page.tsx

@@ -0,0 +1,11 @@
+import Link from 'next/link'
+
+export default function HomePage() {
+  return (
+    <main>
+      <h1>Launch control</h1>
+      <p>Monitor active releases and production rollouts.</p>
+      <Link href="/releases/aurora">Open Aurora release</Link>
+    </main>
+  )
+}

evals/evals/agent-050-optimize-static-shell/app/releases/[release]/layout.tsx

@@ -0,0 +1,25 @@
+import type { ReactNode } from 'react'
+import { getCurrentViewer } from '@/lib/releases'
+import { ViewerControls } from './viewer-controls'
+
+export default async function ReleaseLayout({
+  children,
+}: {
+  children: ReactNode
+}) {
+  const viewer = await getCurrentViewer()
+
+  return (
+    <section data-testid="release-shell">
+      <header>
+        <a href="/">Launch control</a>
+        <nav aria-label="Release">
+          <a href="#overview">Overview</a>
+          <a href="#checks">Checks</a>
+        </nav>
+        <ViewerControls viewer={viewer} />
+      </header>
+      {children}
+    </section>
+  )
+}

evals/evals/agent-050-optimize-static-shell/app/releases/[release]/page.tsx

@@ -0,0 +1,32 @@
+import {
+  getLaunchChecklist,
+  getLiveRollout,
+  isKnownRelease,
+} from '@/lib/releases'
+import { LaunchChecklist, LiveRollout } from './release-panels'
+import { notFound } from 'next/navigation'
+
+export const instant = false
+
+export default async function ReleasePage({
+  params,
+}: PageProps<'/releases/[release]'>) {
+  const { release } = await params
+
+  if (!isKnownRelease(release)) {
+    notFound()
+  }
+
+  const [checklist, rollout] = await Promise.all([
+    getLaunchChecklist(),
+    getLiveRollout(release),
+  ])
+
+  return (
+    <main>
+      <h1 data-testid="release-heading">Release operations</h1>
+      <LaunchChecklist checklist={checklist} />
+      <LiveRollout rollout={rollout} />
+    </main>
+  )
+}

evals/evals/agent-050-optimize-static-shell/app/releases/[release]/release-panels.tsx

@@ -0,0 +1,46 @@
+import type { ReleaseChecklist, Rollout } from '@/lib/releases'
+
+export function LaunchChecklist({
+  checklist,
+}: {
+  checklist: ReleaseChecklist
+}) {
+  return (
+    <section id="checks" data-testid="launch-checklist">
+      <h2>{checklist.title}</h2>
+      <ul>
+        {checklist.items.map((item) => (
+          <li key={item}>{item}</li>
+        ))}
+      </ul>
+    </section>
+  )
+}
+
+export function LaunchChecklistSkeleton() {
+  return (
+    <section aria-label="Loading launch checklist">
+      <h2>Launch checklist</h2>
+      <p>Loading checks…</p>
+    </section>
+  )
+}
+
+export function LiveRollout({ rollout }: { rollout: Rollout }) {
+  return (
+    <section id="overview" data-testid="live-rollout">
+      <h2>Live rollout</h2>
+      <p>{rollout.percent}% deployed</p>
+      <p>{rollout.region}</p>
+    </section>
+  )
+}
+
+export function LiveRolloutSkeleton() {
+  return (
+    <section aria-label="Loading live rollout" data-testid="rollout-skeleton">
+      <h2>Live rollout</h2>
+      <p>Loading current deployment…</p>
+    </section>
+  )
+}

evals/evals/agent-050-optimize-static-shell/app/releases/[release]/viewer-controls.tsx

@@ -0,0 +1,15 @@
+export function ViewerControls({ viewer }: { viewer: string }) {
+  return (
+    <div data-testid="viewer-controls">
+      Viewing as <strong>{viewer}</strong>
+    </div>
+  )
+}
+
+export function ViewerControlsSkeleton() {
+  return (
+    <div aria-label="Loading viewer controls" data-testid="viewer-skeleton">
+      Loading viewer…
+    </div>
+  )
+}

evals/evals/agent-050-optimize-static-shell/data/launch-checklist.json

@@ -0,0 +1,8 @@
+{
+  "title": "Launch checklist",
+  "items": [
+    "Error budget reviewed",
+    "Rollback owner assigned",
+    "Support briefed"
+  ]
+}

evals/evals/agent-050-optimize-static-shell/eval.config.json

@@ -0,0 +1,5 @@
+{
+  "skills": ["next-cache-components-optimizer"],
+  "playwright": true,
+  "timeout": 1800
+}

evals/evals/agent-050-optimize-static-shell/lib/releases.ts

@@ -0,0 +1,39 @@
+import { readFile } from 'node:fs/promises'
+import { cookies } from 'next/headers'
+import { connection } from 'next/server'
+
+export type ReleaseChecklist = {
+  title: string
+  items: string[]
+}
+
+export type Rollout = {
+  percent: number
+  region: string
+}
+
+export function isKnownRelease(release: string): boolean {
+  return release === 'aurora' || release === 'nebula'
+}
+
+export async function getCurrentViewer(): Promise<string> {
+  const cookieStore = await cookies()
+  return cookieStore.get('viewer')?.value ?? 'Release engineer'
+}
+
+export async function getLaunchChecklist(): Promise<ReleaseChecklist> {
+  const source = await readFile(
+    new URL('../data/launch-checklist.json', import.meta.url),
+    'utf8'
+  )
+  return JSON.parse(source) as ReleaseChecklist
+}
+
+export async function getLiveRollout(release: string): Promise<Rollout> {
+  await connection()
+
+  return {
+    percent: release === 'aurora' ? 72 : 18,
+    region: release === 'aurora' ? 'Global' : 'Europe',
+  }
+}

evals/evals/agent-050-optimize-static-shell/next-env.d.ts

@@ -0,0 +1,5 @@
+/// <reference types="next" />
+/// <reference types="next/image-types/global" />
+
+// NOTE: This file should not be edited
+// see https://nextjs.org/docs/app/api-reference/config/typescript for more information.

evals/evals/agent-050-optimize-static-shell/next.config.ts

@@ -0,0 +1,7 @@
+import type { NextConfig } from 'next'
+
+const nextConfig: NextConfig = {
+  cacheComponents: true,
+}
+
+export default nextConfig

evals/evals/agent-050-optimize-static-shell/package.json

@@ -0,0 +1,23 @@
+{
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "dev": "next dev",
+    "build": "next build",
+    "start": "next start"
+  },
+  "dependencies": {
+    "next": "^16",
+    "react": "19.1.0",
+    "react-dom": "19.1.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20",
+    "@types/react": "^19",
+    "@types/react-dom": "^19",
+    "typescript": "^5",
+    "vitest": "^3.1.3",
+    "@vitejs/plugin-react": "^4.4.1",
+    "vite-tsconfig-paths": "^5.1.4"
+  }
+}

evals/evals/agent-050-optimize-static-shell/tsconfig.json

@@ -0,0 +1,27 @@
+{
+  "compilerOptions": {
+    "target": "ES2017",
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "bundler",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "react-jsx",
+    "incremental": true,
+    "plugins": [
+      {
+        "name": "next"
+      }
+    ],
+    "paths": {
+      "@/*": ["./*"]
+    }
+  },
+  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
+  "exclude": ["node_modules", "EVAL.ts"]
+}