fix(design-diff): trace presentation inputs and bound schema 3 reports

Author: Bill Leoutsakos

design-diff.config.json

@@ -82,5 +82,23 @@
     "@/lib/cn"
   ],
   "variantModules": ["class-variance-authority"],
-  "mergeFontSizes": ["micro", "caption", "small", "md"]
+  "mergeFontSizes": ["micro", "caption", "small", "md"],
+  "fileInputs": [
+    {
+      "list": "apps/docs/lib/openapi-specs.ts",
+      "export": "OPENAPI_SPEC_FILES",
+      "root": "apps/docs",
+      "renderer": "apps/docs/lib/openapi.ts"
+    }
+  ],
+  "environmentAdapters": [
+    {
+      "module": "apps/sim/lib/core/config/env-capabilities.server.ts",
+      "export": "wireServerFallback",
+      "environmentModule": "apps/sim/lib/core/config/env.ts",
+      "environmentExport": "env",
+      "implementationModule": "packages/deployment-config/src/env-capabilities.ts",
+      "implementationExport": "wireFallback"
+    }
+  ]
 }

scripts/design-diff/README.md

@@ -26,11 +26,11 @@ duplicating entire function bodies. Reports also include schema, engine and poli
 base/head/merge-base SHAs, and workflow PR/head/engine identity. No timestamp enters the
 deterministic payload. Finding IDs are stable for the same input pair and engine version.
 
-## Binary decisions and grouped report (schema 2)
+## Binary decisions and grouped report (schema 3)
 
 The public decisions are **`flag`** and **`exempt`**. Uncertainty produces `flag`, with
-the obstacle recorded in `limitations`. Schema and policy version `2.0.0`, engine version
-`0.2.0`, replace the earlier public `review` decision and flat consumer findings.
+the obstacle recorded in `limitations`. Schema and policy version `3.0.0`, engine version `0.3.0`, retain binary decisions
+and add bounded values and explicit report truncation metadata.
 
 Each finding groups evidence by **changed source file**. Several changed definitions in
 one file remain individually available in `changes`. Unchanged downstream consumers do not
@@ -42,7 +42,7 @@ one finding anchored to its source, with a representative consumer and usage met
 | `source.before` / `source.after` | Changed source path and Git blob ID, or `null` for addition/deletion |
 | `decision` | `flag` if any grouped evidence requires reporting; otherwise `exempt` |
 | `category`, `symbol`, `reason`, `before`, `after` | Representative evidence; its locations can be in an unchanged consumer of a changed token |
-| `categories` | Categories of the retained direct evidence and representative consumer |
+| `categories` | Categories of all direct evidence and the changed consumer |
 | `changes` | Direct definition changes in the changed file, retaining individual decisions, values and locations |
 | `example` | One consumer evidence record, or `null`; `basis` distinguishes `changed-definition` from `potential-consumer` |
 | `impact.before` / `impact.after` | Separate counts and source locations for resolved direct references in each revision |
@@ -61,8 +61,14 @@ references do not count. Dynamic/ambiguous imports are not presented as exact us
 Zero means no references were enumerated, not proof of no consumers. Overrides, inactive
 variants and runtime conditions can prevent a referenced component from changing visually.
 
-Consumers upgrading from schema 1 must read direct evidence from `changes` and the optional
-`example`, use `source` to identify the changed file, and stop expecting `decision: review`.
+Schema 3 readers must handle either literal values or a summary object containing
+`$truncated`, `preview`, `sha256`, `originalBytes`, `previewBytes` and `omittedBytes`.
+The default preview is at most 4 KiB. Full semantic evidence is compared before presentation
+values are shortened; large opaque helper summaries retain full-value hashes too.
+`truncation` records the 5 MiB serialized-report limit, total/omitted findings, and total/omitted
+entries for sampled detail/reference lists. Sampling retains stable prefixes. The overall
+`flagged` result is computed before sampling, so an omitted finding cannot make a PR clean.
+Read direct evidence from `changes` and optional `example`; use `source` for the changed file.
 
 ## Architecture
 
@@ -89,7 +95,13 @@ scripts/design-diff/
   source.ts                             Source snapshots and path/alias resolution
   dependencies.ts                       Export-aware imports and source reference counts
   ast.ts                                Babel parsing and syntax normalization
+  refactors.ts                          Supported literal/refactor normalization
   resolve.ts                            Bounded expression and import resolution
+  inputs.ts                             Configured file-loaded documentation inputs
+  infrastructure.ts                     Rendering lockfile dependency closure
+  report.ts                             Value previews and bounded JSON serialization
+  benchmark.ts                          Immutable-engine historical replay
+  benchmark/comparisons.json            Frozen original/holdout comparison manifest
   memory.ts                             Bun parser-batch garbage collection
   tailwind.ts                           Pinned compiler and trusted merge convention
   compare.ts                            Stable matching and findings
@@ -147,11 +159,28 @@ through named/star re-exports and import-then-export indexes to the defining mod
 Button edit does not implicate a file merely because it imports an unrelated Icon from the
 same index. Changed top-level bindings and local dependents narrow the first propagation
 step and usage counts within a multi-export file. Both revisions are considered, including
-redirected re-exports. Module effects, side-effect imports, computed namespaces, ambiguous
-exports, cycles and resolution limits retain conservative dependencies. Further transitive
-module effects can still overestimate potential impact.
-
-The resolver reuses the export index, retains at most 128 parsed modules per revision,
+redirected re-exports. The module graph identifies candidates only. Actual findings require
+changed values, guards, referenced implementations or an explicitly unresolved imported input.
+Unrelated imports and dead re-exports cannot flag an unchanged expression. Object properties,
+destructured parameter defaults, and helper return paths are traced separately. Small immutable
+local literals are normalized before expression budgets, preserving supported constant hoists.
+The exact `Object.entries(...).reduce` record-map idiom is normalized to `Object.fromEntries`
+only with an empty accumulator, unchanged key and no accumulator reads in the mapped value. Unknown
+computed namespaces, ambiguous imports, cycles and exhausted resolution remain conservative
+when that particular input feeds rendering. SQL tags and telemetry receivers are not standalone
+rendering definitions; their values can still matter if explicitly consumed by a visual input.
+
+The configured `wireServerFallback` capability adapter traces its declared environment fields,
+provider factories and implementation. Its finite capability schema identifies relevant keys;
+unsupported dynamic schemas use ordinary conservative resolution. Whole-environment backend
+settings therefore do not contaminate the signup page solely through this known adapter.
+
+The configured Fumadocs `OPENAPI_SPEC_FILES` list is parsed from Git in each revision. Listed
+JSON inputs are compared semantically, retaining array order and attributing changes to both
+the spec and configured renderer. Malformed/missing configured inputs flag with a limitation.
+
+The graph reuses up to 10,000 import/export snapshots keyed by source blob, resolving their
+paths again for each revision. The resolver retains at most 128 parsed modules per revision,
 and requests Bun garbage collection between parser batches. These resource controls do
 not change evidence or decisions. The Node-based test runner uses its own garbage collector.
 
@@ -170,7 +199,7 @@ application plugins or every postprocessor have been reproduced.
 
 Desktop support extracts recognized `BrowserWindow` appearance options, native appearance
 setter calls and `nativeTheme.themeSource` assignments. Configured native
-menu/tray/terminal-theme modules use an uncertainty fallback, including their changed dependencies,
+menu/tray/terminal-theme modules use an uncertainty fallback for their own changed definitions,
 because embedded JXA and native operating-system rendering are not executed.
 
 ## Movement proof and remaining limits
@@ -186,20 +215,21 @@ change the result. There is no blanket exemption for translation, margins, gaps
 This engine is conservative, not a runtime equivalence prover:
 
 - Runtime data, arbitrary functions, mutable bindings, dependency cycles, parser failures,
-  unknown props and unsupported rendering syntax produce flags with limitations when affected.
+  unknown props and unsupported rendering syntax produce flags when changed presentation evidence
+  or a traceable unresolved rendering input changes. They are not blanket module-level triggers.
 - Source-order matching after substantial markup edits can pair different elements. Such
   changes remain flagged; findings are evidence for review, not an exact DOM correspondence.
 - Dynamic module/asset paths, inherited/conditional export maps outside the supported forms,
   generated source and arbitrary imperative renderers cannot be fully followed. Directly
   detected DOM/canvas operations and configured native rendering use uncertainty fallbacks.
 - MDX expressions and embedded HTML scripts are flagged, without running MDX components or
   scripts. Plain HTML whitespace is preserved because CSS can make it meaningful.
-- Broad dependency updates and shared runtime expressions can create false positives. All
-  lockfile changes are flagged, including changes to tooling-only dependencies. Inactive
-  variants, unused assets and an apparently inert removed class can also be flagged.
+- Lockfiles compare recognized rendering dependencies and their resolved transitive closure.
+  Unrelated tooling/backend updates are clean. Unknown rendering configs/plugins still flag.
+  Inactive variants, unused assets and an apparently inert removed class can also be flagged.
 - Impact is a conservative approximation. Findings retain direct source changes and one
   consumer example instead of thousands of downstream records. Large changed definitions
-  and usage inventories can still produce substantial JSON reports.
+  and usage inventories are sampled deterministically within the report budget.
 - The default limits are 2 MiB per source file, 256 MiB per source snapshot, 24 resolution
   levels and 5,000 evaluation steps per expression. Per-file/parser/expression limits produce
   flags with limitations; snapshot/Git failures are operational failures, never clean results.
@@ -259,7 +289,7 @@ are whole commits, including ancillary changes, rather than only their headline
 These comparisons validate source-policy behavior, not rendered pixels or recall over all
 historical PRs. Screenshot capture, AI interpretation and Slack delivery are separate stages.
 
-### Incremental smoke test on PR #7742
+### Earlier incremental smoke test on PR #7742 (schema 2)
 
 Five temporary commits were created directly on `458a515cbed7fbcf127ce72348ff755c5308ce13`,
 each changing one existing source file. Comparing that commit with each temporary head
@@ -288,5 +318,26 @@ These are partial source-reference counts, not claims that every reference chang
 Independent changed source files remain separate findings, and all direct definition changes
 within each changed file are retained.
 
-These checks validate local engine behavior, not cloud workflow activation. They are examples,
+These earlier checks do not validate the revised schema 3 engine or cloud workflow activation. They are examples,
 not a measured detection rate across all possible UI changes.
+
+## Frozen historical benchmark
+
+The manifest preserves the original 120 comparisons (baseline: 108 flagged, including 39/49
+source-reviewed nonvisual cases) and the next 60 entries of the original SHA-256 sampling
+order. Holdout source-review labels were frozen before revised engine results: 37 clear
+visual/content, 18 nonvisual and 5 uncertain. Labels describe source edits, not rendered pixel
+ground truth. Corrections must be documented separately rather than rewriting frozen labels.
+
+```sh
+bun --no-env-file scripts/design-diff/benchmark.ts \
+  --engine /path/to/clean/engine-checkout --sha <immutable-engine-commit> \
+  --manifest scripts/design-diff/benchmark/comparisons.json --output /tmp/design-benchmark
+```
+
+Fetch manifest commit objects beforehand; missing history fails explicitly. The runner verifies
+the frozen comparison commits and GitHub file sets. Cache identity includes engine SHA, trusted
+config, lockfile, runtime and comparison commits, with report-content verification before reuse.
+It records per-comparison elapsed time and peak RSS separately from deterministic reports.
+`/usr/bin/time` is required (macOS or Linux); source findings are not printed. Review original
+and holdout rates separately, and inspect every disagreement against the source label.

scripts/design-diff/analyze.ts

@@ -6,6 +6,8 @@ import { extractDocument } from '#design-diff/extract/documents'
 import { extractTsx } from '#design-diff/extract/tsx'
 import { GitReader } from '#design-diff/git'
 import { groupFindings } from '#design-diff/group'
+import { renderingLock } from '#design-diff/infrastructure'
+import { fileLoadedInputs } from '#design-diff/inputs'
 import { reclaimMemory } from '#design-diff/memory'
 import { limitations } from '#design-diff/policy'
 import { Resolver } from '#design-diff/resolve'
@@ -21,9 +23,9 @@ import type { Change, Config, Definition, Report } from '#design-diff/types'
 
 export function emptyReport(): Report {
   return {
-    schemaVersion: '2.0.0',
-    engineVersion: '0.2.0',
-    policyVersion: '2.0.0',
+    schemaVersion: '3.0.0',
+    engineVersion: '0.3.0',
+    policyVersion: '3.0.0',
     commits: null,
     status: 'failed',
     flagged: null,
@@ -151,11 +153,6 @@ export async function analyze(
       const source = tree.texts.get(file)
       if (source === undefined) return []
       const normalizeAll = async (definitions: Definition[]) => {
-        for (const definition of definitions)
-          if (definition.unresolved.length || definition.kind === 'review')
-            definition.dependencies = [
-              ...new Set([...definition.dependencies, ...(tree.dependencies.get(file) ?? [])]),
-            ].sort()
         return Promise.all(definitions.map((definition) => tailwind.normalize(definition)))
       }
       try {
@@ -177,10 +174,6 @@ export async function analyze(
               )
             )
               definition.movement = undefined
-            if (definition.unresolved.length || definition.kind === 'review')
-              definition.dependencies = [
-                ...new Set([...definition.dependencies, ...(tree.dependencies.get(file) ?? [])]),
-              ].sort()
           }
           const normalized: Definition[] = []
           for (const definition of defs) normalized.push(await tailwind.normalize(definition))
@@ -191,7 +184,7 @@ export async function analyze(
           /\.html?$/.test(file) ||
           (/\.mdx?$/.test(file) && config.renderedMarkdown.some((root) => file.startsWith(root)))
         )
-          return normalizeAll(extractDocument(source, file))
+          return normalizeAll(extractDocument(source, file, resolver))
         if (/\.(?:scss|sass|less|vue|svelte)$/.test(file))
           return [review(file, entry.oid, 'Unsupported rendering syntax')]
       } catch {
@@ -249,16 +242,15 @@ export async function analyze(
       if (file !== 'bun.lock' && !file.endsWith('/package.json') && file !== 'package.json')
         continue
       if (file === 'bun.lock') {
-        findings.push(
-          finding(
-            undefined,
-            review(
-              file,
-              after.entries.get(file)?.oid ?? '',
-              'Resolved dependency changes can affect rendering'
+        const a = renderingLock(before.texts.get(file), config.renderingDependencies)
+        const b = renderingLock(after.texts.get(file), config.renderingDependencies)
+        if (JSON.stringify(a) !== JSON.stringify(b))
+          findings.push(
+            finding(
+              { ...review(file, '', 'Resolved dependency changes can affect rendering'), value: a },
+              { ...review(file, '', 'Resolved dependency changes can affect rendering'), value: b }
             )
           )
-        )
         continue
       }
       const project = (source?: string) => {
@@ -298,6 +290,11 @@ export async function analyze(
         )
     }
   }
+  findings.push(...fileLoadedInputs(before, after, config))
+  if (findings.length && !before.graph) {
+    before.buildGraph()
+    after.buildGraph()
+  }
   report.status = 'completed'
   report.findings = groupFindings(findings, causes, before, after, renames)
   report.flagged = report.findings.some((item) => item.decision === 'flag')

scripts/design-diff/ast.ts

@@ -2,6 +2,7 @@ import { createHash } from 'node:crypto'
 import { parse } from '@babel/parser'
 import traverseModule, { type NodePath } from '@babel/traverse'
 import * as t from '@babel/types'
+import { normalizeLiteralAliases, normalizeRefactors } from '#design-diff/refactors'
 import type { Data, Location } from '#design-diff/types'
 
 /** Handles Babel's CommonJS interop consistently in Bun and Vitest. */
@@ -11,13 +12,16 @@ export const traverse: typeof traverseModule =
     : (traverseModule as unknown as { default: typeof traverseModule }).default
 
 export function parseSource(source: string, file: string) {
-  return parse(source, {
+  const ast = parse(source, {
     sourceType: 'unambiguous',
     sourceFilename: file,
     plugins: ['jsx', 'typescript', 'decorators-legacy'],
     errorRecovery: false,
     attachComment: false,
   })
+  normalizeRefactors(ast, traverse)
+  normalizeLiteralAliases(ast, traverse)
+  return ast
 }
 
 /** Removes syntax trivia and erased types, retaining runtime ordering and literal whitespace. */
@@ -67,6 +71,34 @@ export function canonical(value: unknown): Data {
   return result
 }
 
+/** JSON data has no erased AST fields: retain every key and meaningful array order. */
+export function canonicalJson(value: Data): Data {
+  if (Array.isArray(value)) return value.map(canonicalJson)
+  if (!value || typeof value !== 'object') return value
+  return Object.fromEntries(
+    Object.keys(value)
+      .sort()
+      .map((key) => [key, canonicalJson(value[key])])
+  )
+}
+
+/** React's line-wise JSX text whitespace semantics, including explicit single-line spaces. */
+export function jsxText(text: string): string {
+  const lines = text.replace(/\r\n?/g, '\n').split('\n')
+  let last = 0
+  lines.forEach((line, i) => {
+    if (/[^ \t]/.test(line)) last = i
+  })
+  return lines
+    .map((line, i) => {
+      let value = line.replace(/\t/g, ' ')
+      if (i !== 0) value = value.replace(/^ +/, '')
+      if (i !== lines.length - 1) value = value.replace(/ +$/, '')
+      return value ? value + (i !== last ? ' ' : '') : ''
+    })
+    .join('')
+}
+
 export function location(file: string, node?: t.Node | null): Location {
   return { file, line: node?.loc?.start.line ?? 1, column: (node?.loc?.start.column ?? 0) + 1 }
 }
@@ -107,9 +139,15 @@ export function semanticSource(source: string, file: string): string {
   return JSON.stringify(canonical(ast.program))
 }
 
-/** Compact evidence for unsupported syntax without duplicating entire function bodies. */
+const fingerprints = new WeakMap<object, string>()
+
+/** Source ASTs are immutable; repeated references share their complete semantic hash. */
 export function fingerprint(value: unknown): string {
-  return createHash('sha256')
+  const cached = value && typeof value === 'object' ? fingerprints.get(value) : undefined
+  if (cached) return cached
+  const hash = createHash('sha256')
     .update(JSON.stringify(canonical(value)))
     .digest('hex')
+  if (value && typeof value === 'object') fingerprints.set(value, hash)
+  return hash
 }