feat(capture): learn editable motion, sandbox source canvas code, and promote after live verification

docs/generic-behavior-discovery.md

@@ -0,0 +1,63 @@
+# Generic source behavior discovery
+
+Plain URL capture learns diagnosed dynamic behavior from the source itself,
+with no site recipe. For each page `source-interactivity.json` marks
+`unreproduced`, capture:
+
+1. **Observes** the live source twice under controlled `Date` values and
+   timezones. Instrumentation installed before source scripts records
+   timestamped text of leaf and mixed-content elements, visibility changes with
+   their mechanism (`visibility` or `display`), click/pointer listener
+   registrations, and 2D canvas drawing calls. Startup ends when text has been
+   quiet (bounded); each discovered click is probed from a settled page.
+2. **Learns** an editable program in the Blocks Engine motion vocabulary
+   (`learned-motion.ts`):
+   - sequential text reveals: per-character cadence, delay after the previous
+     step, and click replay trigger and delay;
+   - a static pending message with the common cycling-dot suffix;
+   - elements hidden until a reveal starts (`revealSelectors`);
+   - a live clock: which elements carry hour, minute, AM/PM, GMT offset and
+     locale date (proved by the two controlled times), plus its startup and
+     replay frames and date reveal.
+   Values are measured, never embedded editorial copy: the runtime replays the
+   current DOM text, so an edited page replays its edited value.
+3. **Reports** everything the vocabulary cannot express in `unsupported`
+   (unrecognized text or visibility changes, unmatched time-derived text,
+   concurrent reveals, and canvas drawing). Nothing is substituted.
+4. **Reuses source canvas code** (`canvas-sandbox.ts`). A canvas drawing
+   algorithm is not re-derived: the page's own captured, hash-verified scripts
+   that draw on a canvas run behind a DOM membrane. Canvas elements, drawing
+   contexts and event listeners are live; reads return real values; every write
+   to non-canvas content is discarded, so the source can never overwrite the
+   editable page. If the sandboxed copy does not reproduce the source pointer
+   response, promotion falls back to learned motion with canvas as a residual.
+5. **Promotes** (`learned-motion-promotion.ts`) only after a staged portable
+   copy reproduces the learned behavior against the live source at
+   390/768/1440px with a contract derived from the evidence. The markers and
+   the portable interpreter (`motion-runtime.ts`) are then written to
+   `website/`, with a `portable-motion.json` receipt (`origin: "learned"`)
+   carrying the derived contract and the unsupported residuals.
+
+Plain `compare` re-verifies the receipt on every run and fails visibly for each
+residual: `source behavior not translated (<selector>): <reason>`.
+
+## WordPress
+
+The page carries inert `data-blocks-engine-motion-steps` and
+`data-blocks-engine-live-clock` markers. Blocks Engine lowers them to editable
+motion-sequence and live-clock blocks with their own view scripts. The portable
+interpreter is tagged `data-blocks-engine-marker-runtime`, so the importer
+replaces it with those view scripts rather than loading both.
+
+## Current limits
+
+- Canvas: source canvas code runs only if it still works when its content
+  writes are discarded; code whose drawing depends on content it would have
+  created itself stays an explicit residual. In WordPress the sandbox is a
+  preserved runtime island (`runtime_js`); page content remains native blocks.
+- Measured timings carry timer jitter from the observing machine; the fidelity
+  gate compares observable phases and settled state, not exact milliseconds.
+- The vocabulary expresses sequential reveals, pending dots, reveal visibility
+  and one live clock per page; other patterns are reported, not approximated.
+
+Tracker: https://github.com/Automattic/data-liberation-agent/issues/425

src/lib/behavior-discovery.ts

@@ -0,0 +1,70 @@
+import { existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { sourceContextOptions } from './browser-kit/browser-kit.js';
+import { captureSourceBehavior } from './screenshot/behavior-capture.js';
+import { modelSourceBehavior, timeDependentTargets } from './behavior-model.js';
+import { learnMotion, type LearnedMotion } from './learned-motion.js';
+import { promoteLearnedMotion, type LearnedPromotion } from './learned-motion-promotion.js';
+
+const FIRST_TIME = { at: '2031-02-03T04:17:00Z', timezone: 'UTC' };
+const SECOND_TIME = { at: '2032-08-14T19:43:00Z', timezone: 'Europe/Berlin' };
+
+/**
+ * Learn diagnosed behavior from the source itself (bounded site-wide browser
+ * budget), then promote it only if the staged portable site reproduces it.
+ */
+export async function discoverCapturedBehavior( directory: string ): Promise< void > {
+	const path = join( directory, 'source-interactivity.json' );
+	if ( ! existsSync( path ) ) return;
+	const source = JSON.parse( readFileSync( path, 'utf8' ) ) as { pages?: Array< { url: string; status: string } > };
+	const pages = ( source.pages ?? [] ).filter( ( page ) => page.status === 'unreproduced' );
+	if ( ! pages.length ) return;
+	const { chromium } = await import( 'playwright' );
+	const browser = await chromium.launch();
+	const observations: Array< Record< string, unknown > > = [];
+	const learned: Array< { url: string; learned: LearnedMotion } > = [];
+	const failures: Array< { url: string; error: string } > = [];
+	let promotion: LearnedPromotion | null = null;
+	const attempts: Array< { sandboxCanvas: boolean; promoted: boolean; failures: string[] } > = [];
+	try {
+		for ( const { url } of pages.slice( 0, 4 ) ) {
+			const open = async ( timezoneId: string ) => browser.newPage( { ...await sourceContextOptions( browser, url ), timezoneId, viewport: { width: 1440, height: 900 } } );
+			const page = await open( FIRST_TIME.timezone );
+			const counterfactualPage = await open( SECOND_TIME.timezone );
+			try {
+				const evidence = await captureSourceBehavior( page, url, { fixedTime: FIRST_TIME.at } );
+				// A second controlled date and timezone separates time-derived text from editorial text.
+				const counterfactual = await captureSourceBehavior( counterfactualPage, url, { fixedTime: SECOND_TIME.at, maxClicks: 0 } );
+				const motion = learnMotion( evidence, counterfactual );
+				learned.push( { url, learned: motion } );
+				observations.push( {
+					url, evidence, counterfactual, learned: motion,
+					model: { ...modelSourceBehavior( evidence ), dynamicTime: timeDependentTargets( evidence, counterfactual ), timeProbe: { first: FIRST_TIME, second: SECOND_TIME } },
+				} );
+			} catch ( error ) {
+				failures.push( { url, error: error instanceof Error ? error.message : String( error ) } );
+			} finally {
+				await Promise.all( [ page.close(), counterfactualPage.close() ] );
+			}
+		}
+		if ( learned.length ) {
+			// Prefer running the source's own canvas code behind the content membrane;
+			// if that does not verify, promote the learned motion alone with canvas as a residual.
+			const canvas = learned.some( ( row ) => row.learned.unsupported.some( ( item ) => item.kind === 'canvas' ) );
+			if ( canvas ) {
+				const sandboxed = await promoteLearnedMotion( directory, learned, browser, { sandboxCanvas: true } );
+				attempts.push( { sandboxCanvas: true, promoted: sandboxed.promoted, failures: sandboxed.failures } );
+				if ( sandboxed.promoted ) promotion = sandboxed;
+			}
+			promotion ??= await promoteLearnedMotion( directory, learned, browser );
+		}
+	} finally { await browser.close(); }
+	const receipt = existsSync( join( directory, 'portable-motion.json' ) ) ? JSON.parse( readFileSync( join( directory, 'portable-motion.json' ), 'utf8' ) ) as { unsupported?: Record< string, unknown[] > } : null;
+	const residual = Object.values( receipt?.unsupported ?? {} ).some( ( rows ) => rows.length );
+	writeFileSync( join( directory, 'source-behavior.json' ), JSON.stringify( {
+		schema: 'data-liberation/source-behavior-report/v1', observed: observations.length,
+		total: pages.length, omitted: Math.max( 0, pages.length - 4 ), observations, failures,
+		promotion: promotion && { promoted: promotion.promoted, failures: promotion.failures, evidence: promotion.evidence, attempts },
+		status: promotion?.promoted ? ( residual ? 'partially_translated' : 'translated' ) : 'observed_untranslated',
+	}, null, 2 ) );
+}

src/lib/behavior-model.test.ts

@@ -0,0 +1,32 @@
+import { describe, expect, it } from 'vitest';
+import { learnTextReveals } from './behavior-model.js';
+import type { BehaviorTrace } from './screenshot/behavior-capture.js';
+
+const trace = ( text: BehaviorTrace[ 'text' ] ): BehaviorTrace => ( { text, visibility: {}, canvas: {}, events: [], truncated: false } );
+describe( 'observed text reveal semantics', () => {
+	it( 'learns different target IDs, text and timer intervals from evidence', () => {
+		for ( const [ name, value, interval ] of [ [ '#first', 'HELLO', 30 ], [ '#renamed', 'Changed copy', 72 ] ] as const ) {
+			const history = [ { at: 0, text: '' }, ...Array.from( value ).map( ( _, index ) => ( { at: 400 + index * interval, text: value.slice( 0, index + 1 ) } ) ) ];
+			expect( learnTextReveals( trace( { [ name ]: history } ) ) ).toEqual( [ { selector: name, startMs: 400, intervalMs: interval, characters: value.length, confidence: 'observed_prefix_progression' } ] );
+		}
+	} );
+	it( 'does not reinterpret a clock, random counter or final snapshot as a typewriter', () => {
+		expect( learnTextReveals( trace( { '#counter': [ { at: 0, text: '10' }, { at: 50, text: '11' }, { at: 100, text: '12' }, { at: 150, text: '13' } ], '#static': [ { at: 0, text: 'Visible copy' } ] } ) ) ).toEqual( [] );
+	} );
+	it( 'keeps cyclic loading dots distinct from one-shot editorial reveal', () => {
+		const frames = [ '', '.', '..', '...', '', '.', '..', '...' ].map( ( text, index ) => ( { at: index * 100, text } ) );
+		expect( learnTextReveals( trace( { '#cycle': frames } ) ) ).toEqual( [] );
+	} );
+	it( 'learns the rate when coalesced mutation sampling skips characters', () => {
+		const frames = [ { at: 0, text: '' }, { at: 100, text: 'A' }, { at: 160, text: 'ABC' }, { at: 190, text: 'ABCD' }, { at: 250, text: 'ABCDEF' } ];
+		expect( learnTextReveals( trace( { '#coalesced': frames } ) )[ 0 ] ).toMatchObject( { selector: '#coalesced', startMs: 100, intervalMs: 30, characters: 6 } );
+	} );
+	it( 'rejects an instantaneous whole-string swap as a reveal', () => {
+		const frames = [ { at: 0, text: '' }, { at: 100, text: 'A' }, { at: 101, text: 'AB' }, { at: 102, text: 'ABCDEFGHIJKL' } ];
+		expect( learnTextReveals( trace( { '#swap': frames } ) ) ).toEqual( [] );
+	} );
+	it( 'recognizes a click replay resetting existing saved text before revealing it', () => {
+		const frames = [ 'WORD', '', 'W', 'WO', 'WOR', 'WORD' ].map( ( text, index ) => ( { at: index * 30, text } ) );
+		expect( learnTextReveals( trace( { '#saved': frames } ) )[ 0 ] ).toMatchObject( { selector: '#saved', intervalMs: 30, characters: 4 } );
+	} );
+} );

src/lib/behavior-model.ts

@@ -0,0 +1,79 @@
+import type { BehaviorTrace, SourceBehavior } from './screenshot/behavior-capture.js';
+
+export interface LearnedTextReveal {
+	selector: string;
+	/** Relative to phase start; source text is evidence, never runtime replay payload. */
+	startMs: number;
+	intervalMs: number;
+	characters: number;
+	confidence: 'observed_prefix_progression';
+}
+
+/** Only classify observed one-character prefix progressions. Other DOM changes remain unknown. */
+export function learnTextReveals( trace: BehaviorTrace ): LearnedTextReveal[] {
+	const result: LearnedTextReveal[] = [];
+	for ( const [ selector, states ] of Object.entries( trace.text ) ) {
+		if ( states.length < 4 ) continue;
+		const final = states.at( -1 )!.text;
+		if ( final.length < 3 || final.length > 512 ) continue;
+		// Cycling dots and other repeated prefixes are not a one-shot text reveal.
+		const timeline = states[ 0 ].text === final && states[ 1 ]?.text !== final ? states.slice( 1 ) : states;
+		const prefixes = timeline.filter( ( state ) => final.startsWith( state.text ) );
+		if ( prefixes.slice( 1 ).some( ( state, index ) => state.text.length < prefixes[ index ].text.length ) ) continue;
+		const frames = timeline.filter( ( state ) => state.text.length > 0 && final.startsWith( state.text ) );
+		// Browsers coalesce mutation callbacks under load, so observed frames may
+		// skip characters. Accept strictly growing prefixes and measure the
+		// per-character rate over the whole run rather than per frame.
+		let start = frames.length - 1;
+		while ( start > 0 && frames[ start ].text.length > frames[ start - 1 ].text.length && frames[ start ].at > frames[ start - 1 ].at ) start--;
+		const sequence = frames.slice( start );
+		const first = Array.from( sequence[ 0 ]?.text ?? '' ).length;
+		const last = Array.from( final ).length;
+		if ( sequence.length < 3 || first > 2 || sequence.at( -1 )!.text !== final ) continue;
+		const average = ( sequence.at( -1 )!.at - sequence[ 0 ].at ) / Math.max( 1, last - first );
+		// Timer jitter only ever delays a character. The lower quartile of the
+		// single-character steps (excluding catch-up bursts) estimates the
+		// requested rate better than the load-inflated average.
+		const singles = sequence.slice( 1 )
+			.filter( ( frame, index ) => Array.from( frame.text ).length - Array.from( sequence[ index ].text ).length === 1 )
+			.map( ( frame ) => frame.at - sequence[ sequence.indexOf( frame ) - 1 ].at )
+			.filter( ( value ) => value >= average * .5 )
+			.sort( ( a, b ) => a - b );
+		const intervalMs = Math.round( singles.length >= 4 ? singles[ Math.floor( singles.length / 4 ) ] : average );
+		if ( intervalMs < 5 || intervalMs > 1000 ) continue;
+		// Reject bursts: a multi-character jump must have taken plausible time for
+		// that rate. Single-character steps are always valid; timer jitter can
+		// deliver two consecutive ticks only milliseconds apart.
+		if ( sequence.slice( 1 ).some( ( frame, index ) => {
+			const added = Array.from( frame.text ).length - Array.from( sequence[ index ].text ).length;
+			return added > 1 && frame.at - sequence[ index ].at < ( added - 1 ) * intervalMs * .25;
+		} ) ) continue;
+		result.push( { selector, startMs: sequence[ 0 ].at, intervalMs, characters: Array.from( final ).length, confidence: 'observed_prefix_progression' } );
+	}
+	return result.sort( ( a, b ) => a.startMs - b.startMs );
+}
+
+/** Explicit evidence/model separation: a canvas trace is not a translated canvas algorithm. */
+export function modelSourceBehavior( observed: SourceBehavior ) {
+	return {
+		schema: 'data-liberation/behavior-model/v1',
+		url: observed.url,
+		startup: learnTextReveals( observed.startup ),
+		replays: observed.replays.map( ( replay ) => ( { trigger: replay.selector, textReveals: learnTextReveals( replay.trace ) } ) ),
+		unsupported: Object.keys( observed.pointer.trace.canvas ).map( ( selector ) => ( {
+			selector, capability: 'canvas-algorithm', reason: 'Drawing API observations do not reconstruct pointer-dependent source computation',
+			methods: Object.keys( observed.pointer.trace.canvas[ selector ].methods ),
+		} ) ),
+		truncated: observed.startup.truncated || observed.pointer.trace.truncated || observed.replays.some( ( replay ) => replay.trace.truncated ),
+		status: 'observed_untranslated',
+	};
+}
+
+/** A counterfactual Date probe distinguishes dynamic time text from editorial prefixes. */
+export function timeDependentTargets( first: SourceBehavior, second: SourceBehavior ): string[] {
+	return Object.keys( first.startup.text ).filter( ( selector ) => {
+		const initial = first.startup.settledText ? first.startup.settledText[ selector ] : first.startup.text[ selector ].at( -1 )?.text;
+		const counterfactual = second.startup.settledText ? second.startup.settledText[ selector ] : second.startup.text[ selector ]?.at( -1 )?.text;
+		return initial !== undefined && counterfactual !== undefined && initial !== counterfactual;
+	} );
+}

src/lib/canvas-sandbox.test.ts

@@ -0,0 +1,74 @@
+import { chromium, type Browser } from 'playwright';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import { buildCanvasSandbox, drawsOnCanvas } from './canvas-sandbox.js';
+
+// Neutral source code: overwrites copy, hides and restyles elements, rebuilds
+// markup, and draws on pointer movement and clicks.
+const SOURCE = `
+const copy = document.getElementById('copy');
+copy.textContent = 'SOURCE OVERWRITE';
+copy.style.display = 'none';
+copy.classList.add('hidden-by-source');
+copy.setAttribute('data-source', 'yes');
+document.querySelector('#panel').innerHTML = '<b id="injected">x</b>';
+document.getElementById('injected').textContent = 'still inert';
+document.body.appendChild(document.createElement('section'));
+const surface = document.getElementById('surface');
+surface.width = 300; surface.height = 120;
+const context = surface.getContext('2d');
+let strokes = 0;
+window.addEventListener('mousemove', (event) => {
+  context.strokeStyle = '#123456'; context.beginPath();
+  context.moveTo(event.clientX % 300, 10); context.lineTo((event.clientX + 40) % 300, 100); context.stroke(); strokes++;
+});
+document.querySelectorAll('.tile').forEach((tile) => tile.addEventListener('click', (event) => {
+  event.target.textContent = 'clicked';
+  const box = event.currentTarget.getBoundingClientRect();
+  context.fillStyle = '#654321'; context.fillRect(0, 0, Math.max(1, Math.round(box.width) % 50), 20);
+}));
+document.getElementById('panel').onclick = function () { this.textContent = 'handler wrote'; context.fillRect(200, 0, 10, 10); };
+const extra = document.createElement('canvas'); extra.id = 'extra'; document.body.appendChild(extra);
+window.__strokes = () => strokes;
+`;
+
+describe( 'source canvas sandbox', () => {
+	let browser: Browser;
+	beforeAll( async () => { browser = await chromium.launch(); } );
+	afterAll( async () => { await browser.close(); } );
+
+	it( 'selects only drawing code', () => {
+		expect( drawsOnCanvas( SOURCE ) ).toBe( true );
+		expect( drawsOnCanvas( 'navigator.sendBeacon("/rum")' ) ).toBe( false );
+	} );
+
+	it( 'runs source drawing and listeners while every content write is discarded', async () => {
+		const page = await browser.newPage( { viewport: { width: 800, height: 600 } } );
+		const errors: string[] = [];
+		page.on( 'pageerror', ( error ) => errors.push( error.message ) );
+		page.on( 'console', ( message ) => { if ( message.type() === 'error' ) errors.push( message.text() ); } );
+		try {
+			await page.setContent( `<main><p id="copy" class="keep">Edited copy</p><div id="panel"><span class="tile">A tile</span></div></main><canvas id="surface"></canvas>` );
+			const before = await page.evaluate( () => document.body.innerHTML );
+			await page.addScriptTag( { content: buildCanvasSandbox( [ { url: 'https://source.test/app.js', sha256: 'x', body: SOURCE } ] ) } );
+			const pixels = () => page.evaluate( () => ( document.getElementById( 'surface' ) as HTMLCanvasElement ).toDataURL() );
+			const blank = await pixels();
+			await page.mouse.move( 100, 300 );
+			await page.mouse.move( 220, 310 );
+			expect( await page.evaluate( () => ( window as unknown as { __strokes: () => number } ).__strokes() ) ).toBeGreaterThan( 0 );
+			const drawn = await pixels();
+			expect( drawn ).not.toBe( blank );
+			await page.click( '.tile' );
+			await page.click( '#panel', { position: { x: 200, y: 5 } } );
+			expect( await pixels() ).not.toBe( drawn );
+			// Only the source-created canvas joined the page; all editable content is untouched.
+			const after = await page.evaluate( () => {
+				document.getElementById( 'extra' )?.remove();
+				const surface = document.getElementById( 'surface' )!;
+				surface.removeAttribute( 'width' ); surface.removeAttribute( 'height' );
+				return document.body.innerHTML;
+			} );
+			expect( after ).toBe( before );
+			expect( errors ).toEqual( [] );
+		} finally { await page.close(); }
+	} );
+} );