From 3818b89b0da842a1a85d04eb25a0c14a01ed47c6 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Thu, 30 Jul 2026 17:46:11 -0700 Subject: [PATCH 01/28] feat(worlds): number events by slot in the Local and Postgres Worlds Slot identity is only useful if a World can actually keep it, so both first-party Worlds now allocate, honour, and defend dense per-run positions: - `SPEC_VERSION_MAX_SUPPORTED` separates the newest version a World can read from the version it stamps, so turning the flag on somewhere does not make the runs it creates unreadable elsewhere. - `mintedSpecVersion()` gives both Worlds one place to opt new runs in. - The Local World allocates under its storage lock and re-probes on a lost exclusive write; the Postgres World makes the events primary key run-scoped and treats a unique violation as "try the next free position", re-probing every round so contention always makes progress. - A caller-claimed position that is already taken is a 409 carrying the events the caller was missing, and nothing is materialized for it. Co-Authored-By: Claude Opus 5 --- .changeset/slot-event-identity-worlds.md | 8 + .../docs/v5/configuration/runtime-tuning.mdx | 9 + packages/core/src/runtime.ts | 113 +++- packages/core/src/runtime/helpers.test.ts | 43 ++ packages/core/src/runtime/helpers.ts | 34 +- packages/core/src/runtime/start.test.ts | 33 +- .../core/src/runtime/world-compatibility.ts | 27 +- packages/core/src/workflow.ts | 26 +- packages/world-local/src/index.ts | 7 +- .../world-local/src/storage/events-storage.ts | 288 ++++++++- packages/world-local/src/storage/helpers.ts | 92 ++- .../src/storage/slot-identity.test.ts | 281 +++++++++ .../world-local/src/storage/slots.test.ts | 241 +++++++ packages/world-local/src/storage/slots.ts | 247 ++++++++ .../0018_run_scoped_event_and_step_keys.sql | 13 + .../src/drizzle/migrations/meta/_journal.json | 7 + packages/world-postgres/src/drizzle/schema.ts | 18 +- packages/world-postgres/src/index.ts | 7 +- packages/world-postgres/src/slots.ts | 225 +++++++ packages/world-postgres/src/storage.ts | 587 ++++++++++++------ .../world-postgres/test/slot-identity.test.ts | 375 +++++++++++ packages/world/src/index.ts | 3 + packages/world/src/slot-identity.test.ts | 25 +- packages/world/src/spec-version.test.ts | 64 +- packages/world/src/spec-version.ts | 52 +- packages/world/src/ulid.ts | 12 + 26 files changed, 2529 insertions(+), 308 deletions(-) create mode 100644 .changeset/slot-event-identity-worlds.md create mode 100644 packages/world-local/src/storage/slot-identity.test.ts create mode 100644 packages/world-local/src/storage/slots.test.ts create mode 100644 packages/world-local/src/storage/slots.ts create mode 100644 packages/world-postgres/src/drizzle/migrations/0018_run_scoped_event_and_step_keys.sql create mode 100644 packages/world-postgres/src/slots.ts create mode 100644 packages/world-postgres/test/slot-identity.test.ts diff --git a/.changeset/slot-event-identity-worlds.md b/.changeset/slot-event-identity-worlds.md new file mode 100644 index 0000000000..863a2f860b --- /dev/null +++ b/.changeset/slot-event-identity-worlds.md @@ -0,0 +1,8 @@ +--- +'@workflow/world': minor +'@workflow/world-local': minor +'@workflow/world-postgres': minor +'@workflow/core': minor +--- + +Number a run's events by position in the Local and Postgres Worlds when `WORKFLOW_SLOT_IDENTITY` is set, so a reader can prove its copy of an event log is complete. diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index 2f3e68e8c3..110e569d5f 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -48,6 +48,15 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL - Backends that do not support the guard ignore the snapshot; they must not declare the capability, so guard-dependent optimizations stay off against them even when the flag is set. - Set `0` to disable. +### `WORKFLOW_SLOT_IDENTITY` + +- Default: disabled +- Numbers a new run's events by position instead of by ULID: `evnt_…001` is the run's first event, `evnt_…002` its second, with no gaps. A reader can then prove its copy of the log is complete, because the highest number is the event count. +- Contention becomes explicit rather than silent. Two writers proposing the same position cannot both win: the loser gets a 409 ([`SlotConflictError`](/docs/api-reference/workflow-errors/slot-conflict-error)) carrying the events it was missing, and the runtime merges them, replays, and re-proposes at the next free position. +- Applies only to runs created while it is set. A run keeps the identity scheme it was created with for life, so turning the flag on or off never affects runs already in flight. +- Requires a World that supports it. A World that does not rejects the run outright rather than mis-numbering its events. +- Set `1` or `true` to enable. + ## Inline execution ### `WORKFLOW_V2_TIMEOUT_MS` diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index b877d90e0a..95d19866a2 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -4,7 +4,6 @@ import { EntityConflictError, FatalError, MaxEventsExceededError, - PreconditionFailedError, ReplayDivergenceError, RUN_ERROR_CODES, type RunErrorCode, @@ -24,6 +23,7 @@ import { resolveQueueNamespace, SPEC_VERSION_CURRENT, SPEC_VERSION_SUPPORTS_COMPRESSION, + slotFromId, WorkflowInvokePayloadSchema, type WorkflowRun, type World, @@ -57,6 +57,7 @@ import { memoizeEncryptionKey, parseHealthCheckPayload, queueMessage, + requiresFreshReplay, toMutableEventLog, withEventCreateFence, withHealthCheck, @@ -547,6 +548,19 @@ export function workflowEntrypoint( let workflowStartedAt = -1; let preloadedEvents: Event[] | undefined; let preloadedEventsCursor: string | null | undefined; + // Highest slot known to be published on a slot-numbered run, + // for the writes whose snapshot cannot show it: turbo + // backgrounds `run_started` and replays against an empty log, + // so a claim numbered from that log alone would propose a slot + // `run_started` already holds. 0 when the run is not + // slot-numbered — its ids carry no position to compare. + let knownSlotFloor = 0; + const observeSlotFloor = (eventId: string | undefined) => { + const slot = eventId ? slotFromId(eventId) : undefined; + if (slot !== undefined && slot > knownSlotFloor) { + knownSlotFloor = slot; + } + }; // Latency telemetry (TTFS) state — see runtime/step-latency.ts. // Whether this invocation's FIRST event snapshot contained @@ -1021,6 +1035,10 @@ export function workflowEntrypoint( (r) => { const limit = clampMaxEvents(r?.maxEvents); if (limit !== undefined) maxEventsLimit = limit; + // Every write of this invocation is ordered after this + // promise by `runReadyBarrier`, so the slot it reports + // is in hand before the first claim is numbered. + observeSlotFloor(r?.event?.eventId); }, () => {} ); @@ -1088,6 +1106,7 @@ export function workflowEntrypoint( } workflowRun = result.run; maxEventsLimit = clampMaxEvents(result.maxEvents); + observeSlotFloor(result.event?.eventId); // Anchors RSFS — see the declaration above. runStartedReceivedAtMs = Date.now(); @@ -1367,7 +1386,11 @@ export function workflowEntrypoint( // place by the guard's reloads, so a per-iteration // rescan for the slot high-water mark would be wasted // work on an array that never changes identity here. - const waitLog = toMutableEventLog(events, eventsCursor); + const waitLog = toMutableEventLog( + events, + eventsCursor, + knownSlotFloor + ); for (const waitEvent of waitsToComplete) { try { await withEventCreateFence( @@ -1491,6 +1514,20 @@ export function workflowEntrypoint( // point and the inline executeStep mutates eventsCursor. preInlineWriteCursor = eventsCursor; + // One log for everything this replay writes on its way to + // a terminal event: the end-of-run drain inside + // `runWorkflow` (fire-and-forget `*_created` events, and + // the implicit disposal of the abort hooks a completing + // run leaves behind) and the `run_completed` / + // `run_failed` write below. Sharing it is what keeps the + // two from claiming the same slot — the terminal write + // numbers from a snapshot that predates the drain. + const replayWriteLog = toMutableEventLog( + events, + eventsCursor, + knownSlotFloor + ); + // Replay workflow runtimeLogger.debug('Starting workflow replay', { workflowRunId: runId, @@ -1516,7 +1553,8 @@ export function workflowEntrypoint( // `awaitRunReady()` below, so gate those writes on the // backgrounded run_started too. Undefined outside turbo. runReadyBarrier, - world.capabilities + world.capabilities, + replayWriteLog ); await payloadPrewarm; runtimeLogger.debug('Workflow replay completed', { @@ -1527,11 +1565,12 @@ export function workflowEntrypoint( // Workflow completed. Send the snapshot but do NOT // reload-and-retry the create in place: `result` was - // computed by this replay, so a stale (412) rejection must - // force a *fresh replay* (which may observe the new event - // and produce a different result), not re-commit the stale - // result. The catch below lets PreconditionFailedError - // propagate to the queue for re-invocation. + // computed by this replay, so a rejection proving the view + // was incomplete must force a *fresh replay* (which may + // observe the new event and produce a different result), + // not re-commit the stale result. The catch below lets + // `requiresFreshReplay` rejections propagate to the queue + // for re-invocation. try { // Turbo: a workflow that finishes with no steps reaches // here before the backgrounded run_started; order the @@ -1547,7 +1586,7 @@ export function workflowEntrypoint( { requestId, ...eventCreateFenceFor( - toMutableEventLog(events, eventsCursor), + replayWriteLog, workflowRun.specVersion ), } @@ -1637,7 +1676,8 @@ export function workflowEntrypoint( } const suspensionLog = toMutableEventLog( cachedEvents, - eventsCursor + eventsCursor, + knownSlotFloor ); let suspensionResult: Awaited< ReturnType @@ -1653,13 +1693,14 @@ export function workflowEntrypoint( runReadyBarrier, }); } catch (suspensionError) { - // A suspension create whose stale (412) rejection - // survived the in-guard reload retries: schedule an + // A suspension create whose incomplete-view rejection + // (412 stale watermark, or 409 taken slot) survived + // the in-guard reload retries: schedule an // explicit immediate re-invocation (a rethrow relies // on redelivery of a message the turbo path already // acked — the run would stall for the queue's ~300s // default visibility timeout). - if (PreconditionFailedError.is(suspensionError)) { + if (requiresFreshReplay(suspensionError)) { runtimeLogger.warn( 'Suspension event creation rejected as stale after reload retries; re-invoking run for a fresh replay', { workflowRunId: runId, loopIteration } @@ -2273,10 +2314,14 @@ export function workflowEntrypoint( // whole batch so each claim draws its own event slot; // `eventCreateFenceFor` yields undefined for a run // fenced neither way, leaving those claims as they were. - const inlineClaimLog = toMutableEventLog( - cachedEvents ?? [], - eventsCursor - ); + // + // The suspension's own log, not a second one over the + // same snapshot: its reservations are what the hook and + // wait creates just above took, and those events are not + // in `cachedEvents` yet. A fresh log would number these + // claims from the same base and hand the batch's first + // step a slot the suspension already holds. + const inlineClaimLog = suspensionLog; replayBudget.pause(); let stepResults: Awaited< @@ -2416,17 +2461,18 @@ export function workflowEntrypoint( stepExecutionPromises ); } catch (stepErr) { - // A stale (412) rejection of an inline step_started - // claim: the loaded view this batch was scheduled - // from is behind an out-of-band event (e.g. a - // received hook), so the claim was fenced by the + // An incomplete-view rejection of an inline + // step_started claim (412 stale watermark, or 409 + // taken slot): the loaded view this batch was + // scheduled from is behind an out-of-band event (e.g. + // a received hook), so the claim was fenced by the // guard and no step events were written. Abandon the // batch — any optimistic body result is discarded by // executeStep's reconciliation — and re-invoke for a // fresh replay that observes the new event. Wait for // the sibling executions to settle first so no owned // body is in flight when the ack path runs. - if (PreconditionFailedError.is(stepErr)) { + if (requiresFreshReplay(stepErr)) { await Promise.allSettled(stepExecutionPromises); runtimeLogger.warn( 'Inline step claim rejected as stale; re-invoking run for a fresh replay', @@ -2623,17 +2669,18 @@ export function workflowEntrypoint( } } } else { - // Stale-snapshot rejection of a result-bearing create - // (run_completed sends the snapshot but is intentionally - // NOT retried in place), or one that survived the - // in-guard reload retries. Don't fail the run — schedule - // an explicit immediate re-invocation so a fresh replay - // observes the new event. Rethrowing instead would rely - // on redelivery of the CURRENT message, which the turbo - // path has already acked — empirically the run then - // stalls for the queue's ~300s default visibility - // timeout before completing. - if (PreconditionFailedError.is(err)) { + // Incomplete-view rejection of a result-bearing create — + // a stale watermark (412) or a taken slot (409), either + // on `run_completed` (which sends its fence but is + // intentionally NOT retried in place) or on a create + // that survived the in-guard reload retries. Don't fail + // the run — schedule an explicit immediate re-invocation + // so a fresh replay observes the new event. Rethrowing + // instead would rely on redelivery of the CURRENT + // message, which the turbo path has already acked — + // empirically the run then stalls for the queue's ~300s + // default visibility timeout before completing. + if (requiresFreshReplay(err)) { runtimeLogger.warn( 'Event creation rejected as stale; re-invoking run for a fresh replay', { workflowRunId: runId, loopIteration } diff --git a/packages/core/src/runtime/helpers.test.ts b/packages/core/src/runtime/helpers.test.ts index 3b050f0978..ca1a3ff4b5 100644 --- a/packages/core/src/runtime/helpers.test.ts +++ b/packages/core/src/runtime/helpers.test.ts @@ -1,4 +1,5 @@ import { + EntityConflictError, PreconditionFailedError, SlotConflictError, WorkflowWorldError, @@ -29,6 +30,7 @@ import { memoizeEncryptionKey, mergeLoadedEvents, PRECONDITION_MAX_RELOAD_RETRIES, + requiresFreshReplay, reserveSlot, stateUpdatedAtForCreate, toMutableEventLog, @@ -537,6 +539,26 @@ describe('slot bookkeeping', () => { ).toBe(0); }); + it('starts at a floor the snapshot cannot show', () => { + // Turbo replays against an empty log while its `run_started` write is still + // in flight, so the snapshot alone would number the first claim onto a slot + // that write already holds. + const log = toMutableEventLog([], null, 2); + expect(log.maxSlot).toBe(2); + expect(reserveSlot(log)).toBe(3); + }); + + it('ignores a floor the snapshot has already passed', () => { + const log = toMutableEventLog([slotEvent(5)], 'c0', 2); + expect(log.maxSlot).toBe(5); + }); + + it('keeps the floor across a merge', () => { + const log = toMutableEventLog([], null, 2); + mergeLoadedEvents(log, [slotEvent(1)]); + expect(log.maxSlot).toBe(2); + }); + it('never lowers maxSlot when an older delta is merged in', () => { const log = toMutableEventLog([slotEvent(1), slotEvent(3)], 'c0'); mergeLoadedEvents(log, [slotEvent(2)]); @@ -837,6 +859,27 @@ describe('withEventCreateFence', () => { }); }); +describe('requiresFreshReplay', () => { + it('covers both fences, so neither numbering fails the run', () => { + // Each fence reports an incomplete view in its own dialect. A caller that + // recognises only one of them fails runs on the other. + expect(requiresFreshReplay(new PreconditionFailedError('stale'))).toBe( + true + ); + expect( + requiresFreshReplay( + new SlotConflictError('taken', { eventId: slotEventId(3) }) + ) + ).toBe(true); + }); + + it('leaves every other rejection to its own handler', () => { + expect(requiresFreshReplay(new EntityConflictError('exists'))).toBe(false); + expect(requiresFreshReplay(new Error('boom'))).toBe(false); + expect(requiresFreshReplay(undefined)).toBe(false); + }); +}); + describe('withPreconditionRetry', () => { let originalGuard: string | undefined; diff --git a/packages/core/src/runtime/helpers.ts b/packages/core/src/runtime/helpers.ts index cb5e88f9f6..407f202635 100644 --- a/packages/core/src/runtime/helpers.ts +++ b/packages/core/src/runtime/helpers.ts @@ -669,12 +669,26 @@ export interface MutableEventLog { reserved: number; } -/** A `MutableEventLog` over a freshly loaded snapshot. */ +/** + * A `MutableEventLog` over a freshly loaded snapshot. + * + * `slotFloor` is a slot known to be published that the snapshot may not contain + * — the run's own `run_started`, whose write turbo backgrounds while replaying + * against an empty log. Numbering a claim from the snapshot alone would then + * propose a slot that is already taken, so every first write of a turbo + * invocation would conflict and cost the run an extra replay. + */ export function toMutableEventLog( events: Event[], - cursor: string | null + cursor: string | null, + slotFloor = 0 ): MutableEventLog { - return { events, cursor, maxSlot: maxSlotOf(events), reserved: 0 }; + return { + events, + cursor, + maxSlot: Math.max(maxSlotOf(events), slotFloor), + reserved: 0, + }; } /** @@ -982,6 +996,20 @@ export function withEventCreateFence( ); } +/** + * Whether a rejected event create means "this replay's view of the log was + * incomplete", the one condition whose only remedy is replaying from the top. + * + * Both fences report it, one per numbering: a 412 says the snapshot's watermark + * is behind, a 409 says the slot this replay counted to is already occupied. + * Neither is a failure of the run — the run's own decisions may simply need + * revising against the events it did not see — so a caller that gets one + * re-invokes for a fresh replay rather than failing. + */ +export function requiresFreshReplay(error: unknown): boolean { + return PreconditionFailedError.is(error) || SlotConflictError.is(error); +} + /** * CORS headers for health check responses. * Allows the observability UI to check endpoint health from a different origin. diff --git a/packages/core/src/runtime/start.test.ts b/packages/core/src/runtime/start.test.ts index 29a7c80419..d04a415bb7 100644 --- a/packages/core/src/runtime/start.test.ts +++ b/packages/core/src/runtime/start.test.ts @@ -2,6 +2,7 @@ import { WorkflowRuntimeError, WorkflowWorldError } from '@workflow/errors'; import { SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, + SPEC_VERSION_MAX_SUPPORTED, SPEC_VERSION_SUPPORTS_ATTRIBUTES, SPEC_VERSION_SUPPORTS_CBOR_QUEUE_TRANSPORT, } from '@workflow/world'; @@ -136,7 +137,7 @@ describe('start', () => { } as any); await expect(start(validWorkflow, [])).rejects.toThrow( - 'requires a World with matching spec version' + 'requires a World with spec version' ); expect(mockEventsCreate).not.toHaveBeenCalled(); expect(mockQueue).not.toHaveBeenCalled(); @@ -174,7 +175,7 @@ describe('start', () => { } as any); await expect(start(validWorkflow, [])).rejects.toThrow( - 'requires a World with matching spec version' + 'requires a World with spec version' ); expect(mockEventsCreate).not.toHaveBeenCalled(); expect(mockQueue).not.toHaveBeenCalled(); @@ -186,19 +187,43 @@ describe('start', () => { }); setWorld({ - specVersion: SPEC_VERSION_CURRENT + 1, + specVersion: SPEC_VERSION_MAX_SUPPORTED + 1, getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), events: { create: mockEventsCreate }, queue: mockQueue, } as any); await expect(start(validWorkflow, [])).rejects.toThrow( - 'requires a World with matching spec version' + 'requires a World with spec version' ); expect(mockEventsCreate).not.toHaveBeenCalled(); expect(mockQueue).not.toHaveBeenCalled(); }); + it('accepts a world that mints a newer version this runtime supports', async () => { + // A world opted into slot identity stamps a version above the runtime's + // current one. The runtime can read and write those runs, so the + // handshake has to pass and the run has to keep the world's version. + const validWorkflow = Object.assign(() => Promise.resolve('result'), { + workflowId: 'test-workflow', + }); + + setWorld({ + specVersion: SPEC_VERSION_MAX_SUPPORTED, + getDeploymentId: vi.fn().mockResolvedValue('deploy_123'), + events: { create: mockEventsCreate }, + queue: mockQueue, + } as any); + + await start(validWorkflow, []); + + expect(mockEventsCreate).toHaveBeenCalledWith( + expect.stringMatching(/^wrun_/), + expect.objectContaining({ specVersion: SPEC_VERSION_MAX_SUPPORTED }), + expect.anything() + ); + }); + it('should use provided specVersion when passed in options', async () => { const validWorkflow = Object.assign(() => Promise.resolve('result'), { workflowId: 'test-workflow', diff --git a/packages/core/src/runtime/world-compatibility.ts b/packages/core/src/runtime/world-compatibility.ts index c06c26eefb..f3de904359 100644 --- a/packages/core/src/runtime/world-compatibility.ts +++ b/packages/core/src/runtime/world-compatibility.ts @@ -1,19 +1,40 @@ import { WorkflowRuntimeError } from '@workflow/errors'; import type { World } from '@workflow/world'; -import { SPEC_VERSION_CURRENT } from '@workflow/world'; +import { + SPEC_VERSION_CURRENT, + SPEC_VERSION_MAX_SUPPORTED, +} from '@workflow/world'; type WorldSpecVersionMetadata = Pick; +/** + * Rejects a World whose protocol this runtime does not speak. + * + * A World declares the spec version it stamps on the runs it creates. Anything + * from {@link SPEC_VERSION_CURRENT} up to {@link SPEC_VERSION_MAX_SUPPORTED} is + * fine: the upper end covers a World opted into a newer identity scheme that + * this runtime already understands, and only versions this runtime has no code + * for are refused. Below the current version means the World package predates + * this runtime and cannot record what it emits. + */ export function assertWorldSupportsRuntimeProtocol( world: WorldSpecVersionMetadata ): void { - if (world.specVersion === SPEC_VERSION_CURRENT) { + if ( + world.specVersion !== undefined && + world.specVersion >= SPEC_VERSION_CURRENT && + world.specVersion <= SPEC_VERSION_MAX_SUPPORTED + ) { return; } const supportedVersion = world.specVersion ?? 'none'; + const supported = + SPEC_VERSION_CURRENT === SPEC_VERSION_MAX_SUPPORTED + ? `${SPEC_VERSION_CURRENT}` + : `${SPEC_VERSION_CURRENT} to ${SPEC_VERSION_MAX_SUPPORTED}`; throw new WorkflowRuntimeError( - `This Workflow runtime requires a World with matching spec version ${SPEC_VERSION_CURRENT}, ` + + `This Workflow runtime requires a World with spec version ${supported}, ` + `but the configured World declares spec version ${supportedVersion}. ` + 'Install a World package version compatible with the current Workflow runtime.' ); diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index fde4b70250..97c9acee99 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -18,6 +18,7 @@ import { runtimeLogger } from './logger.js'; import type { WorkflowOrchestratorContext } from './private.js'; import { ReplayPayloadCache } from './replay-payload-cache.js'; import { getPortLazy } from './runtime/get-port-lazy.js'; +import type { MutableEventLog } from './runtime/helpers.js'; import { runIdCreatedAt } from './runtime/run-id-time.js'; import { handleSuspension } from './runtime/suspension-handler.js'; import { getWorld } from './runtime/world.js'; @@ -74,7 +75,15 @@ async function drainPendingQueueItems( * In turbo mode, gates final `*_created` writes on backgrounded * `run_started`. Undefined when `run_started` is awaited. */ - runReadyBarrier?: Promise + runReadyBarrier?: Promise, + /** + * The replay's event log, so the drain's writes claim their slots from the + * same source the terminal `run_completed` / `run_failed` write draws from. + * Without it the drain writes unfenced — the World picks the next free slot — + * and the terminal write, numbering from a snapshot taken before the drain, + * proposes the slot the drain just took and loses it. + */ + eventLog?: MutableEventLog ): Promise { if (pendingQueue.size === 0) return; // Implicitly dispose any abort hooks (system hooks) that are still alive at @@ -101,6 +110,7 @@ async function drainPendingQueueItems( world, run: workflowRun, runReadyBarrier, + eventLog, }); } catch (err) { runtimeLogger.warn( @@ -137,7 +147,13 @@ export async function runWorkflow( * Features supported by the World executing this workflow. Missing * capabilities are treated as unsupported. */ - worldCapabilities?: WorldCapabilities + worldCapabilities?: WorldCapabilities, + /** + * The caller's event log for this replay. Its only use here is the end-of-run + * drain, whose writes have to be ordered with the caller's terminal write — + * see {@link drainPendingQueueItems}. + */ + eventLog?: MutableEventLog ): Promise { return trace(`workflow.run ${workflowRun.workflowName}`, async (span) => { span?.setAttributes({ @@ -859,7 +875,8 @@ export async function runWorkflow( vmGlobalThis, workflowRun, 'completed', - runReadyBarrier + runReadyBarrier, + eventLog ); return dehydrated; @@ -876,7 +893,8 @@ export async function runWorkflow( vmGlobalThis, workflowRun, 'failed', - runReadyBarrier + runReadyBarrier, + eventLog ); throw err; diff --git a/packages/world-local/src/index.ts b/packages/world-local/src/index.ts index f8e2166fb7..2014a75652 100644 --- a/packages/world-local/src/index.ts +++ b/packages/world-local/src/index.ts @@ -2,7 +2,7 @@ import { promises as fs } from 'node:fs'; import { rm } from 'node:fs/promises'; import path from 'node:path'; import type { QueuePrefix, World } from '@workflow/world'; -import { reenqueueActiveRuns, SPEC_VERSION_CURRENT } from '@workflow/world'; +import { mintedSpecVersion, reenqueueActiveRuns } from '@workflow/world'; import { warnIfRunningInVercelDeployment } from './build-target-mismatch.js'; import type { Config } from './config.js'; import { config, resolveRecoverActiveRuns } from './config.js'; @@ -72,7 +72,10 @@ export function createWorld(args?: Partial): LocalWorld { ); const recoverActiveRuns = resolveRecoverActiveRuns(mergedConfig); return { - specVersion: SPEC_VERSION_CURRENT, + // What this world stamps on new runs, which is not the newest version it + // can read: slot identity is readable everywhere and minted only where + // WORKFLOW_SLOT_IDENTITY is set. + specVersion: mintedSpecVersion(), ...queue, ...storage, ...instrumentObject('world.streams', { diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index 7efd49cb1e..98e705e98d 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -5,11 +5,13 @@ import { HookNotFoundError, RunExpiredError, RunNotSupportedError, + SlotConflictError, TooEarlyError, WorkflowRunNotFoundError, WorkflowWorldError, } from '@workflow/errors'; import type { + CreateEventParams, Event, EventResult, Hook, @@ -34,8 +36,12 @@ import { isTerminalWorkflowRunStatus, requiresNewerWorld, SPEC_VERSION_CURRENT, + SPEC_VERSION_MAX_SUPPORTED, StepSchema, + slotEventId, + slotFromId, ulidToDate, + usesSlotIdentity, validateAttributeChanges, validateUlidTimestamp, WaitSchema, @@ -85,6 +91,7 @@ import { } from './hooks-storage.js'; import { handleLegacyEvent } from './legacy.js'; import { withRunFileLock } from './runs-storage.js'; +import { createSlotBook, RUN_CREATED_SLOT } from './slots.js'; /** * Per-run event ceiling the Local World reports on run responses (mirrors the @@ -506,6 +513,8 @@ export function createEventsStorage( const cachedPathsByRunId = new Map>(); let totalCachedEventBytes = 0; + const slots = createSlotBook(basedir, tag); + function deleteCachedEvent(eventPath: string): void { const event = eventCache.get(eventPath); if (!event) { @@ -525,6 +534,7 @@ export function createEventsStorage( for (const cachedPath of cachedPathsByRunId.get(runId) ?? []) { deleteCachedEvent(cachedPath); } + slots.forget(runId); } function clearCache(): void { @@ -532,6 +542,7 @@ export function createEventsStorage( cachedEventBytes.clear(); cachedPathsByRunId.clear(); totalCachedEventBytes = 0; + slots.clear(); } function cacheEvent( @@ -592,6 +603,64 @@ export function createEventsStorage( } } + /** + * The events a caller that just lost a slot is missing: one ascending page of + * the run's log after the cursor it wrote from, minus anything at or below the + * highest slot it already held. Because slots are dense, that second filter is + * exact — a caller cannot be missing an event whose position it can name. + * + * Returned inline with the conflict so the common case (a handful of events + * arrived out of band) costs the caller no extra round-trip. `hasMore` is + * forwarded verbatim: an overflowing delta is the caller's signal to page from + * `cursor` instead of treating this as the whole story. + */ + async function eventsAfterClaim( + runId: string, + params: CreateEventParams | undefined + ): Promise<{ events: Event[]; cursor: string | null; hasMore: boolean }> { + const page = await paginatedFileSystemQuery({ + directory: path.join(basedir, 'events'), + schema: EventSchema, + cachedItems: eventCache, + filePrefix: `${runId}-`, + sortOrder: 'asc', + ...(typeof params?.sinceCursor === 'string' + ? { cursor: params.sinceCursor } + : {}), + getCreatedAt: getObjectCreatedAt('evnt'), + getId: (event) => event.eventId, + }); + const maxSlot = params?.maxSlot ?? 0; + const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; + const missing = page.data.filter( + (event) => (slotFromId(event.eventId) ?? 0) > maxSlot + ); + return { + events: + resolveData === 'none' + ? missing.map((event) => stripEventDataRefs(event, resolveData)) + : missing, + cursor: page.cursor, + hasMore: page.hasMore, + }; + } + + /** + * The 409 a caller gets when the slot it named turns out to belong to someone + * else, carrying the events it is missing so it can merge, replay and + * re-propose at a free position. + */ + async function slotConflict( + runId: string, + eventId: string, + params: CreateEventParams | undefined + ): Promise { + return new SlotConflictError( + `Slot ${slotFromId(eventId)} of run "${runId}" is already taken`, + { eventId, ...(await eventsAfterClaim(runId, params)) } + ); + } + async function storeEvent(event: Event): Promise { const eventPath = taggedPath( basedir, @@ -643,6 +712,36 @@ export function createEventsStorage( if ('correlationId' in data && typeof data.correlationId === 'string') { assertSafeEntityId('correlationId', data.correlationId); } + if (params?.eventId !== undefined) { + assertSafeEntityId('eventId', params.eventId); + } + + // A slot-numbered create reserves its position before running the + // validation and materialization that may still reject it. Handing the + // reservation back on the way out is what keeps the log dense: an + // abandoned slot below a sibling's published one is a hole that can never + // be filled, and a log with a hole can no longer prove it is complete. + const reserved = new Set(); + let reservedRunId: string | undefined; + /** + * Hands the slots of a create that never published back to the allocator, + * so an abandoned reservation below a sibling's published slot does not + * become a hole the run can never fill. + */ + async function releasingSlots( + result: Promise + ): Promise { + try { + return await result; + } catch (error) { + if (reservedRunId !== undefined) { + for (const slot of reserved) { + slots.release(reservedRunId, slot); + } + } + throw error; + } + } // Step lifecycle events are serialized per-step via an in-process mutex // so that the "check state, then write" sequence in step_started / @@ -653,7 +752,9 @@ export function createEventsStorage( const lockKey = tag ? `${runId}-${data.correlationId}.${tag}` : `${runId}-${data.correlationId}`; - return withInProcessLock(stepLocks, lockKey, () => createImpl()); + return releasingSlots( + withInProcessLock(stepLocks, lockKey, () => createImpl()) + ); } // `hook_created` is serialized per-(runId, hookId) so the // "claim token, write hook entity, write event" sequence runs to @@ -682,9 +783,11 @@ export function createEventsStorage( const lockKey = tag ? `${runId}-${data.correlationId}.hook.${tag}` : `${runId}-${data.correlationId}.hook`; - return withInProcessLock(hookLocks, lockKey, () => createImpl()); + return releasingSlots( + withInProcessLock(hookLocks, lockKey, () => createImpl()) + ); } - return createImpl(); + return releasingSlots(createImpl()); async function createImpl(): Promise { // Most paths use the freshly-generated candidate eventId. The @@ -719,6 +822,18 @@ export function createEventsStorage( // specVersion is always sent by the runtime, but we provide a fallback for safety const effectiveSpecVersion = data.specVersion ?? SPEC_VERSION_CURRENT; + // Whether this run numbers its events by slot. Read from what was + // persisted, never from this request or this build, so a run stays in + // the mode it was created in for life — a run whose log holds ULID ids + // must never be handed a slot id, and vice versa. `run_created` is the + // one event that decides the mode instead of reading it; the + // resilient-start path below decides it too, on the request that + // creates the run. + let slotMode = + data.eventType === 'run_created' + ? usesSlotIdentity(effectiveSpecVersion) + : await slots.usesSlots(effectiveRunId); + // Get current run state for validation (if not creating a new run) // Skip run validation for step_completed and step_retrying - they only operate // on running steps, and running steps are always allowed to modify regardless @@ -800,8 +915,14 @@ export function createEventsStorage( ); if (created) { - // We created the run — also write the run_created event. - const runCreatedEventId = `evnt_${monotonicUlid()}`; + // We created the run, so this request also decided its mode. + slotMode = usesSlotIdentity(effectiveSpecVersion); + // We created the run — also write the run_created event. Its + // slot needs no allocation: a run's own `run_created` provably + // has nothing before it. + const runCreatedEventId = slotMode + ? slotEventId(RUN_CREATED_SLOT) + : `evnt_${monotonicUlid()}`; const runCreatedEvent: Event = { eventType: 'run_created', runId: effectiveRunId, @@ -820,6 +941,7 @@ export function createEventsStorage( }, }; await storeEvent(runCreatedEvent); + slots.observe(effectiveRunId, runCreatedEventId); currentRun = createdRun; } else { // Run already exists (concurrent run_created won the @@ -836,6 +958,17 @@ export function createEventsStorage( } } + // The run entity we just read is the authority on the mode, and it can + // appear between the probe above and this read: start() issues + // `run_created` and the queue send concurrently, so the delivery's + // `run_started` can arrive while the run is still being published. A + // stale "no" there would number that one event with a ULID on an + // otherwise slot-numbered run, and the hole it leaves in the numbering + // costs the log its completeness proof for life. + if (currentRun && data.eventType !== 'run_created') { + slotMode = usesSlotIdentity(currentRun.specVersion); + } + // run_failed on a non-existent run is rejected to match the // postgres and vercel worlds, which both surface this as a // WorkflowRunNotFoundError rather than silently persisting an @@ -857,7 +990,7 @@ export function createEventsStorage( if (requiresNewerWorld(currentRun.specVersion)) { throw new RunNotSupportedError( currentRun.specVersion!, - SPEC_VERSION_CURRENT + SPEC_VERSION_MAX_SUPPORTED ); } @@ -873,6 +1006,55 @@ export function createEventsStorage( } } + // ============================================================ + // EVENT ID: the caller's slot claim, an allocated slot, or a ULID + // ============================================================ + // A slot-numbered run's ids name positions in its log, so an id is + // either claimed by a caller that holds the log (and is therefore + // asserting the log is complete up to that position) or allocated here + // for a caller that has no log — a step completion reporting in, a + // cancellation from an API call. + if (params?.eventId !== undefined) { + const claimedSlot = slotFromId(params.eventId); + if (!slotMode) { + throw new WorkflowWorldError( + `Event id "${params.eventId}" was supplied for run "${effectiveRunId}", whose events are not numbered by slot`, + { status: 400 } + ); + } + if (claimedSlot === undefined) { + throw new WorkflowWorldError( + `Event id "${params.eventId}" is not a slot id, and run "${effectiveRunId}" numbers its events by slot`, + { status: 400 } + ); + } + eventId = params.eventId; + reservedRunId = effectiveRunId; + reserved.add(claimedSlot); + slots.claim(effectiveRunId, claimedSlot); + if (await slots.isWritten(effectiveRunId, claimedSlot)) { + // Reject a doomed claim before the materialization below creates + // the step, hook or wait this event will now never accompany. A + // caller that re-proposes at the next slot would otherwise + // collide with its own orphan and read that as "my write already + // landed". See SlotBook.isWritten. + throw await slotConflict(effectiveRunId, eventId, params); + } + } else if (slotMode) { + reservedRunId = effectiveRunId; + // A run's own `run_created` owns the first slot — provably, since + // nothing precedes it — so every other event allocates above it, even + // when that event is the first to arrive here. + const slot = await slots.reserve( + effectiveRunId, + data.eventType === 'run_created' + ? RUN_CREATED_SLOT + : RUN_CREATED_SLOT + 1 + ); + reserved.add(slot); + eventId = slotEventId(slot); + } + // ============================================================ // VALIDATION: Terminal state and event ordering checks // ============================================================ @@ -1127,13 +1309,40 @@ export function createEventsStorage( // strictly dominates all visible events of the run guarantees the // terminal event replays last. See mintRunDominantEventKey for // the dominance argument. - const dominantKey = await mintRunDominantEventKey( - basedir, - effectiveRunId, - tag - ); - eventId = dominantKey.eventId; - event = { ...event, eventId, createdAt: dominantKey.createdAt }; + // + // A key the *caller* chose is left alone. Its slot was picked from + // the caller's own log, so a concurrent event either sits below it + // (and already replays first) or takes the slot itself — in which + // case the publish below conflicts and the caller merges and + // re-proposes, which is the stronger answer. Re-numbering it here + // would also be actively wrong: the caller reserves slots for a + // whole flush of concurrent ops at once, so moving this one to + // "highest visible + 1" would steal the slot a sibling op is still + // in flight with. + if (params?.eventId === undefined) { + const dominantKey = await mintRunDominantEventKey( + basedir, + effectiveRunId, + tag, + slotMode + ); + const staleSlot = slotFromId(eventId); + const dominantSlot = slotFromId(dominantKey.eventId); + if (staleSlot !== undefined && staleSlot !== dominantSlot) { + // Only reachable when the log moved under us, which means the + // slot we held is now someone else's written event — handing it + // back leaves no hole. + reserved.delete(staleSlot); + slots.release(effectiveRunId, staleSlot); + } + if (dominantSlot !== undefined) { + reservedRunId = effectiveRunId; + reserved.add(dominantSlot); + slots.claim(effectiveRunId, dominantSlot); + } + eventId = dominantKey.eventId; + event = { ...event, eventId, createdAt: dominantKey.createdAt }; + } } // Create/update entity based on event type (event-sourced architecture) @@ -1505,13 +1714,19 @@ export function createEventsStorage( ); // Write the synthetic step_created event so replay observes it // (the client step consumer sets hasCreatedEvent only on a - // step_created event). Its eventId is a fresh monotonic ULID. + // step_created event). Its eventId is a second slot, or a fresh + // monotonic ULID — one request, two events. // Ordering vs. the step_started event row does not affect // correctness: the step_started consumer is a no-op and only // step_created flips hasCreatedEvent, so the end state is the // same whichever sorts first — this matches the resilient // run_started → run_created precedent in this file. - const stepCreatedEventId = `evnt_${monotonicUlid()}`; + let stepCreatedEventId = `evnt_${monotonicUlid()}`; + if (slotMode) { + const slot = await slots.reserve(effectiveRunId); + reserved.add(slot); + stepCreatedEventId = slotEventId(slot); + } const stepCreatedEvent: Event = { eventType: 'step_created', runId: effectiveRunId, @@ -1533,6 +1748,7 @@ export function createEventsStorage( ), stepCreatedEvent ); + slots.observe(effectiveRunId, stepCreatedEventId); validatedStep = createdStep; stepCreatedLazily = true; } @@ -2215,11 +2431,17 @@ export function createEventsStorage( // race here; whoever links the file first wins, the loser // throws EntityConflictError, and the runtime's existing // concurrent-replay catch path at suspension-handler.ts:142 - // swallows it. For all other event types, eventIds are - // monotonic ULIDs (globally unique by construction) so a - // collision indicates a real bug and EntityConflictError is + // swallows it. For all other event types of a ULID-numbered run, + // eventIds are monotonic ULIDs (globally unique by construction) so + // a collision indicates a real bug and EntityConflictError is // also the right surface — same shape as step_created's // claim-file behavior. + // + // A slot-numbered run collides by design: the id names a position in + // the log, so a loser is not a bug but a writer whose log was missing + // an event. It gets a SlotConflictError carrying that event instead + // (see below), and this write is the authority that decides it — the + // allocator's book is only ever a hint. // Last-instant re-validation for `hook_received` (see the acceptance // check above). The per-hook in-process lock already serializes // resume vs. dispose within one storage instance; this second check @@ -2311,10 +2533,15 @@ export function createEventsStorage( ); const staged = await writeExclusive(stagedPath, serializedEvent); if (!staged) { - // eventId is a freshly generated ULID; its staging path can - // only be occupied by a previous crashed attempt of this very - // event, which never promoted. Surface the same conflict shape - // as a visible-path collision. + // For a ULID-numbered run the eventId is freshly generated, so + // its staging path can only be occupied by a previous crashed + // attempt of this very event, which never promoted. A + // slot-numbered run can also collide here with another instance + // that allocated the same slot from its own book. Either way the + // event is not reader-visible, so there is no delta to hand back + // and nothing for the caller to merge: surface the same conflict + // shape as a visible-path collision, and let the allocator + // re-probe on the retry. throw new EntityConflictError( `Event "${eventId}" already exists for run "${effectiveRunId}"` ); @@ -2364,6 +2591,22 @@ export function createEventsStorage( tag ); } + if (slotMode) { + // Losing a slot means someone else's event occupies this position, + // so the log this event was derived from is missing at least that + // event — the whole proposed event is stale, not just its id. Hand + // back what the caller is missing so it can merge, replay and + // re-propose, and forget the run's book so the next allocation + // re-reads the log this instance evidently does not have. + // + // Reaching here means the slot was taken *after* the pre-check at + // the claim site, so the entity this event was going to describe + // has already been materialized. Only two storage instances + // sharing a directory can do that, since one instance's book + // hands the same slot to nobody else. + slots.forget(effectiveRunId); + throw await slotConflict(effectiveRunId, eventId, params); + } throw new EntityConflictError( `Event "${eventId}" already exists for run "${effectiveRunId}"` ); @@ -2372,6 +2615,7 @@ export function createEventsStorage( // The event is now committed; cache it so an immediate sequential // replay can serve it without rereading from disk. rememberStoredEvent(event, eventPath, serializedEvent); + slots.observe(effectiveRunId, eventId); // Write the hook entity ONLY now that the event publish has // committed. Doing this earlier (in the `hook_created` diff --git a/packages/world-local/src/storage/helpers.ts b/packages/world-local/src/storage/helpers.ts index d7c2cc78bb..2817fb0574 100644 --- a/packages/world-local/src/storage/helpers.ts +++ b/packages/world-local/src/storage/helpers.ts @@ -1,6 +1,7 @@ import { createHash } from 'node:crypto'; import fs from 'node:fs/promises'; import path from 'node:path'; +import { FIRST_SLOT, maxSlotOf, slotEventId } from '@workflow/world'; import { decodeTime, monotonicFactory } from 'ulid'; import { hasTag, @@ -208,6 +209,46 @@ export async function reapPendingHookEvents( } } +/** + * The event ids of `runId` that are visible in the given tag's view, read from + * the event filenames alone — no file contents, so the cost is one `readdir` + * however large the log is. + * + * A missing `events` directory means the run provably has no events yet. Any + * other failure is thrown: callers derive an event key from this scan, and a + * silently short answer would mint a key that collides with, or fails to + * dominate, an event that is actually there. + */ +export async function listRunEventIds( + basedir: string, + runId: string, + tag?: string +): Promise { + let files: string[] = []; + try { + files = await fs.readdir(path.join(basedir, 'events')); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + throw error; + } + } + const prefix = `${runId}-`; + const eventIds: string[] = []; + for (const file of files) { + if (!file.startsWith(prefix) || !file.endsWith('.json')) { + continue; + } + const fileId = file.slice(0, -'.json'.length); + // Mirror read visibility: untagged files are visible to every tag, + // tagged files only to their own tag. + if (!isUntagged(fileId) && !(tag && hasTag(fileId, tag))) { + continue; + } + eventIds.push(stripTag(fileId).slice(prefix.length)); + } + return eventIds; +} + /** * Mint an event key (eventId + createdAt) that sorts strictly AFTER every * reader-visible event of the run in the given tag's view. @@ -229,38 +270,35 @@ export async function reapPendingHookEvents( * >= every visible event's `createdAt`, which was stamped at that event's * `createImpl()` entry — before its publish, and thus before this call. * Equal-`createdAt` ties fall to the strictly-dominant eventId. + * + * A slot-numbered run takes the slot above the highest visible one, which + * dominates by construction, paired with the wall clock — `createdAt` needs + * only to be >= every visible one, by the same argument as above. This is + * the one allocation that deliberately does *not* fill a hole below the max: + * a lower slot would sort before the events it has to follow, and density + * matters less here than replay order, since a hole below a terminal event + * means the run already lost an event it can never write. */ export async function mintRunDominantEventKey( basedir: string, runId: string, - tag?: string + tag: string | undefined, + slotMode: boolean ): Promise<{ eventId: string; createdAt: Date }> { - let files: string[] = []; - try { - files = await fs.readdir(path.join(basedir, 'events')); - } catch (error) { - // Only ENOENT ("no events directory yet") means there is provably - // nothing visible to dominate. Any other failure would silently mint a - // wall-clock key with no dominance guarantee over an already-accepted - // hook — abort the terminal transition instead; its retry re-runs this - // scan. - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { - throw error; - } + const eventIds = await listRunEventIds(basedir, runId, tag); + if (slotMode) { + // Above every event on disk, and above the run's own first slot even when + // that event has not landed yet: only `run_created` may occupy it, and a + // terminal event is never the run's first. + return { + eventId: slotEventId( + Math.max(maxSlotOf(eventIds.map(toEventRef)), FIRST_SLOT) + 1 + ), + createdAt: new Date(), + }; } - const prefix = `${runId}-`; let maxUlid: string | null = null; - for (const file of files) { - if (!file.startsWith(prefix) || !file.endsWith('.json')) { - continue; - } - const fileId = file.slice(0, -'.json'.length); - // Mirror read visibility: untagged files are visible to every tag, - // tagged files only to their own tag. - if (!isUntagged(fileId) && !(tag && hasTag(fileId, tag))) { - continue; - } - const candidate = stripTag(fileId).slice(prefix.length); + for (const candidate of eventIds) { if (!maxUlid || candidate > maxUlid) { maxUlid = candidate; } @@ -279,6 +317,10 @@ export async function mintRunDominantEventKey( return { eventId: `evnt_${monotonicUlid(ts)}`, createdAt: new Date(ts) }; } +function toEventRef(eventId: string): { eventId: string } { + return { eventId }; +} + /** * Path of the exclusive-create claim file that reserves a hook token. */ diff --git a/packages/world-local/src/storage/slot-identity.test.ts b/packages/world-local/src/storage/slot-identity.test.ts new file mode 100644 index 0000000000..00d8b71cce --- /dev/null +++ b/packages/world-local/src/storage/slot-identity.test.ts @@ -0,0 +1,281 @@ +import { promises as fs } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { SlotConflictError } from '@workflow/errors'; +import type { Storage } from '@workflow/world'; +import { + FIRST_SLOT, + maxSlotOf, + SPEC_VERSION_CURRENT, + SPEC_VERSION_SLOT_IDENTITY, + slotEventId, + slotFromId, +} from '@workflow/world'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { createStorage } from './index.js'; + +let testDir: string; +let storage: Storage; + +beforeEach(async () => { + testDir = await fs.mkdtemp(path.join(os.tmpdir(), 'slot-identity-')); + storage = createStorage(testDir); +}); + +afterEach(async () => { + await fs.rm(testDir, { recursive: true, force: true }); +}); + +/** Start a run whose events are numbered by slot, and return its id. */ +async function newSlotRun(): Promise { + const result = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + eventData: { + deploymentId: 'dpl_test', + workflowName: 'test-workflow', + input: new Uint8Array(), + }, + }); + if (!result.run) { + throw new Error('Expected run to be created'); + } + return result.run.runId; +} + +/** + * The slots of the run's log, in list order. The page size is explicit: the + * default would silently truncate a fan-out and make a dense log look sparse. + */ +async function slotsOf(runId: string): Promise { + const { data } = await eventsOf(runId); + return data.map((event) => slotFromId(event.eventId) ?? -1); +} + +function eventsOf(runId: string) { + return storage.events.list({ runId, pagination: { limit: 500 } }); +} + +async function createStep( + runId: string, + stepId: string, + eventId?: string +): Promise { + const result = await storage.events.create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: stepId, + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + eventId === undefined ? undefined : { eventId } + ); + if (!result.event) { + throw new Error('Expected an event'); + } + return result.event.eventId; +} + +describe('numbering', () => { + it('puts run_created in the first slot', async () => { + const runId = await newSlotRun(); + await expect(slotsOf(runId)).resolves.toEqual([FIRST_SLOT]); + }); + + it('allocates dense slots for writers that hold no log', async () => { + // A step completion reporting in, a cancellation from an API call: the + // caller has no event log, so the world numbers the event for it. + const runId = await newSlotRun(); + await createStep(runId, 'step_a'); + await createStep(runId, 'step_b'); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); + + it('honours a slot the caller claims', async () => { + const runId = await newSlotRun(); + const eventId = await createStep(runId, 'step_a', slotEventId(2)); + expect(eventId).toBe(slotEventId(2)); + await expect(slotsOf(runId)).resolves.toEqual([1, 2]); + }); + + it('keeps a burst of concurrent writers dense', async () => { + // The suspension flush issues every op at once. Density is what lets a + // reader prove its log is complete, so a burst must not leave holes. + const runId = await newSlotRun(); + const ids = await Promise.all( + Array.from({ length: 20 }, (_, index) => + createStep(runId, `step_${index}`) + ) + ); + expect(new Set(ids).size).toBe(ids.length); + const slots = await slotsOf(runId); + expect([...slots].sort((a, b) => a - b)).toEqual( + Array.from({ length: ids.length + 1 }, (_, index) => FIRST_SLOT + index) + ); + }); + + it('proves completeness: the highest slot is the event count', async () => { + const runId = await newSlotRun(); + await Promise.all( + Array.from({ length: 5 }, (_, index) => + createStep(runId, `step_${index}`) + ) + ); + const { data } = await eventsOf(runId); + expect(maxSlotOf(data)).toBe(data.length); + }); + + it('leaves no hole behind a rejected write', async () => { + // The rejected op's slot sits below its concurrent sibling's, and a hole + // below a published event can never be filled. + const runId = await newSlotRun(); + const [rejected, accepted] = await Promise.allSettled([ + storage.events.create(runId, { + eventType: 'step_completed', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_never_created', + eventData: { output: new Uint8Array() }, + }), + createStep(runId, 'step_a'), + ]); + expect(rejected.status).toBe('rejected'); + expect(accepted.status).toBe('fulfilled'); + await createStep(runId, 'step_b'); + const slots = await slotsOf(runId); + expect([...slots].sort((a, b) => a - b)).toEqual([1, 2, 3]); + }); +}); + +describe('mode is pinned to the run', () => { + it('rejects a slot id claimed on a ULID-numbered run', async () => { + const created = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'dpl_test', + workflowName: 'test-workflow', + input: new Uint8Array(), + }, + }); + const runId = created.run?.runId as string; + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + /not numbered by slot/ + ); + }); + + it('rejects a ULID id claimed on a slot-numbered run', async () => { + const runId = await newSlotRun(); + await expect( + createStep(runId, 'step_a', 'evnt_01K5Z0000000000000000000AA') + ).rejects.toThrow(/not a slot id/); + }); + + it('ignores the spec version of later requests', async () => { + // A run is in exactly one mode for life; only what was persisted decides. + const runId = await newSlotRun(); + const result = await storage.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_CURRENT, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }); + expect(slotFromId(result.event?.eventId ?? '')).toBe(2); + }); +}); + +describe('conflict', () => { + it('reports the events the loser is missing', async () => { + const runId = await newSlotRun(); + // Out of band: something else takes the slot this caller was about to + // claim, so the caller's log is provably missing an event. + await createStep(runId, 'step_out_of_band'); + + const conflict = await storage.events + .create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(2), maxSlot: 1 } + ) + .catch((error: unknown) => error); + + expect(SlotConflictError.is(conflict)).toBe(true); + const slotConflict = conflict as SlotConflictError; + expect(slotConflict.status).toBe(409); + expect(slotConflict.eventId).toBe(slotEventId(2)); + expect(slotConflict.events?.map((event) => event.eventId)).toEqual([ + slotEventId(2), + ]); + }); + + it('lets the loser re-propose at the next free slot', async () => { + const runId = await newSlotRun(); + await createStep(runId, 'step_out_of_band'); + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + SlotConflictError + ); + // Merging the delta moves the caller's own numbering forward by one. + const eventId = await createStep(runId, 'step_a', slotEventId(3)); + expect(eventId).toBe(slotEventId(3)); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); + + it('excludes events the loser already holds from the delta', async () => { + const runId = await newSlotRun(); + await createStep(runId, 'step_one'); + await createStep(runId, 'step_two'); + + const conflict = await storage.events + .create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(2), maxSlot: 2 } + ) + .catch((error: unknown) => error); + + // Slots 1 and 2 are at or below what the caller had; only 3 is news. + expect( + (conflict as SlotConflictError).events?.map((event) => event.eventId) + ).toEqual([slotEventId(3)]); + }); + + it('conflicts across two storage instances sharing a directory', async () => { + // Two instances keep independent books, so the exclusive write — not the + // book — is what decides who owns a slot. + const runId = await newSlotRun(); + const other = createStorage(testDir); + const [first, second] = await Promise.allSettled([ + storage.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }), + other.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_b', + eventData: { stepName: 'b-step', input: new Uint8Array() }, + }), + ]); + const outcomes = [first, second]; + expect(outcomes.filter((o) => o.status === 'fulfilled')).toHaveLength(1); + const rejection = outcomes.find((o) => o.status === 'rejected'); + expect( + SlotConflictError.is( + (rejection as PromiseRejectedResult | undefined)?.reason + ) + ).toBe(true); + await expect(slotsOf(runId)).resolves.toEqual([1, 2]); + }); +}); diff --git a/packages/world-local/src/storage/slots.test.ts b/packages/world-local/src/storage/slots.test.ts new file mode 100644 index 0000000000..42c962f128 --- /dev/null +++ b/packages/world-local/src/storage/slots.test.ts @@ -0,0 +1,241 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { + FIRST_SLOT, + SPEC_VERSION_CURRENT, + SPEC_VERSION_SLOT_IDENTITY, + slotEventId, +} from '@workflow/world'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { createSlotBook, RUN_CREATED_SLOT } from './slots.js'; + +let basedir: string; + +beforeEach(async () => { + basedir = await fs.mkdtemp(path.join(os.tmpdir(), 'slot-book-')); +}); + +afterEach(async () => { + await fs.rm(basedir, { recursive: true, force: true }); +}); + +const RUN_ID = 'wrun_01K0000000000000000000TEST'; + +async function writeRun(specVersion: number): Promise { + await fs.mkdir(path.join(basedir, 'runs'), { recursive: true }); + await fs.writeFile( + path.join(basedir, 'runs', `${RUN_ID}.json`), + JSON.stringify({ + runId: RUN_ID, + deploymentId: 'dpl_test', + status: 'running', + workflowName: 'test', + specVersion, + input: [], + attributes: {}, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }) + ); +} + +async function writeEvents(...slots: number[]): Promise { + await fs.mkdir(path.join(basedir, 'events'), { recursive: true }); + for (const slot of slots) { + await fs.writeFile( + path.join(basedir, 'events', `${RUN_ID}-${slotEventId(slot)}.json`), + '{}' + ); + } +} + +describe('usesSlots', () => { + it('reads the mode off the persisted run, not the build', async () => { + await writeRun(SPEC_VERSION_SLOT_IDENTITY); + await expect(createSlotBook(basedir).usesSlots(RUN_ID)).resolves.toBe(true); + + await writeRun(SPEC_VERSION_CURRENT); + await expect(createSlotBook(basedir).usesSlots(RUN_ID)).resolves.toBe( + false + ); + }); + + it('re-reads until the run exists', async () => { + // The resilient-start path writes run_started before the run entity, so a + // cached "no" taken from the missing run would strand a slot-numbered run + // on ULID ids for the rest of the process's life. + const book = createSlotBook(basedir); + await expect(book.usesSlots(RUN_ID)).resolves.toBe(false); + await writeRun(SPEC_VERSION_SLOT_IDENTITY); + await expect(book.usesSlots(RUN_ID)).resolves.toBe(true); + }); + + it('prefers its own tagged run over the untagged one', async () => { + await writeRun(SPEC_VERSION_SLOT_IDENTITY); + await fs.rename( + path.join(basedir, 'runs', `${RUN_ID}.json`), + path.join(basedir, 'runs', `${RUN_ID}.mine.json`) + ); + await writeRun(SPEC_VERSION_CURRENT); + + await expect( + createSlotBook(basedir, 'mine').usesSlots(RUN_ID) + ).resolves.toBe(true); + await expect( + createSlotBook(basedir, 'other').usesSlots(RUN_ID) + ).resolves.toBe(false); + }); +}); + +describe('reserve', () => { + it('starts at the first slot for a run with no events', async () => { + await expect(createSlotBook(basedir).reserve(RUN_ID)).resolves.toBe( + FIRST_SLOT + ); + }); + + it('continues above the highest slot already on disk', async () => { + await writeEvents(1, 2, 3); + await expect(createSlotBook(basedir).reserve(RUN_ID)).resolves.toBe(4); + }); + + it('fills a hole left in the persisted log', async () => { + // Density is the whole point of the scheme, so a gap that somehow exists is + // reclaimed rather than skipped over forever. + await writeEvents(1, 3); + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID)).resolves.toBe(2); + await expect(book.reserve(RUN_ID)).resolves.toBe(4); + }); + + it('hands a synchronous burst distinct consecutive slots', async () => { + // The suspension flush issues every op concurrently; with a plain + // "max + 1" they would all pick the same slot and all but one would fail. + const book = createSlotBook(basedir); + const slots = await Promise.all( + Array.from({ length: 20 }, () => book.reserve(RUN_ID)) + ); + expect([...slots].sort((a, b) => a - b)).toEqual( + Array.from({ length: 20 }, (_, index) => FIRST_SLOT + index) + ); + }); + + it('shares one disk scan across concurrent first callers', async () => { + await writeEvents(1); + const book = createSlotBook(basedir); + const slots = await Promise.all([ + book.reserve(RUN_ID), + book.reserve(RUN_ID), + ]); + expect([...slots].sort((a, b) => a - b)).toEqual([2, 3]); + }); + + it('honours a floor, so a concurrent event cannot take run_created’s slot', async () => { + // start() publishes the run entity before its `run_created` event and + // issues the queue send in parallel, so the delivery's `run_started` can + // allocate while slot 1 is still in flight. + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID, RUN_CREATED_SLOT + 1)).resolves.toBe( + RUN_CREATED_SLOT + 1 + ); + }); + + it('leaves slots below the floor allocatable', async () => { + // A floored search skips that range without looking at it, so it proves + // nothing about it — `run_created` must still find its own slot free. + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID, RUN_CREATED_SLOT + 1)).resolves.toBe( + RUN_CREATED_SLOT + 1 + ); + await expect(book.reserve(RUN_ID, RUN_CREATED_SLOT)).resolves.toBe( + RUN_CREATED_SLOT + ); + }); + + it('keeps runs independent', async () => { + await writeEvents(1, 2); + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID)).resolves.toBe(3); + await expect(book.reserve('wrun_01K0000000000000000000OTHR')).resolves.toBe( + FIRST_SLOT + ); + }); +}); + +describe('release', () => { + it('gives an abandoned interior slot to the next caller', async () => { + // A rejected op must not strand the slot below its concurrent siblings': + // that hole can never be filled once a later slot is published. + const book = createSlotBook(basedir); + const [first, second] = await Promise.all([ + book.reserve(RUN_ID), + book.reserve(RUN_ID), + ]); + book.release(RUN_ID, first); + await expect(book.reserve(RUN_ID)).resolves.toBe(first); + await expect(book.reserve(RUN_ID)).resolves.toBe(second + 1); + }); + + it('does not resurrect a slot that was published', async () => { + const book = createSlotBook(basedir); + const slot = await book.reserve(RUN_ID); + book.observe(RUN_ID, slotEventId(slot)); + book.release(RUN_ID, slot); + await expect(book.reserve(RUN_ID)).resolves.toBe(slot + 1); + }); +}); + +describe('isWritten', () => { + it('reads the log to answer for a run it has never seen', async () => { + await writeEvents(1, 2); + const book = createSlotBook(basedir); + await expect(book.isWritten(RUN_ID, 2)).resolves.toBe(true); + await expect(book.isWritten(RUN_ID, 3)).resolves.toBe(false); + }); + + it('is false for a slot that is only reserved', async () => { + // A reservation is not a publish, so a caller claiming the slot has to be + // allowed through to the write that actually decides it. + const book = createSlotBook(basedir); + const slot = await book.reserve(RUN_ID); + await expect(book.isWritten(RUN_ID, slot)).resolves.toBe(false); + }); +}); + +describe('observe', () => { + it('never hands out a slot claimed by the client', async () => { + const book = createSlotBook(basedir); + await book.reserve(RUN_ID); + book.observe(RUN_ID, slotEventId(5)); + const next = await book.reserve(RUN_ID); + expect(next).not.toBe(5); + expect(next).toBe(2); + }); + + it('ignores ULID event ids', async () => { + const book = createSlotBook(basedir); + await book.reserve(RUN_ID); + book.observe(RUN_ID, 'evnt_01K5Z0000000000000000000AA'); + await expect(book.reserve(RUN_ID)).resolves.toBe(2); + }); +}); + +describe('forget', () => { + it("re-reads the log, picking up another writer's events", async () => { + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID)).resolves.toBe(FIRST_SLOT); + await writeEvents(1, 2, 3); + book.forget(RUN_ID); + await expect(book.reserve(RUN_ID)).resolves.toBe(4); + }); + + it('clear() forgets every run', async () => { + await writeEvents(1); + const book = createSlotBook(basedir); + await expect(book.reserve(RUN_ID)).resolves.toBe(2); + await writeEvents(2, 3); + book.clear(); + await expect(book.reserve(RUN_ID)).resolves.toBe(4); + }); +}); diff --git a/packages/world-local/src/storage/slots.ts b/packages/world-local/src/storage/slots.ts new file mode 100644 index 0000000000..c6b2f017f6 --- /dev/null +++ b/packages/world-local/src/storage/slots.ts @@ -0,0 +1,247 @@ +/** + * Slot allocation for the Local World. + * + * A slot-numbered run names its events by position: `evnt_…001` is the first + * event of the run, `evnt_…002` the second, with no gaps. Density is the point + * — it is what lets a reader prove its loaded log is complete — so an allocator + * must never leave a slot permanently unwritten. + * + * Three properties do the work: + * + * - Handing out a slot is a *synchronous* set operation, so concurrent + * callers in one process get distinct slots with no lock. The only await is + * seeding from disk, which is memoized per run. + * - An allocation always picks the lowest slot that is neither written nor + * outstanding, so a reservation that is abandoned (its create threw a + * validation error) is handed to the next caller instead of leaving an + * interior hole. This matters under fan-out: N concurrent step_completed + * writes reserve N consecutive slots, and one rejected op must not strand + * the slot below its siblings'. + * - The event publish is `writeExclusive`, which is the authority. The book is + * a hint: when it turns out to be stale (another process wrote the slot), + * the publish fails and the caller is told so, rather than a duplicate being + * written or a slot being skipped. + * + * The book is per storage instance, and two instances may share a data + * directory (the cross-process convergence tests rely on exactly that). Their + * books are then independent, and the loser of a collision gets a conflict it + * has to resolve by reloading — the same contract as the networked worlds. + */ + +import type { WorkflowRun } from '@workflow/world'; +import { + FIRST_SLOT, + slotFromId, + usesSlotIdentity, + WorkflowRunSchema, +} from '@workflow/world'; +import { readJSONWithFallback } from '../fs.js'; +import { listRunEventIds } from './helpers.js'; + +interface RunSlots { + /** Slots proven to be on disk. */ + written: Set; + /** Slots handed out whose publish has not resolved yet. */ + outstanding: Set; + /** Lowest slot that might still be free; never decreases except on release. */ + searchFrom: number; +} + +export interface SlotBook { + /** + * Whether `runId`'s events are numbered by slot, read from the run's + * persisted `specVersion` — never from the build, so a run stays in the mode + * it was created in for life. A run that does not exist yet is not + * slot-numbered, and that answer is not cached: the resilient-start path + * creates the run moments later, and caching "no" would strand it on ULIDs + * for the rest of this process's life. + */ + usesSlots(runId: string): Promise; + /** + * Reserves the lowest free slot of `runId`, at or above `minSlot`. Distinct + * for every concurrent caller; the publish still has to prove the slot was + * actually free. + * + * `minSlot` is how the run's first slot is kept for its own `run_created`: + * that event's slot needs no allocation, and it may not be on disk yet when a + * concurrent `run_started` allocates (start() issues the creation and the + * queue send in parallel, and the run entity is published before its event). + * Slots below `minSlot` stay allocatable for a later caller, so holding one + * back leaves the log dense. + */ + reserve(runId: string, minSlot?: number): Promise; + /** + * Records that a caller claimed `slot` itself, so an allocation running + * alongside it picks a different one. Reserved and released on the same terms + * as {@link reserve}: the claim is only a hint until the publish proves it. + */ + claim(runId: string, slot: number): void; + /** + * Whether `slot` is already occupied by a published event, seeding from disk + * if this run has not been read yet. + * + * Lets a doomed claim be rejected *before* the create materializes its step, + * hook or wait: the entity mutation runs ahead of the event publish, so a + * claim that only fails at the publish leaves an entity behind with no event, + * and the caller's re-proposal at the next slot then collides with its own + * orphan. A `false` here is not a promise — the publish is still the + * authority — but it turns the case that actually happens (a caller numbering + * from a stale log) into a clean conflict. + */ + isWritten(runId: string, slot: number): Promise; + /** + * Returns a reserved or claimed slot that was never published, so the next + * caller takes it instead of it becoming a hole. + */ + release(runId: string, slot: number): void; + /** Records a published event id, so it is never handed out again. */ + observe(runId: string, eventId: string): void; + /** Drops what is cached for `runId`, so the next reservation re-reads disk. */ + forget(runId: string): void; + /** Drops everything cached (the data directory was cleared out from under us). */ + clear(): void; +} + +export function createSlotBook(basedir: string, tag?: string): SlotBook { + /** runId → whether the run is slot-numbered, memoized once it exists. */ + const modes = new Map(); + const books = new Map(); + /** runId → in-flight seed scan, so concurrent first callers share one scan. */ + const seeds = new Map>(); + + async function readMode(runId: string): Promise { + const run = await readJSONWithFallback( + basedir, + 'runs', + runId, + WorkflowRunSchema, + tag + ); + return run ? usesSlotIdentity(run.specVersion) : false; + } + + async function seed(runId: string): Promise { + const eventIds = await listRunEventIds(basedir, runId, tag); + const written = new Set(); + for (const eventId of eventIds) { + const slot = slotFromId(eventId); + if (slot !== undefined) { + written.add(slot); + } + } + const book: RunSlots = { + written, + outstanding: new Set(), + searchFrom: FIRST_SLOT, + }; + books.set(runId, book); + return book; + } + + /** The run's book, seeding it from disk once for all concurrent callers. */ + function open(runId: string): RunSlots | Promise { + const known = books.get(runId); + if (known) { + return known; + } + let pending = seeds.get(runId); + if (!pending) { + pending = seed(runId).finally(() => seeds.delete(runId)); + seeds.set(runId, pending); + } + return pending; + } + + function take(book: RunSlots, minSlot: number): number { + let slot = Math.max(book.searchFrom, minSlot); + while (book.written.has(slot) || book.outstanding.has(slot)) { + slot += 1; + } + book.outstanding.add(slot); + if (minSlot <= book.searchFrom) { + // Only an unfloored search proves everything below the slot it landed on + // is taken. A floored one skipped that range without looking, and those + // slots are still free for a caller that may take them. + book.searchFrom = slot; + } + return slot; + } + + return { + async usesSlots(runId) { + const cached = modes.get(runId); + if (cached !== undefined) { + return cached; + } + const mode = await readMode(runId); + // `false` here can mean "run not created yet" as well as "ULID run", and + // only the run's own absence is transient — so remember the positive + // answer eagerly and re-read until the run exists. + if (mode) { + modes.set(runId, true); + } + return mode; + }, + + async reserve(runId, minSlot = FIRST_SLOT) { + const opened = open(runId); + // Awaiting a book that is already in hand would yield to the microtask + // queue and let a concurrent caller take the same slot. + return take(opened instanceof Promise ? await opened : opened, minSlot); + }, + + claim(runId, slot) { + // No book means nothing is allocating for this run in this instance yet, + // and the seed scan that starts one reads the claim off disk if it landed. + books.get(runId)?.outstanding.add(slot); + }, + + async isWritten(runId, slot) { + const book = await open(runId); + return book.written.has(slot); + }, + + release(runId, slot) { + const book = books.get(runId); + if (!book) { + return; + } + book.outstanding.delete(slot); + if (slot < book.searchFrom) { + book.searchFrom = slot; + } + }, + + observe(runId, eventId) { + const book = books.get(runId); + if (!book) { + // Nothing to keep consistent: the slot is on disk by the time this is + // called, so the eventual seed scan picks it up. + return; + } + const slot = slotFromId(eventId); + if (slot === undefined) { + return; + } + book.written.add(slot); + book.outstanding.delete(slot); + }, + + forget(runId) { + modes.delete(runId); + books.delete(runId); + }, + + clear() { + modes.clear(); + books.clear(); + }, + }; +} + +/** + * The slot a run's first event occupies. A run's own `run_created` is the only + * event that can be numbered without consulting the log, because there is + * provably nothing before it. + */ +export const RUN_CREATED_SLOT = FIRST_SLOT; diff --git a/packages/world-postgres/src/drizzle/migrations/0018_run_scoped_event_and_step_keys.sql b/packages/world-postgres/src/drizzle/migrations/0018_run_scoped_event_and_step_keys.sql new file mode 100644 index 0000000000..0764dee02d --- /dev/null +++ b/packages/world-postgres/src/drizzle/migrations/0018_run_scoped_event_and_step_keys.sql @@ -0,0 +1,13 @@ +-- Event ids and step ids are unique per run, not globally. Under slot identity +-- (spec 6) every run numbers its own log from 1, so "evnt_0...001" and +-- "step_0...001" exist once per run and the old global primary keys would make +-- the second run to reach slot 1 collide with the first. +-- +-- The run leads both keys so the existing run-scoped range scans stay a single +-- index seek; that also makes the standalone run_id indexes redundant. +ALTER TABLE "workflow"."workflow_events" DROP CONSTRAINT IF EXISTS "workflow_events_pkey";--> statement-breakpoint +ALTER TABLE "workflow"."workflow_events" ADD CONSTRAINT "workflow_events_run_id_id_pk" PRIMARY KEY("run_id","id");--> statement-breakpoint +DROP INDEX IF EXISTS "workflow"."workflow_events_run_id_index";--> statement-breakpoint +ALTER TABLE "workflow"."workflow_steps" DROP CONSTRAINT IF EXISTS "workflow_steps_pkey";--> statement-breakpoint +ALTER TABLE "workflow"."workflow_steps" ADD CONSTRAINT "workflow_steps_run_id_step_id_pk" PRIMARY KEY("run_id","step_id");--> statement-breakpoint +DROP INDEX IF EXISTS "workflow"."workflow_steps_run_id_index"; diff --git a/packages/world-postgres/src/drizzle/migrations/meta/_journal.json b/packages/world-postgres/src/drizzle/migrations/meta/_journal.json index b7fb5d8215..9dca967a5b 100644 --- a/packages/world-postgres/src/drizzle/migrations/meta/_journal.json +++ b/packages/world-postgres/src/drizzle/migrations/meta/_journal.json @@ -127,6 +127,13 @@ "when": 1785283200000, "tag": "0017_add_hook_resume_context", "breakpoints": true + }, + { + "idx": 18, + "version": "7", + "when": 1785801600000, + "tag": "0018_run_scoped_event_and_step_keys", + "breakpoints": true } ] } diff --git a/packages/world-postgres/src/drizzle/schema.ts b/packages/world-postgres/src/drizzle/schema.ts index 6ffb21abcb..df58f73eb8 100644 --- a/packages/world-postgres/src/drizzle/schema.ts +++ b/packages/world-postgres/src/drizzle/schema.ts @@ -133,7 +133,7 @@ export const runs = schema.table( export const events = schema.table( 'workflow_events', { - eventId: varchar('id').primaryKey(), + eventId: varchar('id').notNull(), eventType: varchar('type').$type().notNull(), correlationId: varchar('correlation_id'), createdAt: timestamp('created_at').defaultNow().notNull(), @@ -146,7 +146,11 @@ export const events = schema.table( Cborized & { eventData?: undefined }, 'eventData'> >, (tb) => [ - index().on(tb.runId), + // Event ids are only unique within their run: under slot identity every run + // numbers its own log from 1, so `evnt_0…001` exists once per run. The run + // leads the key so the range scans in `list` stay a single index seek, and + // it subsumes the plain `run_id` index the table used to carry. + primaryKey({ columns: [tb.runId, tb.eventId] }), index().on(tb.correlationId), // Runtime-correlated one-shot events must be unique per (run, correlation) // — without @@ -167,7 +171,7 @@ export const steps = schema.table( 'workflow_steps', { runId: varchar('run_id').notNull(), - stepId: varchar('step_id').primaryKey(), + stepId: varchar('step_id').notNull(), stepName: varchar('step_name').notNull(), status: stepStatus('status').notNull(), /** @deprecated */ @@ -203,7 +207,13 @@ export const steps = schema.table( 'output' | 'input' | 'error' > >, - (tb) => [index().on(tb.runId), index().on(tb.status)] + (tb) => [ + // A step id is a correlation id, which under slot identity is only unique + // within its run — same reasoning as `workflow_events`. Every step query in + // this world is already run-scoped, so the run leads the key. + primaryKey({ columns: [tb.runId, tb.stepId] }), + index().on(tb.status), + ] ); export const hooks = schema.table( diff --git a/packages/world-postgres/src/index.ts b/packages/world-postgres/src/index.ts index 430cea0812..0db47b89e2 100644 --- a/packages/world-postgres/src/index.ts +++ b/packages/world-postgres/src/index.ts @@ -1,5 +1,5 @@ import type { Storage, World } from '@workflow/world'; -import { reenqueueActiveRuns, SPEC_VERSION_CURRENT } from '@workflow/world'; +import { mintedSpecVersion, reenqueueActiveRuns } from '@workflow/world'; import { Pool } from 'pg'; import type { PostgresWorldConfig } from './config.js'; import { createClient, type Drizzle } from './drizzle/index.js'; @@ -63,7 +63,10 @@ export function createWorld( const streamer = createStreamer(pool, drizzle); return { - specVersion: SPEC_VERSION_CURRENT, + // What this world stamps on new runs, which is not the newest version it + // can read: slot identity is readable everywhere and minted only where + // WORKFLOW_SLOT_IDENTITY is set. + specVersion: mintedSpecVersion(), ...storage, ...streamer, ...queue, diff --git a/packages/world-postgres/src/slots.ts b/packages/world-postgres/src/slots.ts new file mode 100644 index 0000000000..89435810a9 --- /dev/null +++ b/packages/world-postgres/src/slots.ts @@ -0,0 +1,225 @@ +/** + * Slot identity for the postgres world. + * + * A slot-numbered run names its events by position: `evnt_…001` is the first + * event of the run, `evnt_…002` the second, with no gaps. Density is the point + * — it is what lets a reader prove its copy of a log is complete — so the two + * things this module has to get right are that a position is written at most + * once and that a lost position is never left behind as a hole. + * + * The authority for both is the events table's primary key, `(run_id, id)`: the + * INSERT either lands or raises a unique violation, and a writer that loses the + * race is retried at a position that is still free rather than abandoning the + * one it lost. The probe below is only ever a hint about where to try next. + */ + +import { WorkflowWorldError } from '@workflow/errors'; +import { FIRST_SLOT, slotEventId, slotFromId } from '@workflow/world'; +import { and, desc, eq } from 'drizzle-orm'; +import { type Drizzle, Schema } from './drizzle/index.js'; + +/** + * The slot a run's own `run_created` occupies. Nothing in a run precedes its + * creation, so this one position needs no allocation, and every other event of + * the run searches above it — including the event that happens to reach storage + * first, which on the start path is routinely `run_started`. + */ +export const RUN_CREATED_SLOT = FIRST_SLOT; + +/** First backoff after losing a position; doubled each round. */ +export const SLOT_RETRY_BASE_MS = 5; + +/** Ceiling for a single backoff, so a contended run keeps making attempts. */ +export const SLOT_RETRY_MAX_DELAY_MS = 250; + +/** + * How long a writer keeps looking for a free position before giving up. + * Exhausting it surfaces as a 503, so the caller — in practice a queue + * delivery — retries the whole operation instead of the run stalling on it. + */ +export const SLOT_RETRY_BUDGET_MS = 30_000; + +/** Postgres unique-violation code. */ +const UNIQUE_VIOLATION = '23505'; + +/** + * Whether an error says the position a write aimed at is already occupied. + * + * Drizzle wraps the pg error, so the code can sit on the error or on its cause. + * Both the name drizzle generates for the composite key and the name postgres + * gives an inline `PRIMARY KEY` are accepted, so a database whose key predates + * the run-scoped migration still classifies correctly. + */ +export function isEventKeyViolation(error: unknown): boolean { + const pg = (error as { code?: string; constraint?: string }).code + ? (error as { code?: string; constraint?: string }) + : ((error as { cause?: { code?: string; constraint?: string } }).cause ?? + {}); + return ( + pg.code === UNIQUE_VIOLATION && + (pg.constraint === 'workflow_events_run_id_id_pk' || + pg.constraint === 'workflow_events_pkey') + ); +} + +/** + * The highest event id in a run's log, or undefined when the log is empty. + * + * One backwards scan of the `(run_id, id)` primary key. Ids are fixed-width + * within a scheme, so for a slot-numbered run the highest id names the highest + * written position — and because a log holds ids of exactly one scheme, that id + * also reports which scheme the run was created with. + */ +export async function highestEventId( + drizzle: Drizzle, + runId: string +): Promise { + const [row] = await drizzle + .select({ eventId: Schema.events.eventId }) + .from(Schema.events) + .where(eq(Schema.events.runId, runId)) + .orderBy(desc(Schema.events.eventId)) + .limit(1); + return row?.eventId; +} + +/** The position an id names, or 0 for an empty log or a ULID-numbered one. */ +export function highestSlotOf(eventId: string | undefined): number { + return eventId === undefined ? 0 : (slotFromId(eventId) ?? 0); +} + +/** Whether a run's log already holds `eventId`. */ +export async function eventExists( + drizzle: Drizzle, + runId: string, + eventId: string +): Promise { + const [row] = await drizzle + .select({ eventId: Schema.events.eventId }) + .from(Schema.events) + .where( + and(eq(Schema.events.runId, runId), eq(Schema.events.eventId, eventId)) + ) + .limit(1); + return row !== undefined; +} + +/** Full jitter over an exponentially growing, capped window. */ +export function slotRetryDelay(round: number): number { + return ( + Math.random() * + Math.min(SLOT_RETRY_BASE_MS * 2 ** round, SLOT_RETRY_MAX_DELAY_MS) + ); +} + +/** The event ids a single create publishes. */ +export interface EventIds { + /** + * The id of the event this create returns. Taken on demand: a ULID-numbered + * write mints it inside its transaction, once the row lock that orders it is + * held. + */ + primary: () => string; + /** + * An additional event written in the same breath — the synthetic + * `step_created` of a lazy step start. Its position is allocated here even + * when the caller named its own for the primary event, because a caller that + * defers a `step_created` cannot know it will be synthesized. + */ + extra: () => Promise; +} + +export interface PlaceEventOptions { + /** + * Position the caller named, when it holds the log and claimed one. A claim + * asserts the log is complete up to that position, so losing it is a conflict + * the caller has to resolve rather than something to retry here. + */ + claimedSlot?: number; + /** Lowest position this write may take when allocating. */ + minSlot: number; + /** + * Result of a probe the caller has already made, used for the first attempt + * instead of probing again. Later rounds always re-probe: the log has + * demonstrably moved. + */ + seedHighestEventId?: string | undefined; + /** The conflict raised when a claimed position turns out to be taken. */ + onClaimTaken: () => Promise; + /** Performs the write with the ids it should publish under. */ + write: (ids: EventIds) => Promise; +} + +/** + * Writes an event of a slot-numbered run, at the position the caller claimed or + * at the next free one. + * + * Every round re-probes rather than incrementing a local counter: each round at + * least one writer wins, so re-probing guarantees progress under any amount of + * contention. `write` must leave nothing behind when it raises a unique + * violation — the callers here either write only the event row or wrap their + * materialization in the same transaction, so a lost round rolls back whole. + */ +export async function placeEvent( + drizzle: Drizzle, + runId: string, + options: PlaceEventOptions +): Promise { + const deadline = Date.now() + SLOT_RETRY_BUDGET_MS; + for (let round = 0; ; round++) { + let cursor: number | undefined; + /** + * Positions for this attempt, consecutive from one probe. Deferred so a + * claimed write with no extra event never probes at all. + */ + const take = async (): Promise => { + if (cursor === undefined) { + const highest = + round === 0 && options.seedHighestEventId !== undefined + ? options.seedHighestEventId + : await highestEventId(drizzle, runId); + cursor = Math.max( + highestSlotOf(highest) + 1, + options.minSlot, + // The claimed position is this write's own; an extra event must not + // be handed it. + (options.claimedSlot ?? 0) + 1 + ); + } + return cursor++; + }; + + const primary = + options.claimedSlot === undefined + ? slotEventId(await take()) + : slotEventId(options.claimedSlot); + try { + return await options.write({ + primary: () => primary, + extra: async () => slotEventId(await take()), + }); + } catch (error) { + if (!isEventKeyViolation(error)) { + throw error; + } + // A claimed write can also lose on its extra event's position, which is + // this world's to reallocate — only a claim that is itself taken is the + // caller's problem. + if ( + options.claimedSlot !== undefined && + (await eventExists(drizzle, runId, primary)) + ) { + throw await options.onClaimTaken(); + } + if (Date.now() >= deadline) { + throw new WorkflowWorldError( + `Could not place an event in run "${runId}" within ${SLOT_RETRY_BUDGET_MS}ms of contention`, + { status: 503 } + ); + } + await new Promise((resolve) => + setTimeout(resolve, slotRetryDelay(round)) + ); + } + } +} diff --git a/packages/world-postgres/src/storage.ts b/packages/world-postgres/src/storage.ts index 140597a6a9..878f39758b 100644 --- a/packages/world-postgres/src/storage.ts +++ b/packages/world-postgres/src/storage.ts @@ -3,12 +3,14 @@ import { HookNotFoundError, RunExpiredError, RunNotSupportedError, + SlotConflictError, TooEarlyError, WorkflowRunNotFoundError, WorkflowWorldError, } from '@workflow/errors'; import type { AttributeChange, + CreateEventParams, Event, EventResult, ExperimentalSetAttributesResult, @@ -35,15 +37,20 @@ import { isChildEntityCreationEventType, isHookEventRequiringExistence, isLegacySpecVersion, + isSlotId, isTerminalRunEventType, isTerminalStepStatus, isTerminalWorkflowRunStatus, requiresNewerWorld, SPEC_VERSION_CURRENT, + SPEC_VERSION_MAX_SUPPORTED, StepSchema, + slotEventId, + slotFromId, stripEventDataRefs, TERMINAL_STEP_STATUSES, TERMINAL_WORKFLOW_RUN_STATUSES, + usesSlotIdentity, validateAttributeChanges, validateUlidTimestamp, WorkflowRunSchema, @@ -62,6 +69,13 @@ import { import { monotonicFactory } from 'ulid'; import { type Drizzle, Schema } from './drizzle/index.js'; import type { SerializedContent } from './drizzle/schema.js'; +import { + type EventIds, + eventExists, + highestEventId, + placeEvent, + RUN_CREATED_SLOT, +} from './slots.js'; import { compact } from './util.js'; /** @@ -464,6 +478,64 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { .limit(1) .prepare('events_get_wait_for_validation'); + /** + * The events a caller that just lost a slot is missing: one ascending page of + * the run's log after the cursor it wrote from, minus anything at or below the + * highest slot it already held. Because slots are dense, that second filter is + * exact — a caller cannot be missing an event whose position it can name. + * + * Returned inline with the conflict so the common case (a handful of events + * arrived out of band) costs the caller no extra round-trip. `hasMore` is + * forwarded verbatim: an overflowing delta is the caller's signal to page from + * `cursor` instead of treating this as the whole story. + */ + async function eventsAfterClaim( + runId: string, + params: CreateEventParams | undefined + ): Promise<{ events: Event[]; cursor: string | null; hasMore: boolean }> { + const limit = 100; + const all = await drizzle + .select() + .from(events) + .where( + and( + eq(events.runId, runId), + map(params?.sinceCursor, (c) => gt(events.eventId, c)) + ) + ) + .orderBy(events.eventId) + .limit(limit + 1); + const page = all.slice(0, limit); + const maxSlot = params?.maxSlot ?? 0; + const resolveData = params?.resolveData ?? 'all'; + return { + events: page + .filter((v) => (slotFromId(v.eventId) ?? 0) > maxSlot) + .map((v) => { + v.eventData ||= v.eventDataJson; + return stripEventDataRefs(EventSchema.parse(compact(v)), resolveData); + }), + cursor: page.at(-1)?.eventId ?? null, + hasMore: all.length > limit, + }; + } + + /** + * The 409 a caller gets when the slot it named turns out to belong to someone + * else, carrying the events it is missing so it can merge, replay and + * re-propose at a free position. + */ + async function slotConflict( + runId: string, + eventId: string, + params: CreateEventParams | undefined + ): Promise { + return new SlotConflictError( + `Slot ${slotFromId(eventId)} of run "${runId}" is already taken`, + { eventId, ...(await eventsAfterClaim(runId, params)) } + ); + } + return { async create(runId, data, params): Promise { let eventId: string | undefined; @@ -490,6 +562,20 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // specVersion is always sent by the runtime, but we provide a fallback for safety const effectiveSpecVersion = data.specVersion ?? SPEC_VERSION_CURRENT; + // Whether this run numbers its events by slot. Decided from what was + // persisted, never from this request or this build, so a run stays in the + // mode it was created in for life — a run whose log holds ULID ids must + // never be handed a slot id, and vice versa. `run_created` is the one + // event that decides the mode instead of reading it; the resilient-start + // path below decides it too, on the request that creates the run. + let slotMode: boolean | undefined = + data.eventType === 'run_created' + ? usesSlotIdentity(effectiveSpecVersion) + : undefined; + // The run's highest event id, when it was read before the write. Seeds the + // allocator's first attempt so a probe is never made twice. + let seedHighestEventId: string | undefined; + // Track entity created/updated for EventResult let run: WorkflowRun | undefined; let step: Step | undefined; @@ -585,7 +671,13 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { .returning(); if (inserted) { - const runCreatedEventId = `wevt_${ulid()}`; + // We created the run, so this request also decided its mode. + slotMode = usesSlotIdentity(effectiveSpecVersion); + // A run's own `run_created` provably has nothing before it, so its + // slot needs no allocation. + const runCreatedEventId = slotMode + ? slotEventId(RUN_CREATED_SLOT) + : `wevt_${ulid()}`; await drizzle.insert(events).values({ runId: effectiveRunId, eventId: runCreatedEventId, @@ -631,7 +723,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { if (requiresNewerWorld(currentRun.specVersion)) { throw new RunNotSupportedError( currentRun.specVersion!, - SPEC_VERSION_CURRENT + SPEC_VERSION_MAX_SUPPORTED ); } @@ -651,6 +743,92 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { throw new WorkflowRunNotFoundError(effectiveRunId); } + // The run entity is the authority on the mode, and it can appear between + // the resilient-start insert above and this point: start() issues + // `run_created` and the queue send concurrently, so a delivery's + // `run_started` can arrive while the run is still being published. A stale + // "no" would number that one event with a ULID on an otherwise + // slot-numbered run, and the hole it leaves in the numbering costs the log + // its completeness proof for life. + if (currentRun && data.eventType !== 'run_created') { + slotMode = usesSlotIdentity(currentRun.specVersion); + } + if (slotMode === undefined) { + // step_completed and step_retrying skip the run read above. The log's + // own highest id reports the scheme, since a log holds ids of exactly + // one, and it is the probe the allocator needs anyway — so a + // slot-numbered run pays nothing extra for this query. + seedHighestEventId = await highestEventId(drizzle, effectiveRunId); + slotMode = isSlotId(seedHighestEventId ?? ''); + } + + // ============================================================ + // EVENT ID: the caller's slot claim, an allocated slot, or a ULID + // ============================================================ + // A slot-numbered run's ids name positions in its log, so an id is either + // claimed by a caller that holds the log (and is therefore asserting the + // log is complete up to that position) or allocated at write time for a + // caller that has no log — a step completion reporting in, a cancellation + // from an API call. + let claimedSlot: number | undefined; + if (params?.eventId !== undefined) { + if (!slotMode) { + throw new WorkflowWorldError( + `Event id "${params.eventId}" was supplied for run "${effectiveRunId}", whose events are not numbered by slot`, + { status: 400 } + ); + } + claimedSlot = slotFromId(params.eventId); + if (claimedSlot === undefined) { + throw new WorkflowWorldError( + `Event id "${params.eventId}" is not a slot id, and run "${effectiveRunId}" numbers its events by slot`, + { status: 400 } + ); + } + eventId = params.eventId; + if (await eventExists(drizzle, effectiveRunId, eventId)) { + // Reject a doomed claim before the materialization below creates the + // step, hook or wait this event will now never accompany. A caller + // that re-proposes at the next slot would otherwise collide with its + // own orphan and read that as "my write already landed". + throw await slotConflict(effectiveRunId, eventId, params); + } + } + + /** + * Runs one of the event writes below under this run's id discipline: in + * slot mode it places the event at the position the caller claimed or at + * the next free one, retrying a position lost to a concurrent writer; + * otherwise it mints a ULID. + */ + const publish = async ( + write: (ids: EventIds) => Promise + ): Promise => + slotMode + ? placeEvent(drizzle, effectiveRunId, { + ...(claimedSlot !== undefined ? { claimedSlot } : {}), + // A run's own `run_created` owns the first slot — provably, since + // nothing precedes it — so every other event allocates above it, + // even when that event is the first to arrive here. + minSlot: + data.eventType === 'run_created' + ? RUN_CREATED_SLOT + : RUN_CREATED_SLOT + 1, + seedHighestEventId, + onClaimTaken: () => + slotConflict(effectiveRunId, eventId as string, params), + write: (ids) => { + // Each attempt publishes under its own id, and the result and + // error messages below read it back from here. + eventId = ids.primary(); + return write(ids); + }, + }) + : write({ + primary: getEventId, + extra: async () => `wevt_${ulid()}`, + }); + // Lazy step start: a step_started carrying step-creation data // (stepName + input) may arrive with no prior step_created — it creates // the step on the fly (see the materialization block below). This @@ -676,17 +854,19 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { .limit(1); // Create the event (still record it) - const [value] = await drizzle - .insert(Schema.events) - .values({ - runId: effectiveRunId, - eventId: getEventId(), - correlationId: data.correlationId, - eventType: data.eventType, - eventData: 'eventData' in data ? data.eventData : undefined, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: Schema.events.createdAt }); + const [value] = await publish((ids) => + drizzle + .insert(Schema.events) + .values({ + runId: effectiveRunId, + eventId: ids.primary(), + correlationId: data.correlationId, + eventType: data.eventType, + eventData: 'eventData' in data ? data.eventData : undefined, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: Schema.events.createdAt }) + ); const result = { ...data, @@ -1192,41 +1372,52 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // event INSERT behind that lock prevents a late step_started from being // ordered after a concurrent terminal event that already won the row. if (data.eventType === 'step_started') { - value = await drizzle.transaction(async (tx) => { - // Lazy step start: no prior step_created exists, but this - // step_started carries the step-creation data. The step INSERT is - // the ownership claim: only the caller that inserts the row gets to - // run the step body inline. - if (lazyStepStart && !validatedStep) { - const lazyData = data.eventData; - const [inserted] = await tx - .insert(Schema.steps) - .values({ - runId: effectiveRunId, - stepId: data.correlationId, - stepName: lazyData.stepName, - input: lazyData.input as SerializedContent, - status: 'pending', - attempt: 0, - specVersion: effectiveSpecVersion, - }) - .onConflictDoNothing() - .returning({ stepId: Schema.steps.stepId }); - - if (!inserted) { - throw new EntityConflictError( - `Step "${data.correlationId}" already created` - ); - } + // The whole transaction is the retry unit here: a step_started that + // loses its slot has to roll the step row and the synthetic + // step_created back with it, or the next attempt would trip its own + // orphaned step and read that as "a concurrent handler won the create". + value = await publish((ids) => + drizzle.transaction(async (tx) => { + // Lazy step start: no prior step_created exists, but this + // step_started carries the step-creation data. The step INSERT is + // the ownership claim: only the caller that inserts the row gets to + // run the step body inline. + if (lazyStepStart && !validatedStep) { + const lazyData = data.eventData; + const [inserted] = await tx + .insert(Schema.steps) + .values({ + runId: effectiveRunId, + stepId: data.correlationId, + stepName: lazyData.stepName, + input: lazyData.input as SerializedContent, + status: 'pending', + attempt: 0, + specVersion: effectiveSpecVersion, + }) + .onConflictDoNothing() + .returning({ stepId: Schema.steps.stepId }); + + if (!inserted) { + throw new EntityConflictError( + `Step "${data.correlationId}" already created` + ); + } - // Replay still needs to observe step_created before - // step_started. Because this synthetic event is in the same - // transaction as the lazy step row and step_started event, we - // cannot leave behind only one side of that materialization. - const stepCreatedEventId = `wevt_${ulid()}`; - await tx - .insert(events) - .values({ + // Replay still needs to observe a step_created at all: the + // client's step consumer sets hasCreatedEvent only on that event + // type. Which of the pair sorts first does not matter — the + // step_started consumer is a no-op — but leaving behind only one + // side of the materialization would, hence the shared + // transaction. + // + // It takes a position of its own, which on a claimed write is + // necessarily one this world allocated: a caller that defers a + // step_created cannot know it will be synthesized, so it never + // claims a slot for it. Losing that position rolls the transaction + // back for a retry, so the insert must not swallow the collision. + const stepCreatedEventId = await ids.extra(); + const insertStepCreated = tx.insert(events).values({ runId: effectiveRunId, eventId: stepCreatedEventId, correlationId: data.correlationId, @@ -1236,99 +1427,104 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { input: lazyData.input, }, specVersion: effectiveSpecVersion, - }) - .onConflictDoNothing(); - stepCreatedLazily = true; - } - - // Retried steps may be scheduled for later. Keep this check inside - // the transaction so the step_started write cannot slip past it. - if ( - validatedStep?.retryAfter && - validatedStep.retryAfter.getTime() > Date.now() - ) { - throw new TooEarlyError( - `Cannot start step "${data.correlationId}": retryAfter timestamp has not been reached yet`, - { - retryAfter: Math.ceil( - (validatedStep.retryAfter.getTime() - Date.now()) / 1000 - ), - } - ); - } + }); + await (slotMode + ? insertStepCreated + : insertStepCreated.onConflictDoNothing()); + stepCreatedLazily = true; + } - // The terminal-state guard is part of the UPDATE, not just the - // earlier validation read. That closes the race where another - // writer completes/fails the step between validation and start. - const [stepValue] = await tx - .update(Schema.steps) - .set({ - status: 'running', - attempt: sql`${Schema.steps.attempt} + 1`, - // Preserve the original first-start timestamp across retries or - // overlapping starts. - startedAt: sql`COALESCE(${Schema.steps.startedAt}, ${now.toISOString()})`, - retryAfter: null, - }) - .where( - and( - eq(Schema.steps.runId, effectiveRunId), - eq(Schema.steps.stepId, data.correlationId!), - notInArray(Schema.steps.status, terminalStepStatuses) - ) - ) - .returning(); + // Retried steps may be scheduled for later. Keep this check inside + // the transaction so the step_started write cannot slip past it. + if ( + validatedStep?.retryAfter && + validatedStep.retryAfter.getTime() > Date.now() + ) { + throw new TooEarlyError( + `Cannot start step "${data.correlationId}": retryAfter timestamp has not been reached yet`, + { + retryAfter: Math.ceil( + (validatedStep.retryAfter.getTime() - Date.now()) / 1000 + ), + } + ); + } - if (stepValue) { - step = deserializeStepError(compact(stepValue)); - } else { - const [existing] = await tx - .select({ status: Schema.steps.status }) - .from(Schema.steps) + // The terminal-state guard is part of the UPDATE, not just the + // earlier validation read. That closes the race where another + // writer completes/fails the step between validation and start. + const [stepValue] = await tx + .update(Schema.steps) + .set({ + status: 'running', + attempt: sql`${Schema.steps.attempt} + 1`, + // Preserve the original first-start timestamp across retries or + // overlapping starts. + startedAt: sql`COALESCE(${Schema.steps.startedAt}, ${now.toISOString()})`, + retryAfter: null, + }) .where( and( eq(Schema.steps.runId, effectiveRunId), - eq(Schema.steps.stepId, data.correlationId!) + eq(Schema.steps.stepId, data.correlationId!), + notInArray(Schema.steps.status, terminalStepStatuses) ) ) - .limit(1); - if (!existing) { - throw new WorkflowWorldError( - `Step "${data.correlationId}" not found` - ); + .returning(); + + if (stepValue) { + step = deserializeStepError(compact(stepValue)); + } else { + const [existing] = await tx + .select({ status: Schema.steps.status }) + .from(Schema.steps) + .where( + and( + eq(Schema.steps.runId, effectiveRunId), + eq(Schema.steps.stepId, data.correlationId!) + ) + ) + .limit(1); + if (!existing) { + throw new WorkflowWorldError( + `Step "${data.correlationId}" not found` + ); + } + if (isTerminalStepStatus(existing.status)) { + throw new EntityConflictError( + `Cannot modify step in terminal state "${existing.status}"` + ); + } } - if (isTerminalStepStatus(existing.status)) { + + // A ULID-numbered step_started takes its id only after the guarded + // step UPDATE has acquired and passed the row lock. Without a + // sequence, this is the local ordering guarantee we can provide: a + // writer blocked on the step row will not carry an older event id + // into a later insert. A slot id is exempt — it names a position in + // the log, not a time — and the caller may have claimed it already. + const stepStartedEventId = ids.primary(); + eventId = stepStartedEventId; + const [eventValue] = await tx + .insert(events) + .values({ + runId: effectiveRunId, + eventId: stepStartedEventId, + correlationId: data.correlationId, + eventType: data.eventType, + eventData: storedEventData, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: events.createdAt }); + + if (!eventValue) { throw new EntityConflictError( - `Cannot modify step in terminal state "${existing.status}"` + `Event ${stepStartedEventId} could not be created` ); } - } - - // Allocate the step_started ULID only after the guarded step UPDATE - // has acquired and passed the row lock. Without a sequence, this is - // the local ordering guarantee we can provide: a writer blocked on - // the step row will not carry an older event id into a later insert. - const stepStartedEventId = `wevt_${ulid()}`; - eventId = stepStartedEventId; - const [eventValue] = await tx - .insert(events) - .values({ - runId: effectiveRunId, - eventId: stepStartedEventId, - correlationId: data.correlationId, - eventType: data.eventType, - eventData: storedEventData, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: events.createdAt }); - - if (!eventValue) { - throw new EntityConflictError( - `Event ${stepStartedEventId} could not be created` - ); - } - return eventValue; - }); + return eventValue; + }) + ); } // Handle step_completed event: update step status @@ -1531,20 +1727,21 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { token: eventData.token, conflictingRunId: existingHook.runId, }; + const [conflictValue] = await publish((ids) => + drizzle + .insert(events) + .values({ + runId: effectiveRunId, + eventId: ids.primary(), + correlationId: data.correlationId, + eventType: 'hook_conflict', + eventData: conflictEventData, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: events.createdAt }) + ); const conflictEventId = getEventId(); - const [conflictValue] = await drizzle - .insert(events) - .values({ - runId: effectiveRunId, - eventId: conflictEventId, - correlationId: data.correlationId, - eventType: 'hook_conflict', - eventData: conflictEventData, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: events.createdAt }); - if (!conflictValue) { throw new EntityConflictError( `Event ${conflictEventId} could not be created` @@ -1622,47 +1819,49 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // guarded UPDATE linearizes against a concurrent terminal step // event. if (data.eventType === 'hook_received') { - value = await drizzle.transaction(async (tx) => { - const [runRow] = await tx - .select({ status: Schema.runs.status }) - .from(Schema.runs) - .where(eq(Schema.runs.runId, effectiveRunId)) - .for('update') - .limit(1); - if (!runRow) { - throw new WorkflowRunNotFoundError(effectiveRunId); - } - if (isTerminalWorkflowRunStatus(runRow.status)) { - throw new RunExpiredError( - `Workflow run "${effectiveRunId}" is already in terminal state "${runRow.status}"` - ); - } + value = await publish((ids) => + drizzle.transaction(async (tx) => { + const [runRow] = await tx + .select({ status: Schema.runs.status }) + .from(Schema.runs) + .where(eq(Schema.runs.runId, effectiveRunId)) + .for('update') + .limit(1); + if (!runRow) { + throw new WorkflowRunNotFoundError(effectiveRunId); + } + if (isTerminalWorkflowRunStatus(runRow.status)) { + throw new RunExpiredError( + `Workflow run "${effectiveRunId}" is already in terminal state "${runRow.status}"` + ); + } - // Allocate the ULID only after the row lock is acquired, - // matching step_started's ordering guarantee: a writer blocked - // on the run row must not carry an older event id into a later - // insert. - const hookReceivedEventId = `wevt_${ulid()}`; - eventId = hookReceivedEventId; - const [eventValue] = await tx - .insert(events) - .values({ - runId: effectiveRunId, - eventId: hookReceivedEventId, - correlationId: data.correlationId, - eventType: data.eventType, - eventData: storedEventData, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: events.createdAt }); + // Take the ULID only after the row lock is acquired, matching + // step_started's ordering guarantee: a writer blocked on the run + // row must not carry an older event id into a later insert. A slot + // id names a position rather than a time, so it is exempt. + const hookReceivedEventId = ids.primary(); + eventId = hookReceivedEventId; + const [eventValue] = await tx + .insert(events) + .values({ + runId: effectiveRunId, + eventId: hookReceivedEventId, + correlationId: data.correlationId, + eventType: data.eventType, + eventData: storedEventData, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: events.createdAt }); - if (!eventValue) { - throw new EntityConflictError( - `Event ${hookReceivedEventId} could not be created` - ); - } - return eventValue; - }); + if (!eventValue) { + throw new EntityConflictError( + `Event ${hookReceivedEventId} could not be created` + ); + } + return eventValue; + }) + ); } // Handle wait_created event: create wait entity @@ -1748,17 +1947,23 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { try { if (!value) { - [value] = await drizzle - .insert(events) - .values({ - runId: effectiveRunId, - eventId: getEventId(), - correlationId: data.correlationId, - eventType: data.eventType, - eventData: storedEventData, - specVersion: effectiveSpecVersion, - }) - .returning({ createdAt: events.createdAt }); + // Only the event row is retried here: the entity this event describes + // was materialized above, outside any transaction, and re-inserting + // the event at a higher position leaves the log dense and still + // consistent with that entity. + [value] = await publish((ids) => + drizzle + .insert(events) + .values({ + runId: effectiveRunId, + eventId: ids.primary(), + correlationId: data.correlationId, + eventType: data.eventType, + eventData: storedEventData, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: events.createdAt }) + ); } } catch (err) { // Translate unique-violation on the correlated-event partial index diff --git a/packages/world-postgres/test/slot-identity.test.ts b/packages/world-postgres/test/slot-identity.test.ts new file mode 100644 index 0000000000..d8b0578a7d --- /dev/null +++ b/packages/world-postgres/test/slot-identity.test.ts @@ -0,0 +1,375 @@ +import { execSync } from 'node:child_process'; +import { PostgreSqlContainer } from '@testcontainers/postgresql'; +import { SlotConflictError } from '@workflow/errors'; +import { + FIRST_SLOT, + maxSlotOf, + SPEC_VERSION_CURRENT, + SPEC_VERSION_SLOT_IDENTITY, + slotEventId, + slotFromId, +} from '@workflow/world'; +import { Pool } from 'pg'; +import { + afterAll, + beforeAll, + beforeEach, + describe, + expect, + test, +} from 'vitest'; +import { createClient } from '../src/drizzle/index.js'; +import { createEventsStorage } from '../src/storage.js'; + +describe('Slot identity (Postgres integration)', () => { + if (process.platform === 'win32') { + test.skip('skipped on Windows since it relies on a docker container', () => {}); + return; + } + + let container: Awaited>; + let pool: Pool; + let events: ReturnType; + + beforeAll(async () => { + container = await new PostgreSqlContainer('postgres:15-alpine').start(); + const dbUrl = container.getConnectionUri(); + process.env.DATABASE_URL = dbUrl; + process.env.WORKFLOW_POSTGRES_URL = dbUrl; + execSync('pnpm db:push', { + stdio: 'inherit', + cwd: process.cwd(), + env: process.env, + }); + // Contention is the point of these tests, so the pool has to be able to + // hold every writer of a burst at once. + pool = new Pool({ connectionString: dbUrl, max: 20 }); + events = createEventsStorage(createClient(pool)); + }, 120_000); + + beforeEach(async () => { + await pool.query( + 'TRUNCATE TABLE workflow.workflow_events, workflow.workflow_steps, workflow.workflow_hooks, workflow.workflow_runs RESTART IDENTITY CASCADE' + ); + }); + + afterAll(async () => { + await pool.end(); + await container.stop(); + }); + + /** Start a run whose events are numbered by slot, and return its id. */ + async function newSlotRun(): Promise { + const result = await events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + eventData: { + deploymentId: 'dpl_test', + workflowName: 'test-workflow', + input: new Uint8Array(), + }, + }); + if (!result.run) { + throw new Error('Expected run to be created'); + } + return result.run.runId; + } + + function eventsOf(runId: string) { + // The page size is explicit: the default would silently truncate a fan-out + // and make a dense log look sparse. + return events.list({ runId, pagination: { limit: 500 } }); + } + + /** The slots of the run's log, in list order. */ + async function slotsOf(runId: string): Promise { + const { data } = await eventsOf(runId); + return data.map((event) => slotFromId(event.eventId) ?? -1); + } + + function ascending(slots: number[]): number[] { + return [...slots].sort((a, b) => a - b); + } + + function denseFrom(count: number): number[] { + return Array.from({ length: count }, (_, index) => FIRST_SLOT + index); + } + + async function createStep( + runId: string, + stepId: string, + eventId?: string + ): Promise { + const result = await events.create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: stepId, + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + eventId === undefined ? undefined : { eventId } + ); + if (!result.event) { + throw new Error('Expected an event'); + } + return result.event.eventId; + } + + describe('numbering', () => { + test('puts run_created in the first slot', async () => { + const runId = await newSlotRun(); + await expect(slotsOf(runId)).resolves.toEqual([FIRST_SLOT]); + }); + + test('allocates dense slots for writers that hold no log', async () => { + // A step completion reporting in, a cancellation from an API call: the + // caller has no event log, so the world numbers the event for it. + const runId = await newSlotRun(); + await createStep(runId, 'step_a'); + await createStep(runId, 'step_b'); + await expect(slotsOf(runId)).resolves.toEqual(denseFrom(3)); + }); + + test('honours a slot the caller claims', async () => { + const runId = await newSlotRun(); + const eventId = await createStep(runId, 'step_a', slotEventId(2)); + expect(eventId).toBe(slotEventId(2)); + await expect(slotsOf(runId)).resolves.toEqual(denseFrom(2)); + }); + + test('numbers a ULID-mode run the way it always did', async () => { + const created = await events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'dpl_test', + workflowName: 'test-workflow', + input: new Uint8Array(), + }, + }); + const runId = created.run?.runId as string; + const eventId = await createStep(runId, 'step_a'); + expect(eventId).toMatch(/^wevt_/); + expect(slotFromId(eventId)).toBeUndefined(); + }); + + test('gives a lazy step start two consecutive slots', async () => { + // One request, two events: the step_started the caller sent and the + // step_created it deferred. Which sorts first does not matter — only + // step_created flips the client's hasCreatedEvent — but both have to + // land, and neither may leave a hole. + const runId = await newSlotRun(); + const started = await events.create(runId, { + eventType: 'step_started', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }); + expect(slotFromId(started.event?.eventId ?? '')).toBe(2); + const { data } = await eventsOf(runId); + expect( + data.map((event) => `${slotFromId(event.eventId)} ${event.eventType}`) + ).toEqual(['1 run_created', '2 step_started', '3 step_created']); + }); + + test('allocates the deferred step_created above a claimed slot', async () => { + // A caller that defers a step_created cannot know it will be synthesized, + // so it never claims a slot for it — and the slot it did claim is its own. + const runId = await newSlotRun(); + const started = await events.create( + runId, + { + eventType: 'step_started', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(2) } + ); + expect(started.event?.eventId).toBe(slotEventId(2)); + expect(ascending(await slotsOf(runId))).toEqual(denseFrom(3)); + }); + + test('numbers events of runs it never created', async () => { + // `step_completed` and `step_retrying` deliberately skip the run read, so + // the mode comes from the log rather than from a run row in hand. + const runId = await newSlotRun(); + await createStep(runId, 'step_a'); + const completed = await events.create(runId, { + eventType: 'step_completed', + specVersion: SPEC_VERSION_CURRENT, + correlationId: 'step_a', + eventData: { output: new Uint8Array() }, + }); + expect(slotFromId(completed.event?.eventId ?? '')).toBe(3); + }); + }); + + describe('contention', () => { + // The primary key is the authority, and a writer that loses a position is + // retried at one that is still free. Nothing here may leave a hole: density + // is what lets a reader prove its copy of a log is complete. + for (const writers of [2, 8, 50]) { + test(`keeps ${writers} concurrent writers dense`, async () => { + const runId = await newSlotRun(); + const ids = await Promise.all( + Array.from({ length: writers }, (_, index) => + createStep(runId, `step_${index}`) + ) + ); + expect(new Set(ids).size).toBe(writers); + expect(ascending(await slotsOf(runId))).toEqual(denseFrom(writers + 1)); + }, 60_000); + } + + test('proves completeness: the highest slot is the event count', async () => { + const runId = await newSlotRun(); + await Promise.all( + Array.from({ length: 5 }, (_, index) => + createStep(runId, `step_${index}`) + ) + ); + const { data } = await eventsOf(runId); + expect(maxSlotOf(data)).toBe(data.length); + }); + + test('leaves no hole behind a rejected write', async () => { + // The rejected op's slot sits below its concurrent sibling's, and a hole + // below a published event can never be filled. + const runId = await newSlotRun(); + const [rejected, accepted] = await Promise.allSettled([ + events.create(runId, { + eventType: 'step_completed', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_never_created', + eventData: { output: new Uint8Array() }, + }), + createStep(runId, 'step_a'), + ]); + expect(rejected.status).toBe('rejected'); + expect(accepted.status).toBe('fulfilled'); + await createStep(runId, 'step_b'); + expect(ascending(await slotsOf(runId))).toEqual(denseFrom(3)); + }); + }); + + describe('mode is pinned to the run', () => { + test('rejects a slot id claimed on a ULID-numbered run', async () => { + const created = await events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: 'dpl_test', + workflowName: 'test-workflow', + input: new Uint8Array(), + }, + }); + const runId = created.run?.runId as string; + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + /not numbered by slot/ + ); + }); + + test('rejects a ULID id claimed on a slot-numbered run', async () => { + const runId = await newSlotRun(); + await expect( + createStep(runId, 'step_a', 'evnt_01K5Z0000000000000000000AA') + ).rejects.toThrow(/not a slot id/); + }); + + test('ignores the spec version of later requests', async () => { + // A run is in exactly one mode for life; only what was persisted decides. + const runId = await newSlotRun(); + const result = await events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_CURRENT, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }); + expect(slotFromId(result.event?.eventId ?? '')).toBe(2); + }); + }); + + describe('conflict', () => { + test('reports the events the loser is missing', async () => { + const runId = await newSlotRun(); + // Out of band: something else takes the slot this caller was about to + // claim, so the caller's log is provably missing an event. + await createStep(runId, 'step_out_of_band'); + + const conflict = await events + .create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(2), maxSlot: 1 } + ) + .catch((error: unknown) => error); + + expect(SlotConflictError.is(conflict)).toBe(true); + const slotConflict = conflict as SlotConflictError; + expect(slotConflict.status).toBe(409); + expect(slotConflict.eventId).toBe(slotEventId(2)); + expect(slotConflict.events?.map((event) => event.eventId)).toEqual([ + slotEventId(2), + ]); + }); + + test('excludes events the loser already holds from the delta', async () => { + const runId = await newSlotRun(); + await createStep(runId, 'step_one'); + await createStep(runId, 'step_two'); + + const conflict = await events + .create( + runId, + { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(2), maxSlot: 2 } + ) + .catch((error: unknown) => error); + + // Slots 1 and 2 are at or below what the caller had; only 3 is news. + expect( + (conflict as SlotConflictError).events?.map((event) => event.eventId) + ).toEqual([slotEventId(3)]); + }); + + test('lets the loser re-propose at the next free slot', async () => { + const runId = await newSlotRun(); + await createStep(runId, 'step_out_of_band'); + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + SlotConflictError + ); + // Merging the delta moves the caller's own numbering forward by one. + const eventId = await createStep(runId, 'step_a', slotEventId(3)); + expect(eventId).toBe(slotEventId(3)); + await expect(slotsOf(runId)).resolves.toEqual(denseFrom(3)); + }); + + test('materializes nothing for a claim that is already taken', async () => { + // The re-post is what the guard protects: a step row left behind by the + // losing attempt would make the retry trip its own orphan and read that + // as "a concurrent handler won the create". + const runId = await newSlotRun(); + await createStep(runId, 'step_out_of_band'); + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + SlotConflictError + ); + const { rows } = await pool.query( + 'SELECT step_id FROM workflow.workflow_steps WHERE run_id = $1', + [runId] + ); + expect(rows.map((row) => row.step_id)).toEqual(['step_out_of_band']); + }); + }); +}); diff --git a/packages/world/src/index.ts b/packages/world/src/index.ts index 7317b340ba..5a8c185f2a 100644 --- a/packages/world/src/index.ts +++ b/packages/world/src/index.ts @@ -113,9 +113,12 @@ export { export type { SpecVersion } from './spec-version.js'; export { isLegacySpecVersion, + mintedSpecVersion, requiresNewerWorld, + SLOT_IDENTITY_ENV_VAR, SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, + SPEC_VERSION_MAX_SUPPORTED, SPEC_VERSION_SLOT_IDENTITY, SPEC_VERSION_SUPPORTS_ATTRIBUTES, SPEC_VERSION_SUPPORTS_CBOR_QUEUE_TRANSPORT, diff --git a/packages/world/src/slot-identity.test.ts b/packages/world/src/slot-identity.test.ts index c99d2aa930..acdfada627 100644 --- a/packages/world/src/slot-identity.test.ts +++ b/packages/world/src/slot-identity.test.ts @@ -1,5 +1,6 @@ import { ulid } from 'ulid'; import { describe, expect, it } from 'vitest'; +import { z } from 'zod'; import { FIRST_SLOT, isSlotId, @@ -17,9 +18,9 @@ describe('slotIdBody', () => { expect(body).toHaveLength(SLOT_ID_WIDTH); expect(`evnt_${body}`).toHaveLength(`evnt_${ulid()}`.length); // Crockford base32 starts with the decimal digits, so the padded body - // parses as a ULID — this is what keeps every existing schema, sort key and - // range fence working unchanged. - expect(ulidToDate(body)).not.toBeNull(); + // satisfies the ULID syntax — this is what keeps every existing schema, + // sort key and range fence working unchanged. + expect(z.string().ulid().safeParse(body).success).toBe(true); }); it('orders lexicographically by slot at a fixed width', () => { @@ -60,6 +61,24 @@ describe('slotFromId', () => { }); }); +describe('a slot carries no timestamp', () => { + it('reports no time rather than epoch 0', () => { + // Passing the ULID syntax check is what makes a slot portable; decoding a + // *time* out of one is always a bug. Two that this guards: the sandbox + // clock is set from the events it consumes, so epoch 0 would rewind a + // replaying workflow's `Date.now()` to 1970; and world-local prefilters + // cursor pagination on the time in the filename, so epoch 0 would hide + // every slot-numbered event from an ascending page. + expect(ulidToDate(slotIdBody(FIRST_SLOT))).toBeNull(); + expect(ulidToDate(slotEventId(FIRST_SLOT))).toBeNull(); + expect(ulidToDate(slotIdBody(123_456))).toBeNull(); + }); + + it('still reads the time out of a ULID', () => { + expect(ulidToDate(ulid())?.getTime()).toBeGreaterThan(0); + }); +}); + describe('maxSlotOf', () => { it('finds the highest slot regardless of position', () => { // A log is merged from several loads and is not sorted, so the last element diff --git a/packages/world/src/spec-version.test.ts b/packages/world/src/spec-version.test.ts index 880c43180d..f21596f951 100644 --- a/packages/world/src/spec-version.test.ts +++ b/packages/world/src/spec-version.test.ts @@ -1,9 +1,13 @@ import { describe, expect, it } from 'vitest'; import { isLegacySpecVersion, + mintedSpecVersion, requiresNewerWorld, + SLOT_IDENTITY_ENV_VAR, SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, + SPEC_VERSION_MAX_SUPPORTED, + SPEC_VERSION_SLOT_IDENTITY, SPEC_VERSION_SUPPORTS_ATTRIBUTES, SPEC_VERSION_SUPPORTS_COMPRESSION, } from './spec-version.js'; @@ -13,10 +17,21 @@ describe('spec version constants', () => { expect(SPEC_VERSION_CURRENT).toBe(SPEC_VERSION_SUPPORTS_COMPRESSION); expect(SPEC_VERSION_SUPPORTS_COMPRESSION).toBe(5); }); + + it('can read a newer spec version than it mints', () => { + // Slot identity is readable by every world before any world mints it, so + // that turning it on for new runs cannot make those same worlds reject + // them. Once slots are the default the two constants coincide again. + expect(SPEC_VERSION_MAX_SUPPORTED).toBe(SPEC_VERSION_SLOT_IDENTITY); + expect(SPEC_VERSION_MAX_SUPPORTED).toBeGreaterThanOrEqual( + SPEC_VERSION_CURRENT + ); + }); }); describe('requiresNewerWorld', () => { - it('accepts runs at or below the current spec version', () => { + it('accepts runs at or below the newest readable spec version', () => { + expect(requiresNewerWorld(SPEC_VERSION_MAX_SUPPORTED)).toBe(false); expect(requiresNewerWorld(SPEC_VERSION_CURRENT)).toBe(false); expect(requiresNewerWorld(SPEC_VERSION_SUPPORTS_ATTRIBUTES)).toBe(false); expect(requiresNewerWorld(SPEC_VERSION_LEGACY)).toBe(false); @@ -24,13 +39,20 @@ describe('requiresNewerWorld', () => { expect(requiresNewerWorld(null)).toBe(false); }); - it('rejects runs newer than the current spec version', () => { + it('accepts a slot-identity run', () => { + // Gates the flag rollout: a world that rejected spec-6 would reject the + // runs it had just stamped spec-6 itself, at their first event after + // run_created. + expect(requiresNewerWorld(SPEC_VERSION_SLOT_IDENTITY)).toBe(false); + }); + + it('rejects runs newer than the newest readable spec version', () => { // This is the contract that protects older SDKs from compressed // payloads they cannot decode: a spec-5 run read by an SDK whose - // SPEC_VERSION_CURRENT is 4 fails this check up front (with - // RunNotSupportedError at the storage layer) instead of failing on - // individual compressed payloads. - expect(requiresNewerWorld(SPEC_VERSION_CURRENT + 1)).toBe(true); + // ceiling is 4 fails this check up front (with RunNotSupportedError at + // the storage layer) instead of failing on individual compressed + // payloads. + expect(requiresNewerWorld(SPEC_VERSION_MAX_SUPPORTED + 1)).toBe(true); }); it('simulates a v4 reader rejecting a compression-era run', () => { @@ -51,3 +73,33 @@ describe('isLegacySpecVersion', () => { expect(isLegacySpecVersion(5)).toBe(false); }); }); + +describe('mintedSpecVersion', () => { + it('mints the current version by default', () => { + expect(mintedSpecVersion({})).toBe(SPEC_VERSION_CURRENT); + }); + + it('mints slot identity when the flag is set', () => { + for (const value of ['1', 'true']) { + expect(mintedSpecVersion({ [SLOT_IDENTITY_ENV_VAR]: value })).toBe( + SPEC_VERSION_SLOT_IDENTITY + ); + } + }); + + it('treats any other value as off', () => { + // An unset-but-present variable is the shape a shell leaves behind, and it + // must not silently switch a deployment's event identity scheme. + for (const value of ['', '0', 'false', 'yes']) { + expect(mintedSpecVersion({ [SLOT_IDENTITY_ENV_VAR]: value })).toBe( + SPEC_VERSION_CURRENT + ); + } + }); + + it('mints nothing a world cannot read', () => { + expect( + requiresNewerWorld(mintedSpecVersion({ [SLOT_IDENTITY_ENV_VAR]: '1' })) + ).toBe(false); + }); +}); diff --git a/packages/world/src/spec-version.ts b/packages/world/src/spec-version.ts index d9be743be6..a86f4f3e90 100644 --- a/packages/world/src/spec-version.ts +++ b/packages/world/src/spec-version.ts @@ -42,20 +42,60 @@ export const SPEC_VERSION_SUPPORTS_COMPRESSION = 5 as SpecVersion; * persisted `specVersion`, never from the build. A run started under ULID * correlation ids and replayed by a slot-capable build would otherwise propose * `step_…001` where its log holds `step_01K…`, matching no existing entity. - * - * Deliberately not `SPEC_VERSION_CURRENT` yet: `requiresNewerWorld()` is what - * makes a world reject runs it cannot read, so bumping current before the - * worlds can allocate slots would have them reject their own new runs. */ export const SPEC_VERSION_SLOT_IDENTITY = 6 as SpecVersion; /** * Current spec version (event-sourced architecture with native attributes * and compressed payloads). + * + * This is the version new runs are stamped with, which is a *lower* bar than + * the newest version this build can read — see + * {@link SPEC_VERSION_MAX_SUPPORTED}. Slot identity ships behind a flag, so it + * is readable everywhere before it is minted anywhere. */ export const SPEC_VERSION_CURRENT = SPEC_VERSION_SUPPORTS_COMPRESSION as SpecVersion; +/** + * Newest spec version this build can read. Runs above it are rejected outright + * by {@link requiresNewerWorld} rather than misread. + * + * Distinct from {@link SPEC_VERSION_CURRENT} because a world has to be able to + * read a version before anything may mint it: the flag that turns slot identity + * on for new runs would otherwise make every world reject the runs it had just + * created. Worlds opt into minting individually, via the `specVersion` they + * declare. + */ +export const SPEC_VERSION_MAX_SUPPORTED = + SPEC_VERSION_SLOT_IDENTITY as SpecVersion; + +/** + * Environment variable that opts new runs into slot identity. + * + * Read per `createWorld()` call rather than at module load, so a test or a + * single process can create worlds in both modes. + */ +export const SLOT_IDENTITY_ENV_VAR = 'WORKFLOW_SLOT_IDENTITY'; + +/** + * The spec version a world should stamp on the runs it creates: slot identity + * when {@link SLOT_IDENTITY_ENV_VAR} is set, otherwise + * {@link SPEC_VERSION_CURRENT}. + * + * Every world reads runs up to {@link SPEC_VERSION_MAX_SUPPORTED} whatever this + * returns, so turning the flag on in one place does not make the runs it creates + * unreadable elsewhere. + */ +export function mintedSpecVersion( + env: Record = process.env +): SpecVersion { + const value = env[SLOT_IDENTITY_ENV_VAR]; + return value === '1' || value === 'true' + ? SPEC_VERSION_SLOT_IDENTITY + : SPEC_VERSION_CURRENT; +} + /** * Check if a spec version is legacy (<= SPEC_VERSION_LEGACY or undefined). * Legacy runs require different handling - they use direct entity mutation @@ -73,7 +113,7 @@ export function isLegacySpecVersion(v: number | undefined | null): boolean { } /** - * Check if a spec version requires a newer world (> SPEC_VERSION_CURRENT). + * Check if a spec version requires a newer world (> SPEC_VERSION_MAX_SUPPORTED). * This happens when a run was created by a newer SDK version. * * @param v - The spec version number, or undefined/null for legacy runs @@ -81,7 +121,7 @@ export function isLegacySpecVersion(v: number | undefined | null): boolean { */ export function requiresNewerWorld(v: number | undefined | null): boolean { if (v === undefined || v === null) return false; - return v > SPEC_VERSION_CURRENT; + return v > SPEC_VERSION_MAX_SUPPORTED; } /** diff --git a/packages/world/src/ulid.ts b/packages/world/src/ulid.ts index 1ee1b7b47c..cc893834c3 100644 --- a/packages/world/src/ulid.ts +++ b/packages/world/src/ulid.ts @@ -1,5 +1,6 @@ import { decodeTime } from 'ulid'; import { z } from 'zod'; +import { isSlotId } from './slot-identity.js'; const UlidSchema = z.string().ulid(); @@ -36,8 +37,19 @@ export const DEFAULT_TIMESTAMP_THRESHOLD_MS = /** * Extracts a Date from a ULID string, or null if the string is not a valid ULID. + * + * Slot ids are not ULIDs even though they pass the ULID *syntax* check: their + * body is all decimal digits, which Crockford base32 accepts, and it would + * decode to a timestamp of epoch 0 instead of failing. A slot encodes a + * position, not a time, so it is reported here as having no time at all — + * callers must read the object's own `createdAt`. Silently returning 1970 + * instead would, among other things, rewind a replaying workflow's clock and + * make cursor pagination skip every slot-numbered event. */ export function ulidToDate(maybeUlid: string): Date | null { + if (isSlotId(maybeUlid)) { + return null; + } const ulid = UlidSchema.safeParse(maybeUlid); if (!ulid.success) { return null; From e5af056d607e539a7122ce6442c7ee7c8b08f0ff Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Thu, 30 Jul 2026 21:48:59 -0700 Subject: [PATCH 02/28] feat(worlds): mint slot identity by default, including from world-vercel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `mintedSpecVersion()` now returns spec 6 unless `WORKFLOW_SLOT_IDENTITY` is explicitly `0`/`false`, and world-vercel reads it instead of hardcoding spec 5. world-vercel was the one World that never called it, so a deployment could not mint a slot-numbered run however the flag was set — the flag only reached the Local and Postgres Worlds. Every World still reads both schemes, and mode stays pinned to a run's persisted specVersion, so flipping the default only changes how runs created from here on are numbered. --- .changeset/slot-identity-default-on.md | 8 +++++ .../docs/v5/configuration/runtime-tuning.mdx | 6 ++-- packages/world-local/src/index.ts | 6 ++-- packages/world-postgres/src/index.ts | 6 ++-- packages/world-vercel/src/index.ts | 11 ++++--- packages/world/src/spec-version.test.ts | 23 +++++++-------- packages/world/src/spec-version.ts | 29 +++++++++---------- 7 files changed, 49 insertions(+), 40 deletions(-) create mode 100644 .changeset/slot-identity-default-on.md diff --git a/.changeset/slot-identity-default-on.md b/.changeset/slot-identity-default-on.md new file mode 100644 index 0000000000..55e3973586 --- /dev/null +++ b/.changeset/slot-identity-default-on.md @@ -0,0 +1,8 @@ +--- +'@workflow/world': minor +'@workflow/world-vercel': minor +'@workflow/world-local': minor +'@workflow/world-postgres': minor +--- + +Number new runs' events by position by default, in every World including the Vercel one. Set `WORKFLOW_SLOT_IDENTITY=0` to keep minting ULID event ids. diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index 110e569d5f..ba4a05b397 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -50,12 +50,12 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL ### `WORKFLOW_SLOT_IDENTITY` -- Default: disabled +- Default: enabled - Numbers a new run's events by position instead of by ULID: `evnt_…001` is the run's first event, `evnt_…002` its second, with no gaps. A reader can then prove its copy of the log is complete, because the highest number is the event count. - Contention becomes explicit rather than silent. Two writers proposing the same position cannot both win: the loser gets a 409 ([`SlotConflictError`](/docs/api-reference/workflow-errors/slot-conflict-error)) carrying the events it was missing, and the runtime merges them, replays, and re-proposes at the next free position. -- Applies only to runs created while it is set. A run keeps the identity scheme it was created with for life, so turning the flag on or off never affects runs already in flight. +- Applies only to runs created while it is enabled. A run keeps the identity scheme it was created with for life, so turning the flag on or off never affects runs already in flight. - Requires a World that supports it. A World that does not rejects the run outright rather than mis-numbering its events. -- Set `1` or `true` to enable. +- Set `0` or `false` to disable, which numbers new runs by ULID as before. ## Inline execution diff --git a/packages/world-local/src/index.ts b/packages/world-local/src/index.ts index 2014a75652..aa77546732 100644 --- a/packages/world-local/src/index.ts +++ b/packages/world-local/src/index.ts @@ -72,9 +72,9 @@ export function createWorld(args?: Partial): LocalWorld { ); const recoverActiveRuns = resolveRecoverActiveRuns(mergedConfig); return { - // What this world stamps on new runs, which is not the newest version it - // can read: slot identity is readable everywhere and minted only where - // WORKFLOW_SLOT_IDENTITY is set. + // What this world stamps on new runs: slot identity, unless + // WORKFLOW_SLOT_IDENTITY switches it off. Every world reads both schemes + // whatever this says. specVersion: mintedSpecVersion(), ...queue, ...storage, diff --git a/packages/world-postgres/src/index.ts b/packages/world-postgres/src/index.ts index 0db47b89e2..926d752d3b 100644 --- a/packages/world-postgres/src/index.ts +++ b/packages/world-postgres/src/index.ts @@ -63,9 +63,9 @@ export function createWorld( const streamer = createStreamer(pool, drizzle); return { - // What this world stamps on new runs, which is not the newest version it - // can read: slot identity is readable everywhere and minted only where - // WORKFLOW_SLOT_IDENTITY is set. + // What this world stamps on new runs: slot identity, unless + // WORKFLOW_SLOT_IDENTITY switches it off. Every world reads both schemes + // whatever this says. specVersion: mintedSpecVersion(), ...storage, ...streamer, diff --git a/packages/world-vercel/src/index.ts b/packages/world-vercel/src/index.ts index 42ff155fda..6a68e6845a 100644 --- a/packages/world-vercel/src/index.ts +++ b/packages/world-vercel/src/index.ts @@ -1,5 +1,5 @@ import type { World } from '@workflow/world'; -import { SPEC_VERSION_SUPPORTS_COMPRESSION } from '@workflow/world'; +import { mintedSpecVersion } from '@workflow/world'; import { createAnalytics } from './analytics.js'; import { createRunId, describeRun } from './create-run-id.js'; import { createGetEncryptionKeyForRun } from './encryption.js'; @@ -29,9 +29,12 @@ export function createWorld(config?: APIConfig): World { config?.projectConfig?.projectId || process.env.VERCEL_PROJECT_ID; return { - // Spec v5 adds client-side zstd/gzip payload compression. The server stores - // those payloads opaquely, and v5 remains a superset of v4 attributes. - specVersion: SPEC_VERSION_SUPPORTS_COMPRESSION, + // What this world stamps on new runs: slot identity (spec v6) unless + // WORKFLOW_SLOT_IDENTITY switches it off, in which case v5 — client-side + // zstd/gzip payload compression over a superset of the v4 attributes. + // Either way this world reads both, so the stamp only decides how the runs + // it creates from here on are numbered. + specVersion: mintedSpecVersion(), capabilities: { // workflow-server enforces the `stateUpdatedAt` optimistic-concurrency // guard: creations carrying a stale snapshot are rejected with 412 diff --git a/packages/world/src/spec-version.test.ts b/packages/world/src/spec-version.test.ts index f21596f951..0125e2fb1c 100644 --- a/packages/world/src/spec-version.test.ts +++ b/packages/world/src/spec-version.test.ts @@ -75,31 +75,30 @@ describe('isLegacySpecVersion', () => { }); describe('mintedSpecVersion', () => { - it('mints the current version by default', () => { - expect(mintedSpecVersion({})).toBe(SPEC_VERSION_CURRENT); + it('mints slot identity by default', () => { + expect(mintedSpecVersion({})).toBe(SPEC_VERSION_SLOT_IDENTITY); }); - it('mints slot identity when the flag is set', () => { - for (const value of ['1', 'true']) { + it('mints the previous version when the flag is switched off', () => { + for (const value of ['0', 'false']) { expect(mintedSpecVersion({ [SLOT_IDENTITY_ENV_VAR]: value })).toBe( - SPEC_VERSION_SLOT_IDENTITY + SPEC_VERSION_CURRENT ); } }); - it('treats any other value as off', () => { + it('treats any other value as on', () => { // An unset-but-present variable is the shape a shell leaves behind, and it - // must not silently switch a deployment's event identity scheme. - for (const value of ['', '0', 'false', 'yes']) { + // must not silently switch a deployment's event identity scheme. Opting + // out takes an explicit `0`/`false`. + for (const value of ['', '1', 'true', 'yes']) { expect(mintedSpecVersion({ [SLOT_IDENTITY_ENV_VAR]: value })).toBe( - SPEC_VERSION_CURRENT + SPEC_VERSION_SLOT_IDENTITY ); } }); it('mints nothing a world cannot read', () => { - expect( - requiresNewerWorld(mintedSpecVersion({ [SLOT_IDENTITY_ENV_VAR]: '1' })) - ).toBe(false); + expect(requiresNewerWorld(mintedSpecVersion({}))).toBe(false); }); }); diff --git a/packages/world/src/spec-version.ts b/packages/world/src/spec-version.ts index a86f4f3e90..4b830fd4a8 100644 --- a/packages/world/src/spec-version.ts +++ b/packages/world/src/spec-version.ts @@ -49,10 +49,10 @@ export const SPEC_VERSION_SLOT_IDENTITY = 6 as SpecVersion; * Current spec version (event-sourced architecture with native attributes * and compressed payloads). * - * This is the version new runs are stamped with, which is a *lower* bar than - * the newest version this build can read — see - * {@link SPEC_VERSION_MAX_SUPPORTED}. Slot identity ships behind a flag, so it - * is readable everywhere before it is minted anywhere. + * The floor a world stamps on new runs, and a *lower* bar than the newest + * version this build can read — see {@link SPEC_VERSION_MAX_SUPPORTED}. What a + * world actually stamps comes from {@link mintedSpecVersion}; this is what it + * falls back to when slot identity is switched off. */ export const SPEC_VERSION_CURRENT = SPEC_VERSION_SUPPORTS_COMPRESSION as SpecVersion; @@ -62,16 +62,15 @@ export const SPEC_VERSION_CURRENT = * by {@link requiresNewerWorld} rather than misread. * * Distinct from {@link SPEC_VERSION_CURRENT} because a world has to be able to - * read a version before anything may mint it: the flag that turns slot identity - * on for new runs would otherwise make every world reject the runs it had just - * created. Worlds opt into minting individually, via the `specVersion` they - * declare. + * read a version before anything may mint it, and because a world that mints + * slot identity still has to read the spec-5 runs it created before the switch. + * Worlds opt into minting individually, via the `specVersion` they declare. */ export const SPEC_VERSION_MAX_SUPPORTED = SPEC_VERSION_SLOT_IDENTITY as SpecVersion; /** - * Environment variable that opts new runs into slot identity. + * Environment variable that opts new runs out of slot identity. * * Read per `createWorld()` call rather than at module load, so a test or a * single process can create worlds in both modes. @@ -80,20 +79,20 @@ export const SLOT_IDENTITY_ENV_VAR = 'WORKFLOW_SLOT_IDENTITY'; /** * The spec version a world should stamp on the runs it creates: slot identity - * when {@link SLOT_IDENTITY_ENV_VAR} is set, otherwise + * unless {@link SLOT_IDENTITY_ENV_VAR} disables it, in which case * {@link SPEC_VERSION_CURRENT}. * * Every world reads runs up to {@link SPEC_VERSION_MAX_SUPPORTED} whatever this - * returns, so turning the flag on in one place does not make the runs it creates - * unreadable elsewhere. + * returns, so turning the flag off in one place does not make the runs another + * process created unreadable here. */ export function mintedSpecVersion( env: Record = process.env ): SpecVersion { const value = env[SLOT_IDENTITY_ENV_VAR]; - return value === '1' || value === 'true' - ? SPEC_VERSION_SLOT_IDENTITY - : SPEC_VERSION_CURRENT; + return value === '0' || value === 'false' + ? SPEC_VERSION_CURRENT + : SPEC_VERSION_SLOT_IDENTITY; } /** From df22f35ebbf302373bb67b16440a98f9ea0fee0c Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Thu, 30 Jul 2026 23:20:10 -0700 Subject: [PATCH 03/28] fix(world-local): re-probe a lost position instead of conflicting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A writer with no event log — a step completion reporting in, a hook being received, a cancellation from an API call — has nothing to reconcile when the position it allocated turns out to be taken, so surfacing a slot conflict to it strands the caller with an error it cannot act on. Two world instances sharing a data directory keep independent books, which makes that the common case rather than a rarity. The publish is now one attempt in a loop: an allocated position that loses tops the book up from disk, backs off with full jitter, and takes the next free one, within a 30s budget after which the write fails retryably. A position the caller claimed still conflicts on the first loss — the claim asserts a complete log, so only the caller can resolve it. The contention profile moves to @workflow/world so the Local and Postgres Worlds share one definition. --- .../world-local/src/storage/events-storage.ts | 301 +++++++++++------- .../src/storage/slot-identity.test.ts | 39 ++- packages/world-local/src/storage/slots.ts | 28 ++ packages/world-postgres/src/slots.ts | 29 +- packages/world/src/index.ts | 4 + packages/world/src/slot-identity.ts | 25 ++ 6 files changed, 273 insertions(+), 153 deletions(-) diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index 98e705e98d..72c8757a46 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -37,9 +37,11 @@ import { requiresNewerWorld, SPEC_VERSION_CURRENT, SPEC_VERSION_MAX_SUPPORTED, + SLOT_RETRY_BUDGET_MS, StepSchema, slotEventId, slotFromId, + slotRetryDelay, ulidToDate, usesSlotIdentity, validateAttributeChanges, @@ -1009,6 +1011,13 @@ export function createEventsStorage( // ============================================================ // EVENT ID: the caller's slot claim, an allocated slot, or a ULID // ============================================================ + // A run's own `run_created` owns the first slot — provably, since + // nothing precedes it — so every other event allocates above it, even + // when that event is the first to arrive here. + const allocationFloor = + data.eventType === 'run_created' + ? RUN_CREATED_SLOT + : RUN_CREATED_SLOT + 1; // A slot-numbered run's ids name positions in its log, so an id is // either claimed by a caller that holds the log (and is therefore // asserting the log is complete up to that position) or allocated here @@ -1042,15 +1051,7 @@ export function createEventsStorage( } } else if (slotMode) { reservedRunId = effectiveRunId; - // A run's own `run_created` owns the first slot — provably, since - // nothing precedes it — so every other event allocates above it, even - // when that event is the first to arrive here. - const slot = await slots.reserve( - effectiveRunId, - data.eventType === 'run_created' - ? RUN_CREATED_SLOT - : RUN_CREATED_SLOT + 1 - ); + const slot = await slots.reserve(effectiveRunId, allocationFloor); reserved.add(slot); eventId = slotEventId(slot); } @@ -2457,120 +2458,172 @@ export function createEventsStorage( throw new HookNotFoundError(data.correlationId); } - const compositeKey = `${effectiveRunId}-${eventId}`; - const eventPath = taggedPath(basedir, 'events', compositeKey, tag); - // Capture the serialized payload before the write's `await` so the - // cached snapshot can't observe a later mutation (see - // rememberStoredEvent). - const serializedEvent = JSON.stringify(event, jsonReplacer, 2); - - // Cross-process terminal-run guard for `hook_received`. A terminal - // transition (run_completed / run_failed / run_cancelled) in ANY - // process (1) publishes a durable `runTerminalMarkerPath` marker and - // (2) reaps the run's staged hook_received events, both BEFORE it - // writes the terminal run state or appends its terminal event (see - // the terminal-transition block earlier in this function). In-memory - // locks cannot close the shared-filesystem race this backend - // explicitly supports, and a published event file is immediately - // visible to `events.list()` in other processes — so it can never be - // "rolled back" after the fact. Instead, the event stays INVISIBLE - // to readers until a single atomic filesystem operation decides its - // fate: - // - // 1. (fast path) reject if the run is already terminal — by - // marker, or by run state for runs that predate the marker — - // so the common case never creates a file. - // 2. STAGE the event at a non-reader-visible path under `.locks`. - // 3. re-CHECK the terminal marker; reject if present. - // 4. PROMOTE the staged file into `events/` with an atomic hard - // link; reject if the staged file was reaped (`'missing'`). - // - // Correctness: the reap's `unlink` and step 4's `link` target the - // same staged file, so the filesystem serializes them — exactly one - // wins. If the link wins, the event was reader-visible before the - // reap completed, and therefore before the terminal state and - // terminal event were written: acceptance happened-before the - // termination and legitimately precedes it. If the unlink wins, - // promotion fails and the event is never visible to any reader — - // there is nothing to roll back. A resume that stages after the - // reap has passed necessarily stages after the marker was - // committed, so step 3 rejects it. Rejections before step 4 unlink - // a file no reader can see. - let eventPublished: boolean; - if (data.eventType === 'hook_received') { - // Step 1: fast path. The marker is the authoritative durable - // signal; the run-state read additionally rejects runs whose - // terminal state was written without a marker (e.g. runs that - // terminated on an older storage version). - const terminalByMarker = await isRunTerminalCommitted( - basedir, - effectiveRunId, - tag - ); - const runNow = terminalByMarker - ? null - : await readJSONWithFallback( - basedir, - 'runs', - effectiveRunId, - WorkflowRunSchema, - tag - ); - if ( - terminalByMarker || - (runNow && isTerminalWorkflowRunStatus(runNow.status)) - ) { - throw new RunExpiredError( - `Workflow run "${effectiveRunId}" is already in a terminal state` - ); - } - - const stagedPath = pendingHookEventPath( - basedir, - effectiveRunId, - eventId, - tag - ); - const staged = await writeExclusive(stagedPath, serializedEvent); - if (!staged) { - // For a ULID-numbered run the eventId is freshly generated, so - // its staging path can only be occupied by a previous crashed - // attempt of this very event, which never promoted. A - // slot-numbered run can also collide here with another instance - // that allocated the same slot from its own book. Either way the - // event is not reader-visible, so there is no delta to hand back - // and nothing for the caller to merge: surface the same conflict - // shape as a visible-path collision, and let the allocator - // re-probe on the retry. - throw new EntityConflictError( - `Event "${eventId}" already exists for run "${effectiveRunId}"` + /** + * One attempt at publishing the event at the position `eventId` + * currently names: `true` when this call made it reader-visible, + * `false` when the position was already taken. What a loss means is the + * loop's decision — a position this world allocated is simply retried + * one higher, a position the caller claimed is a conflict it has to + * resolve. + */ + async function publishOnce(): Promise { + // Cross-process terminal-run guard for `hook_received`. A terminal + // transition (run_completed / run_failed / run_cancelled) in ANY + // process (1) publishes a durable `runTerminalMarkerPath` marker and + // (2) reaps the run's staged hook_received events, both BEFORE it + // writes the terminal run state or appends its terminal event (see + // the terminal-transition block earlier in this function). In-memory + // locks cannot close the shared-filesystem race this backend + // explicitly supports, and a published event file is immediately + // visible to `events.list()` in other processes — so it can never be + // "rolled back" after the fact. Instead, the event stays INVISIBLE + // to readers until a single atomic filesystem operation decides its + // fate: + // + // 1. (fast path) reject if the run is already terminal — by + // marker, or by run state for runs that predate the marker — + // so the common case never creates a file. + // 2. STAGE the event at a non-reader-visible path under `.locks`. + // 3. re-CHECK the terminal marker; reject if present. + // 4. PROMOTE the staged file into `events/` with an atomic hard + // link; reject if the staged file was reaped (`'missing'`). + // + // Correctness: the reap's `unlink` and step 4's `link` target the + // same staged file, so the filesystem serializes them — exactly one + // wins. If the link wins, the event was reader-visible before the + // reap completed, and therefore before the terminal state and + // terminal event were written: acceptance happened-before the + // termination and legitimately precedes it. If the unlink wins, + // promotion fails and the event is never visible to any reader — + // there is nothing to roll back. A resume that stages after the + // reap has passed necessarily stages after the marker was + // committed, so step 3 rejects it. Rejections before step 4 unlink + // a file no reader can see. + if (data.eventType === 'hook_received') { + // Step 1: fast path. The marker is the authoritative durable + // signal; the run-state read additionally rejects runs whose + // terminal state was written without a marker (e.g. runs that + // terminated on an older storage version). + const terminalByMarker = await isRunTerminalCommitted( + basedir, + effectiveRunId, + tag ); - } - try { - if (await isRunTerminalCommitted(basedir, effectiveRunId, tag)) { + const runNow = terminalByMarker + ? null + : await readJSONWithFallback( + basedir, + 'runs', + effectiveRunId, + WorkflowRunSchema, + tag + ); + if ( + terminalByMarker || + (runNow && isTerminalWorkflowRunStatus(runNow.status)) + ) { throw new RunExpiredError( `Workflow run "${effectiveRunId}" is already in a terminal state` ); } - const promoted = await promoteExclusive(stagedPath, eventPath); - if (promoted === 'missing') { - // A terminal transition reaped the staged file between the - // check and the link — the atomic loss of the arbitration. - throw new RunExpiredError( - `Workflow run "${effectiveRunId}" is already in a terminal state` + + const stagedPath = pendingHookEventPath( + basedir, + effectiveRunId, + eventId, + tag + ); + const staged = await writeExclusive(stagedPath, serializedEvent); + if (!staged) { + // For a ULID-numbered run the eventId is freshly generated, so + // its staging path can only be occupied by a previous crashed + // attempt of this very event, which never promoted. A + // slot-numbered run can also collide here with another instance + // that allocated the same slot from its own book. Either way the + // event is not reader-visible, so there is no delta to hand back + // and nothing for the caller to merge: surface the same conflict + // shape as a visible-path collision — or, when this world + // allocated the position itself, let the loop below re-probe and + // take the next one. + if (reallocatesSlot) { + return false; + } + throw new EntityConflictError( + `Event "${eventId}" already exists for run "${effectiveRunId}"` ); } - eventPublished = promoted === 'linked'; - } finally { - // The staged path is not reader-visible; removing it is pure - // cleanup on every outcome (already gone when reaped). - await deleteJSON(stagedPath).catch(() => {}); + try { + if (await isRunTerminalCommitted(basedir, effectiveRunId, tag)) { + throw new RunExpiredError( + `Workflow run "${effectiveRunId}" is already in a terminal state` + ); + } + const promoted = await promoteExclusive(stagedPath, eventPath); + if (promoted === 'missing') { + // A terminal transition reaped the staged file between the + // check and the link — the atomic loss of the arbitration. + throw new RunExpiredError( + `Workflow run "${effectiveRunId}" is already in a terminal state` + ); + } + return promoted === 'linked'; + } finally { + // The staged path is not reader-visible; removing it is pure + // cleanup on every outcome (already gone when reaped). + await deleteJSON(stagedPath).catch(() => {}); + } } - } else { - eventPublished = await writeExclusive(eventPath, serializedEvent); + return await writeExclusive(eventPath, serializedEvent); } - if (!eventPublished) { + // A write that allocated its own position may take the next free one + // when it loses: nothing outside this world named the slot, so which + // position the event lands on is this world's business, and the caller + // — a step reporting its completion, a hook being received — has no log + // to reconcile. A write whose position the *caller* claimed may not: + // the claim asserts a log complete up to that position, so losing it + // means that log is stale and only the caller can resolve it. + const reallocatesSlot = slotMode && params?.eventId === undefined; + const slotDeadline = Date.now() + SLOT_RETRY_BUDGET_MS; + let compositeKey = ''; + let eventPath = ''; + let serializedEvent = ''; + let eventPublished = false; + + for (let round = 0; ; round++) { + compositeKey = `${effectiveRunId}-${eventId}`; + eventPath = taggedPath(basedir, 'events', compositeKey, tag); + // Capture the serialized payload before the write's `await` so the + // cached snapshot can't observe a later mutation (see + // rememberStoredEvent). + serializedEvent = JSON.stringify(event, jsonReplacer, 2); + eventPublished = await publishOnce(); + if (eventPublished) { + break; + } + if (reallocatesSlot && Date.now() < slotDeadline) { + // The position is someone else's — either published there or + // staged for it. Record that, top the book up from disk, and try + // again one position higher rather than surfacing a conflict the + // caller cannot act on. Re-reading rather than incrementing keeps + // the log dense: every round at least one writer wins, so the + // search never runs away from the log it is filling. + const lost = slotFromId(eventId); + if (lost !== undefined) { + reserved.delete(lost); + } + slots.observe(effectiveRunId, eventId); + await slots.refresh(effectiveRunId); + await new Promise((resolve) => + setTimeout(resolve, slotRetryDelay(round)) + ); + const slot = await slots.reserve(effectiveRunId, allocationFloor); + reservedRunId = effectiveRunId; + reserved.add(slot); + eventId = slotEventId(slot); + event = { ...event, eventId }; + continue; + } // For `hook_created`, losing the event publish means the // event was already committed at this exact (canonical) // path. The original publisher may have crashed between @@ -2591,13 +2644,23 @@ export function createEventsStorage( tag ); } + if (reallocatesSlot) { + // Out of budget: every position this writer tried was taken by + // someone else. Surfacing it as a 503 puts the whole operation + // back on the queue rather than stalling the run here. + throw new WorkflowWorldError( + `Could not place an event in run "${effectiveRunId}" within ${SLOT_RETRY_BUDGET_MS}ms of contention`, + { status: 503 } + ); + } if (slotMode) { - // Losing a slot means someone else's event occupies this position, - // so the log this event was derived from is missing at least that - // event — the whole proposed event is stale, not just its id. Hand - // back what the caller is missing so it can merge, replay and - // re-propose, and forget the run's book so the next allocation - // re-reads the log this instance evidently does not have. + // Losing a claimed slot means someone else's event occupies this + // position, so the log this event was derived from is missing at + // least that event — the whole proposed event is stale, not just + // its id. Hand back what the caller is missing so it can merge, + // replay and re-propose, and forget the run's book so the next + // allocation re-reads the log this instance evidently does not + // have. // // Reaching here means the slot was taken *after* the pre-check at // the claim site, so the entity this event was going to describe diff --git a/packages/world-local/src/storage/slot-identity.test.ts b/packages/world-local/src/storage/slot-identity.test.ts index 00d8b71cce..f936b55718 100644 --- a/packages/world-local/src/storage/slot-identity.test.ts +++ b/packages/world-local/src/storage/slot-identity.test.ts @@ -249,12 +249,31 @@ describe('conflict', () => { ).toEqual([slotEventId(3)]); }); - it('conflicts across two storage instances sharing a directory', async () => { + it('conflicts when another instance takes a claimed slot', async () => { // Two instances keep independent books, so the exclusive write — not the - // book — is what decides who owns a slot. + // book — is what decides who owns a slot. A claim asserts a complete log, + // so its loser has to reload rather than move over. const runId = await newSlotRun(); const other = createStorage(testDir); - const [first, second] = await Promise.allSettled([ + await other.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_b', + eventData: { stepName: 'b-step', input: new Uint8Array() }, + }); + await expect(createStep(runId, 'step_a', slotEventId(2))).rejects.toThrow( + SlotConflictError + ); + await expect(slotsOf(runId)).resolves.toEqual([1, 2]); + }); + + it('reallocates around another instance holding the slot it picked', async () => { + // Neither writer holds a log, so neither has anything to reconcile: the + // loser takes the next free position instead of surfacing a conflict its + // caller could not act on. + const runId = await newSlotRun(); + const other = createStorage(testDir); + const outcomes = await Promise.allSettled([ storage.events.create(runId, { eventType: 'step_created', specVersion: SPEC_VERSION_SLOT_IDENTITY, @@ -268,14 +287,10 @@ describe('conflict', () => { eventData: { stepName: 'b-step', input: new Uint8Array() }, }), ]); - const outcomes = [first, second]; - expect(outcomes.filter((o) => o.status === 'fulfilled')).toHaveLength(1); - const rejection = outcomes.find((o) => o.status === 'rejected'); - expect( - SlotConflictError.is( - (rejection as PromiseRejectedResult | undefined)?.reason - ) - ).toBe(true); - await expect(slotsOf(runId)).resolves.toEqual([1, 2]); + expect(outcomes.map((outcome) => outcome.status)).toEqual([ + 'fulfilled', + 'fulfilled', + ]); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); }); }); diff --git a/packages/world-local/src/storage/slots.ts b/packages/world-local/src/storage/slots.ts index c6b2f017f6..632e31c1ee 100644 --- a/packages/world-local/src/storage/slots.ts +++ b/packages/world-local/src/storage/slots.ts @@ -96,6 +96,16 @@ export interface SlotBook { release(runId: string, slot: number): void; /** Records a published event id, so it is never handed out again. */ observe(runId: string, eventId: string): void; + /** + * Merges the run's published positions from disk into the book kept for it, + * leaving the reservations other writers in this instance still hold. + * + * A writer whose publish lost its position calls this before trying again: + * the book is demonstrably behind another instance's writes, and dropping it + * wholesale ({@link SlotBook.forget}) would hand a sibling's outstanding + * position to the next caller and cost that sibling its own publish. + */ + refresh(runId: string): Promise; /** Drops what is cached for `runId`, so the next reservation re-reads disk. */ forget(runId: string): void; /** Drops everything cached (the data directory was cleared out from under us). */ @@ -227,6 +237,24 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { book.outstanding.delete(slot); }, + async refresh(runId) { + const book = books.get(runId); + if (!book) { + // Nothing cached to correct; the next reservation seeds from disk. + return; + } + for (const eventId of await listRunEventIds(basedir, runId, tag)) { + const slot = slotFromId(eventId); + if (slot !== undefined) { + book.written.add(slot); + book.outstanding.delete(slot); + } + } + // Positions released while this scan ran may sit below where the search + // had reached, and they are free again. + book.searchFrom = FIRST_SLOT; + }, + forget(runId) { modes.delete(runId); books.delete(runId); diff --git a/packages/world-postgres/src/slots.ts b/packages/world-postgres/src/slots.ts index 89435810a9..2f30ad9f31 100644 --- a/packages/world-postgres/src/slots.ts +++ b/packages/world-postgres/src/slots.ts @@ -14,7 +14,13 @@ */ import { WorkflowWorldError } from '@workflow/errors'; -import { FIRST_SLOT, slotEventId, slotFromId } from '@workflow/world'; +import { + FIRST_SLOT, + SLOT_RETRY_BUDGET_MS, + slotEventId, + slotFromId, + slotRetryDelay, +} from '@workflow/world'; import { and, desc, eq } from 'drizzle-orm'; import { type Drizzle, Schema } from './drizzle/index.js'; @@ -26,19 +32,6 @@ import { type Drizzle, Schema } from './drizzle/index.js'; */ export const RUN_CREATED_SLOT = FIRST_SLOT; -/** First backoff after losing a position; doubled each round. */ -export const SLOT_RETRY_BASE_MS = 5; - -/** Ceiling for a single backoff, so a contended run keeps making attempts. */ -export const SLOT_RETRY_MAX_DELAY_MS = 250; - -/** - * How long a writer keeps looking for a free position before giving up. - * Exhausting it surfaces as a 503, so the caller — in practice a queue - * delivery — retries the whole operation instead of the run stalling on it. - */ -export const SLOT_RETRY_BUDGET_MS = 30_000; - /** Postgres unique-violation code. */ const UNIQUE_VIOLATION = '23505'; @@ -104,14 +97,6 @@ export async function eventExists( return row !== undefined; } -/** Full jitter over an exponentially growing, capped window. */ -export function slotRetryDelay(round: number): number { - return ( - Math.random() * - Math.min(SLOT_RETRY_BASE_MS * 2 ** round, SLOT_RETRY_MAX_DELAY_MS) - ); -} - /** The event ids a single create publishes. */ export interface EventIds { /** diff --git a/packages/world/src/index.ts b/packages/world/src/index.ts index 5a8c185f2a..1ecae0457d 100644 --- a/packages/world/src/index.ts +++ b/packages/world/src/index.ts @@ -106,9 +106,13 @@ export { isSlotId, maxSlotOf, SLOT_ID_WIDTH, + SLOT_RETRY_BASE_MS, + SLOT_RETRY_BUDGET_MS, + SLOT_RETRY_MAX_DELAY_MS, slotEventId, slotFromId, slotIdBody, + slotRetryDelay, } from './slot-identity.js'; export type { SpecVersion } from './spec-version.js'; export { diff --git a/packages/world/src/slot-identity.ts b/packages/world/src/slot-identity.ts index d9ff11b65f..d761f91452 100644 --- a/packages/world/src/slot-identity.ts +++ b/packages/world/src/slot-identity.ts @@ -67,6 +67,31 @@ export function slotEventId(slot: number): string { return `evnt_${slotIdBody(slot)}`; } +/** First backoff after losing a position; doubled each round. */ +export const SLOT_RETRY_BASE_MS = 5; + +/** Ceiling for a single backoff, so a contended run keeps making attempts. */ +export const SLOT_RETRY_MAX_DELAY_MS = 250; + +/** + * How long a writer that allocates its own position keeps looking for a free + * one before giving up. Exhausting it is a retryable failure for the caller — + * in practice a queue delivery — rather than something the run stalls on. + */ +export const SLOT_RETRY_BUDGET_MS = 30_000; + +/** + * Full jitter over an exponentially growing, capped window. Shared by every + * world that allocates positions, so contention behaves the same wherever a run + * is stored. + */ +export function slotRetryDelay(round: number): number { + return ( + Math.random() * + Math.min(SLOT_RETRY_BASE_MS * 2 ** round, SLOT_RETRY_MAX_DELAY_MS) + ); +} + /** * The highest slot named by any of `events`, or 0 when none is slot-numbered. * From 531b15ec082fb269f477a5ff1fdad0f68aacd05d Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 00:37:37 -0700 Subject: [PATCH 04/28] fix(worlds): number the deferred step_created below the claim it rides with A lazy start publishes two events, and only the caller knows which positions are free: it hands out a whole batch's slots synchronously, before any of them land. So a claim names the top of the pair and the deferred `step_created` takes the position immediately below it, which the caller reserved for exactly that. A start that claimed nothing has both positions allocated here instead, and a claim with no room below it is rejected rather than allowed to overwrite the run's creation. Co-Authored-By: Claude Opus 5 --- .../world-local/src/storage/events-storage.ts | 82 +++++++++++++++---- .../src/storage/slot-identity.test.ts | 78 ++++++++++++++++++ packages/world-postgres/src/slots.ts | 44 +++++++--- packages/world-postgres/src/storage.ts | 9 +- .../world-postgres/test/slot-identity.test.ts | 63 ++++++++++++-- 5 files changed, 235 insertions(+), 41 deletions(-) diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index 72c8757a46..558d5f7d03 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -1008,6 +1008,17 @@ export function createEventsStorage( } } + // Lazy step start: a step_started carrying step-creation data + // (stepName + input) is allowed to arrive with no prior step_created + // — it creates the step on the fly (see the materialization block + // below). This mirrors the resilient run_started path. Detect it here + // so the second event it publishes can be numbered alongside the + // first, the entity-creation terminal-run guard treats it like a + // creation, and the "step must exist" ordering guard doesn't reject it. + const createsChildEntity = isChildEntityCreationEvent(data); + const lazyStepStart = + createsChildEntity && data.eventType === 'step_started'; + // ============================================================ // EVENT ID: the caller's slot claim, an allocated slot, or a ULID // ============================================================ @@ -1023,6 +1034,11 @@ export function createEventsStorage( // asserting the log is complete up to that position) or allocated here // for a caller that has no log — a step completion reporting in, a // cancellation from an API call. + // + // The position of the second event a lazy start publishes, when this + // one publishes two. Consumed by the materialization below; released + // again if that block turns out not to need it. + let companionSlot: number | undefined; if (params?.eventId !== undefined) { const claimedSlot = slotFromId(params.eventId); if (!slotMode) { @@ -1041,13 +1057,38 @@ export function createEventsStorage( reservedRunId = effectiveRunId; reserved.add(claimedSlot); slots.claim(effectiveRunId, claimedSlot); - if (await slots.isWritten(effectiveRunId, claimedSlot)) { - // Reject a doomed claim before the materialization below creates - // the step, hook or wait this event will now never accompany. A - // caller that re-proposes at the next slot would otherwise - // collide with its own orphan and read that as "my write already - // landed". See SlotBook.isWritten. - throw await slotConflict(effectiveRunId, eventId, params); + // One request, two events: a lazy start also publishes the + // `step_created` it deferred. A claim names the *top* of the pair, + // so the second event takes the slot immediately below it — the + // caller reserved both positions and named only one, which is what + // keeps the pair from landing on a position another write in the + // same batch is already holding. + if (lazyStepStart) { + companionSlot = claimedSlot - 1; + if (companionSlot < RUN_CREATED_SLOT + 1) { + throw new WorkflowWorldError( + `Event id "${params.eventId}" leaves no slot below it in run "${effectiveRunId}" for the "step_created" published alongside it`, + { status: 400 } + ); + } + reserved.add(companionSlot); + slots.claim(effectiveRunId, companionSlot); + } + // Reject a doomed claim before the materialization below creates + // the step, hook or wait this event will now never accompany. A + // caller that re-proposes at the next slot would otherwise + // collide with its own orphan and read that as "my write already + // landed". See SlotBook.isWritten. + for (const slot of companionSlot === undefined + ? [claimedSlot] + : [companionSlot, claimedSlot]) { + if (await slots.isWritten(effectiveRunId, slot)) { + throw await slotConflict( + effectiveRunId, + slotEventId(slot), + params + ); + } } } else if (slotMode) { reservedRunId = effectiveRunId; @@ -1060,16 +1101,6 @@ export function createEventsStorage( // VALIDATION: Terminal state and event ordering checks // ============================================================ - // Lazy step start: a step_started carrying step-creation data - // (stepName + input) is allowed to arrive with no prior step_created - // — it creates the step on the fly (see the materialization block - // below). This mirrors the resilient run_started path. Detect it here - // so the entity-creation terminal-run guard treats it like a creation - // and the "step must exist" ordering guard doesn't reject it. - const createsChildEntity = isChildEntityCreationEvent(data); - const lazyStepStart = - createsChildEntity && data.eventType === 'step_started'; - // Run terminal state validation if (currentRun && isTerminalWorkflowRunStatus(currentRun.status)) { // Idempotent operation: run_cancelled on already cancelled run is allowed @@ -1724,7 +1755,13 @@ export function createEventsStorage( // run_started → run_created precedent in this file. let stepCreatedEventId = `evnt_${monotonicUlid()}`; if (slotMode) { - const slot = await slots.reserve(effectiveRunId); + // A claimed start numbers this event one below its own + // position, which the caller reserved for exactly this. A start + // that allocated takes the next free slot instead: nothing + // outside this world named either position. + const slot = + companionSlot ?? (await slots.reserve(effectiveRunId)); + companionSlot = undefined; reserved.add(slot); stepCreatedEventId = slotEventId(slot); } @@ -2679,6 +2716,15 @@ export function createEventsStorage( // replay can serve it without rereading from disk. rememberStoredEvent(event, eventPath, serializedEvent); slots.observe(effectiveRunId, eventId); + if (companionSlot !== undefined) { + // A start that carried creation data for a step that already existed + // synthesized no `step_created`, so the position below it went + // unused. Hand it back instead of leaving it outstanding for the life + // of the process, where it would block the allocator from ever + // filling that position. + reserved.delete(companionSlot); + slots.release(effectiveRunId, companionSlot); + } // Write the hook entity ONLY now that the event publish has // committed. Doing this earlier (in the `hook_created` diff --git a/packages/world-local/src/storage/slot-identity.test.ts b/packages/world-local/src/storage/slot-identity.test.ts index f936b55718..885126e5fa 100644 --- a/packages/world-local/src/storage/slot-identity.test.ts +++ b/packages/world-local/src/storage/slot-identity.test.ts @@ -147,6 +147,84 @@ describe('numbering', () => { }); }); +/** + * A lazy step start: a `step_started` carrying the step's creation data, which + * the world materializes into a step plus the `step_created` event the caller + * deferred — one request, two events. + */ +async function startStepLazily( + runId: string, + stepId: string, + eventId?: string +): Promise { + const result = await storage.events.create( + runId, + { + eventType: 'step_started', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: stepId, + eventData: { stepName: 'a-step', input: new Uint8Array(), attempt: 0 }, + }, + eventId === undefined ? undefined : { eventId } + ); + if (!result.event) { + throw new Error('Expected an event'); + } + return result.event.eventId; +} + +describe('a write that publishes two events', () => { + it('numbers the deferred step_created below the claim', async () => { + // The caller reserves both positions and names only the top one, so the + // pair is fixed before either lands — which is what keeps it off the slot + // the next write of the same batch is holding. + const runId = await newSlotRun(); + const startedEventId = await startStepLazily( + runId, + 'step_a', + slotEventId(3) + ); + expect(startedEventId).toBe(slotEventId(3)); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); + + it('allocates both positions for a start that claims neither', async () => { + const runId = await newSlotRun(); + await startStepLazily(runId, 'step_a'); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); + + it('keeps every claim in a burst of lazy starts', async () => { + // The suspension flush issues its lazy starts at once, each having reserved + // two positions. A second event numbered off the log as this world sees it + // would take the slot the next start in the batch claimed, and cost every + // start after the first its claim — collapsing the fan-out to one step. + const runId = await newSlotRun(); + const claims = Array.from({ length: 10 }, (_, index) => + slotEventId(FIRST_SLOT + 2 * (index + 1)) + ); + const ids = await Promise.all( + claims.map((eventId, index) => + startStepLazily(runId, `step_${index}`, eventId) + ) + ); + expect(ids).toEqual(claims); + const slots = await slotsOf(runId); + expect([...slots].sort((a, b) => a - b)).toEqual( + Array.from({ length: 2 * claims.length + 1 }, (_, i) => FIRST_SLOT + i) + ); + }); + + it('rejects a claim that leaves no room for the second event', async () => { + // The run's own run_created holds the first slot, so a claim of the second + // means the caller reserved one position for a write that publishes two. + const runId = await newSlotRun(); + await expect( + startStepLazily(runId, 'step_a', slotEventId(FIRST_SLOT + 1)) + ).rejects.toThrow(/leaves no slot below it/); + }); +}); + describe('mode is pinned to the run', () => { it('rejects a slot id claimed on a ULID-numbered run', async () => { const created = await storage.events.create(null, { diff --git a/packages/world-postgres/src/slots.ts b/packages/world-postgres/src/slots.ts index 2f30ad9f31..cd0a54f6f5 100644 --- a/packages/world-postgres/src/slots.ts +++ b/packages/world-postgres/src/slots.ts @@ -107,9 +107,12 @@ export interface EventIds { primary: () => string; /** * An additional event written in the same breath — the synthetic - * `step_created` of a lazy step start. Its position is allocated here even - * when the caller named its own for the primary event, because a caller that - * defers a `step_created` cannot know it will be synthesized. + * `step_created` of a lazy step start. + * + * A claim names the *top* of the pair, so this event takes the position + * immediately below it: the caller reserved both and named one. Numbering it + * off the log instead would hand it a position another write of the same + * concurrent batch is already holding, and cost that write its claim. */ extra: () => Promise; } @@ -178,23 +181,40 @@ export async function placeEvent( options.claimedSlot === undefined ? slotEventId(await take()) : slotEventId(options.claimedSlot); + /** Positions the caller named, which are the caller's to resolve. */ + const claimed = options.claimedSlot === undefined ? [] : [primary]; try { return await options.write({ primary: () => primary, - extra: async () => slotEventId(await take()), + extra: async () => { + if (options.claimedSlot === undefined) { + return slotEventId(await take()); + } + const slot = options.claimedSlot - 1; + if (slot <= FIRST_SLOT) { + // The run's own `run_created` holds the first slot, so a claim of + // the second leaves nowhere for a second event to go: the caller + // reserved one position for a write that publishes two. + throw new WorkflowWorldError( + `Event id "${primary}" leaves no slot below it in run "${runId}" for the second event published alongside it`, + { status: 400 } + ); + } + const id = slotEventId(slot); + claimed.push(id); + return id; + }, }); } catch (error) { if (!isEventKeyViolation(error)) { throw error; } - // A claimed write can also lose on its extra event's position, which is - // this world's to reallocate — only a claim that is itself taken is the - // caller's problem. - if ( - options.claimedSlot !== undefined && - (await eventExists(drizzle, runId, primary)) - ) { - throw await options.onClaimTaken(); + // Only a position the caller named is the caller's problem; one this + // world allocated is reallocated below without ever surfacing. + for (const id of claimed) { + if (await eventExists(drizzle, runId, id)) { + throw await options.onClaimTaken(); + } } if (Date.now() >= deadline) { throw new WorkflowWorldError( diff --git a/packages/world-postgres/src/storage.ts b/packages/world-postgres/src/storage.ts index 878f39758b..ab752f7de5 100644 --- a/packages/world-postgres/src/storage.ts +++ b/packages/world-postgres/src/storage.ts @@ -1411,11 +1411,10 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // side of the materialization would, hence the shared // transaction. // - // It takes a position of its own, which on a claimed write is - // necessarily one this world allocated: a caller that defers a - // step_created cannot know it will be synthesized, so it never - // claims a slot for it. Losing that position rolls the transaction - // back for a retry, so the insert must not swallow the collision. + // It takes a position of its own — the one below the claim on a + // claimed write, the next free one otherwise. Losing that position + // rolls the transaction back for a retry, so the insert must not + // swallow the collision. const stepCreatedEventId = await ids.extra(); const insertStepCreated = tx.insert(events).values({ runId: effectiveRunId, diff --git a/packages/world-postgres/test/slot-identity.test.ts b/packages/world-postgres/test/slot-identity.test.ts index d8b0578a7d..8e3f04c8a4 100644 --- a/packages/world-postgres/test/slot-identity.test.ts +++ b/packages/world-postgres/test/slot-identity.test.ts @@ -173,9 +173,10 @@ describe('Slot identity (Postgres integration)', () => { ).toEqual(['1 run_created', '2 step_started', '3 step_created']); }); - test('allocates the deferred step_created above a claimed slot', async () => { - // A caller that defers a step_created cannot know it will be synthesized, - // so it never claims a slot for it — and the slot it did claim is its own. + test('numbers the deferred step_created below a claimed slot', async () => { + // A claim names the top of the pair: the caller reserved both positions + // before either landed, which is what keeps the second event off the slot + // the next write of the same batch claimed. const runId = await newSlotRun(); const started = await events.create( runId, @@ -185,10 +186,60 @@ describe('Slot identity (Postgres integration)', () => { correlationId: 'step_a', eventData: { stepName: 'a-step', input: new Uint8Array() }, }, - { eventId: slotEventId(2) } + { eventId: slotEventId(3) } ); - expect(started.event?.eventId).toBe(slotEventId(2)); - expect(ascending(await slotsOf(runId))).toEqual(denseFrom(3)); + expect(started.event?.eventId).toBe(slotEventId(3)); + const { data } = await eventsOf(runId); + expect( + data.map((event) => `${slotFromId(event.eventId)} ${event.eventType}`) + ).toEqual(['1 run_created', '2 step_created', '3 step_started']); + }); + + test('keeps every claim in a burst of lazy starts', async () => { + // The suspension flush issues its lazy starts at once, each having + // reserved two positions. A second event numbered off the log as this + // world sees it would take the slot the next start in the batch claimed, + // costing every start after the first its claim. + const runId = await newSlotRun(); + const claims = Array.from({ length: 10 }, (_, index) => + slotEventId(FIRST_SLOT + 2 * (index + 1)) + ); + const started = await Promise.all( + claims.map((eventId, index) => + events.create( + runId, + { + eventType: 'step_started', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: `step_${index}`, + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId } + ) + ) + ); + expect(started.map((result) => result.event?.eventId)).toEqual(claims); + expect(ascending(await slotsOf(runId))).toEqual( + denseFrom(2 * claims.length + 1) + ); + }); + + test('rejects a claim that leaves no room for the second event', async () => { + // The run's own run_created holds the first slot, so a claim of the second + // means the caller reserved one position for a write that publishes two. + const runId = await newSlotRun(); + await expect( + events.create( + runId, + { + eventType: 'step_started', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_a', + eventData: { stepName: 'a-step', input: new Uint8Array() }, + }, + { eventId: slotEventId(FIRST_SLOT + 1) } + ) + ).rejects.toThrow(/leaves no slot below it/); }); test('numbers events of runs it never created', async () => { From bcd872ec455a6afdddc93e1c83baab995e63388d Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 01:00:48 -0700 Subject: [PATCH 05/28] fix(core): floor a turbo run's slot claims above its own positions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Turbo skips the initial event-log load and replays against an empty snapshot, so the log a claim is numbered from shows neither `run_created` nor the `run_started` still in flight. The floor was only raised from the backgrounded `run_started` response, which is too late to matter: a suspension reserves its whole batch of positions synchronously, so the first batch numbered from an empty log claims the two positions the run already holds, both ops lose their claims, and after the bounded reclaim retries one of them is dropped for good — the run then waits forever on an event that will never be written. Both positions are certain the moment turbo engages, so seed the floor there instead. Co-Authored-By: Claude Opus 5 --- packages/core/src/runtime.test.ts | 44 ++++++++++++++++++++++++++++--- packages/core/src/runtime.ts | 16 +++++++++++ 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/packages/core/src/runtime.test.ts b/packages/core/src/runtime.test.ts index aafec00075..3a42435b01 100644 --- a/packages/core/src/runtime.test.ts +++ b/packages/core/src/runtime.test.ts @@ -6,7 +6,10 @@ import { } from '@workflow/errors'; import { type Event, + FIRST_SLOT, SPEC_VERSION_CURRENT, + SPEC_VERSION_SLOT_IDENTITY, + slotFromId, type WorkflowRun, } from '@workflow/world'; import { ulid } from 'ulid'; @@ -1472,12 +1475,15 @@ describe('workflowEntrypoint turbo mode', () => { return r; }${xform('workflow')}`; - async function makeRunInput(runId: string) { + async function makeRunInput( + runId: string, + specVersion = SPEC_VERSION_CURRENT + ) { return { input: await dehydrateWorkflowArguments([], runId, undefined, []), deploymentId: 'test-deployment', workflowName: 'workflow', - specVersion: SPEC_VERSION_CURRENT, + specVersion, executionContext: {}, }; } @@ -1493,8 +1499,10 @@ describe('workflowEntrypoint turbo mode', () => { attempt: number; source: string; runStartedGate?: Promise; + specVersion?: typeof SPEC_VERSION_CURRENT; }) { const { runId, attempt, source } = opts; + const specVersion = opts.specVersion ?? SPEC_VERSION_CURRENT; const order = turboOrder; const durable: Event[] = []; let seq = 0; @@ -1513,6 +1521,7 @@ describe('workflowEntrypoint turbo mode', () => { runId, workflowName: 'workflow', status: 'running', + specVersion, input: await dehydrateWorkflowArguments([], runId, undefined, []), createdAt: new Date('2024-01-01T00:00:00.000Z'), updatedAt: new Date('2024-01-01T00:00:00.000Z'), @@ -1558,7 +1567,7 @@ describe('workflowEntrypoint turbo mode', () => { }); setWorld({ - specVersion: SPEC_VERSION_CURRENT, + specVersion, createQueueHandler: vi.fn( (_p: string, handler: (m: unknown, md: unknown) => Promise) => async () => { @@ -1566,7 +1575,7 @@ describe('workflowEntrypoint turbo mode', () => { { runId, requestedAt: new Date('2024-01-01T00:00:00.000Z'), - runInput: await makeRunInput(runId), + runInput: await makeRunInput(runId, specVersion), }, { requestId: 'req_turbo', @@ -1652,6 +1661,33 @@ describe('workflowEntrypoint turbo mode', () => { ); }); + it('claims slots above the run own positions on a first delivery', async () => { + // Turbo replays against an empty snapshot, so the log the claims are + // numbered from cannot show `run_created` or the in-flight `run_started`. + // Both positions are nonetheless taken, and the mocked `run_started` + // response reports no event — the same shape as a World that skips the + // preload — so nothing but the floor seeded at turbo entry keeps the first + // batch of claims off them. + const { handlerPromise, eventsCreate } = await driveTurbo({ + runId: 'wrun_turbo_slots', + attempt: 1, + source: stepAndSleepWorkflow, + specVersion: SPEC_VERSION_SLOT_IDENTITY, + }); + + const res = await handlerPromise; + expect(res.status).toBe(204); + + const claimed = eventsCreate.mock.calls + .map((c) => (c[2] as { eventId?: unknown } | undefined)?.eventId) + .filter((id): id is string => typeof id === 'string'); + // The sleep's `wait_created` is claimed, so there is something to assert on. + expect(claimed.length).toBeGreaterThan(0); + for (const eventId of claimed) { + expect(slotFromId(eventId)).toBeGreaterThan(FIRST_SLOT + 1); + } + }); + it('does not turbo when WORKFLOW_TURBO=0 (parity with the awaited path)', async () => { process.env.WORKFLOW_TURBO = '0'; const { handlerPromise, order } = await driveTurbo({ diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 95d19866a2..45928f36aa 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -17,6 +17,7 @@ import { } from '@workflow/utils/parse-name'; import { type Event, + FIRST_SLOT, getQueueTopicPrefix, isLegacySpecVersion, ROOT_RUN_ID_ATTRIBUTE, @@ -24,6 +25,7 @@ import { SPEC_VERSION_CURRENT, SPEC_VERSION_SUPPORTS_COMPRESSION, slotFromId, + usesSlotIdentity, WorkflowInvokePayloadSchema, type WorkflowRun, type World, @@ -1055,6 +1057,20 @@ export function workflowEntrypoint( // intentionally truthy here — do not change the load // branches' `if (preloadedEvents)` checks to test length. preloadedEvents = []; + // A slot-numbered run's first two positions are the run's + // own: `run_created` from start(), then the `run_started` + // in flight above. Both are certain before any write of + // this invocation, and turbo replays against the empty + // snapshot skipped just above — so seed the floor with + // them here rather than waiting for the backgrounded + // response to report it. Waiting loses the race: a + // suspension reserves its whole batch of positions + // synchronously, so a batch that starts numbering from an + // empty log claims the two the run already holds and the + // ops holding them lose their claims. + if (usesSlotIdentity(runInput.specVersion)) { + knownSlotFloor = FIRST_SLOT + 1; + } const now = new Date(); workflowRun = { runId, From 992da0f05a6019fa78681a2ca72e2af07c1e7931 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 03:17:26 -0700 Subject: [PATCH 06/28] fix(world-local,world-postgres): undo an unpublished write's duplicate-suppression claims A create materializes its entity before it publishes the event, so a claimed slot that loses its position left the entity and its `.created` lock behind. The caller re-proposes the same op one slot higher and trips its own orphan, receiving a duplicate-entity error the runtime reads as "my write already landed" - the wait is never created and the run re-invokes forever. world-local now unwinds those claims when a create ends without publishing, and a synchronous claim is held even before the run has a slot book, so a concurrent allocation in the same process cannot hand the position away. world-postgres adopts an orphaned wait row the way it already adopts a hook. --- packages/world-local/src/fs.ts | 5 ++ .../world-local/src/storage/events-storage.ts | 45 ++++++++++++++--- .../src/storage/slot-identity.test.ts | 48 +++++++++++++++++++ .../world-local/src/storage/slots.test.ts | 33 +++++++++++++ packages/world-local/src/storage/slots.ts | 46 +++++++++++++++--- packages/world-postgres/src/storage.ts | 46 +++++++++++++++--- 6 files changed, 203 insertions(+), 20 deletions(-) diff --git a/packages/world-local/src/fs.ts b/packages/world-local/src/fs.ts index 8ae765d0f0..5827f36896 100644 --- a/packages/world-local/src/fs.ts +++ b/packages/world-local/src/fs.ts @@ -461,6 +461,11 @@ export async function deleteJSON(filePath: string): Promise { await fs.unlink(filePath); } catch (error) { if ((error as any).code !== 'ENOENT') throw error; + } finally { + // The cache stands in for an `fs.access` on the write path, so a path that + // no longer exists may not stay in it: a later create-if-absent write of the + // same path would be rejected as a duplicate of a file that is gone. + createdFilesCache.delete(filePath); } } diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index 558d5f7d03..ba4180c405 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -725,6 +725,15 @@ export function createEventsStorage( // be filled, and a log with a hole can no longer prove it is complete. const reserved = new Set(); let reservedRunId: string | undefined; + /** + * Undo actions for the duplicate-suppression claims a create takes before + * its event exists, newest last. Run only when the create ends without + * publishing anything: the answer to a lost position is to propose the + * same operation one position higher, and a claim left behind is what + * would reject that retry as a duplicate of a write that never landed. + */ + const abandonedClaims: Array<() => Promise> = []; + let eventCommitted = false; /** * Hands the slots of a create that never published back to the allocator, * so an abandoned reservation below a sibling's published slot does not @@ -741,6 +750,14 @@ export function createEventsStorage( slots.release(reservedRunId, slot); } } + if (!eventCommitted) { + for (const undo of abandonedClaims.reverse()) { + // Best effort: the throw the caller sees is the one that matters, + // and a claim that outlives its create is a duplicate suppressed + // for a write that is not coming back. + await undo().catch(() => {}); + } + } throw error; } } @@ -1657,6 +1674,7 @@ export function createEventsStorage( `Step "${data.correlationId}" already created` ); } + abandonedClaims.push(() => fs.unlink(stepCreatedLockPath)); const stepData = data.eventData as { stepName: string; input: any; @@ -1678,10 +1696,14 @@ export function createEventsStorage( specVersion: effectiveSpecVersion, }; const stepCompositeKey = `${effectiveRunId}-${data.correlationId}`; - await writeJSON( - taggedPath(basedir, 'steps', stepCompositeKey, tag), - step + const stepEntityPath = taggedPath( + basedir, + 'steps', + stepCompositeKey, + tag ); + await writeJSON(stepEntityPath, step); + abandonedClaims.push(() => deleteJSON(stepEntityPath)); } else if (data.eventType === 'step_started') { // step_started: Increments attempt, sets status to 'running' // Sets startedAt only on the first start (not updated on retries) @@ -2391,6 +2413,7 @@ export function createEventsStorage( `Wait "${data.correlationId}" already exists` ); } + abandonedClaims.push(() => fs.unlink(waitCreatedLockPath)); const waitData = data.eventData as { resumeAt?: Date; }; @@ -2404,10 +2427,14 @@ export function createEventsStorage( updatedAt: now, specVersion: effectiveSpecVersion, }; - await writeJSON( - taggedPath(basedir, 'waits', waitCompositeKey, tag), - wait + const waitEntityPath = taggedPath( + basedir, + 'waits', + waitCompositeKey, + tag ); + await writeJSON(waitEntityPath, wait); + abandonedClaims.push(() => deleteJSON(waitEntityPath)); } else if (data.eventType === 'wait_completed') { // wait_completed: Transitions wait to 'completed', rejects duplicates. // Uses writeExclusive on a lock file to atomically prevent concurrent @@ -2713,7 +2740,11 @@ export function createEventsStorage( } // The event is now committed; cache it so an immediate sequential - // replay can serve it without rereading from disk. + // replay can serve it without rereading from disk. Nothing this create + // claimed may be undone from here on: readers can see the event, so the + // entity it describes has to keep existing even if a later step of this + // call fails. + eventCommitted = true; rememberStoredEvent(event, eventPath, serializedEvent); slots.observe(effectiveRunId, eventId); if (companionSlot !== undefined) { diff --git a/packages/world-local/src/storage/slot-identity.test.ts b/packages/world-local/src/storage/slot-identity.test.ts index 885126e5fa..e364584989 100644 --- a/packages/world-local/src/storage/slot-identity.test.ts +++ b/packages/world-local/src/storage/slot-identity.test.ts @@ -77,6 +77,27 @@ async function createStep( return result.event.eventId; } +async function createWait( + runId: string, + waitId: string, + eventId?: string +): Promise { + const result = await storage.events.create( + runId, + { + eventType: 'wait_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: waitId, + eventData: { resumeAt: new Date('2030-01-01T00:00:00.000Z') }, + }, + eventId === undefined ? undefined : { eventId } + ); + if (!result.event) { + throw new Error('Expected an event'); + } + return result.event.eventId; +} + describe('numbering', () => { it('puts run_created in the first slot', async () => { const runId = await newSlotRun(); @@ -345,6 +366,33 @@ describe('conflict', () => { await expect(slotsOf(runId)).resolves.toEqual([1, 2]); }); + it('lets a lost claim re-propose an entity it had already materialized', async () => { + // A claim only reaches its exclusive write after the entity it describes + // exists, so a claim that loses leaves that entity behind. The caller's + // whole answer to a conflict is to merge, replay and propose the same + // operation one position higher — which it cannot do if its own leftover + // entity is what rejects the retry. + const runId = await newSlotRun(); + // Seed this instance's book, then let another instance take the position + // the book will hand out next. The claim below passes the book's + // "is it written?" check because the book has not seen that write. + await createStep(runId, 'step_seed'); + const other = createStorage(testDir); + await other.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_out_of_band', + eventData: { stepName: 'b-step', input: new Uint8Array() }, + }); + + await expect(createWait(runId, 'wait_a', slotEventId(3))).rejects.toThrow( + SlotConflictError + ); + const eventId = await createWait(runId, 'wait_a', slotEventId(4)); + expect(eventId).toBe(slotEventId(4)); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3, 4]); + }); + it('reallocates around another instance holding the slot it picked', async () => { // Neither writer holds a log, so neither has anything to reconcile: the // loser takes the next free position instead of surfacing a conflict its diff --git a/packages/world-local/src/storage/slots.test.ts b/packages/world-local/src/storage/slots.test.ts index 42c962f128..0e60b2ff74 100644 --- a/packages/world-local/src/storage/slots.test.ts +++ b/packages/world-local/src/storage/slots.test.ts @@ -203,6 +203,39 @@ describe('isWritten', () => { }); }); +describe('claim', () => { + it('holds a slot claimed before anything allocated for the run', async () => { + // A claim is synchronous and the first allocation's log scan is not, so a + // claim that only registered against an existing book would be invisible to + // the very allocation it races — and a single-process app would hand the + // caller's own position away. + await writeEvents(1); + const book = createSlotBook(basedir); + book.claim(RUN_ID, 2); + await expect(book.reserve(RUN_ID)).resolves.toBe(3); + }); + + it('frees the slot again once the claim resolves', async () => { + await writeEvents(1); + const book = createSlotBook(basedir); + book.claim(RUN_ID, 2); + book.release(RUN_ID, 2); + // Nothing is allocating for the run yet, so the book the next caller seeds + // has to start from the log alone. + await expect(book.reserve(RUN_ID)).resolves.toBe(2); + }); + + it('keeps holding a claim across a forget', async () => { + // `forget` follows a lost publish: the book is behind another writer, but + // the claims other writes in this instance still hold are not. + await writeEvents(1); + const book = createSlotBook(basedir); + book.claim(RUN_ID, 2); + book.forget(RUN_ID); + await expect(book.reserve(RUN_ID)).resolves.toBe(3); + }); +}); + describe('observe', () => { it('never hands out a slot claimed by the client', async () => { const book = createSlotBook(basedir); diff --git a/packages/world-local/src/storage/slots.ts b/packages/world-local/src/storage/slots.ts index 632e31c1ee..58275f3d57 100644 --- a/packages/world-local/src/storage/slots.ts +++ b/packages/world-local/src/storage/slots.ts @@ -118,6 +118,14 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { const books = new Map(); /** runId → in-flight seed scan, so concurrent first callers share one scan. */ const seeds = new Map>(); + /** + * runId → slots claimed while the run had no book yet, so the book the next + * allocation seeds starts out holding them. A claim is synchronous and a seed + * scan is not: without this, the first allocation of a run would read the log + * from disk and hand out a position a caller in this very instance had already + * claimed — the case that makes a claim lose in a single-process app. + */ + const claims = new Map>(); async function readMode(runId: string): Promise { const run = await readJSONWithFallback( @@ -141,7 +149,7 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { } const book: RunSlots = { written, - outstanding: new Set(), + outstanding: new Set(claims.get(runId)), searchFrom: FIRST_SLOT, }; books.set(runId, book); @@ -162,6 +170,21 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { return pending; } + /** + * Drops a claim once its publish resolved, either way: a claim left behind + * would be handed to no one and become a hole in a log seeded later. + */ + function forgetClaim(runId: string, slot: number): void { + const claimed = claims.get(runId); + if (!claimed) { + return; + } + claimed.delete(slot); + if (claimed.size === 0) { + claims.delete(runId); + } + } + function take(book: RunSlots, minSlot: number): number { let slot = Math.max(book.searchFrom, minSlot); while (book.written.has(slot) || book.outstanding.has(slot)) { @@ -201,8 +224,12 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { }, claim(runId, slot) { - // No book means nothing is allocating for this run in this instance yet, - // and the seed scan that starts one reads the claim off disk if it landed. + const claimed = claims.get(runId); + if (claimed) { + claimed.add(slot); + } else { + claims.set(runId, new Set([slot])); + } books.get(runId)?.outstanding.add(slot); }, @@ -212,6 +239,7 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { }, release(runId, slot) { + forgetClaim(runId, slot); const book = books.get(runId); if (!book) { return; @@ -223,16 +251,17 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { }, observe(runId, eventId) { + const slot = slotFromId(eventId); + if (slot === undefined) { + return; + } + forgetClaim(runId, slot); const book = books.get(runId); if (!book) { // Nothing to keep consistent: the slot is on disk by the time this is // called, so the eventual seed scan picks it up. return; } - const slot = slotFromId(eventId); - if (slot === undefined) { - return; - } book.written.add(slot); book.outstanding.delete(slot); }, @@ -258,11 +287,14 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { forget(runId) { modes.delete(runId); books.delete(runId); + // Claims outlive the book on purpose: they belong to writes still in + // flight, and the book a later allocation seeds has to hold them back. }, clear() { modes.clear(); books.clear(); + claims.clear(); }, }; } diff --git a/packages/world-postgres/src/storage.ts b/packages/world-postgres/src/storage.ts index ab752f7de5..b62a17ec6f 100644 --- a/packages/world-postgres/src/storage.ts +++ b/packages/world-postgres/src/storage.ts @@ -456,7 +456,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // hook row left behind by a process / database interruption between // the hook INSERT and the events INSERT below (see the recovery // logic in the hook_created branch). - const getHookCreatedEvent = drizzle + const getCorrelatedEvent = drizzle .select({ eventId: events.eventId }) .from(events) .where( @@ -467,7 +467,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { ) ) .limit(1) - .prepare('events_get_hook_created_for_run_correlation'); + .prepare('events_get_correlated_event'); const getWaitForValidation = drizzle .select({ @@ -1691,7 +1691,7 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { existingHook.runId === effectiveRunId && existingHook.hookId === data.correlationId ) { - const [existingEvent] = await getHookCreatedEvent.execute({ + const [existingEvent] = await getCorrelatedEvent.execute({ runId: effectiveRunId, correlationId: data.correlationId, eventType: 'hook_created', @@ -1892,9 +1892,43 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { specVersion: waitValue.specVersion ?? undefined, }; } else { - throw new EntityConflictError( - `Wait "${data.correlationId}" already exists` - ); + // The wait row exists but this call did not write it. Which of the two + // reasons it is decides everything, and only the event log knows: + // - the `wait_created` event exists → a real duplicate, so throw and + // let the runtime's concurrent-replay catch path swallow it. + // - it does not → an orphaned row from an attempt that materialized + // the wait and then lost its event write (a crash, or a slot + // claimed by someone else). The caller re-proposing the same + // operation one position higher is exactly what has to succeed + // here, so adopt the row and complete the partial write. Mirrors + // hook_created's handling of the same window. + const [existingEvent] = await getCorrelatedEvent.execute({ + runId: effectiveRunId, + correlationId: data.correlationId, + eventType: 'wait_created', + }); + if (existingEvent) { + throw new EntityConflictError( + `Wait "${data.correlationId}" already exists` + ); + } + const [orphan] = await drizzle + .select() + .from(Schema.waits) + .where(eq(Schema.waits.waitId, waitId)) + .limit(1); + if (orphan) { + wait = { + waitId: orphan.waitId, + runId: orphan.runId, + status: orphan.status, + resumeAt: orphan.resumeAt ?? undefined, + completedAt: orphan.completedAt ?? undefined, + createdAt: orphan.createdAt, + updatedAt: orphan.updatedAt, + specVersion: orphan.specVersion ?? undefined, + }; + } } } From 3cecc5290a0ba5414d1397cab9d5ef0537dd1537 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 03:36:45 -0700 Subject: [PATCH 07/28] fix(world-local): publish a deferred step_created on the same terms as its start The synthetic `step_created` a lazy step start materializes was written with a plain create-if-absent write, so losing its position surfaced as a duplicate- entity error - which the runtime reads as "another handler owns this step" and skips, leaving the step claimed but never run. It now takes the position the same way every other event does, and a loss is reported as a slot conflict the caller can merge and re-propose. --- .../world-local/src/storage/events-storage.ts | 60 ++++++++++++++----- .../src/storage/slot-identity.test.ts | 23 +++++++ 2 files changed, 67 insertions(+), 16 deletions(-) diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index ba4180c405..97376f5aab 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -733,6 +733,11 @@ export function createEventsStorage( * would reject that retry as a duplicate of a write that never landed. */ const abandonedClaims: Array<() => Promise> = []; + /** + * Whether any event of this create became reader-visible. Once one has, + * nothing the create claimed may be undone: the entity an event describes + * has to keep existing even if a later step of the same call fails. + */ let eventCommitted = false; /** * Hands the slots of a create that never published back to the allocator, @@ -1742,6 +1747,7 @@ export function createEventsStorage( `Step "${data.correlationId}" already created` ); } else { + abandonedClaims.push(() => fs.unlink(stepCreatedLockPath)); const createdStep: Step = { runId: effectiveRunId, stepId: data.correlationId, @@ -1757,15 +1763,14 @@ export function createEventsStorage( updatedAt: now, specVersion: effectiveSpecVersion, }; - await writeJSON( - taggedPath( - basedir, - 'steps', - `${effectiveRunId}-${data.correlationId}`, - tag - ), - createdStep + const lazyStepEntityPath = taggedPath( + basedir, + 'steps', + `${effectiveRunId}-${data.correlationId}`, + tag ); + await writeJSON(lazyStepEntityPath, createdStep); + abandonedClaims.push(() => deleteJSON(lazyStepEntityPath)); // Write the synthetic step_created event so replay observes it // (the client step consumer sets hasCreatedEvent only on a // step_created event). Its eventId is a second slot, or a fresh @@ -1799,15 +1804,38 @@ export function createEventsStorage( input: lazyData.input, }, }; - await writeJSON( - taggedPath( - basedir, - 'events', - `${effectiveRunId}-${stepCreatedEventId}`, - tag - ), - stepCreatedEvent + const stepCreatedEventPath = taggedPath( + basedir, + 'events', + `${effectiveRunId}-${stepCreatedEventId}`, + tag ); + if (slotMode) { + // The position decides this event as much as it decides the + // start it rides with, so it is published the same way: whoever + // links the file first owns the slot. A loss here is the + // caller's to resolve — it named this position — and the undo + // list above takes the step entity and its claim back out, so + // the re-proposal one position higher starts the step lazily + // again instead of tripping its own leftovers. + const published = await writeExclusive( + stepCreatedEventPath, + JSON.stringify(stepCreatedEvent, jsonReplacer, 2) + ); + if (!published) { + throw await slotConflict( + effectiveRunId, + stepCreatedEventId, + params + ); + } + } else { + await writeJSON(stepCreatedEventPath, stepCreatedEvent); + } + // Readers can see this event from here on, so the step entity it + // describes has to keep existing even if the start it rides with + // goes on to lose its own position. + eventCommitted = true; slots.observe(effectiveRunId, stepCreatedEventId); validatedStep = createdStep; stepCreatedLazily = true; diff --git a/packages/world-local/src/storage/slot-identity.test.ts b/packages/world-local/src/storage/slot-identity.test.ts index e364584989..c21e215afa 100644 --- a/packages/world-local/src/storage/slot-identity.test.ts +++ b/packages/world-local/src/storage/slot-identity.test.ts @@ -393,6 +393,29 @@ describe('conflict', () => { await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3, 4]); }); + it('lets a lazy start lose the position of the event it defers', async () => { + // The deferred `step_created` is published on the same terms as the start + // itself, so it is the pair's first position that can be lost. The retry has + // to be able to start the step lazily all over again — its own claim file + // and step entity would otherwise answer for a write that never landed. + const runId = await newSlotRun(); + await createStep(runId, 'step_seed'); + const other = createStorage(testDir); + await other.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_SLOT_IDENTITY, + correlationId: 'step_out_of_band', + eventData: { stepName: 'b-step', input: new Uint8Array() }, + }); + + await expect( + startStepLazily(runId, 'step_a', slotEventId(4)) + ).rejects.toThrow(SlotConflictError); + const eventId = await startStepLazily(runId, 'step_a', slotEventId(5)); + expect(eventId).toBe(slotEventId(5)); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3, 4, 5]); + }); + it('reallocates around another instance holding the slot it picked', async () => { // Neither writer holds a log, so neither has anything to reconcile: the // loser takes the next free position instead of surfacing a conflict its From f9bd77c2deb8503ae9cd614d42bf743ff764d44c Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 05:10:03 -0700 Subject: [PATCH 08/28] fix(core): send the instant a turbo start synthesizes its run from An optimistic start runs against a locally synthesized run row, so the workflow start time this invocation reports comes from the client clock while every later replay reads the persisted run. Sending that instant as the run_started event's occurredAt lets a backend record it as the run's startedAt, so the two agree instead of differing by the round-trip. Co-Authored-By: Claude Opus 5 --- .changeset/turbo-run-started-occurred-at.md | 5 +++ packages/core/src/runtime.test.ts | 36 +++++++++++++++++++++ packages/core/src/runtime.ts | 13 ++++++-- 3 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 .changeset/turbo-run-started-occurred-at.md diff --git a/.changeset/turbo-run-started-occurred-at.md b/.changeset/turbo-run-started-occurred-at.md new file mode 100644 index 0000000000..bfbfbaf2d6 --- /dev/null +++ b/.changeset/turbo-run-started-occurred-at.md @@ -0,0 +1,5 @@ +--- +'@workflow/core': patch +--- + +Report the same workflow start time on an optimistically started run's first pass and its replays diff --git a/packages/core/src/runtime.test.ts b/packages/core/src/runtime.test.ts index 3a42435b01..8c2c6e4eec 100644 --- a/packages/core/src/runtime.test.ts +++ b/packages/core/src/runtime.test.ts @@ -15,6 +15,7 @@ import { import { ulid } from 'ulid'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { registerStepFunction } from './private.js'; +import { getWorkflowMetadata } from './step/get-workflow-metadata.js'; import { REPLAY_DIVERGENCE_MAX_RETRIES } from './runtime/constants.js'; import { setWorld } from './runtime/world.js'; import { workflowEntrypoint } from './runtime.js'; @@ -1463,6 +1464,17 @@ describe('workflowEntrypoint turbo mode', () => { return undefined; }); + // Records the workflow start time the step body observes, which is the one + // the synthesized run row carries under turbo. + let turboObservedStartedAt: Date | undefined; + registerStepFunction('turboMetadataStep', async () => { + turboObservedStartedAt = getWorkflowMetadata().workflowStartedAt; + return undefined; + }); + + const oneMetadataStepWorkflow = `const s = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("turboMetadataStep"); + async function workflow() { return await s(); }${xform('workflow')}`; + const oneStepWorkflow = `const s = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("turboStep"); async function workflow() { return await s(); }${xform('workflow')}`; @@ -1734,6 +1746,30 @@ describe('workflowEntrypoint turbo mode', () => { expect((redeliverRunStarted?.[2] as any)?.skipPreload).toBeUndefined(); }); + it('sends run_started the same instant it synthesizes the run from', async () => { + // Turbo starts the run against a locally synthesized run row, so the start + // time this invocation reports comes from the client clock. Backends that + // persist `occurredAt` record the run's `startedAt` from it, so sending it + // is what makes a later replay — which reads the persisted run — report the + // same `workflowStartedAt` this pass already captured into its steps. + turboObservedStartedAt = undefined; + const { handlerPromise, eventsCreate } = await driveTurbo({ + runId: 'wrun_turbo_occurred_at', + attempt: 1, + source: oneMetadataStepWorkflow, + specVersion: SPEC_VERSION_SLOT_IDENTITY, + }); + expect((await handlerPromise).status).toBe(204); + + const runStarted = eventsCreate.mock.calls.find( + (c) => (c[1] as any).eventType === 'run_started' + ); + const occurredAt = (runStarted?.[2] as { occurredAt?: Date } | undefined) + ?.occurredAt; + expect(occurredAt).toBeInstanceOf(Date); + expect(turboObservedStartedAt).toEqual(occurredAt); + }); + it('exits turbo (no forced optimistic) when the suspension creates a wait', async () => { const { handlerPromise, order } = await driveTurbo({ runId: 'wrun_turbo_wait', diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 45928f36aa..19581650d2 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -1014,6 +1014,16 @@ export function workflowEntrypoint( // handler, optimistic step_started, terminal run writes) so // nothing is written before the run exists. recordRunStartedCreateStart(true); + // The instant this invocation calls the run started, sent + // with the event and reused for the synthesized run row + // below. Backends that persist `occurredAt` record the + // run's `startedAt` from it, which is what keeps + // `workflowStartedAt` identical between this optimistic + // pass and every later replay that reads the persisted + // run. Without it the two disagree by the round-trip, and + // a step's captured metadata no longer matches the + // workflow's on the next replay. + const now = new Date(); const startedPromise = world.events.create( runId, runStartedEvent, @@ -1024,7 +1034,7 @@ export function workflowEntrypoint( // run_started request the chained first step_started // waits on — shortening time-to-second-step — and the // wasted list+resolve it would otherwise compute. - { requestId, skipPreload: true } + { requestId, skipPreload: true, occurredAt: now } ); runReadyBarrier = startedPromise; // Turbo backgrounds run_started, so the non-turbo assignment @@ -1071,7 +1081,6 @@ export function workflowEntrypoint( if (usesSlotIdentity(runInput.specVersion)) { knownSlotFloor = FIRST_SLOT + 1; } - const now = new Date(); workflowRun = { runId, status: 'running', From b5ea66588693475cea031264641c559c9f83b4df Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 06:58:39 -0700 Subject: [PATCH 09/28] fix(world-local): order a slot-numbered event log by slot, not by start time `createdAt` is stamped when a write begins, before its final slot is known, so it disagrees with slot order in two ways: a writer that loses a slot re-proposes above the winner while keeping its earlier stamp, and a caller that reserves a range of slots for one flush commits them in whatever order the network returns. Replay consumes the log in list order and never sorts, so listing by `createdAt` hands it an order no execution produced. Slot events now report one shared order time and let the existing event-id tie-break do the ordering, matching the Postgres World's `orderBy(eventId)` and the Vercel World's sort key. ULID runs keep their wall-clock order, which their ids agree with anyway. --- .changeset/local-slot-order.md | 5 +++++ packages/world-local/src/fs.ts | 17 ++++++++++++---- .../world-local/src/storage/events-storage.ts | 20 +++++++++++++++++++ .../src/storage/slot-identity.test.ts | 11 ++++++++++ 4 files changed, 49 insertions(+), 4 deletions(-) create mode 100644 .changeset/local-slot-order.md diff --git a/.changeset/local-slot-order.md b/.changeset/local-slot-order.md new file mode 100644 index 0000000000..148f7df032 --- /dev/null +++ b/.changeset/local-slot-order.md @@ -0,0 +1,5 @@ +--- +'@workflow/world-local': patch +--- + +Read a position-numbered event log in position order, so a replay sees the log the order it was written diff --git a/packages/world-local/src/fs.ts b/packages/world-local/src/fs.ts index 5827f36896..bc370a2ca0 100644 --- a/packages/world-local/src/fs.ts +++ b/packages/world-local/src/fs.ts @@ -579,6 +579,14 @@ interface PaginatedFileSystemQueryConfig { cursor?: string; getCreatedAt(filename: string): Date | null; getId?(item: T): string; + /** + * The time an item sorts and paginates by, when that is not its `createdAt`. + * A slot-numbered event log orders by slot — the position is the order — and + * a writer that loses a slot re-proposes above the winner while keeping the + * stamp it started with, so `createdAt` there disagrees with the log. Such an + * item reports one shared time and lets the `getId` tie-break order it. + */ + getOrderTime?: (item: NoInfer) => number; } // Cursor format: "timestamp|id" for tie-breaking interface ParsedCursor { @@ -615,6 +623,7 @@ export async function paginatedFileSystemQuery( cursor, getCreatedAt, getId, + getOrderTime = (item: T) => item.createdAt.getTime(), } = config; // Validate filePrefix (typically `${runId}-`) so request-derived prefixes @@ -718,7 +727,7 @@ export async function paginatedFileSystemQuery( // Double-check cursor filtering with actual createdAt from JSON // (in case ULID timestamp differs from stored createdAt) if (parsedCursor) { - const itemTime = item.createdAt.getTime(); + const itemTime = getOrderTime(item); const cursorTime = parsedCursor.timestamp.getTime(); if (sortOrder === 'desc') { @@ -746,8 +755,8 @@ export async function paginatedFileSystemQuery( // 5. Sort by createdAt (and by ID for tie-breaking if getId is provided) validItems.sort((a, b) => { - const aTime = a.createdAt.getTime(); - const bTime = b.createdAt.getTime(); + const aTime = getOrderTime(a); + const bTime = getOrderTime(b); const timeComparison = sortOrder === 'asc' ? aTime - bTime : bTime - aTime; // If timestamps are equal and we have getId, use ID for stable sorting @@ -768,7 +777,7 @@ export async function paginatedFileSystemQuery( const nextCursor = items.length > 0 ? createCursor( - items[items.length - 1].createdAt, + new Date(getOrderTime(items[items.length - 1])), getId?.(items[items.length - 1]) ) : null; diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index 97376f5aab..0e6c0ce39d 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -131,6 +131,20 @@ function getMaxEventsPerRun(): number { // but a shared filesystem), exactly matching the cross-process // semantics without spawning subprocesses. +/** + * The time an event orders and paginates by. A slot-numbered run's order is its + * slot order — the position *is* the order, the way the sort key is for the + * other backends — so every such event reports the same time and lets the + * event-id tie-break do the ordering. Ordering those by `createdAt` reads the + * log in an order no replay produced: a writer that loses a slot re-proposes + * above the winner while keeping the stamp it started with, and a caller that + * reserves slots for a whole flush commits them in whatever order the network + * returns. A ULID-numbered run keeps its wall-clock order, which its ids agree + * with anyway. + */ +const eventOrderTime = (event: { eventId: string; createdAt: Date }): number => + slotFromId(event.eventId) === undefined ? event.createdAt.getTime() : 0; + const HookTokenClaimSchema = z.object({ // The token-claim writer below has always persisted `hookId`, but // this read schema previously omitted it, which is the bug fixed @@ -268,6 +282,7 @@ async function findExistingHookCreatedEventId( event.correlationId === correlationId, limit: 1, getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, getId: (event) => event.eventId, }); return result.data[0]?.eventId ?? null; @@ -630,6 +645,7 @@ export function createEventsStorage( ? { cursor: params.sinceCursor } : {}), getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, getId: (event) => event.eventId, }); const maxSlot = params?.maxSlot ?? 0; @@ -2821,6 +2837,7 @@ export function createEventsStorage( sortOrder: 'asc', limit: 1000, getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, getId: (e) => e.eventId, }); events = allEvents.data; @@ -2869,6 +2886,7 @@ export function createEventsStorage( sortOrder: 'asc', cursor: params.sinceCursor, getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, getId: (e) => e.eventId, }); events = @@ -2929,6 +2947,7 @@ export function createEventsStorage( limit: params.pagination?.limit, cursor: params.pagination?.cursor, getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, getId: (event) => event.eventId, }); @@ -2961,6 +2980,7 @@ export function createEventsStorage( limit: params.pagination?.limit, cursor: params.pagination?.cursor, getCreatedAt: getObjectCreatedAt('evnt'), + getOrderTime: eventOrderTime, getId: (event) => event.eventId, }); diff --git a/packages/world-local/src/storage/slot-identity.test.ts b/packages/world-local/src/storage/slot-identity.test.ts index c21e215afa..5455e0a52f 100644 --- a/packages/world-local/src/storage/slot-identity.test.ts +++ b/packages/world-local/src/storage/slot-identity.test.ts @@ -120,6 +120,17 @@ describe('numbering', () => { await expect(slotsOf(runId)).resolves.toEqual([1, 2]); }); + it('lists the log in slot order, not in the order writes started', async () => { + // A writer that loses its slot re-proposes above the winner while keeping + // the wall-clock stamp it started with, so `createdAt` order and slot order + // disagree. Replay consumes the log in list order, so list order has to be + // slot order — what the sort key gives the other backends for free. + const runId = await newSlotRun(); + await createStep(runId, 'step_late', slotEventId(3)); + await createStep(runId, 'step_early', slotEventId(2)); + await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + }); + it('keeps a burst of concurrent writers dense', async () => { // The suspension flush issues every op at once. Density is what lets a // reader prove its log is complete, so a burst must not leave holes. From 41bc65ad0ae491a405dbef975be17f129fcb72b4 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 11:17:34 -0700 Subject: [PATCH 10/28] fix(core): stop rejecting healthy logs on late deliveries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two independent sources of `CorruptedEventLogError` on well-formed event logs, both found by dumping the logs of runs that failed that way. A hook delivery is ordered by when its event row commits, not by when the payload arrived, so a delivery that races a disposal — arriving first, committing second — lands after its own `hook_disposed` in the log. The hook's consumer retired on the disposal, so nothing consumed that event on any replay: a divergence that recurs identically every attempt and escalates to a terminal error. The consumer now stays registered as a tombstone and discards the late delivery, which is what `disposeHook` already assumed when it settled every awaiter. Separately, the unconsumed-event check gave the VM a flat 100ms of wall clock to register the next event's consumer. Real logs routinely need more: a hook payload fanning out into steps measures 254-717ms between the delivery and the first `step_created` it causes. The check now re-arms while a delivery is still in flight — the condition `scheduleWhenIdle` already polls — bounded by `WORKFLOW_DEFERRED_CHECK_MAX_GRACE_MS` so an abandoned delivery cannot park it forever. --- .changeset/late-hook-delivery-divergence.md | 6 + .../docs/v5/configuration/runtime-tuning.mdx | 6 + packages/core/src/events-consumer.test.ts | 69 +++++++++- packages/core/src/events-consumer.ts | 118 ++++++++++++++---- packages/core/src/private.ts | 15 ++- packages/core/src/workflow.ts | 13 +- packages/core/src/workflow/hook.test.ts | 66 +++++++++- packages/core/src/workflow/hook.ts | 41 +++++- 8 files changed, 292 insertions(+), 42 deletions(-) create mode 100644 .changeset/late-hook-delivery-divergence.md diff --git a/.changeset/late-hook-delivery-divergence.md b/.changeset/late-hook-delivery-divergence.md new file mode 100644 index 0000000000..ab62353ca8 --- /dev/null +++ b/.changeset/late-hook-delivery-divergence.md @@ -0,0 +1,6 @@ +--- +'@workflow/core': patch +'workflow': patch +--- + +Stop failing runs with a corrupted-event-log error when a hook delivery arrives after the hook was disposed, or when a step result is still being fetched diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index ba4a05b397..9569bb6f3d 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -198,6 +198,12 @@ These variables are primarily for tests, debugging, or unusual deployments. - Delay before the unconsumed-event check fires. - Minimum: `10`. +### `WORKFLOW_DEFERRED_CHECK_MAX_GRACE_MS` + +- Default: `15000` +- How long the unconsumed-event check keeps waiting while a step result or hook payload is still on its way to the workflow. An event whose consumer has not been registered yet looks exactly like an orphaned one, so the check waits rather than failing the run. +- Once this budget is spent the check reports regardless, so a delivery that never lands cannot keep a genuinely orphaned event from being detected. + ### `WORKFLOW_LOCK_POLL_INTERVAL_MS` - Default: `10` diff --git a/packages/core/src/events-consumer.test.ts b/packages/core/src/events-consumer.test.ts index ecf828f728..6294803363 100644 --- a/packages/core/src/events-consumer.test.ts +++ b/packages/core/src/events-consumer.test.ts @@ -1,7 +1,15 @@ import { withResolvers } from '@workflow/utils'; import type { Event } from '@workflow/world'; -import { describe, expect, it, vi } from 'vitest'; -import { EventConsumerResult, EventsConsumer } from './events-consumer.js'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + DEFERRED_CHECK_DELAY_MS, + EventConsumerResult, + EventsConsumer, +} from './events-consumer.js'; + +afterEach(() => { + vi.unstubAllEnvs(); +}); // Helper function to create mock events function createMockEvent(overrides: Partial = {}): Event { @@ -468,12 +476,63 @@ describe('EventsConsumer', () => { expect(consumer.eventIndex).toBe(1); }); - // Wait past the internal 100ms unconsumed-event setTimeout window to - // ensure the cancelled check truly does not fire. - await new Promise((resolve) => setTimeout(resolve, 150)); + // Wait past the internal unconsumed-event setTimeout window to ensure the + // cancelled check truly does not fire. + await new Promise((resolve) => + setTimeout(resolve, DEFERRED_CHECK_DELAY_MS * 1.5) + ); // The new callback consumed the event, so onUnconsumedEvent should NOT be called expect(onUnconsumedEvent).not.toHaveBeenCalled(); }); + + it('waits while a delivery is in flight, then reports once it lands', async () => { + vi.stubEnv('WORKFLOW_DEFERRED_CHECK_DELAY_MS', '10'); + const event = createMockEvent(); + const onUnconsumedEvent = vi.fn(); + let inFlight = true; + const consumer = new EventsConsumer([event], { + onUnconsumedEvent, + getPromiseQueue: () => Promise.resolve(), + isDeliveryInFlight: () => inFlight, + }); + + consumer.subscribe( + vi.fn().mockReturnValue(EventConsumerResult.NotConsumed) + ); + + // Many delay windows pass. A delivery still on its way to the workflow + // means the consumer for this event has not been registered YET — which + // is not the same thing as the event being orphaned. + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(onUnconsumedEvent).not.toHaveBeenCalled(); + + inFlight = false; + await vi.waitFor(() => { + expect(onUnconsumedEvent).toHaveBeenCalledWith(event); + }); + }); + + it('reports once the grace budget runs out even if a delivery never lands', async () => { + vi.stubEnv('WORKFLOW_DEFERRED_CHECK_DELAY_MS', '10'); + vi.stubEnv('WORKFLOW_DEFERRED_CHECK_MAX_GRACE_MS', '50'); + const event = createMockEvent(); + const onUnconsumedEvent = vi.fn(); + const consumer = new EventsConsumer([event], { + onUnconsumedEvent, + getPromiseQueue: () => Promise.resolve(), + // Never clears: a delivery that is abandoned must not park the check + // forever, or a genuinely orphaned event would never be reported. + isDeliveryInFlight: () => true, + }); + + consumer.subscribe( + vi.fn().mockReturnValue(EventConsumerResult.NotConsumed) + ); + + await vi.waitFor(() => { + expect(onUnconsumedEvent).toHaveBeenCalledWith(event); + }); + }); }); }); diff --git a/packages/core/src/events-consumer.ts b/packages/core/src/events-consumer.ts index 4b7cf0742e..5db54b4169 100644 --- a/packages/core/src/events-consumer.ts +++ b/packages/core/src/events-consumer.ts @@ -26,6 +26,26 @@ const getDeferredCheckDelayMs = (): number => min: 10, }); +/** + * Upper bound on how long the unconsumed-event check keeps re-arming while a + * data delivery is still in flight (see `isDeliveryInFlight`). The delay above + * is a margin for a microtask chain; this is a margin for real async work — + * decrypting a hook payload, fetching a remote ref — that has to finish before + * the VM can resume the branch that registers the next event's consumer. + * + * Bounded rather than unbounded so a genuinely orphaned event still reports, + * and so a delivery that never lands cannot park the check forever. + */ +export const DEFERRED_CHECK_MAX_GRACE_MS = 15_000; + +/** Override: `WORKFLOW_DEFERRED_CHECK_MAX_GRACE_MS`. */ +const getDeferredCheckMaxGraceMs = (): number => + envNumber( + 'WORKFLOW_DEFERRED_CHECK_MAX_GRACE_MS', + DEFERRED_CHECK_MAX_GRACE_MS, + { integer: true, min: 0 } + ); + export enum EventConsumerResult { /** * Callback consumed the event, but should not be removed from the callbacks list @@ -65,6 +85,16 @@ export interface EventsConsumerOptions { * deserialization delays the resolve() that triggers the next subscribe(). */ getPromiseQueue: () => Promise; + /** + * Whether a data delivery (step result, hook payload) is still on its way to + * the workflow. The unconsumed-event check re-arms while this holds instead + * of reporting: an event whose consumer has not been registered yet is + * indistinguishable from an orphaned one by log inspection alone, and the + * promise-queue drain does not cover the gap between a delivery's `resolve()` + * and the VM body reaching its next `subscribe()`. Defaults to never in + * flight, which is the plain wall-clock behaviour. + */ + isDeliveryInFlight?: () => boolean; } export class EventsConsumer { @@ -74,6 +104,7 @@ export class EventsConsumer { private onConsumedEvent?: (event: Event) => void; private onUnconsumedEvent: (event: Event) => void; private getPromiseQueue: () => Promise; + private isDeliveryInFlight: () => boolean; private pendingUnconsumedCheck: Promise | null = null; private pendingUnconsumedTimeout: ReturnType | null = null; private unconsumedCheckVersion = 0; @@ -84,6 +115,7 @@ export class EventsConsumer { this.onConsumedEvent = options.onConsumedEvent; this.onUnconsumedEvent = options.onUnconsumedEvent; this.getPromiseQueue = options.getPromiseQueue; + this.isDeliveryInFlight = options.isDeliveryInFlight ?? (() => false); } /** @@ -200,32 +232,66 @@ export class EventsConsumer { // is still unconsumed after the queue drains, it's truly orphaned. if (currentEvent !== null) { const checkVersion = ++this.unconsumedCheckVersion; - this.pendingUnconsumedCheck = this.getPromiseQueue() - .then( - // Yield once after the first queue drain so promise chains resumed by - // that drain can run across the VM boundary and append any follow-up - // async work (for example: step_completed resolves -> for-await loop - // resumes -> the next hook payload starts hydrating). - () => new Promise((resolve) => setTimeout(resolve, 0)) - ) - .then(() => this.getPromiseQueue()) - .then(() => { - // Use a delayed setTimeout after the queue drains. The delay must be - // long enough for promise chains to propagate across the VM boundary - // (from resolve() in the host context through to the workflow code - // calling subscribe() in the VM context). Node.js does not guarantee - // that setTimeout(0) fires after all cross-context microtasks settle, - // so we use a small but non-zero delay. Any subscribe() call that - // arrives during this window will cancel the check via version - // invalidation + clearTimeout. - this.pendingUnconsumedTimeout = setTimeout(() => { - this.pendingUnconsumedTimeout = null; - if (this.unconsumedCheckVersion === checkVersion) { - this.pendingUnconsumedCheck = null; - this.onUnconsumedEvent(currentEvent); - } - }, getDeferredCheckDelayMs()); - }); + this.armUnconsumedCheck( + currentEvent, + checkVersion, + getDeferredCheckMaxGraceMs() + ); } } + + /** + * Wait for the promise queue to drain, then a short delay, then report + * `currentEvent` as unconsumed — unless a `subscribe()` invalidated + * `checkVersion` in the meantime, or a delivery is still in flight, in which + * case re-arm with `graceRemainingMs` reduced by the delay just spent. + */ + private armUnconsumedCheck( + currentEvent: Event, + checkVersion: number, + graceRemainingMs: number + ) { + const delay = getDeferredCheckDelayMs(); + this.pendingUnconsumedCheck = this.getPromiseQueue() + .then( + // Yield once after the first queue drain so promise chains resumed by + // that drain can run across the VM boundary and append any follow-up + // async work (for example: step_completed resolves -> for-await loop + // resumes -> the next hook payload starts hydrating). + () => new Promise((resolve) => setTimeout(resolve, 0)) + ) + .then(() => this.getPromiseQueue()) + .then(() => { + // Use a delayed setTimeout after the queue drains. The delay must be + // long enough for promise chains to propagate across the VM boundary + // (from resolve() in the host context through to the workflow code + // calling subscribe() in the VM context). Node.js does not guarantee + // that setTimeout(0) fires after all cross-context microtasks settle, + // so we use a small but non-zero delay. Any subscribe() call that + // arrives during this window will cancel the check via version + // invalidation + clearTimeout. + this.pendingUnconsumedTimeout = setTimeout(() => { + this.pendingUnconsumedTimeout = null; + if (this.unconsumedCheckVersion !== checkVersion) { + return; + } + if (graceRemainingMs > 0 && this.isDeliveryInFlight()) { + // A delivery is hydrating, or has resolved but is parked behind its + // deferral. The workflow body has not had the chance to register + // this event's consumer yet, so reporting now would reject a + // healthy run: the resulting `ReplayDivergenceError` recurs on + // every replay that is unlucky in the same way and escalates to a + // terminal `CorruptedEventLogError`. + this.armUnconsumedCheck( + currentEvent, + checkVersion, + graceRemainingMs - delay + ); + return; + } + this.pendingUnconsumedCheck = null; + this.onUnconsumedEvent(currentEvent); + }, delay); + }); + } } diff --git a/packages/core/src/private.ts b/packages/core/src/private.ts index a4498b3179..89b2e86a61 100644 --- a/packages/core/src/private.ts +++ b/packages/core/src/private.ts @@ -555,12 +555,25 @@ function hasParkedCommittedDelivery(ctx: WorkflowOrchestratorContext): boolean { * delivery still in flight. Empirically, replacing it with `queueMicrotask` * breaks hook/sleep `Promise.race` ordering (CorruptedEventLogError). */ +/** + * Whether some data delivery is still on its way to the workflow — the same + * two windows {@link scheduleWhenIdle} polls on, exposed for callers that need + * to test the condition without waiting on it. + * + * While this holds, the VM has not yet run the continuation that registers the + * next event's consumer, so "no consumer for this event" says nothing about + * whether the event log is well-formed. + */ +export function hasInFlightDelivery(ctx: WorkflowOrchestratorContext): boolean { + return ctx.pendingDeliveries > 0 || hasParkedCommittedDelivery(ctx); +} + export function scheduleWhenIdle( ctx: WorkflowOrchestratorContext, fn: () => void ): void { const check = () => { - if (ctx.pendingDeliveries > 0 || hasParkedCommittedDelivery(ctx)) { + if (hasInFlightDelivery(ctx)) { // A delivery is still hydrating, or is committed but parked behind its // deferral (whose resolve runs on a detached timer, not this queue). // Either way: let the queue drain, then re-check a timer tick later. diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 97c9acee99..03f8d1e203 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -15,7 +15,10 @@ import { EventConsumerResult, EventsConsumer } from './events-consumer.js'; import type { QueueItem } from './global.js'; import { ENOTSUP, WorkflowSuspension } from './global.js'; import { runtimeLogger } from './logger.js'; -import type { WorkflowOrchestratorContext } from './private.js'; +import { + hasInFlightDelivery, + type WorkflowOrchestratorContext, +} from './private.js'; import { ReplayPayloadCache } from './replay-payload-cache.js'; import { getPortLazy } from './runtime/get-port-lazy.js'; import type { MutableEventLog } from './runtime/helpers.js'; @@ -208,6 +211,10 @@ export async function runWorkflow( // by step/hook/sleep callbacks as events are processed. const promiseQueueHolder = { current: Promise.resolve() }; + // Assigned immediately below. The consumer needs to test the context's + // delivery state, and the context needs the consumer. + let workflowContext: WorkflowOrchestratorContext; + const eventsConsumer = new EventsConsumer(events, { onConsumedEvent: (event) => { updateTimestamp(+event.createdAt); @@ -221,9 +228,11 @@ export async function runWorkflow( ); }, getPromiseQueue: () => promiseQueueHolder.current, + isDeliveryInFlight: () => + workflowContext !== undefined && hasInFlightDelivery(workflowContext), }); - const workflowContext: WorkflowOrchestratorContext = { + workflowContext = { runId: workflowRun.runId, encryptionKey, worldCapabilities, diff --git a/packages/core/src/workflow/hook.test.ts b/packages/core/src/workflow/hook.test.ts index 86cf8b54a3..fa913ff4e7 100644 --- a/packages/core/src/workflow/hook.test.ts +++ b/packages/core/src/workflow/hook.test.ts @@ -24,7 +24,10 @@ import { createWebhook } from './create-hook.js'; import { createCreateHook } from './hook.js'; // Helper to setup context to simulate a workflow run -function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { +function setupWorkflowContext( + events: Event[], + onUnconsumedEvent: (event: Event) => void = () => {} +): WorkflowOrchestratorContext { const context = createContext({ seed: 'test', fixedTimestamp: 1753481739458, @@ -43,7 +46,7 @@ function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { replayPayloadCache: new ReplayPayloadCache(undefined), globalThis: context.globalThis, eventsConsumer: new EventsConsumer(events, { - onUnconsumedEvent: () => {}, + onUnconsumedEvent, getPromiseQueue: () => Promise.resolve(), }), invocationsQueue: new Map(), @@ -471,6 +474,65 @@ describe('createCreateHook', () => { expect(runtimeErrors).toHaveLength(0); }); + it('should discard a hook_received ordered after the hook_disposed', async () => { + // The world orders a delivery by when its event row commits, not by when + // the payload arrived, so a delivery that raced the disposal can land after + // it in the log. Nothing else in the run can consume that event, so the + // hook's own consumer has to swallow it — otherwise the events consumer + // reports an orphan and the replay diverges on a well-formed log. + const ops: Promise[] = []; + const onUnconsumedEvent = vi.fn(); + const ctx = setupWorkflowContext( + [ + { + eventId: 'evnt_0', + runId: 'wrun_123', + eventType: 'hook_created', + correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: { token: 'test-token' }, + createdAt: new Date(), + }, + { + eventId: 'evnt_1', + runId: 'wrun_123', + eventType: 'hook_disposed', + correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: { token: 'test-token' }, + createdAt: new Date(), + }, + { + eventId: 'evnt_2', + runId: 'wrun_123', + eventType: 'hook_received', + correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: { + token: 'test-token', + payload: await dehydrateStepReturnValue( + { message: 'lost the race' }, + 'wrun_test', + undefined, + ops + ), + }, + createdAt: new Date(), + }, + ], + onUnconsumedEvent + ); + + const createHook = createCreateHook(ctx); + createHook({ token: 'test-token' }); + + // The whole log is consumed: the disposal retires the hook and the late + // delivery is dropped on the floor. + await vi.waitFor(() => { + expect(ctx.eventsConsumer.eventIndex).toBe(3); + }); + expect(onUnconsumedEvent).not.toHaveBeenCalled(); + expect(ctx.onWorkflowError).not.toHaveBeenCalled(); + expect(ctx.invocationsQueue.size).toBe(0); + }); + it('should handle multiple hook_received events with iterator', async () => { const ops: Promise[] = []; const ctx = setupWorkflowContext([ diff --git a/packages/core/src/workflow/hook.ts b/packages/core/src/workflow/hook.ts index e079827051..f7d1f209ac 100644 --- a/packages/core/src/workflow/hook.ts +++ b/packages/core/src/workflow/hook.ts @@ -154,8 +154,9 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { eventLogEmpty = true; if ( - (promises.length > 0 && payloadsQueue.length === 0) || - (getConflictPromises.length > 0 && !hasCreated && !hasConflict) + !hasDisposedEvent && + ((promises.length > 0 && payloadsQueue.length === 0) || + (getConflictPromises.length > 0 && !hasCreated && !hasConflict)) ) { scheduleWhenIdle(ctx, () => { ctx.onWorkflowError( @@ -171,6 +172,31 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { return EventConsumerResult.NotConsumed; } + if (hasDisposedEvent) { + // A delivery ordered AFTER this hook's own `hook_disposed`. The world + // orders a delivery by when its event row is written, not by when the + // payload arrived, so a delivery that raced the disposal — arriving + // first, committing second — lands here. Swallow it: the hook is gone, + // there is no consumer to hand the payload to, and every awaiter was + // already settled by `disposeHook`. + // + // This consumer must stay registered to do that. Retiring it on + // `hook_disposed` leaves the late delivery with no consumer at all, + // which the events consumer reports as an orphaned event — + // a `ReplayDivergenceError` that recurs on every replay of a log that + // is otherwise perfectly well-formed, escalating to a terminal + // `CorruptedEventLogError`. + webhookLogger.warn( + 'Discarding a hook delivery ordered after disposal', + { + correlationId, + eventId: event.eventId, + eventType: event.eventType, + } + ); + return EventConsumerResult.Consumed; + } + const eventToken = 'eventData' in event && event.eventData && 'token' in event.eventData ? event.eventData.token @@ -418,8 +444,10 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { ctx.invocationsQueue.delete(correlationId); // Mark that the event log confirms disposal happened hasDisposedEvent = true; - // We're done processing any more events for this hook - return EventConsumerResult.Finished; + // Stay registered as a tombstone rather than retiring: a delivery that + // raced this disposal can still be ordered after it, and nothing else + // in the run can consume it. See the `hasDisposedEvent` branch above. + return EventConsumerResult.Consumed; } // This replay installed a different consumer than the stored event needs. @@ -539,8 +567,9 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { // Drain any pending promises that are waiting for payloads. // Without this, promises created by `await hook` or the async iterator's - // `yield await this` would hang forever since the event consumer will - // never deliver another hook_received after disposal. + // `yield await this` would hang forever: a hook_received ordered after + // the disposal is discarded rather than handed to an awaiter, so nothing + // will ever settle them. if (promises.length > 0) { promises.length = 0; scheduleWhenIdle(ctx, () => { From a02b180881053aa95de4c0e54391a5fb07f0c8a0 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 11:27:13 -0700 Subject: [PATCH 11/28] docs(world): stop calling a dense log a completeness proof Contiguous allocation is not the same thing as a gap-free published log: a slot claimed by an operation that then fails for a reason of its own is never filled, and once a later slot is published that gap is permanent. What the scheme actually buys is explicit contention and a log that reads in write order. Nothing consumed the proof, so this is a comment and docs correction. --- .../docs/v5/configuration/runtime-tuning.mdx | 2 +- packages/world-postgres/src/slots.ts | 9 +++++---- .../world-postgres/test/slot-identity.test.ts | 6 +++--- packages/world-vercel/src/events-v4.test.ts | 2 +- packages/world/src/slot-identity.ts | 16 +++++++++++----- 5 files changed, 21 insertions(+), 14 deletions(-) diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index 9569bb6f3d..254b0d4815 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -51,7 +51,7 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL ### `WORKFLOW_SLOT_IDENTITY` - Default: enabled -- Numbers a new run's events by position instead of by ULID: `evnt_…001` is the run's first event, `evnt_…002` its second, with no gaps. A reader can then prove its copy of the log is complete, because the highest number is the event count. +- Numbers a new run's events by position instead of by ULID: `evnt_…001` is the run's first event, `evnt_…002` its second. Positions are allocated in order, so the log reads in the order it was written regardless of clock skew between writers. - Contention becomes explicit rather than silent. Two writers proposing the same position cannot both win: the loser gets a 409 ([`SlotConflictError`](/docs/api-reference/workflow-errors/slot-conflict-error)) carrying the events it was missing, and the runtime merges them, replays, and re-proposes at the next free position. - Applies only to runs created while it is enabled. A run keeps the identity scheme it was created with for life, so turning the flag on or off never affects runs already in flight. - Requires a World that supports it. A World that does not rejects the run outright rather than mis-numbering its events. diff --git a/packages/world-postgres/src/slots.ts b/packages/world-postgres/src/slots.ts index cd0a54f6f5..6a0366a636 100644 --- a/packages/world-postgres/src/slots.ts +++ b/packages/world-postgres/src/slots.ts @@ -2,10 +2,11 @@ * Slot identity for the postgres world. * * A slot-numbered run names its events by position: `evnt_…001` is the first - * event of the run, `evnt_…002` the second, with no gaps. Density is the point - * — it is what lets a reader prove its copy of a log is complete — so the two - * things this module has to get right are that a position is written at most - * once and that a lost position is never left behind as a hole. + * event of the run, `evnt_…002` the second. Contention on a position is the + * point — it is what makes a concurrent write detectable rather than silent — + * so the two things this module has to get right are that a position is written + * at most once and that a position this allocator loses is retried rather than + * abandoned as a hole. * * The authority for both is the events table's primary key, `(run_id, id)`: the * INSERT either lands or raises a unique violation, and a writer that loses the diff --git a/packages/world-postgres/test/slot-identity.test.ts b/packages/world-postgres/test/slot-identity.test.ts index 8e3f04c8a4..a6cf498651 100644 --- a/packages/world-postgres/test/slot-identity.test.ts +++ b/packages/world-postgres/test/slot-identity.test.ts @@ -259,8 +259,8 @@ describe('Slot identity (Postgres integration)', () => { describe('contention', () => { // The primary key is the authority, and a writer that loses a position is - // retried at one that is still free. Nothing here may leave a hole: density - // is what lets a reader prove its copy of a log is complete. + // retried at one that is still free rather than abandoning the one it lost, + // so a burst of concurrent writers still numbers itself densely. for (const writers of [2, 8, 50]) { test(`keeps ${writers} concurrent writers dense`, async () => { const runId = await newSlotRun(); @@ -274,7 +274,7 @@ describe('Slot identity (Postgres integration)', () => { }, 60_000); } - test('proves completeness: the highest slot is the event count', async () => { + test('numbers a burst so the highest slot is the event count', async () => { const runId = await newSlotRun(); await Promise.all( Array.from({ length: 5 }, (_, index) => diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index 2139d8c3eb..5c3bda47dd 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -726,7 +726,7 @@ describe('createWorkflowRunEventV4 over HTTP', () => { // A slot-numbered run names its own event ids, so the id has to reach the // wire: the backend reads it from the frame meta and inserts it // conditionally. Dropped, the backend mints a ULID instead and the run - // silently loses the density its completeness check depends on. + // silently reverts to server-assigned identity mid-log. const origin = WORKFLOW_SERVER_URL_OVERRIDE || 'https://vercel-workflow.com'; const agent = new MockAgent(); diff --git a/packages/world/src/slot-identity.ts b/packages/world/src/slot-identity.ts index d761f91452..203005c719 100644 --- a/packages/world/src/slot-identity.ts +++ b/packages/world/src/slot-identity.ts @@ -7,11 +7,17 @@ * cursor that accepted a ULID keeps accepting a slot — and because the width is * fixed, lexicographic order is numeric order. * - * Slots are dense and start at 1. Density is the whole point of the scheme: it - * makes `events.length === maxSlot` a proof that a loaded log has no holes, - * which is something a server-minted ULID log can never offer. Zero is left - * unused because the inclusive lower fence for range queries over a run's - * events is the all-zero id. + * Slots start at 1 and are allocated contiguously, which is what makes + * contention explicit: two writers proposing one position cannot both win, and + * the loser is told which events it was missing. Zero is left unused because + * the inclusive lower fence for range queries over a run's events is the + * all-zero id. + * + * Allocation being contiguous does not make a published log gap-free, so + * `events.length === maxSlot` is not a completeness proof. A slot claimed by an + * operation that then fails for a reason of its own is never filled, and if a + * later slot has already been published the gap is permanent. Nothing may treat + * a missing slot as an event still on its way. * * A slot body decodes as a ULID *timestamp* of epoch 0 without erroring, so * nothing may read a time out of one. Use the event's own `createdAt` / From 435e52583639b85b4c7769d0cc35b8e12fcc28c8 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 06:01:09 -0700 Subject: [PATCH 12/28] fix(core): render structured error messages and name what a divergence waited for MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three diagnostic gaps that together made replay divergence unreadable: - `composeLogLine` dropped `errorMessage` whenever the message did not already contain it, so a warn carrying an error alongside its own summary line logged the symptom and none of the diagnosis. - An unconsumable event named only itself. It is almost always an event whose entity this replay never issued, so the pending invocation queue is what distinguishes "never issued" from "issued under another id". - An inline step batch abandoned on a fenced claim logged neither which member was fenced nor how the others settled. The fence is per-write, so a batch can split: the rejected claim writes nothing while a sibling on a different slot commits. Also read a failed run's error through `returnValue()` in the race-repro harness — `runs.get` returns the raw serialized payload, so every corruption in the report carried a code and no message. Co-Authored-By: Claude Opus 5 --- .changeset/tidy-moons-observe.md | 5 +++ .../core/e2e/event-log-race-repro.test.ts | 35 +++++++++++++++++-- packages/core/src/log-format.test.ts | 26 +++++++++++++- packages/core/src/log-format.ts | 20 ++++++++--- packages/core/src/runtime.ts | 26 ++++++++++++-- packages/core/src/workflow.ts | 8 ++++- 6 files changed, 110 insertions(+), 10 deletions(-) create mode 100644 .changeset/tidy-moons-observe.md diff --git a/.changeset/tidy-moons-observe.md b/.changeset/tidy-moons-observe.md new file mode 100644 index 0000000000..9c6c5b7692 --- /dev/null +++ b/.changeset/tidy-moons-observe.md @@ -0,0 +1,5 @@ +--- +'@workflow/core': patch +--- + +Name the divergent event's pending invocations and the fenced member of an inline step batch in replay-divergence logs diff --git a/packages/core/e2e/event-log-race-repro.test.ts b/packages/core/e2e/event-log-race-repro.test.ts index e98f6fd10a..b40fed2ec3 100644 --- a/packages/core/e2e/event-log-race-repro.test.ts +++ b/packages/core/e2e/event-log-race-repro.test.ts @@ -373,6 +373,30 @@ function validateStormReturn(value: unknown): { return { stragglers }; } +/** + * Reads a terminal-failed run's error through `returnValue()`, which hydrates + * the stored payload into an Error. Returns undefined when the read itself + * fails — the outcome is already known from `errorCode`, so a missing message + * degrades the report rather than the classification. + */ +async function readFailureMessage( + run: Run +): Promise<{ name?: string; message?: string } | undefined> { + try { + await run.returnValue(); + return undefined; + } catch (err) { + if (WorkflowRunFailedError.is(err)) { + const cause = err.cause; + return { + name: cause instanceof Error ? cause.name : err.name, + message: cause instanceof Error ? cause.message : err.message, + }; + } + return undefined; + } +} + async function pollTerminalRun( run: Run, startedAt: number, @@ -412,13 +436,20 @@ async function pollTerminalRun( errorCode?: string; error?: { name?: string; message?: string }; }; + // `runs.get` hands back the raw serialized error payload, not an Error, so + // reading `.message` off it yields undefined and the report records the + // code with no diagnosis. Read the failure through the public + // return-value path, which hydrates it. For a corruption that message + // carries the divergent event and what the replay was waiting for, which + // is the whole reason to keep the report. + const hydrated = await readFailureMessage(run); return { ...base, outcome: classifyFailure(failure.errorCode), status: runData.status, errorCode: failure.errorCode, - errorMessage: failure.error?.message, - errorName: failure.error?.name, + errorMessage: hydrated?.message ?? failure.error?.message, + errorName: hydrated?.name ?? failure.error?.name, durationMs: Date.now() - startedAt, }; } diff --git a/packages/core/src/log-format.test.ts b/packages/core/src/log-format.test.ts index c673893f06..3b78208744 100644 --- a/packages/core/src/log-format.test.ts +++ b/packages/core/src/log-format.test.ts @@ -113,6 +113,29 @@ describe('composeLogLine', () => { `); }); + test('renders errorMessage when the message does not already carry it', () => { + // The replay-divergence warn writes its own summary line and passes the + // error only as metadata, so this is the sole place the divergent event's + // identity appears. Dropping it leaves the log naming a symptom with no + // way to tell which event diverged. + const out = composeLogLine( + PREFIX, + 'Workflow replay diverged; queueing a recovery replay before declaring the event log corrupted', + { + errorCode: 'REPLAY_DIVERGENCE', + errorMessage: + 'Replay could not consume event: eventType=step_created, correlationId=step_00000000000000000000000025.', + divergenceCount: 1, + } + ); + expect(out).toMatchInlineSnapshot(` + "[workflow-sdk] Workflow replay diverged; queueing a recovery replay before declaring the event log corrupted + code REPLAY_DIVERGENCE + error Replay could not consume event: eventType=step_created, correlationId=step_00000000000000000000000025. + divergenceCount 1" + `); + }); + test('falls back gracefully on machine names it cannot parse', () => { const out = composeLogLine(PREFIX, 'msg', { workflowRunId: 'wrun_X', @@ -156,7 +179,8 @@ describe('composeLogLine', () => { user error · Error run wrun_01ABC · myWorkflow (./workflows/x) step step_01XYZ · add (./workflows/x) - retry 4 attempts · 3 max retries" + retry 4 attempts · 3 max retries + error Transient failure" `); }); }); diff --git a/packages/core/src/log-format.ts b/packages/core/src/log-format.ts index d63e847e3f..b3daa3263c 100644 --- a/packages/core/src/log-format.ts +++ b/packages/core/src/log-format.ts @@ -36,7 +36,7 @@ export function composeLogLine( ): string { const [framing, ...rest] = message.split('\n'); const body = rest.join('\n'); - const fields = renderStructuredFields(framing ?? '', metadata); + const fields = renderStructuredFields(message, metadata); const trimmedBody = trimStackBody(body); const lines: string[] = [`${prefix} ${framing ?? ''}`]; @@ -46,19 +46,21 @@ export function composeLogLine( } function renderStructuredFields( - framing: string, + message: string, metadata: Record | undefined ): string | null { if (!metadata || Object.keys(metadata).length === 0) return null; // Drop fields that the message already encodes. We render framings and // stacks into the message string itself in step executor / combined runtime, so - // repeating them here would be pure noise. + // repeating them here would be pure noise. The whole message counts, not just + // its first line: callers that pass `${framing}\n${stack}` put the error's + // text in the stack's leading `Name: message` line. const redundant = new Set(); redundant.add('errorStack'); if ( typeof metadata.errorMessage === 'string' && - framing.includes(metadata.errorMessage as string) + message.includes(metadata.errorMessage as string) ) { redundant.add('errorMessage'); } @@ -130,6 +132,16 @@ function renderStructuredFields( lines.push(` ${kvKey('code')} ${Ansi.dim(errorCode)}`); } + // The message only duplicates the framing when the framing was built from + // the error itself (step executor, terminal run failures), and that case is + // already marked redundant above. Everywhere else — a warn that carries an + // error alongside its own summary line — this is the only place the error's + // own text appears, so dropping it loses the diagnosis. + const errorMessage = pickString(metadata, 'errorMessage'); + if (errorMessage && !redundant.has('errorMessage')) { + lines.push(` ${kvKey('error')} ${errorMessage}`); + } + const hint = pickString(metadata, 'hint'); if (hint) { lines.push(` ${Ansi.hint(hint)}`); diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 19581650d2..6534fbfe15 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -2498,10 +2498,32 @@ export function workflowEntrypoint( // the sibling executions to settle first so no owned // body is in flight when the ack path runs. if (requiresFreshReplay(stepErr)) { - await Promise.allSettled(stepExecutionPromises); + const settled = await Promise.allSettled( + stepExecutionPromises + ); runtimeLogger.warn( 'Inline step claim rejected as stale; re-invoking run for a fresh replay', - { workflowRunId: runId, loopIteration } + { + workflowRunId: runId, + loopIteration, + // Which members of the batch were fenced and + // which committed. A fence is per-write, so a + // batch can be split: the rejected claim wrote + // nothing, but a sibling holding a different + // slot may have committed its step. That + // asymmetry is the shape to look for when a + // later replay cannot consume a step event. + batchSteps: inlineExecutions + .map( + (s, i) => + `${s.correlationId}:${settled[i]?.status === 'rejected' ? 'rejected' : 'settled'}` + ) + .join(', '), + errorMessage: + stepErr instanceof Error + ? stepErr.message + : String(stepErr), + } ); // The finally below resumes the replay budget // before this return completes. diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 03f8d1e203..76fd5e00dd 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -220,9 +220,15 @@ export async function runWorkflow( updateTimestamp(+event.createdAt); }, onUnconsumedEvent: (event) => { + // Name what the replay was waiting for instead. An unconsumable event + // is almost always one whose entity this replay never issued, or + // issued under a different correlation ID; the pending invocation + // queue is the only place that distinction is visible, and without it + // the log names a symptom with no way to reach the cause. + const pending = [...workflowContext.invocationsQueue.keys()]; workflowDiscontinuation.reject( new ReplayDivergenceError( - `Replay could not consume event: eventType=${event.eventType}, correlationId=${event.correlationId}, eventId=${event.eventId}.`, + `Replay could not consume event: eventType=${event.eventType}, correlationId=${event.correlationId}, eventId=${event.eventId}. Pending invocations: ${pending.length > 0 ? pending.join(', ') : '(none)'}.`, { eventId: event.eventId } ) ); From 5cfc07dcdf6551d1401008bbc3e8500944bdfbd9 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 06:34:59 -0700 Subject: [PATCH 13/28] fix(core): reclaim a lost inline step slot instead of splitting the batch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An inline step's step_started claim is fenced per-write under slot identity, so a 409 only proves another writer took that write's number — routinely true, since the backend allocates outside events from the same next-free pointer the client reserves from. Abandoning the whole batch on it left the loser's events landing seconds later, after a whole later phase, in an order no single replay could consume. stepClaimFence keeps a watermark-guarded run on its single shared fence (a 412 does mean the view is stale, and the batch is meant to fail as a unit) and gives a slot-numbered run an in-place reclaim: merge the delta, reserve past it, re-claim. The reservation pointer is now absolute and only moves forward, so a merge cannot hand the retrying writer a slot a sibling is still in flight on. --- .changeset/inline-claim-reclaim.md | 5 + packages/core/src/logger.test.ts | 4 +- packages/core/src/runtime.ts | 35 +++-- packages/core/src/runtime/helpers.test.ts | 116 +++++++++++++++-- packages/core/src/runtime/helpers.ts | 121 ++++++++++++++---- .../core/src/runtime/step-executor.test.ts | 38 +++++- packages/core/src/runtime/step-executor.ts | 109 +++++++++------- 7 files changed, 333 insertions(+), 95 deletions(-) create mode 100644 .changeset/inline-claim-reclaim.md diff --git a/.changeset/inline-claim-reclaim.md b/.changeset/inline-claim-reclaim.md new file mode 100644 index 0000000000..9d57541efc --- /dev/null +++ b/.changeset/inline-claim-reclaim.md @@ -0,0 +1,5 @@ +--- +'@workflow/core': patch +--- + +Keep a batch of inline steps together when one of its event writes loses a race, instead of discarding the batch diff --git a/packages/core/src/logger.test.ts b/packages/core/src/logger.test.ts index 5560c9e213..78066923ca 100644 --- a/packages/core/src/logger.test.ts +++ b/packages/core/src/logger.test.ts @@ -148,6 +148,7 @@ describe('logger', () => { user error · FatalError run wrun_123 step step_456 + error boom hint: Move the call to a step function.", ], ] @@ -178,7 +179,8 @@ describe('logger', () => { user error · Error run wrun_abc step step_xyz - retry 4 attempts · 3 max retries", + retry 4 attempts · 3 max retries + error Transient failure", ], ] `); diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 6534fbfe15..7f04514754 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -51,6 +51,7 @@ import { countStepStartedEvents } from './runtime/count-step-started-events.js'; import { appendUniqueEvents, eventCreateFenceFor, + stepClaimFence, getQueueOverhead, getWorkflowQueueName, handleHealthCheckMessage, @@ -2187,10 +2188,14 @@ export function workflowEntrypoint( // (retried over the reloaded log, or exhausted into // a queue re-invocation), AND the lazy step_started // claim of its next inline step, which is fenced too - // (threaded below via - // `eventCreateFence`; on rejection the batch is - // abandoned and re-invoked for a fresh replay, so a - // stale view can never commit a step). Hooks created + // (threaded below via `stepClaimFence`; on rejection + // the batch is abandoned and re-invoked for a fresh + // replay, so a stale view can never commit a step). + // A slot-numbered run gets there differently — the + // claim merges the missed events and retries in + // place, so the same events are observed without + // discarding the batch. See stepClaimFence. + // Hooks created // by THIS suspension are inside the delta (their // `hook_created` lands before the step-terminal // write), so only their `hook_received` responses @@ -2337,8 +2342,8 @@ export function workflowEntrypoint( // could claim — and commit — a step scheduled off a view // that misses an out-of-band event. One log for the // whole batch so each claim draws its own event slot; - // `eventCreateFenceFor` yields undefined for a run - // fenced neither way, leaving those claims as they were. + // `stepClaimFence` leaves the claims of a run fenced + // neither way exactly as they were. // // The suspension's own log, not a second one over the // same snapshot: its reservations are what the hook and @@ -2359,7 +2364,8 @@ export function workflowEntrypoint( // positional, so it has to be assigned before these // executions start racing each other, and the order // it is assigned in has to be replay-stable. - const eventCreateFence = eventCreateFenceFor( + const claimFence = stepClaimFence( + runId, inlineClaimLog, workflowRun.specVersion, { @@ -2445,7 +2451,7 @@ export function workflowEntrypoint( // see suppressOptimisticStart above. suppressOptimisticStart, runReadyBarrier, - eventCreateFence, + claimFence, ...(stepIndex === 0 && s.lazyStepInput !== undefined && latencyTracking @@ -2487,12 +2493,15 @@ export function workflowEntrypoint( ); } catch (stepErr) { // An incomplete-view rejection of an inline - // step_started claim (412 stale watermark, or 409 - // taken slot): the loaded view this batch was + // step_started claim: the loaded view this batch was // scheduled from is behind an out-of-band event (e.g. - // a received hook), so the claim was fenced by the - // guard and no step events were written. Abandon the - // batch — any optimistic body result is discarded by + // a received hook), so the claim was fenced and no + // step events were written. Under the watermark that + // is every 412; under slot identity `stepClaimFence` + // first merges the missed events and re-claims in + // place, so a 409 only arrives here once those + // retries are exhausted. Abandon the batch — any + // optimistic body result is discarded by // executeStep's reconciliation — and re-invoke for a // fresh replay that observes the new event. Wait for // the sibling executions to settle first so no owned diff --git a/packages/core/src/runtime/helpers.test.ts b/packages/core/src/runtime/helpers.test.ts index ca1a3ff4b5..1462804bd9 100644 --- a/packages/core/src/runtime/helpers.test.ts +++ b/packages/core/src/runtime/helpers.test.ts @@ -33,6 +33,7 @@ import { requiresFreshReplay, reserveSlot, stateUpdatedAtForCreate, + stepClaimFence, toMutableEventLog, withEventCreateFence, withPreconditionRetry, @@ -529,7 +530,7 @@ describe('slot bookkeeping', () => { // element is not necessarily its newest event. const log = toMutableEventLog([slotEvent(3), slotEvent(1)], 'c0'); expect(log.maxSlot).toBe(3); - expect(log.reserved).toBe(0); + expect(log.nextSlot).toBe(4); }); it('reports maxSlot 0 for an empty or ULID-numbered log', () => { @@ -566,21 +567,33 @@ describe('slot bookkeeping', () => { expect(log.events).toHaveLength(3); }); - it('raises maxSlot and drops reservations when a newer delta is merged in', () => { + it('raises the reservation pointer past a newer delta', () => { const log = toMutableEventLog([slotEvent(1)], 'c0'); reserveSlot(log); reserveSlot(log); - expect(log.reserved).toBe(2); + expect(log.nextSlot).toBe(4); mergeLoadedEvents(log, [slotEvent(2), slotEvent(5)]); expect(log.maxSlot).toBe(5); - // The merged events are the authority on which slots are taken, so the - // outstanding reservations (slots 2 and 3) are void. - expect(log.reserved).toBe(0); expect(reserveSlot(log)).toBe(6); }); + it('never rewinds the reservation pointer onto an outstanding slot', () => { + // A writer that loses its slot merges the delta and reserves again while + // its siblings are still in flight on theirs. Rewinding to `maxSlot + 1` + // would hand it slot 4, which a sibling already holds. + const log = toMutableEventLog([slotEvent(1)], 'c0'); + expect(reserveSlot(log)).toBe(2); + expect(reserveSlot(log)).toBe(3); + expect(reserveSlot(log)).toBe(4); + + mergeLoadedEvents(log, [slotEvent(2)]); + + expect(log.maxSlot).toBe(2); + expect(reserveSlot(log)).toBe(5); + }); + it('deduplicates merged events by id', () => { const log = toMutableEventLog([slotEvent(1)], 'c0'); mergeLoadedEvents(log, [slotEvent(1), slotEvent(2)]); @@ -639,7 +652,7 @@ describe('slot bookkeeping', () => { eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY - 1, { extraEvents: 1, }); - expect(log.reserved).toBe(0); + expect(log.nextSlot).toBe(1); }); it('proposes no event id for a ULID-numbered run', () => { @@ -647,7 +660,7 @@ describe('slot bookkeeping', () => { const log = toMutableEventLog([], null); const fence = eventCreateFenceFor(log, SPEC_VERSION_SLOT_IDENTITY - 1); expect(fence?.eventId).toBeUndefined(); - expect(log.reserved).toBe(0); + expect(log.nextSlot).toBe(1); }); }); @@ -859,6 +872,93 @@ describe('withEventCreateFence', () => { }); }); +describe('stepClaimFence', () => { + const slotEvent = (slot: number) => makeEvent(slotEventId(slot)); + + beforeEach(() => { + eventsListMock.mockReset(); + }); + + it('numbers a batch in the order its claims were built, not the order they fire', async () => { + // The batch is built during replay and only starts racing afterwards, so + // the slot of each member has to be fixed at build time for the numbering + // to be replay-stable. + const log = toMutableEventLog([slotEvent(1)], 'c0'); + const first = stepClaimFence('wrun_test', log, SPEC_VERSION_SLOT_IDENTITY, { + extraEvents: 1, + }); + const second = stepClaimFence('wrun_test', log, SPEC_VERSION_SLOT_IDENTITY); + + const claimed: (string | undefined)[] = []; + const record = (fence?: { eventId?: string }) => { + claimed.push(fence?.eventId); + return Promise.resolve('ok'); + }; + // Fired in reverse: the numbering must not depend on it. + await second(record); + await first(record); + + expect(claimed).toEqual([slotEventId(4), slotEventId(3)]); + }); + + it('reclaims a lost slot in place, keeping the batch adjacent', async () => { + // The server allocates an outside event from the same next-free pointer + // the client reserves from, so losing a claim is routine. Abandoning the + // batch on it would leave this step's events far later in the log than its + // siblings' — an order no single replay can consume — and leave the lost + // slot permanently empty. + const log = toMutableEventLog([slotEvent(1)], 'c0'); + const claim = stepClaimFence('wrun_test', log, SPEC_VERSION_SLOT_IDENTITY); + const sibling = stepClaimFence( + 'wrun_test', + log, + SPEC_VERSION_SLOT_IDENTITY + ); + + const claimed: string[] = []; + const op = vi.fn(async (fence?: { eventId?: string }) => { + claimed.push(fence?.eventId as string); + if (claimed.length === 1) { + // An out-of-band hook took slot 2 while the batch was being built. + throw new SlotConflictError('taken', { + eventId: fence?.eventId as string, + events: [slotEvent(2)], + cursor: 'c1', + }); + } + return 'done'; + }); + + await expect(claim(op)).resolves.toBe('done'); + await expect(sibling(async (f) => f?.eventId)).resolves.toBe( + slotEventId(3) + ); + // Reclaimed above the merged event rather than propagating the conflict. + expect(claimed).toEqual([slotEventId(2), slotEventId(4)]); + }); + + it('leaves a ULID-numbered batch on one shared watermark, unretried', async () => { + // A 412 compares time, so every member of the batch carries the same fence + // value and the batch fails as a unit — which is what the caller's + // fresh-replay path expects. + const time = 1_700_000_000_000; + const log = toMutableEventLog([makeUlidEvent(time)], 'c0'); + const claim = stepClaimFence( + 'wrun_test', + log, + SPEC_VERSION_SLOT_IDENTITY - 1 + ); + const op = vi.fn(async () => { + throw new PreconditionFailedError('stale'); + }); + + await expect(claim(op)).rejects.toBeInstanceOf(PreconditionFailedError); + expect(op).toHaveBeenCalledTimes(1); + expect(op).toHaveBeenCalledWith({ stateUpdatedAt: time }); + expect(eventsListMock).not.toHaveBeenCalled(); + }); +}); + describe('requiresFreshReplay', () => { it('covers both fences, so neither numbering fails the run', () => { // Each fence reports an incomplete view in its own dialect. A caller that diff --git a/packages/core/src/runtime/helpers.ts b/packages/core/src/runtime/helpers.ts index 407f202635..c1a9248f32 100644 --- a/packages/core/src/runtime/helpers.ts +++ b/packages/core/src/runtime/helpers.ts @@ -662,11 +662,11 @@ export interface MutableEventLog { */ maxSlot: number; /** - * Slots handed out by `reserveSlot` past `maxSlot` whose events have not been - * merged back yet. Reset whenever the log is merged into, because the merged - * events are the authority on which slots are taken. + * Next slot `reserveSlot` will hand out. Absolute, and it only ever moves + * forward: a merge can raise it past the events it brought in, but must never + * lower it onto a slot already handed to a writer that is still in flight. */ - reserved: number; + nextSlot: number; } /** @@ -683,17 +683,18 @@ export function toMutableEventLog( cursor: string | null, slotFloor = 0 ): MutableEventLog { + const maxSlot = Math.max(maxSlotOf(events), slotFloor); return { events, cursor, - maxSlot: Math.max(maxSlotOf(events), slotFloor), - reserved: 0, + maxSlot, + nextSlot: maxSlot + 1, }; } /** * Merges loaded events into `log` in place, keeping `maxSlot` current and - * dropping outstanding reservations (the merged events supersede them). + * advancing the reservation pointer past the events merged in. */ export function mergeLoadedEvents( log: MutableEventLog, @@ -701,7 +702,7 @@ export function mergeLoadedEvents( ): void { appendUniqueEvents(log.events, events); log.maxSlot = Math.max(log.maxSlot, maxSlotOf(events)); - log.reserved = 0; + log.nextSlot = Math.max(log.nextSlot, log.maxSlot + 1); } /** @@ -713,10 +714,16 @@ export function mergeLoadedEvents( * in the flush would propose the same slot and all but one would conflict, on * every single flush. Operations are built in deterministic replay order, so * the slot each one draws is replay-stable too. + * + * The pointer is never rewound by a merge, only pushed forward. A writer that + * loses its slot merges the delta and reserves again while its siblings still + * hold theirs; rewinding to `maxSlot + 1` would hand it a sibling's slot and + * turn one conflict into a chain of them. */ export function reserveSlot(log: MutableEventLog): number { - log.reserved += 1; - return log.maxSlot + log.reserved; + const slot = log.nextSlot; + log.nextSlot = slot + 1; + return slot; } /** @@ -859,7 +866,7 @@ export interface EventCreateFence { /** * The fence for a create that is deliberately **not** retried in place, because * a rejection means the committed decision itself is stale and only a fresh - * replay can revise it (`run_completed`, an inline `step_started` claim). + * replay can revise it (`run_completed`). * * Claims a slot off `log` for a slot-numbered run — which counts as a * reservation, so a caller that fences several creates from one log gets a @@ -885,19 +892,80 @@ export function eventCreateFenceFor( options?: { extraEvents?: number } ): EventCreateFence | undefined { if (usesSlotIdentity(specVersion)) { - const maxSlot = log.maxSlot; - // The extra events sit below the one being created, matching the order a - // reader expects (a step is created before it starts) — so their slots are - // reserved first and the claim names the last of the run. - for (let i = 0; i < (options?.extraEvents ?? 0); i++) { - reserveSlot(log); - } - return { eventId: slotEventId(reserveSlot(log)), maxSlot }; + return reserveSlotFence(log, options?.extraEvents ?? 0); } const stateUpdatedAt = stateUpdatedAtForCreate(log.events, specVersion); return stateUpdatedAt !== undefined ? { stateUpdatedAt } : undefined; } +/** + * Reserves this write's slots off `log` and names the one the event itself + * takes. + * + * `extraEvents` sit below the one being created, matching the order a reader + * expects (a step is created before it starts), so their slots are reserved + * first and the claim names the last of the run — a World that writes a pair + * derives the lower id from the one it was given. + */ +function reserveSlotFence( + log: MutableEventLog, + extraEvents: number +): EventCreateFence { + const maxSlot = log.maxSlot; + for (let i = 0; i < extraEvents; i++) { + reserveSlot(log); + } + return { eventId: slotEventId(reserveSlot(log)), maxSlot }; +} + +/** + * Runs one event create under whichever fence its run uses, handling a lost + * claim however that run's scheme requires. + */ +export type FencedCreate = ( + op: (fence: EventCreateFence | undefined) => Promise +) => Promise; + +/** + * The fence for an inline step's `step_started` claim. + * + * A slot-numbered run retries a lost claim in place; a watermark-guarded run + * does not, and lets the rejection abandon the batch for a fresh replay. The + * asymmetry is in what a rejection proves, and it is the difference between a + * batch that stays contiguous and one that splits: + * + * - A 412 compares the *time* of the newest outside event, so every claim in a + * batch carries the same fence value and a stale view fails all of them. The + * batch is abandoned as a unit, nothing is written, and the fresh replay + * reschedules from a complete view. + * - A 409 only proves another writer took this write's *number*. That happens + * routinely without any staleness: the server allocates outside events from + * the same next-free pointer the client reserves from, so any outside event + * landing mid-batch takes the slot the batch's next claim is holding. Fencing + * the batch on it splits it — the loser writes nothing while its siblings + * commit, and the loser's events land far later in the log (or never), leaving + * an order no single replay can consume. Taking another number instead keeps + * the batch's events adjacent and its slots dense. + */ +export function stepClaimFence( + runId: string, + log: MutableEventLog, + specVersion: number | undefined, + options?: { extraEvents?: number } +): FencedCreate { + if (usesSlotIdentity(specVersion)) { + // Reserved here, synchronously, rather than when the claim fires: a batch's + // claims have to be numbered in replay order, and they only start racing + // each other afterwards. Retries re-reserve at that point by necessity — + // the merged delta has moved the log — but by then this write is the only + // one of the batch still choosing a slot. + const initialFence = reserveSlotFence(log, options?.extraEvents ?? 0); + return (op) => withSlotRetry(runId, log, op, { ...options, initialFence }); + } + const fence = eventCreateFenceFor(log, specVersion, options); + return (op) => op(fence); +} + /** * Runs a replay-context event creation that claims its own event slot. * @@ -917,15 +985,20 @@ export function eventCreateFenceFor( export async function withSlotRetry( runId: string, log: MutableEventLog, - op: (fence: EventCreateFence) => Promise + op: (fence: EventCreateFence) => Promise, + options?: { extraEvents?: number; initialFence?: EventCreateFence } ): Promise { for (let attempt = 0; ; attempt++) { // Claimed per attempt, not once up front: a merged delta moves the log's - // high-water mark, so the previous claim is stale by definition. - const maxSlot = log.maxSlot; - const eventId = slotEventId(reserveSlot(log)); + // high-water mark, so the previous claim is stale by definition. The first + // attempt can carry a slot the caller reserved earlier, for a caller whose + // numbering has to be assigned in a particular order (see stepClaimFence). + const fence = + (attempt === 0 ? options?.initialFence : undefined) ?? + reserveSlotFence(log, options?.extraEvents ?? 0); + const eventId = fence.eventId; try { - return await op({ eventId, maxSlot }); + return await op(fence); } catch (error) { if ( !SlotConflictError.is(error) || diff --git a/packages/core/src/runtime/step-executor.test.ts b/packages/core/src/runtime/step-executor.test.ts index 4cb6f6b8f1..8b4d99e746 100644 --- a/packages/core/src/runtime/step-executor.test.ts +++ b/packages/core/src/runtime/step-executor.test.ts @@ -4,7 +4,7 @@ import { join } from 'node:path'; import type { Event, World } from '@workflow/world'; import { SPEC_VERSION_CURRENT } from '@workflow/world'; import { createWorld } from '@workflow/world-local'; -import { afterEach, describe, expect, it } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import { registerStepFunction } from '../private.js'; import { dehydrateStepArguments } from '../serialization.js'; import { executeStep } from './step-executor.js'; @@ -164,3 +164,39 @@ describe('executeStep — retry ceiling (authoritativeAttempt)', () => { expect(ceilingFailures).toHaveLength(0); }); }); + +describe('executeStep — claim fence plumbing', () => { + afterEach(() => { + counter += 1; + }); + + it("runs the step_started claim under the caller's fence", async () => { + const world = makeWorld(); + const stepName = uniqueStepName(); + const { runId, stepId } = await setupRunningStep({ + world, + stepName, + onBody: () => {}, + }); + + // The fence rides in CreateEventParams, which world-local does not + // persist — so observe the call itself rather than the stored event. + const createSpy = vi.spyOn(world.events, 'create'); + + await executeStep({ + world, + workflowRunId: runId, + workflowName: 'wf', + workflowStartedAt: Date.now(), + stepId, + stepName, + claimFence: (op) => op({ stateUpdatedAt: 1_700_000_000_000 }), + }); + + const started = createSpy.mock.calls.filter( + ([, data]) => data.eventType === 'step_started' + ); + expect(started).toHaveLength(1); + expect(started[0]?.[2]?.stateUpdatedAt).toBe(1_700_000_000_000); + }); +}); diff --git a/packages/core/src/runtime/step-executor.ts b/packages/core/src/runtime/step-executor.ts index e1823cfb24..89a46947ff 100644 --- a/packages/core/src/runtime/step-executor.ts +++ b/packages/core/src/runtime/step-executor.ts @@ -43,7 +43,7 @@ import { isOptimisticInlineStartExplicitlyDisabled, } from './constants.js'; import { getPortLazy } from './get-port-lazy.js'; -import { type EventCreateFence, memoizeEncryptionKey } from './helpers.js'; +import { type FencedCreate, memoizeEncryptionKey } from './helpers.js'; import { computeStepLatencyEventData, type StepLatencyEventData, @@ -129,23 +129,27 @@ export interface StepExecutorParams { */ inlineDeltaSinceCursor?: string; /** - * Concurrency fence to attach to this step's `step_started` claim: the event - * slot the claim occupies, or the caller's replay snapshot (`stateUpdatedAt`, - * epoch ms of the latest event it loaded) for a run on the older numbering. + * Runs this step's `step_started` claim under its run's concurrency fence: + * the event slot the claim occupies, or the caller's replay snapshot + * (`stateUpdatedAt`, epoch ms of the latest event it loaded) for a run on the + * older numbering. * * On the lazy inline path the claim is the step's FIRST durable write (its * `step_created` is deferred), so without a fence it would be unguarded * entirely: a replay working from a stale view could claim — and then commit — * a step scheduled without observing an out-of-band event. A fencing World * rejects such a claim with `SlotConflictError` (409) or - * `PreconditionFailedError` (412); executeStep does NOT translate either - * rejection (re-claiming in place would still commit the stale schedule), so - * it propagates for the caller to abandon the batch and force a fresh replay. + * `PreconditionFailedError` (412). + * + * Whether a rejection is retried in place is the caller's decision, made per + * scheme — see `stepClaimFence`. Either way executeStep does NOT translate a + * rejection that reaches it, so an unretried one propagates for the caller to + * abandon the batch and force a fresh replay. * * Undefined when the caller has no snapshot, or when the watermark guard is * disabled on a run that uses it; Worlds that fence neither way ignore it. */ - eventCreateFence?: EventCreateFence; + claimFence?: FencedCreate; /** * Suppress optimistic inline start for this step regardless of * `WORKFLOW_OPTIMISTIC_INLINE_START` / `forceOptimisticStart`: take the @@ -253,6 +257,10 @@ export async function executeStep( stepName, } = params; const isVercel = process.env.VERCEL_URL !== undefined; + // Unfenced when the caller passes no fence — every World that fences + // ignores the field it does not understand, so this is the same create it + // was before either mechanism existed. + const runClaim: FencedCreate = params.claimFence ?? ((op) => op(undefined)); // Gate payload compression on the run's specVersion. const compression = (params.runSpecVersion ?? 0) >= SPEC_VERSION_SUPPORTS_COMPRESSION; @@ -543,27 +551,29 @@ export async function executeStep( // RSFS measures the run_started-to-POST stretch, and the barrier // wait IS part of that stretch under turbo. stepStartPostSentAtMs = Date.now(); - return world.events.create( - workflowRunId, - { - eventType: 'step_started', - specVersion: SPEC_VERSION_CURRENT, - correlationId: stepId, - eventData: { - stepName, - workflowName, - input: params.lazyStepInput, - // Inline-ownership stamp — see StepExecutorParams.ownerMessageId. - ...(params.ownerMessageId !== undefined - ? { ownerMessageId: params.ownerMessageId } - : {}), + // Fence the claim — see StepExecutorParams.claimFence. A rejection + // the fence does not retry surfaces via reconcileOptimisticStart as a + // non-translatable error: the body result is discarded and the + // rejection propagates to the caller. + return runClaim((fence) => + world.events.create( + workflowRunId, + { + eventType: 'step_started', + specVersion: SPEC_VERSION_CURRENT, + correlationId: stepId, + eventData: { + stepName, + workflowName, + input: params.lazyStepInput, + // Inline-ownership stamp — see StepExecutorParams.ownerMessageId. + ...(params.ownerMessageId !== undefined + ? { ownerMessageId: params.ownerMessageId } + : {}), + }, }, - }, - // Fence the claim — see StepExecutorParams.eventCreateFence. A 409/412 - // rejection surfaces via reconcileOptimisticStart as a - // non-translatable error: the body result is discarded and the - // rejection propagates to the caller. - params.eventCreateFence + fence + ) ); } ); @@ -604,26 +614,29 @@ export async function executeStep( ? { ownerMessageId: params.ownerMessageId } : {}; stepStartPostSentAtMs = Date.now(); - const startResult = await world.events.create( - workflowRunId, - { - eventType: 'step_started', - specVersion: SPEC_VERSION_CURRENT, - correlationId: stepId, - eventData: - params.lazyStepInput !== undefined - ? { - stepName, - workflowName, - input: params.lazyStepInput, - ...ownershipStamp, - } - : { stepName, ...ownershipStamp }, - }, - // Fence the claim — see StepExecutorParams.eventCreateFence. A 409/412 - // rejection is intentionally NOT translated by startErrorToResult - // below, so it propagates to the caller for a fresh replay. - params.eventCreateFence + // Fence the claim — see StepExecutorParams.claimFence. A rejection the + // fence does not retry is intentionally NOT translated by + // startErrorToResult below, so it propagates to the caller for a fresh + // replay. + const startResult = await runClaim((fence) => + world.events.create( + workflowRunId, + { + eventType: 'step_started', + specVersion: SPEC_VERSION_CURRENT, + correlationId: stepId, + eventData: + params.lazyStepInput !== undefined + ? { + stepName, + workflowName, + input: params.lazyStepInput, + ...ownershipStamp, + } + : { stepName, ...ownershipStamp }, + }, + fence + ) ); if (!startResult.step) { From f66196cdedd44156b0f4a08ae69726e4ffa005ca Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 13:16:22 -0700 Subject: [PATCH 14/28] chore: untrack local e2e diagnostics dump --- .gitignore | 3 + e2e-diagnostics-nextjs-turbopack-local.json | 848 -------------------- 2 files changed, 3 insertions(+), 848 deletions(-) delete mode 100644 e2e-diagnostics-nextjs-turbopack-local.json diff --git a/.gitignore b/.gitignore index 8b7641f7d7..28b5575028 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,6 @@ packages/swc-plugin-workflow/build-hash.json workbench/nextjs-*/public/.well-known/workflow workbench/sveltekit/static/.well-known/workflow + +# Local e2e diagnostics dumps +e2e-diagnostics-*.json diff --git a/e2e-diagnostics-nextjs-turbopack-local.json b/e2e-diagnostics-nextjs-turbopack-local.json deleted file mode 100644 index 73a3cda9df..0000000000 --- a/e2e-diagnostics-nextjs-turbopack-local.json +++ /dev/null @@ -1,848 +0,0 @@ -[ - { - "testName": "addTenWorkflow", - "runId": "wrun_01KYWTGT53YMY8BX4JKWHYVPD9", - "timestamp": "2026-07-31T19:30:49.649Z", - "dashboardUrl": null - }, - { - "testName": "addTenWorkflow", - "runId": "wrun_01KYWTGWX1Z0ZV3HYPFH7SKYCC", - "timestamp": "2026-07-31T19:30:52.452Z", - "dashboardUrl": null - }, - { - "testName": "deploymentId: 'latest' is a no-op in non-Vercel worlds", - "runId": "wrun_01KYWTGZE8309234ZWEJNK29QB", - "timestamp": "2026-07-31T19:30:55.051Z", - "dashboardUrl": null - }, - { - "testName": "wellKnownAgentWorkflow (.well-known/agent)", - "runId": "wrun_01KYWTH207G4MECBEGA07TJ08W", - "timestamp": "2026-07-31T19:30:57.674Z", - "dashboardUrl": null - }, - { - "testName": "should work with react rendering in step", - "runId": "wrun_01KYWTH2ZQCE82XFR5GER6HMNA", - "timestamp": "2026-07-31T19:30:58.687Z", - "dashboardUrl": null - }, - { - "testName": "promiseAllWorkflow", - "runId": "wrun_01KYWTH3ZATA1AMTRNKR26AEEJ", - "timestamp": "2026-07-31T19:30:59.693Z", - "dashboardUrl": null - }, - { - "testName": "promiseRaceWorkflow", - "runId": "wrun_01KYWTH6XDT15ZVPWGYW1W1J46", - "timestamp": "2026-07-31T19:31:02.708Z", - "dashboardUrl": null - }, - { - "testName": "promiseAnyWorkflow", - "runId": "wrun_01KYWTHVHXDYNC7JTAF25QGT3X", - "timestamp": "2026-07-31T19:31:23.876Z", - "dashboardUrl": null - }, - { - "testName": "importedStepOnlyWorkflow", - "runId": "wrun_01KYWTJ2F4SDBP0WAEE3FQXTA3", - "timestamp": "2026-07-31T19:31:30.970Z", - "dashboardUrl": null - }, - { - "testName": "readableStreamWorkflow", - "runId": "wrun_01KYWTJ3GCRERJ80FMM6C9WK61", - "timestamp": "2026-07-31T19:31:32.027Z", - "dashboardUrl": null - }, - { - "testName": "hookWorkflow", - "runId": "wrun_01KYWTJCGAQTY2S8DHXNKMTGAH", - "timestamp": "2026-07-31T19:31:41.249Z", - "dashboardUrl": null - }, - { - "testName": "hookWorkflow is not resumable via public webhook endpoint", - "runId": "wrun_01KYWTJE0N3Q65FDEMJ60486FC", - "timestamp": "2026-07-31T19:31:42.767Z", - "dashboardUrl": null - }, - { - "testName": "webhookWorkflow", - "runId": "wrun_01KYWTJHWC6H21T1PBSHHG11R0", - "timestamp": "2026-07-31T19:31:46.745Z", - "dashboardUrl": null - }, - { - "testName": "parallelStepsThenWebhookWorkflow - no hook_conflict from same-tick replay race", - "runId": "wrun_01KYWTJNEDJWC6TY0NXR6XNH62", - "timestamp": "2026-07-31T19:31:50.398Z", - "dashboardUrl": null - }, - { - "testName": "sleepingWorkflow", - "runId": "wrun_01KYWTK9MFDYNBSAX5CQ61HCY0", - "timestamp": "2026-07-31T19:32:11.048Z", - "dashboardUrl": null - }, - { - "testName": "parallelSleepWorkflow", - "runId": "wrun_01KYWTKMEEB8W3007D7Q9APF9Z", - "timestamp": "2026-07-31T19:32:22.098Z", - "dashboardUrl": null - }, - { - "testName": "sleepWinsRaceWorkflow", - "runId": "wrun_01KYWTKPD8KK1TP8KRMX97HCTW", - "timestamp": "2026-07-31T19:32:24.108Z", - "dashboardUrl": null - }, - { - "testName": "stepWinsRaceWorkflow", - "runId": "wrun_01KYWTKRBZHA0PAHBYH9MTMHX4", - "timestamp": "2026-07-31T19:32:26.115Z", - "dashboardUrl": null - }, - { - "testName": "nullByteWorkflow", - "runId": "wrun_01KYWTKTASR8KS1S97MFKHH6AC", - "timestamp": "2026-07-31T19:32:28.130Z", - "dashboardUrl": null - }, - { - "testName": "workflowAndStepMetadataWorkflow", - "runId": "wrun_01KYWTKVADTSZD3QFQ1QDYFX01", - "timestamp": "2026-07-31T19:32:29.137Z", - "dashboardUrl": null - }, - { - "testName": "no startIndex (reads all chunks)", - "runId": "wrun_01KYWTKW9XYD8K1B7TK1PWFVRD", - "timestamp": "2026-07-31T19:32:30.147Z", - "dashboardUrl": null - }, - { - "testName": "positive startIndex (skips first chunk)", - "runId": "wrun_01KYWTM2WJ4E2VAKKBA3RFB6KS", - "timestamp": "2026-07-31T19:32:36.885Z", - "dashboardUrl": null - }, - { - "testName": "negative startIndex (reads from end)", - "runId": "wrun_01KYWTM8RQQFS4KYD92DXDQVJA", - "timestamp": "2026-07-31T19:32:42.909Z", - "dashboardUrl": null - }, - { - "testName": "getTailIndex returns correct index after stream completes", - "runId": "wrun_01KYWTMEMXEM02429V1HGVWJSW", - "timestamp": "2026-07-31T19:32:48.929Z", - "dashboardUrl": null - }, - { - "testName": "getTailIndex returns -1 before any chunks are written", - "runId": "wrun_01KYWTMMGZ0EXXTPC7Q0CP4NVK", - "timestamp": "2026-07-31T19:32:54.951Z", - "dashboardUrl": null - }, - { - "testName": "getChunks returns same content as reading the stream", - "runId": "wrun_01KYWTMMH9ZKJ3RMQ06H6GVKYF", - "timestamp": "2026-07-31T19:32:54.961Z", - "dashboardUrl": null - }, - { - "testName": "outputStreamInsideStepWorkflow - getWritable() called inside step functions", - "runId": "wrun_01KYWTMVCPH5ZQ127WFA7DR4DQ", - "timestamp": "2026-07-31T19:33:01.977Z", - "dashboardUrl": null - }, - { - "testName": "utf8StreamWorkflow", - "runId": "wrun_01KYWTN1018KV6MGDG65CF52G8", - "timestamp": "2026-07-31T19:33:07.716Z", - "dashboardUrl": null - }, - { - "testName": "writableForwardedFromWorkflowWorkflow", - "runId": "wrun_01KYWTN3BBG0RVQJWB7YCEW80B", - "timestamp": "2026-07-31T19:33:10.126Z", - "dashboardUrl": null - }, - { - "testName": "writableForwardedFromStepWorkflow", - "runId": "wrun_01KYWTN4H8D8Y40MNV6MK9JA72", - "timestamp": "2026-07-31T19:33:11.338Z", - "dashboardUrl": null - }, - { - "testName": "fetchWorkflow", - "runId": "wrun_01KYWTN5KWPKK4552YGNZ8D672", - "timestamp": "2026-07-31T19:33:12.447Z", - "dashboardUrl": null - }, - { - "testName": "promiseRaceStressTestWorkflow", - "runId": "wrun_01KYWTN6K9NCKYF6F4HZDTYJ6P", - "timestamp": "2026-07-31T19:33:13.454Z", - "dashboardUrl": null - }, - { - "testName": "nested function calls preserve message and stack trace", - "runId": "wrun_01KYWTNV5B5Q920TCE4HKQA1P5", - "timestamp": "2026-07-31T19:33:34.528Z", - "dashboardUrl": null - }, - { - "testName": "cross-file imports preserve message and stack trace", - "runId": "wrun_01KYWTP0HG449F33YGC8N3VZ15", - "timestamp": "2026-07-31T19:33:40.020Z", - "dashboardUrl": null - }, - { - "testName": "basic step error preserves message and stack trace", - "runId": "wrun_01KYWTP8DMRKPH4YVREB8883ZX", - "timestamp": "2026-07-31T19:33:48.093Z", - "dashboardUrl": null - }, - { - "testName": "cross-file step error preserves message and function names in stack", - "runId": "wrun_01KYWTPT1K314ZS28VXBAMJD49", - "timestamp": "2026-07-31T19:34:06.140Z", - "dashboardUrl": null - }, - { - "testName": "regular Error retries until success", - "runId": "wrun_01KYWTQ136BMTVANXKY45RB3FC", - "timestamp": "2026-07-31T19:34:13.361Z", - "dashboardUrl": null - }, - { - "testName": "FatalError fails immediately without retries", - "runId": "wrun_01KYWTQ7DQ6AHDBPAAQ08074P7", - "timestamp": "2026-07-31T19:34:19.835Z", - "dashboardUrl": null - }, - { - "testName": "RetryableError respects custom retryAfter delay", - "runId": "wrun_01KYWTQF8E1T2R49Z0PA7ECJ9M", - "timestamp": "2026-07-31T19:34:27.866Z", - "dashboardUrl": null - }, - { - "testName": "maxRetries=0 disables retries", - "runId": "wrun_01KYWTQT136FT5047CJEC29SH2", - "timestamp": "2026-07-31T19:34:38.887Z", - "dashboardUrl": null - }, - { - "testName": "FatalError can be caught and detected with FatalError.is()", - "runId": "wrun_01KYWTQV0H9R02DPQZ4NNTBYGF", - "timestamp": "2026-07-31T19:34:39.893Z", - "dashboardUrl": null - }, - { - "testName": "step throw round-trips FatalError with cause chain to workflow catch", - "runId": "wrun_01KYWTR0JNPEGQWP23K0860MND", - "timestamp": "2026-07-31T19:34:45.595Z", - "dashboardUrl": null - }, - { - "testName": "workflow throw round-trips FatalError + cause through run_failed event", - "runId": "wrun_01KYWTR57Z8SFEDC2F2E1AB53V", - "timestamp": "2026-07-31T19:34:50.371Z", - "dashboardUrl": null - }, - { - "testName": "workflow throw of a non-Error value round-trips verbatim as cause", - "runId": "wrun_01KYWTR95MW35V8DZZ79BQHEVX", - "timestamp": "2026-07-31T19:34:54.393Z", - "dashboardUrl": null - }, - { - "testName": "step throw of a non-Error value preserves it as cause on the wrapping FatalError", - "runId": "wrun_01KYWTRCYWR0B9RXDF3QZ676E6", - "timestamp": "2026-07-31T19:34:58.273Z", - "dashboardUrl": null - }, - { - "testName": "WorkflowNotRegisteredError fails the run when workflow does not exist", - "runId": "wrun_01KYWTRG1WPWABASTGR7PVFG8H", - "timestamp": "2026-07-31T19:35:01.442Z", - "dashboardUrl": null - }, - { - "testName": "StepNotRegisteredError fails the step but workflow can catch it", - "runId": "wrun_01KYWTRK81ZETDCHGTAMAERV0P", - "timestamp": "2026-07-31T19:35:04.710Z", - "dashboardUrl": null - }, - { - "testName": "StepNotRegisteredError fails the run when not caught in workflow", - "runId": "wrun_01KYWTRRGPRHS7AVTGTG71MA4D", - "timestamp": "2026-07-31T19:35:10.110Z", - "dashboardUrl": null - }, - { - "testName": "hookCleanupTestWorkflow - hook token reuse after workflow completion", - "runId": "wrun_01KYWTRSJJ14ZPKGZZ1NM10R2P", - "timestamp": "2026-07-31T19:35:11.190Z", - "dashboardUrl": null - }, - { - "testName": "hookCleanupTestWorkflow - hook token reuse after workflow completion", - "runId": "wrun_01KYWTRTT9KG8R6PSJPZTKDZ70", - "timestamp": "2026-07-31T19:35:12.461Z", - "dashboardUrl": null - }, - { - "testName": "concurrent hook token conflict - two workflows cannot use the same hook token simultaneously", - "runId": "wrun_01KYWTS12RRCJQMRWZVNFFRXB8", - "timestamp": "2026-07-31T19:35:18.877Z", - "dashboardUrl": null - }, - { - "testName": "concurrent hook token conflict - two workflows cannot use the same hook token simultaneously", - "runId": "wrun_01KYWTS1AT7M9Z6YN7K946BNT3", - "timestamp": "2026-07-31T19:35:19.133Z", - "dashboardUrl": null - }, - { - "testName": "hookGetConflictWorkflow - awaiting hook.getConflict() registers hook without payload", - "runId": "wrun_01KYWTS8ETNFDTFHGVBT3962PK", - "timestamp": "2026-07-31T19:35:26.430Z", - "dashboardUrl": null - }, - { - "testName": "'hookGetConflictWithPriorStepWorkflow' - hook.getConflict() does not block step execution", - "runId": "wrun_01KYWTS9E9N19K42B08N4QGTF6", - "timestamp": "2026-07-31T19:35:27.437Z", - "dashboardUrl": null - }, - { - "testName": "'hookGetConflictWithParallelStepWorkfl…' - hook.getConflict() does not block step execution", - "runId": "wrun_01KYWTSADVMTC1KA9JSJDGVP7N", - "timestamp": "2026-07-31T19:35:28.449Z", - "dashboardUrl": null - }, - { - "testName": "hookGetConflictThenStepParallelWorkflow - hook.getConflict() continuation step runs alongside other steps", - "runId": "wrun_01KYWTSBDEHP6T5E2QTV1HT105", - "timestamp": "2026-07-31T19:35:29.458Z", - "dashboardUrl": null - }, - { - "testName": "hookGetConflictWorkflow - hook.getConflict() resolves with the conflicting run when token is already registered", - "runId": "wrun_01KYWTSP5RG0TRGZHK55B6PZY5", - "timestamp": "2026-07-31T19:35:40.481Z", - "dashboardUrl": null - }, - { - "testName": "hookGetConflictWorkflow - hook.getConflict() resolves with the conflicting run when token is already registered", - "runId": "wrun_01KYWTSPDXXQ0MBYVY2BH2Y1VY", - "timestamp": "2026-07-31T19:35:40.739Z", - "dashboardUrl": null - }, - { - "testName": "hookClaimOnlyMutexWorkflow - hook works as a pure run mutex without payload data", - "runId": "wrun_01KYWTSTYCHBMGS8S7FTEW4V81", - "timestamp": "2026-07-31T19:35:45.367Z", - "dashboardUrl": null - }, - { - "testName": "hookClaimOnlyMutexWorkflow - hook works as a pure run mutex without payload data", - "runId": "wrun_01KYWTSV6M0G7DT6A05NY0HYFJ", - "timestamp": "2026-07-31T19:35:45.625Z", - "dashboardUrl": null - }, - { - "testName": "hookClaimOnlyMutexWorkflow - hook works as a pure run mutex without payload data", - "runId": "wrun_01KYWTT9W6X75NJTAQ8CYVR6G4", - "timestamp": "2026-07-31T19:36:00.649Z", - "dashboardUrl": null - }, - { - "testName": "hookAdoptOwnerResultWorkflow - duplicate adopts the owner result via conflict.returnValue", - "runId": "wrun_01KYWTTAVK7FBSX4D3MRQAMGKQ", - "timestamp": "2026-07-31T19:36:01.654Z", - "dashboardUrl": null - }, - { - "testName": "hookAdoptOwnerResultWorkflow - duplicate adopts the owner result via conflict.returnValue", - "runId": "wrun_01KYWTTB3H2DH8X898J6HT87YK", - "timestamp": "2026-07-31T19:36:01.908Z", - "dashboardUrl": null - }, - { - "testName": "hookSignalOwnerWorkflow - duplicate forwards its payload to the owner via resumeHook", - "runId": "wrun_01KYWTTCJZFQQVYTFC3DGQ9AZ6", - "timestamp": "2026-07-31T19:36:03.428Z", - "dashboardUrl": null - }, - { - "testName": "hookSignalOwnerWorkflow - duplicate forwards its payload to the owner via resumeHook", - "runId": "wrun_01KYWTTCTZASVMDC9MZ9ZGZKMT", - "timestamp": "2026-07-31T19:36:03.682Z", - "dashboardUrl": null - }, - { - "testName": "hookSupersedeOwnerWorkflow - duplicate cancels the owner and claims the released token", - "runId": "wrun_01KYWTTDTC0X9R0J8Y6NQN9GY6", - "timestamp": "2026-07-31T19:36:04.689Z", - "dashboardUrl": null - }, - { - "testName": "hookSupersedeOwnerWorkflow - duplicate cancels the owner and claims the released token", - "runId": "wrun_01KYWTTE2D2AZMN7S11B7NHQVD", - "timestamp": "2026-07-31T19:36:04.945Z", - "dashboardUrl": null - }, - { - "testName": "resume-or-start route pattern - resumeHook retried after start() reaches the new run", - "runId": "wrun_01KYWTTHVYA70TF6TMD5VA5TMY", - "timestamp": "2026-07-31T19:36:08.834Z", - "dashboardUrl": null - }, - { - "testName": "hookDisposeTestWorkflow - hook token reuse after explicit disposal while workflow still running", - "runId": "wrun_01KYWTTK3KSBK34YQG0AKVDB5S", - "timestamp": "2026-07-31T19:36:10.104Z", - "dashboardUrl": null - }, - { - "testName": "hookDisposeTestWorkflow - hook token reuse after explicit disposal while workflow still running", - "runId": "wrun_01KYWTTMBBY2H750S8BFZMW4NA", - "timestamp": "2026-07-31T19:36:11.376Z", - "dashboardUrl": null - }, - { - "testName": "hookTokenReuseLoopWorkflow - same run recreates a hook with the same token after dispose()", - "runId": "wrun_01KYWTTZ681MXD232MBCFC31QQ", - "timestamp": "2026-07-31T19:36:22.478Z", - "dashboardUrl": null - }, - { - "testName": "stepFunctionPassingWorkflow - step function references can be passed as arguments (without closure vars)", - "runId": "wrun_01KYWTV36WNVT61JF7RHTRMFD6", - "timestamp": "2026-07-31T19:36:26.592Z", - "dashboardUrl": null - }, - { - "testName": "stepFunctionWithClosureWorkflow - step function with closure variables passed as argument", - "runId": "wrun_01KYWTV81NV7Y86Q8BBNARSF4M", - "timestamp": "2026-07-31T19:36:31.545Z", - "dashboardUrl": null - }, - { - "testName": "closureVariableWorkflow - nested step functions with closure variables", - "runId": "wrun_01KYWTVASK303FKWMH710NR50N", - "timestamp": "2026-07-31T19:36:34.358Z", - "dashboardUrl": null - }, - { - "testName": "spawnWorkflowFromStepWorkflow - spawning a child workflow using start() inside a step", - "runId": "wrun_01KYWTVBS46NC2RW292EM5CQ4E", - "timestamp": "2026-07-31T19:36:35.379Z", - "dashboardUrl": null - }, - { - "testName": "runClassSerializationWorkflow - Run instances serialize across workflow/step boundaries", - "runId": "wrun_01KYWTVH8PVZGW79CR72G7QCAA", - "timestamp": "2026-07-31T19:36:40.988Z", - "dashboardUrl": null - }, - { - "testName": "startFromWorkflow - calling start() directly inside a workflow function with hook communication", - "runId": "wrun_01KYWTVP84HMHTD6WHA2YEYG82", - "timestamp": "2026-07-31T19:36:46.087Z", - "dashboardUrl": null - }, - { - "testName": "startFromWorkflow - calling start() directly inside a workflow function with hook communication", - "runId": "wrun_01KYWTVP84HMHTD6WHA2YEYG82", - "timestamp": "2026-07-31T19:36:46.087Z", - "dashboardUrl": null - }, - { - "testName": "startFromWorkflow - calling start() directly inside a workflow function with hook communication", - "runId": "wrun_01KYWTVP9FPHZH0VBC29PX675R", - "timestamp": "2026-07-31T19:36:47.090Z", - "dashboardUrl": null - }, - { - "testName": "fibonacciWorkflow - recursive workflow composition via start()", - "runId": "wrun_01KYWTVQ7KVHN8DNS4MS7EB3CS", - "timestamp": "2026-07-31T19:36:47.098Z", - "dashboardUrl": null - }, - { - "testName": "fibonacciWorkflow - recursive workflow composition via start()", - "runId": "wrun_01KYWTVQ7KVHN8DNS4MS7EB3CS", - "timestamp": "2026-07-31T19:36:47.098Z", - "dashboardUrl": null - }, - { - "testName": "pathsAliasWorkflow - TypeScript path aliases resolve correctly", - "runId": "wrun_01KYWTVZRP0MMM4ZY52VNJCK2X", - "timestamp": "2026-07-31T19:36:55.838Z", - "dashboardUrl": null - }, - { - "testName": "Calculator.calculate - static workflow method using static step methods from another class", - "runId": "wrun_01KYWTW2WD30KDSERSAEEMYQTZ", - "timestamp": "2026-07-31T19:36:59.025Z", - "dashboardUrl": null - }, - { - "testName": "AllInOneService.processNumber - static workflow method using sibling static step methods", - "runId": "wrun_01KYWTW6VKT6FVD6A9A5WZPBG7", - "timestamp": "2026-07-31T19:37:03.104Z", - "dashboardUrl": null - }, - { - "testName": "ChainableService.processWithThis - static step methods using `this` to reference the class", - "runId": "wrun_01KYWTWAG87PPD6TGE58KN3YYA", - "timestamp": "2026-07-31T19:37:06.833Z", - "dashboardUrl": null - }, - { - "testName": "thisSerializationWorkflow - step function invoked with .call() and .apply()", - "runId": "wrun_01KYWTWFGCDV5YRGRBC3CHHSVH", - "timestamp": "2026-07-31T19:37:11.955Z", - "dashboardUrl": null - }, - { - "testName": "customSerializationWorkflow - custom class serialization with WORKFLOW_SERIALIZE/WORKFLOW_DESERIALIZE", - "runId": "wrun_01KYWTWKHFD2BNB5PYSM0QTMC1", - "timestamp": "2026-07-31T19:37:16.083Z", - "dashboardUrl": null - }, - { - "testName": "instanceMethodStepWorkflow - instance methods with \"use step\" directive", - "runId": "wrun_01KYWTWQ4FNK1DV94S72HF5NSJ", - "timestamp": "2026-07-31T19:37:19.763Z", - "dashboardUrl": null - }, - { - "testName": "crossContextSerdeWorkflow - classes defined in step code are deserializable in workflow context", - "runId": "wrun_01KYWTWWSRA4RRQ9BN4XK33S0W", - "timestamp": "2026-07-31T19:37:25.566Z", - "dashboardUrl": null - }, - { - "testName": "errorSubclassRoundTripWorkflow - first-class Error subclasses survive every serialization boundary", - "runId": "wrun_01KYWTX02EAN0P7Z6XPZME7AQX", - "timestamp": "2026-07-31T19:37:28.919Z", - "dashboardUrl": null - }, - { - "testName": "stepFunctionAsStartArgWorkflow - step function reference passed as start() argument", - "runId": "wrun_01KYWTX1265YFEX0EDDAH4YSB9", - "timestamp": "2026-07-31T19:37:29.935Z", - "dashboardUrl": null - }, - { - "testName": "cancelRun - cancelling a running workflow", - "runId": "wrun_01KYWTX4M4ZH02XMM25GGVAE1B", - "timestamp": "2026-07-31T19:37:33.577Z", - "dashboardUrl": null - }, - { - "testName": "cancelRun via CLI - cancelling a running workflow", - "runId": "wrun_01KYWTX7DNCPQA2GPFQNGQNRJ0", - "timestamp": "2026-07-31T19:37:36.444Z", - "dashboardUrl": null - }, - { - "testName": "addTenWorkflow via pages router", - "runId": "wrun_01KYWTXD09P8XHTQGMXVWTPEED", - "timestamp": "2026-07-31T19:37:42.269Z", - "dashboardUrl": null - }, - { - "testName": "promiseAllWorkflow via pages router", - "runId": "wrun_01KYWTXE3GJ22QE6RRK1NQ1VPB", - "timestamp": "2026-07-31T19:37:43.330Z", - "dashboardUrl": null - }, - { - "testName": "sleepingWorkflow via pages router", - "runId": "wrun_01KYWTXH33GH4JBD33DK7WEMKA", - "timestamp": "2026-07-31T19:37:46.367Z", - "dashboardUrl": null - }, - { - "testName": "plainModuleDoneHook resumed via plain API route (o2flow shape)", - "runId": "wrun_01KYWTXVWA8C8CVXJ95V5N3QR4", - "timestamp": "2026-07-31T19:37:57.394Z", - "dashboardUrl": null - }, - { - "testName": "hookWithSleepWorkflow - hook payloads delivered correctly with concurrent sleep", - "runId": "wrun_01KYWTXX5N5M947PY128TVP0V6", - "timestamp": "2026-07-31T19:37:58.719Z", - "dashboardUrl": null - }, - { - "testName": "hookWithSleepFinalStepWorkflow - step only on final payload", - "runId": "wrun_01KYWTY1470C7T0W6DPY1WB2M8", - "timestamp": "2026-07-31T19:38:02.764Z", - "dashboardUrl": null - }, - { - "testName": "sleepInLoopWorkflow - sleep inside loop with steps actually delays each iteration", - "runId": "wrun_01KYWTY4MXFRQH3D08N6B33NZ4", - "timestamp": "2026-07-31T19:38:06.370Z", - "dashboardUrl": null - }, - { - "testName": "sleepWithSequentialStepsWorkflow - sequential steps work with concurrent sleep (control)", - "runId": "wrun_01KYWTYBG5HGHG076R85GN5M8G", - "timestamp": "2026-07-31T19:38:13.386Z", - "dashboardUrl": null - }, - { - "testName": "abortTimeoutWorkflow: timeout cancels long-running step", - "runId": "wrun_01KYWTYEMCSGRDE6ZP43T1AG5T", - "timestamp": "2026-07-31T19:38:16.592Z", - "dashboardUrl": null - }, - { - "testName": "abortParallelWorkflow: abort cancels all parallel steps", - "runId": "wrun_01KYWTYJHWZQF85FQ2P5KHK14J", - "timestamp": "2026-07-31T19:38:20.615Z", - "dashboardUrl": null - }, - { - "testName": "abortFromStepWorkflow: step abort cancels an in-flight sibling step", - "runId": "wrun_01KYWTYPFFY1YN6MG939K7WJTC", - "timestamp": "2026-07-31T19:38:24.627Z", - "dashboardUrl": null - }, - { - "testName": "abortAlreadyAbortedWorkflow: pre-aborted signal seen by step", - "runId": "wrun_01KYWTYRE8CVM07RBRP5JT3Z7R", - "timestamp": "2026-07-31T19:38:26.641Z", - "dashboardUrl": null - }, - { - "testName": "abortReasonWorkflow: abort reason preserved across boundaries", - "runId": "wrun_01KYWTYSDW8FJE1ZK195Y2PDKW", - "timestamp": "2026-07-31T19:38:27.654Z", - "dashboardUrl": null - }, - { - "testName": "abortAfterCompletionWorkflow: abort after step completes is a no-op", - "runId": "wrun_01KYWTYWC3STAPMJQ4QTFSMGTV", - "timestamp": "2026-07-31T19:38:30.663Z", - "dashboardUrl": null - }, - { - "testName": "abortViaHookWorkflow: external hook triggers abort on in-flight step", - "runId": "wrun_01KYWTYXBKMDANRQ5SSG0JSY2C", - "timestamp": "2026-07-31T19:38:31.677Z", - "dashboardUrl": null - }, - { - "testName": "abortExternalSignalWorkflow: signal passed as workflow input", - "runId": "wrun_01KYWTYYKTDFEQTWF98360VJJ8", - "timestamp": "2026-07-31T19:38:32.962Z", - "dashboardUrl": null - }, - { - "testName": "abortExternalSignalInFlightWorkflow: external abort fires mid-flight, propagates to nested steps", - "runId": "wrun_01KYWTYZKD92D0N50HTAJVAMC1", - "timestamp": "2026-07-31T19:38:33.968Z", - "dashboardUrl": null - }, - { - "testName": "abortAnyInWorkflowWorkflow: AbortSignal.any composes signals inside the workflow VM", - "runId": "wrun_01KYWTZ2HEFYASWMGQC6Z520RT", - "timestamp": "2026-07-31T19:38:36.979Z", - "dashboardUrl": null - }, - { - "testName": "abortAnyInStepWorkflow: AbortSignal.any inside a step composes deserialized signals", - "runId": "wrun_01KYWTZ3GZ00AADP380JWS0DBN", - "timestamp": "2026-07-31T19:38:37.987Z", - "dashboardUrl": null - }, - { - "testName": "abortSurvivesReplayWorkflow: controller state consistent across replay", - "runId": "wrun_01KYWTZ6F3TPX1Z9AT73MR6WMC", - "timestamp": "2026-07-31T19:38:41.007Z", - "dashboardUrl": null - }, - { - "testName": "abortThrowIfAbortedWorkflow: throwIfAborted causes FatalError, no retries", - "runId": "wrun_01KYWTZ8E3DPT451Z6HARJRBDR", - "timestamp": "2026-07-31T19:38:43.021Z", - "dashboardUrl": null - }, - { - "testName": "abortReasonTypesWorkflow: various abort reason types propagate correctly", - "runId": "wrun_01KYWTZCBM5KS9R4AC2M3Y3EYV", - "timestamp": "2026-07-31T19:38:47.036Z", - "dashboardUrl": null - }, - { - "testName": "abortFetchUncaughtWorkflow: uncaught fetch AbortError is FatalError, no retries", - "runId": "wrun_01KYWTZDB8G3YN8HJ4BSPT01C1", - "timestamp": "2026-07-31T19:38:48.052Z", - "dashboardUrl": null - }, - { - "testName": "abortFetchInFlightWorkflow: aborting cancels an in-flight fetch", - "runId": "wrun_01KYWTZEB2KSFD2EMNZGNZ4BGZ", - "timestamp": "2026-07-31T19:38:49.062Z", - "dashboardUrl": null - }, - { - "testName": "abortVoidSleepTimeoutWorkflow: documented `void sleep().then(abort)` pattern works", - "runId": "wrun_01KYWTZH94PDJPH3HYH0XJDG4K", - "timestamp": "2026-07-31T19:38:52.076Z", - "dashboardUrl": null - }, - { - "testName": "abortDeterministicBranchWorkflow: if-check takes same path on first-run and replay", - "runId": "wrun_01KYWTZM792J3PKPHFZKENNVG8", - "timestamp": "2026-07-31T19:38:55.085Z", - "dashboardUrl": null - }, - { - "testName": "abortListenerWorkflow: signal.addEventListener fires on the deserialized step signal", - "runId": "wrun_01KYWTZN6RRKXGEYV2HRW6SHQA", - "timestamp": "2026-07-31T19:38:56.093Z", - "dashboardUrl": null - }, - { - "testName": "abortThrowIfAbortedMidFlightWorkflow: throwIfAborted in a polling loop bails when abort fires", - "runId": "wrun_01KYWTZQ5G9EGEJ01G39GBC035", - "timestamp": "2026-07-31T19:38:58.102Z", - "dashboardUrl": null - }, - { - "testName": "abortDeterministicBranchFromStepWorkflow: branches stay consistent when abort comes from a step", - "runId": "wrun_01KYWTZY0REZV6A0JBA96YQKW1", - "timestamp": "2026-07-31T19:39:05.119Z", - "dashboardUrl": null - }, - { - "testName": "abortHookOrderingWorkflow [listener-first-abort-first]: addEventListener → hook.then → abort() → resumeHook", - "runId": "wrun_01KYWTZZZMQSFATHQ2JF07W8F9", - "timestamp": "2026-07-31T19:39:07.129Z", - "dashboardUrl": null - }, - { - "testName": "abortHookOrderingWorkflow [listener-first-hook-first]: addEventListener → hook.then → resumeHook → abort()", - "runId": "wrun_01KYWV0A1EKBET35VXE29YRRFX", - "timestamp": "2026-07-31T19:39:17.427Z", - "dashboardUrl": null - }, - { - "testName": "abortHookOrderingWorkflow [hook-first-abort-first]: hook.then → addEventListener → abort() → resumeHook", - "runId": "wrun_01KYWV0M2MQTPN9ESAFQWWAC91", - "timestamp": "2026-07-31T19:39:27.708Z", - "dashboardUrl": null - }, - { - "testName": "abortHookOrderingWorkflow [hook-first-hook-first]: hook.then → addEventListener → resumeHook → abort()", - "runId": "wrun_01KYWV0Y44PJA3SE5CVGDGPENP", - "timestamp": "2026-07-31T19:39:37.993Z", - "dashboardUrl": null - }, - { - "testName": "importMetaUrlWorkflow - import.meta.url is available in step bundles", - "runId": "wrun_01KYWV185GKFMWCA3GC747DWE8", - "timestamp": "2026-07-31T19:39:48.279Z", - "dashboardUrl": null - }, - { - "testName": "metadataFromHelperWorkflow - getWorkflowMetadata/getStepMetadata work from module-level helper (#1577)", - "runId": "wrun_01KYWV1952Q0DYGYYN4E59M80S", - "timestamp": "2026-07-31T19:39:49.287Z", - "dashboardUrl": null - }, - { - "testName": "resilient start: addTenWorkflow completes when run_created returns 500", - "runId": "wrun_01KYWV1A4M5PY9CRW6HW065NWK", - "timestamp": "2026-07-31T19:39:50.293Z", - "dashboardUrl": null - }, - { - "testName": "getterStepWorkflow - getter functions with \"use step\" directive", - "runId": "wrun_01KYWV1B41YKQFRRG2NTQNF6SY", - "timestamp": "2026-07-31T19:39:51.304Z", - "dashboardUrl": null - }, - { - "testName": "distributedAbortController - manual abort triggers signal", - "runId": "wrun_01KYWV1FCJ5M3ZMWDXB24C1Z84", - "timestamp": "2026-07-31T19:39:55.673Z", - "dashboardUrl": null - }, - { - "testName": "distributedAbortController - TTL expiration triggers signal", - "runId": "wrun_01KYWV1FQVXB3XDH89FW2ZWVYV", - "timestamp": "2026-07-31T19:39:56.034Z", - "dashboardUrl": null - }, - { - "testName": "distributedAbortController - reconnect to existing controller", - "runId": "wrun_01KYWV1JWW1RRCEKGZFS6R36V6", - "timestamp": "2026-07-31T19:39:59.266Z", - "dashboardUrl": null - }, - { - "testName": "start: initial attributes are seeded on run creation", - "runId": "wrun_01KYWV1M4MDRWPTX1JYJJDZM3V", - "timestamp": "2026-07-31T19:40:00.539Z", - "dashboardUrl": null - }, - { - "testName": "start: reserved-prefix initial attributes are seeded with allowReservedAttributes", - "runId": "wrun_01KYWV1N47HTHZP7ESPX0VTT5E", - "timestamp": "2026-07-31T19:40:01.550Z", - "dashboardUrl": null - }, - { - "testName": "setAttributesWorkflow: workflow-body calls append native attr_set events and merge correctly", - "runId": "wrun_01KYWV1P3SYAGGMPK5HRXQN64X", - "timestamp": "2026-07-31T19:40:02.562Z", - "dashboardUrl": null - }, - { - "testName": "setAttributesInsideStepWorkflow: step-body calls append attributed native events", - "runId": "wrun_01KYWV1Q3JW86C750J66RZJWPF", - "timestamp": "2026-07-31T19:40:03.576Z", - "dashboardUrl": null - }, - { - "testName": "fire-and-forget: void setAttributes lands without awaiting", - "runId": "wrun_01KYWV1R3BW1VYWADWJBFY12D9", - "timestamp": "2026-07-31T19:40:04.594Z", - "dashboardUrl": null - }, - { - "testName": "Promise.all of disjoint-key writes: every key lands", - "runId": "wrun_01KYWV1V1ESEQ5VPA14CQ9573Y", - "timestamp": "2026-07-31T19:40:07.603Z", - "dashboardUrl": null - }, - { - "testName": "workflow throws after awaited setAttributes: attribute still persists on the failed run", - "runId": "wrun_01KYWV1W18742A0498EFG2D27V", - "timestamp": "2026-07-31T19:40:08.658Z", - "dashboardUrl": null - }, - { - "testName": "validation DX: invalid writes throw catchable FatalErrors naming rule and limit", - "runId": "wrun_01KYWV1X20BBMKJ2HPTDM8FQDA", - "timestamp": "2026-07-31T19:40:09.669Z", - "dashboardUrl": null - } -] \ No newline at end of file From 34b8554b6e6b93a65aa03aef1d8578042e8bca22 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 13:33:39 -0700 Subject: [PATCH 15/28] fix(world-local): allocate slots append-only A slot names a position in the replay order, so allocation has to hand out a position no published event sits above. Handing out the lowest free position instead let a late `step_completed` drop into a hole beneath its own `step_created`/`step_started`, and every replay of that run then met a completion for a step it had not started. The book now keeps a monotonic ceiling: a reservation goes above every position the book has ever seen, and releasing one does not lower it. `run_created` takes the first slot outright rather than allocating it, since a `run_started` racing it can already have moved the book past that position. Co-Authored-By: Claude Opus 5 --- .../world-local/src/storage/events-storage.ts | 39 +++++--- .../src/storage/slot-identity.test.ts | 16 +++- .../world-local/src/storage/slots.test.ts | 88 +++++++++--------- packages/world-local/src/storage/slots.ts | 93 ++++++++++--------- 4 files changed, 133 insertions(+), 103 deletions(-) diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index 0e6c0ce39d..24d2f787c4 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -1061,12 +1061,13 @@ export function createEventsStorage( // EVENT ID: the caller's slot claim, an allocated slot, or a ULID // ============================================================ // A run's own `run_created` owns the first slot — provably, since - // nothing precedes it — so every other event allocates above it, even - // when that event is the first to arrive here. - const allocationFloor = - data.eventType === 'run_created' - ? RUN_CREATED_SLOT - : RUN_CREATED_SLOT + 1; + // nothing precedes it — so it takes that position outright instead of + // allocating one, and every other event allocates above it even when it + // is the first to arrive here. Allocation is append-only (see SlotBook), + // so `run_created` cannot get the first position by asking for the lowest + // free one: a `run_started` racing it (start() issues the creation and + // the queue send in parallel) may already have moved the book past it. + const ownsFirstSlot = data.eventType === 'run_created'; // A slot-numbered run's ids name positions in its log, so an id is // either claimed by a caller that holds the log (and is therefore // asserting the log is complete up to that position) or allocated here @@ -1130,7 +1131,13 @@ export function createEventsStorage( } } else if (slotMode) { reservedRunId = effectiveRunId; - const slot = await slots.reserve(effectiveRunId, allocationFloor); + let slot: number; + if (ownsFirstSlot) { + slot = RUN_CREATED_SLOT; + slots.claim(effectiveRunId, slot); + } else { + slot = await slots.reserve(effectiveRunId); + } reserved.add(slot); eventId = slotEventId(slot); } @@ -2691,7 +2698,12 @@ export function createEventsStorage( // to reconcile. A write whose position the *caller* claimed may not: // the claim asserts a log complete up to that position, so losing it // means that log is stale and only the caller can resolve it. - const reallocatesSlot = slotMode && params?.eventId === undefined; + // `run_created` is excluded even though it named its own position: the + // first slot is the only position it can ever occupy, so losing it means + // the run already has a creation event, and appending a second one above + // it would be worse than the duplicate the publish is reporting. + const reallocatesSlot = + slotMode && params?.eventId === undefined && !ownsFirstSlot; const slotDeadline = Date.now() + SLOT_RETRY_BUDGET_MS; let compositeKey = ''; let eventPath = ''; @@ -2712,10 +2724,11 @@ export function createEventsStorage( if (reallocatesSlot && Date.now() < slotDeadline) { // The position is someone else's — either published there or // staged for it. Record that, top the book up from disk, and try - // again one position higher rather than surfacing a conflict the - // caller cannot act on. Re-reading rather than incrementing keeps - // the log dense: every round at least one writer wins, so the - // search never runs away from the log it is filling. + // again above whatever the log has reached rather than surfacing a + // conflict the caller cannot act on. Re-reading rather than + // incrementing bounds the search: every round at least one writer + // wins, so the top of the log is never further than the number of + // writers still contending for it. const lost = slotFromId(eventId); if (lost !== undefined) { reserved.delete(lost); @@ -2725,7 +2738,7 @@ export function createEventsStorage( await new Promise((resolve) => setTimeout(resolve, slotRetryDelay(round)) ); - const slot = await slots.reserve(effectiveRunId, allocationFloor); + const slot = await slots.reserve(effectiveRunId); reservedRunId = effectiveRunId; reserved.add(slot); eventId = slotEventId(slot); diff --git a/packages/world-local/src/storage/slot-identity.test.ts b/packages/world-local/src/storage/slot-identity.test.ts index 5455e0a52f..b7c579e557 100644 --- a/packages/world-local/src/storage/slot-identity.test.ts +++ b/packages/world-local/src/storage/slot-identity.test.ts @@ -158,9 +158,11 @@ describe('numbering', () => { expect(maxSlotOf(data)).toBe(data.length); }); - it('leaves no hole behind a rejected write', async () => { - // The rejected op's slot sits below its concurrent sibling's, and a hole - // below a published event can never be filled. + it('leaves a rejected write’s position unused instead of recycling it', async () => { + // The rejected op's position sits below its concurrent sibling's, so handing + // it to the next writer would order that writer's event below one that + // already published. The hole costs a reader the density proof; the + // inversion would cost the run. const runId = await newSlotRun(); const [rejected, accepted] = await Promise.allSettled([ storage.events.create(runId, { @@ -175,7 +177,13 @@ describe('numbering', () => { expect(accepted.status).toBe('fulfilled'); await createStep(runId, 'step_b'); const slots = await slotsOf(runId); - expect([...slots].sort((a, b) => a - b)).toEqual([1, 2, 3]); + expect(slots).toHaveLength(3); + expect(slots[0]).toBe(FIRST_SLOT); + // Both concurrent writers took a position, one abandoned its own, and the + // third write went above them both. + expect(slots[2]).toBe(FIRST_SLOT + 3); + expect(slots[1]).toBeGreaterThan(slots[0]); + expect(slots[1]).toBeLessThan(slots[2]); }); }); diff --git a/packages/world-local/src/storage/slots.test.ts b/packages/world-local/src/storage/slots.test.ts index 0e60b2ff74..d7b87f9e10 100644 --- a/packages/world-local/src/storage/slots.test.ts +++ b/packages/world-local/src/storage/slots.test.ts @@ -2,7 +2,6 @@ import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; import { - FIRST_SLOT, SPEC_VERSION_CURRENT, SPEC_VERSION_SLOT_IDENTITY, slotEventId, @@ -89,9 +88,11 @@ describe('usesSlots', () => { }); describe('reserve', () => { - it('starts at the first slot for a run with no events', async () => { + it('starts above the slot the run’s own creation event owns', async () => { + // `run_created` takes the first slot outright — nothing can precede it — so + // an allocation never hands that position out. await expect(createSlotBook(basedir).reserve(RUN_ID)).resolves.toBe( - FIRST_SLOT + RUN_CREATED_SLOT + 1 ); }); @@ -100,24 +101,36 @@ describe('reserve', () => { await expect(createSlotBook(basedir).reserve(RUN_ID)).resolves.toBe(4); }); - it('fills a hole left in the persisted log', async () => { - // Density is the whole point of the scheme, so a gap that somehow exists is - // reclaimed rather than skipped over forever. + it('leaves a hole in the persisted log unfilled', async () => { + // Allocation is append-only: the free position sits below a published event, + // and an event placed there would order before one that already happened. await writeEvents(1, 3); const book = createSlotBook(basedir); - await expect(book.reserve(RUN_ID)).resolves.toBe(2); await expect(book.reserve(RUN_ID)).resolves.toBe(4); + await expect(book.reserve(RUN_ID)).resolves.toBe(5); + }); + + it('stays above published events when a lower position comes free', async () => { + // The corruption this rules out: a step completion allocating late, dropping + // into a hole, and landing below the step_started it reports on. Replay reads + // the log in slot order and cannot consume that. + const book = createSlotBook(basedir); + const abandoned = await book.reserve(RUN_ID); + const published = await book.reserve(RUN_ID); + book.observe(RUN_ID, slotEventId(published)); + book.release(RUN_ID, abandoned); + await expect(book.reserve(RUN_ID)).resolves.toBeGreaterThan(published); }); it('hands a synchronous burst distinct consecutive slots', async () => { - // The suspension flush issues every op concurrently; with a plain - // "max + 1" they would all pick the same slot and all but one would fail. + // The suspension flush issues every op concurrently; a book that only moved + // on publish would give them all the same position and fail all but one. const book = createSlotBook(basedir); const slots = await Promise.all( Array.from({ length: 20 }, () => book.reserve(RUN_ID)) ); expect([...slots].sort((a, b) => a - b)).toEqual( - Array.from({ length: 20 }, (_, index) => FIRST_SLOT + index) + Array.from({ length: 20 }, (_, index) => RUN_CREATED_SLOT + 1 + index) ); }); @@ -131,26 +144,13 @@ describe('reserve', () => { expect([...slots].sort((a, b) => a - b)).toEqual([2, 3]); }); - it('honours a floor, so a concurrent event cannot take run_created’s slot', async () => { - // start() publishes the run entity before its `run_created` event and - // issues the queue send in parallel, so the delivery's `run_started` can - // allocate while slot 1 is still in flight. - const book = createSlotBook(basedir); - await expect(book.reserve(RUN_ID, RUN_CREATED_SLOT + 1)).resolves.toBe( - RUN_CREATED_SLOT + 1 - ); - }); - - it('leaves slots below the floor allocatable', async () => { - // A floored search skips that range without looking at it, so it proves - // nothing about it — `run_created` must still find its own slot free. + it('honours a floor above where the book has reached', async () => { + // start() publishes the run entity before its `run_created` event and issues + // the queue send in parallel, so the delivery's `run_started` can allocate + // while slot 1 is still in flight. const book = createSlotBook(basedir); - await expect(book.reserve(RUN_ID, RUN_CREATED_SLOT + 1)).resolves.toBe( - RUN_CREATED_SLOT + 1 - ); - await expect(book.reserve(RUN_ID, RUN_CREATED_SLOT)).resolves.toBe( - RUN_CREATED_SLOT - ); + await expect(book.reserve(RUN_ID, 5)).resolves.toBe(5); + await expect(book.reserve(RUN_ID)).resolves.toBe(6); }); it('keeps runs independent', async () => { @@ -158,22 +158,22 @@ describe('reserve', () => { const book = createSlotBook(basedir); await expect(book.reserve(RUN_ID)).resolves.toBe(3); await expect(book.reserve('wrun_01K0000000000000000000OTHR')).resolves.toBe( - FIRST_SLOT + RUN_CREATED_SLOT + 1 ); }); }); describe('release', () => { - it('gives an abandoned interior slot to the next caller', async () => { - // A rejected op must not strand the slot below its concurrent siblings': - // that hole can never be filled once a later slot is published. + it('does not recycle an abandoned interior slot', async () => { + // The position may already sit below a sibling that published, and no + // caller can tell from here. A hole costs a reader its completeness proof; + // an inversion costs the run. const book = createSlotBook(basedir); const [first, second] = await Promise.all([ book.reserve(RUN_ID), book.reserve(RUN_ID), ]); book.release(RUN_ID, first); - await expect(book.reserve(RUN_ID)).resolves.toBe(first); await expect(book.reserve(RUN_ID)).resolves.toBe(second + 1); }); @@ -215,13 +215,15 @@ describe('claim', () => { await expect(book.reserve(RUN_ID)).resolves.toBe(3); }); - it('frees the slot again once the claim resolves', async () => { + it('stops holding back a claim that resolved', async () => { + // Released before anything allocated for the run, so no position in this + // instance was ever handed out above it and the log — which is the authority + // on what published — reaches only slot 1. Nothing can be inverted by + // seeding the book from disk alone. await writeEvents(1); const book = createSlotBook(basedir); book.claim(RUN_ID, 2); book.release(RUN_ID, 2); - // Nothing is allocating for the run yet, so the book the next caller seeds - // has to start from the log alone. await expect(book.reserve(RUN_ID)).resolves.toBe(2); }); @@ -237,27 +239,25 @@ describe('claim', () => { }); describe('observe', () => { - it('never hands out a slot claimed by the client', async () => { + it('moves allocation above a position the client claimed', async () => { const book = createSlotBook(basedir); await book.reserve(RUN_ID); book.observe(RUN_ID, slotEventId(5)); - const next = await book.reserve(RUN_ID); - expect(next).not.toBe(5); - expect(next).toBe(2); + await expect(book.reserve(RUN_ID)).resolves.toBe(6); }); it('ignores ULID event ids', async () => { const book = createSlotBook(basedir); - await book.reserve(RUN_ID); + const slot = await book.reserve(RUN_ID); book.observe(RUN_ID, 'evnt_01K5Z0000000000000000000AA'); - await expect(book.reserve(RUN_ID)).resolves.toBe(2); + await expect(book.reserve(RUN_ID)).resolves.toBe(slot + 1); }); }); describe('forget', () => { it("re-reads the log, picking up another writer's events", async () => { const book = createSlotBook(basedir); - await expect(book.reserve(RUN_ID)).resolves.toBe(FIRST_SLOT); + await expect(book.reserve(RUN_ID)).resolves.toBe(RUN_CREATED_SLOT + 1); await writeEvents(1, 2, 3); book.forget(RUN_ID); await expect(book.reserve(RUN_ID)).resolves.toBe(4); diff --git a/packages/world-local/src/storage/slots.ts b/packages/world-local/src/storage/slots.ts index 58275f3d57..25b2b8e522 100644 --- a/packages/world-local/src/storage/slots.ts +++ b/packages/world-local/src/storage/slots.ts @@ -2,21 +2,29 @@ * Slot allocation for the Local World. * * A slot-numbered run names its events by position: `evnt_…001` is the first - * event of the run, `evnt_…002` the second, with no gaps. Density is the point - * — it is what lets a reader prove its loaded log is complete — so an allocator - * must never leave a slot permanently unwritten. + * event of the run, `evnt_…002` the second. A replay reads the log in slot + * order, so the order slots are handed out in has to be an order some execution + * could have produced — which makes allocation strictly *append-only*: a slot is + * only ever handed out above every position this book has seen. + * + * Filling a hole is what that rules out, and it is worth naming why, because the + * alternative looks appealing (it keeps the log dense). A position left unwritten + * by an abandoned reservation sits below events that are already published. Hand + * it to the next caller and a `step_completed` lands below its own + * `step_started`; the replay reaches a completion for a step it has not started + * and diverges, and every later replay diverges the same way. A hole costs a + * reader the ability to prove its copy of the log is complete. An inversion + * costs the run. * * Three properties do the work: * * - Handing out a slot is a *synchronous* set operation, so concurrent * callers in one process get distinct slots with no lock. The only await is * seeding from disk, which is memoized per run. - * - An allocation always picks the lowest slot that is neither written nor - * outstanding, so a reservation that is abandoned (its create threw a - * validation error) is handed to the next caller instead of leaving an - * interior hole. This matters under fan-out: N concurrent step_completed - * writes reserve N consecutive slots, and one rejected op must not strand - * the slot below its siblings'. + * - An allocation picks the position above the highest one the book knows of, + * written or outstanding, and that ceiling never descends. A reservation that + * is abandoned (its create threw a validation error) leaves its position + * unused rather than being recycled below a sibling that already published. * - The event publish is `writeExclusive`, which is the authority. The book is * a hint: when it turns out to be stale (another process wrote the slot), * the publish fails and the caller is told so, rather than a duplicate being @@ -43,8 +51,12 @@ interface RunSlots { written: Set; /** Slots handed out whose publish has not resolved yet. */ outstanding: Set; - /** Lowest slot that might still be free; never decreases except on release. */ - searchFrom: number; + /** + * The highest position this book has ever seen written, claimed or handed out. + * Allocation goes above it and it never descends, which is what keeps a + * released position from being recycled below events already published. + */ + ceiling: number; } export interface SlotBook { @@ -58,16 +70,15 @@ export interface SlotBook { */ usesSlots(runId: string): Promise; /** - * Reserves the lowest free slot of `runId`, at or above `minSlot`. Distinct - * for every concurrent caller; the publish still has to prove the slot was - * actually free. + * Reserves the position above every one this book knows of for `runId`, and at + * or above `minSlot`. Distinct for every concurrent caller; the publish still + * has to prove the position was actually free. * - * `minSlot` is how the run's first slot is kept for its own `run_created`: - * that event's slot needs no allocation, and it may not be on disk yet when a - * concurrent `run_started` allocates (start() issues the creation and the - * queue send in parallel, and the run entity is published before its event). - * Slots below `minSlot` stay allocatable for a later caller, so holding one - * back leaves the log dense. + * `minSlot` defaults to the position above the run's first slot, which is + * reserved for its own `run_created`: that event needs no allocation, and it + * may not be on disk yet when a concurrent `run_started` allocates (start() + * issues the creation and the queue send in parallel, and the run entity is + * published before its event). */ reserve(runId: string, minSlot?: number): Promise; /** @@ -90,8 +101,10 @@ export interface SlotBook { */ isWritten(runId: string, slot: number): Promise; /** - * Returns a reserved or claimed slot that was never published, so the next - * caller takes it instead of it becoming a hole. + * Forgets a reserved or claimed slot whose publish is never going to happen, + * so nothing waits on it. The position itself is not handed out again: it may + * already sit below a sibling that published, and recycling it there would put + * a later event below an earlier one. */ release(runId: string, slot: number): void; /** Records a published event id, so it is never handed out again. */ @@ -147,10 +160,11 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { written.add(slot); } } + const outstanding = new Set(claims.get(runId)); const book: RunSlots = { written, - outstanding: new Set(claims.get(runId)), - searchFrom: FIRST_SLOT, + outstanding, + ceiling: Math.max(FIRST_SLOT - 1, ...written, ...outstanding), }; books.set(runId, book); return book; @@ -186,17 +200,9 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { } function take(book: RunSlots, minSlot: number): number { - let slot = Math.max(book.searchFrom, minSlot); - while (book.written.has(slot) || book.outstanding.has(slot)) { - slot += 1; - } + const slot = Math.max(book.ceiling + 1, minSlot); book.outstanding.add(slot); - if (minSlot <= book.searchFrom) { - // Only an unfloored search proves everything below the slot it landed on - // is taken. A floored one skipped that range without looking, and those - // slots are still free for a caller that may take them. - book.searchFrom = slot; - } + book.ceiling = slot; return slot; } @@ -216,7 +222,7 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { return mode; }, - async reserve(runId, minSlot = FIRST_SLOT) { + async reserve(runId, minSlot = RUN_CREATED_SLOT + 1) { const opened = open(runId); // Awaiting a book that is already in hand would yield to the microtask // queue and let a concurrent caller take the same slot. @@ -230,7 +236,11 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { } else { claims.set(runId, new Set([slot])); } - books.get(runId)?.outstanding.add(slot); + const book = books.get(runId); + if (book) { + book.outstanding.add(slot); + book.ceiling = Math.max(book.ceiling, slot); + } }, async isWritten(runId, slot) { @@ -244,10 +254,10 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { if (!book) { return; } + // The ceiling stays where it is: this position may already sit below one a + // sibling published, and handing it out again would order a later event + // before an earlier one. book.outstanding.delete(slot); - if (slot < book.searchFrom) { - book.searchFrom = slot; - } }, observe(runId, eventId) { @@ -264,6 +274,7 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { } book.written.add(slot); book.outstanding.delete(slot); + book.ceiling = Math.max(book.ceiling, slot); }, async refresh(runId) { @@ -277,11 +288,9 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { if (slot !== undefined) { book.written.add(slot); book.outstanding.delete(slot); + book.ceiling = Math.max(book.ceiling, slot); } } - // Positions released while this scan ran may sit below where the search - // had reached, and they are free again. - book.searchFrom = FIRST_SLOT; }, forget(runId) { From 41ddc933fa1c85424e40e3e8531847912923e79e Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 13:40:14 -0700 Subject: [PATCH 16/28] chore: sort imports --- packages/core/src/runtime.test.ts | 2 +- packages/core/src/runtime.ts | 2 +- packages/world-local/src/storage/events-storage.ts | 2 +- packages/world-vercel/src/events-v4.test.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/core/src/runtime.test.ts b/packages/core/src/runtime.test.ts index 92dc696419..ff4c0830c9 100644 --- a/packages/core/src/runtime.test.ts +++ b/packages/core/src/runtime.test.ts @@ -15,7 +15,6 @@ import { import { ulid } from 'ulid'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { registerStepFunction } from './private.js'; -import { getWorkflowMetadata } from './step/get-workflow-metadata.js'; import { REPLAY_DIVERGENCE_MAX_RETRIES } from './runtime/constants.js'; import { setWorld } from './runtime/world.js'; import { workflowEntrypoint } from './runtime.js'; @@ -23,6 +22,7 @@ import { dehydrateStepReturnValue, dehydrateWorkflowArguments, } from './serialization.js'; +import { getWorkflowMetadata } from './step/get-workflow-metadata.js'; // Capture every promise handed to `waitUntil` so tests can assert that // progress-critical sends are never registered on a detached, unconsumed diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 3e4915b678..5e711b9182 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -53,7 +53,6 @@ import { appendUniqueEvents, type EventCreator, eventCreateFenceFor, - stepClaimFence, getQueueOverhead, getWorkflowQueueName, handleHealthCheckMessage, @@ -63,6 +62,7 @@ import { parseHealthCheckPayload, queueMessage, requiresFreshReplay, + stepClaimFence, toMutableEventLog, withEventCreateFence, withHealthCheck, diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index 24d2f787c4..c84ffbf7c7 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -35,9 +35,9 @@ import { isTerminalStepStatus, isTerminalWorkflowRunStatus, requiresNewerWorld, + SLOT_RETRY_BUDGET_MS, SPEC_VERSION_CURRENT, SPEC_VERSION_MAX_SUPPORTED, - SLOT_RETRY_BUDGET_MS, StepSchema, slotEventId, slotFromId, diff --git a/packages/world-vercel/src/events-v4.test.ts b/packages/world-vercel/src/events-v4.test.ts index 5c3bda47dd..09bdf3a716 100644 --- a/packages/world-vercel/src/events-v4.test.ts +++ b/packages/world-vercel/src/events-v4.test.ts @@ -7,9 +7,9 @@ import { WorkflowWorldError, } from '@workflow/errors'; import { + SPEC_VERSION_SLOT_IDENTITY, slotEventId, slotIdBody, - SPEC_VERSION_SLOT_IDENTITY, } from '@workflow/world'; import { decode, encode } from 'cbor-x'; import { MockAgent } from 'undici'; From a67b34547e3b79c4db7257311cbc5a78fbb8864a Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 13:43:00 -0700 Subject: [PATCH 17/28] docs(world): state the append-only rule for slot allocation --- packages/world/src/slot-identity.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/packages/world/src/slot-identity.ts b/packages/world/src/slot-identity.ts index 203005c719..f5cfa50367 100644 --- a/packages/world/src/slot-identity.ts +++ b/packages/world/src/slot-identity.ts @@ -7,13 +7,17 @@ * cursor that accepted a ULID keeps accepting a slot — and because the width is * fixed, lexicographic order is numeric order. * - * Slots start at 1 and are allocated contiguously, which is what makes - * contention explicit: two writers proposing one position cannot both win, and - * the loser is told which events it was missing. Zero is left unused because - * the inclusive lower fence for range queries over a run's events is the - * all-zero id. + * Slots start at 1 and are handed out above every position the allocator has + * seen, never into a lower one that happens to be free. That makes contention + * explicit — two writers proposing one position cannot both win, and the loser + * is told which events it was missing — and it keeps slot order, which is the + * order a replay reads the log in, a linear extension of what actually + * happened. Filling a hole would place an event below ones that preceded it, + * and a replay reaching a `step_completed` below its own `step_started` diverges + * for good. Zero is left unused because the inclusive lower fence for + * range queries over a run's events is the all-zero id. * - * Allocation being contiguous does not make a published log gap-free, so + * Allocation being append-only does not make a published log gap-free, so * `events.length === maxSlot` is not a completeness proof. A slot claimed by an * operation that then fails for a reason of its own is never filled, and if a * later slot has already been published the gap is permanent. Nothing may treat From 44b20a1ca9847390ef942d21b589ae729cf35678 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 14:16:48 -0700 Subject: [PATCH 18/28] docs(worlds): describe a mixed id log by what it breaks in replay order --- packages/world-local/src/storage/events-storage.ts | 5 +++-- packages/world-local/src/storage/slot-identity.test.ts | 4 ++-- packages/world-local/src/storage/slots.test.ts | 4 ++-- packages/world-postgres/src/storage.ts | 5 +++-- packages/world/src/events.ts | 8 ++++---- 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index c84ffbf7c7..1339a2e0bd 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -1003,8 +1003,9 @@ export function createEventsStorage( // `run_created` and the queue send concurrently, so the delivery's // `run_started` can arrive while the run is still being published. A // stale "no" there would number that one event with a ULID on an - // otherwise slot-numbered run, and the hole it leaves in the numbering - // costs the log its completeness proof for life. + // otherwise slot-numbered run, and a ULID names no position: the replay + // would read it wherever its timestamp happens to sort rather than where + // the writer meant it to go. if (currentRun && data.eventType !== 'run_created') { slotMode = usesSlotIdentity(currentRun.specVersion); } diff --git a/packages/world-local/src/storage/slot-identity.test.ts b/packages/world-local/src/storage/slot-identity.test.ts index b7c579e557..5b9eaf2626 100644 --- a/packages/world-local/src/storage/slot-identity.test.ts +++ b/packages/world-local/src/storage/slot-identity.test.ts @@ -147,7 +147,7 @@ describe('numbering', () => { ); }); - it('proves completeness: the highest slot is the event count', async () => { + it('numbers a run densely when every write it makes lands', async () => { const runId = await newSlotRun(); await Promise.all( Array.from({ length: 5 }, (_, index) => @@ -161,7 +161,7 @@ describe('numbering', () => { it('leaves a rejected write’s position unused instead of recycling it', async () => { // The rejected op's position sits below its concurrent sibling's, so handing // it to the next writer would order that writer's event below one that - // already published. The hole costs a reader the density proof; the + // already published. The hole costs a reader nothing it was promised; the // inversion would cost the run. const runId = await newSlotRun(); const [rejected, accepted] = await Promise.allSettled([ diff --git a/packages/world-local/src/storage/slots.test.ts b/packages/world-local/src/storage/slots.test.ts index d7b87f9e10..06f2986b71 100644 --- a/packages/world-local/src/storage/slots.test.ts +++ b/packages/world-local/src/storage/slots.test.ts @@ -166,8 +166,8 @@ describe('reserve', () => { describe('release', () => { it('does not recycle an abandoned interior slot', async () => { // The position may already sit below a sibling that published, and no - // caller can tell from here. A hole costs a reader its completeness proof; - // an inversion costs the run. + // caller can tell from here. A hole is a position nothing ever wrote; an + // inversion is an event a replay reads before the one it followed. const book = createSlotBook(basedir); const [first, second] = await Promise.all([ book.reserve(RUN_ID), diff --git a/packages/world-postgres/src/storage.ts b/packages/world-postgres/src/storage.ts index b62a17ec6f..ccb70bb73c 100644 --- a/packages/world-postgres/src/storage.ts +++ b/packages/world-postgres/src/storage.ts @@ -748,8 +748,9 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { // `run_created` and the queue send concurrently, so a delivery's // `run_started` can arrive while the run is still being published. A stale // "no" would number that one event with a ULID on an otherwise - // slot-numbered run, and the hole it leaves in the numbering costs the log - // its completeness proof for life. + // slot-numbered run, and a ULID names no position: the replay would read + // it wherever its timestamp happens to sort rather than where the writer + // meant it to go. if (currentRun && data.eventType !== 'run_created') { slotMode = usesSlotIdentity(currentRun.specVersion); } diff --git a/packages/world/src/events.ts b/packages/world/src/events.ts index f87af16d54..c8cd0bd0f0 100644 --- a/packages/world/src/events.ts +++ b/packages/world/src/events.ts @@ -801,10 +801,10 @@ export interface CreateEventParams { * taken, reject the write with `SlotConflictError` (HTTP 409) instead of * minting a different id — a lost slot means the client replayed against an * event log missing at least one event, so its whole proposed event, not just - * its id, is suspect. Reject a mismatch in either direction with a 400: an id - * of this shape on a run that does not use slot identity, or an absent or - * ULID-shaped id on a run that does, would leave the log unable to prove its - * own completeness. + * its id, is suspect. Reject a mismatch in either direction with a 400: a + * ULID names a time and a slot names a position, so a log holding both sorts + * partly by one and partly by the other and no replay can read it in the order + * it was written. * * A World that ignores this field keeps minting ids itself, which is correct * only for runs that were never stamped with slot identity in the first From 7dd98637bccc792f98daa136a22fe8915e51be96 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 31 Jul 2026 15:12:09 -0700 Subject: [PATCH 19/28] test(e2e): capture the committed log around a corruption's divergent event A divergence is a disagreement between the order the log records and the order a replay reconstructs it in, so the log's own ordering is the only evidence that separates the candidate causes. The runs live on an ephemeral preview deployment, so the window has to be read while the job is still running. Sampled (6 runs) and kept out of the sticky comment, which has a size limit; the results artifact carries them. --- .changeset/slimy-weeks-act.md | 2 + .../render-event-log-race-repro-results.js | 5 + .../core/e2e/event-log-race-repro.test.ts | 103 +++++++++++++++++- 3 files changed, 108 insertions(+), 2 deletions(-) create mode 100644 .changeset/slimy-weeks-act.md diff --git a/.changeset/slimy-weeks-act.md b/.changeset/slimy-weeks-act.md new file mode 100644 index 0000000000..a845151cc8 --- /dev/null +++ b/.changeset/slimy-weeks-act.md @@ -0,0 +1,2 @@ +--- +--- diff --git a/.github/scripts/render-event-log-race-repro-results.js b/.github/scripts/render-event-log-race-repro-results.js index bbead520b2..c0cc1aadfd 100644 --- a/.github/scripts/render-event-log-race-repro-results.js +++ b/.github/scripts/render-event-log-race-repro-results.js @@ -407,6 +407,11 @@ function renderLatestFailures(entry) { `\nShowing 20 of ${entry.failing.length + entry.truncatedFailingCount} non-completed runs.` ); } + // Deliberately not inlined here: the slices are large and this comment has a + // size limit, while the artifact has none. + console.log( + '\nThe `event-log-race-repro-results` artifact carries a window of the committed log around the divergent event, for a sample of the corruptions.' + ); console.log(''); } diff --git a/packages/core/e2e/event-log-race-repro.test.ts b/packages/core/e2e/event-log-race-repro.test.ts index b40fed2ec3..0135b16747 100644 --- a/packages/core/e2e/event-log-race-repro.test.ts +++ b/packages/core/e2e/event-log-race-repro.test.ts @@ -128,6 +128,31 @@ interface ReproRunResult { resumesFailed: number; stragglers?: number; }; + /** + * The committed log around the divergent event, for corruptions only. A + * divergence is a disagreement between the order the log records and the + * order a replay reconstructs, so the log's own ordering is the only + * evidence that distinguishes the candidate causes — and the run is on an + * ephemeral preview deployment, so it has to be captured while the job is + * still running rather than read back afterwards. + */ + logSlice?: LogSliceEntry[]; +} + +/** + * One committed event, projected to the fields that decide replay order: + * its position (`slot`), what it resolves (`eventType`/`correlationId`), and + * both clock domains. `occurredAt` is the client/VM moment and `createdAt` + * the persisted event time the sandbox clock is driven from; a race between a + * `sleep` and a step is decided by that clock, so the two have to be + * comparable side by side. + */ +interface LogSliceEntry { + slot: number | string; + eventType: string; + correlationId?: string; + occurredAt?: string; + createdAt?: string; } function envNumber(name: string, fallback: number) { @@ -397,6 +422,74 @@ async function readFailureMessage( } } +/** + * How many events either side of the divergent one to keep. The window has to + * span a whole round of the storm — width branches, each with a step create, + * start and completion, plus the round's waits — or it can miss the very + * event whose position explains the divergence. + */ +const LOG_SLICE_RADIUS = envNumber('EVENT_LOG_RACE_REPRO_LOG_SLICE_RADIUS', 45); + +/** + * Cap on how many corruptions carry a slice. The results JSON is rendered into + * a PR comment, and a body over GitHub's limit is rejected outright, so the + * slices are a sample rather than a complete record. + */ +const LOG_SLICE_MAX_RUNS = envNumber('EVENT_LOG_RACE_REPRO_LOG_SLICE_RUNS', 6); + +let logSlicesCaptured = 0; + +/** Worlds hand timestamps back as a Date or as the stored ISO string. */ +function isoOrUndefined(value: unknown): string | undefined { + if (value instanceof Date) return value.toISOString(); + return typeof value === 'string' ? value : undefined; +} + +/** Ordinal of a slot-numbered id, or the raw id when it is a ULID. */ +function idOrdinal(id: string): number | string { + const body = id.slice(id.indexOf('_') + 1); + return /^\d+$/.test(body) ? Number(body) : id; +} + +/** + * Reads the committed log and returns the window around the divergent event + * named in `message`. Best-effort: the report is a measurement, so a failed + * read costs a slice rather than the attempt's classification. + */ +async function readLogSlice( + runId: string, + message: string | undefined +): Promise { + if (logSlicesCaptured >= LOG_SLICE_MAX_RUNS) return undefined; + try { + const world = await getWorld(); + const { data: events } = await world.events.list({ runId }); + const projected: LogSliceEntry[] = events.map((event) => ({ + slot: idOrdinal(event.eventId), + eventType: event.eventType, + correlationId: event.correlationId, + occurredAt: isoOrUndefined(event.occurredAt), + createdAt: isoOrUndefined(event.createdAt), + })); + + // The corruption message names the last divergent event; centre on it when + // it is there, and otherwise keep the tail, where a divergence that ran out + // of recovery replays ends up. + const divergent = message?.match(/evnt_[0-9A-Z]+/)?.[0]; + const at = divergent + ? projected.findIndex((entry) => entry.slot === idOrdinal(divergent)) + : -1; + const centre = at >= 0 ? at : projected.length - 1; + logSlicesCaptured += 1; + return projected.slice( + Math.max(0, centre - LOG_SLICE_RADIUS), + centre + LOG_SLICE_RADIUS + 1 + ); + } catch { + return undefined; + } +} + async function pollTerminalRun( run: Run, startedAt: number, @@ -443,14 +536,20 @@ async function pollTerminalRun( // carries the divergent event and what the replay was waiting for, which // is the whole reason to keep the report. const hydrated = await readFailureMessage(run); + const outcome = classifyFailure(failure.errorCode); + const errorMessage = hydrated?.message ?? failure.error?.message; return { ...base, - outcome: classifyFailure(failure.errorCode), + outcome, status: runData.status, errorCode: failure.errorCode, - errorMessage: hydrated?.message ?? failure.error?.message, + errorMessage, errorName: hydrated?.name ?? failure.error?.name, durationMs: Date.now() - startedAt, + logSlice: + outcome === 'CORRUPTED_EVENT_LOG' + ? await readLogSlice(run.runId, errorMessage) + : undefined, }; } From 2d9af9ad92234e74d90eb31dc7a664c32de2d113 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Sat, 1 Aug 2026 10:59:07 -0700 Subject: [PATCH 20/28] [ci] Shrink the event-log race repro job 100x and add a local world-postgres runner (#3273) --- .changeset/event-log-race-repro-local.md | 4 + .changeset/local-empty-vercel-url.md | 5 + .github/workflows/event-log-race-repro.yml | 56 +-- .gitignore | 7 + AGENTS.md | 73 ++++ package.json | 1 + .../core/e2e/event-log-race-repro.test.ts | 47 ++- packages/core/src/runtime/step-executor.ts | 6 +- packages/core/src/workflow.ts | 6 +- scripts/event-log-race-repro-local.sh | 385 ++++++++++++++++++ vitest.config.ts | 9 +- 11 files changed, 557 insertions(+), 42 deletions(-) create mode 100644 .changeset/event-log-race-repro-local.md create mode 100644 .changeset/local-empty-vercel-url.md create mode 100755 scripts/event-log-race-repro-local.sh diff --git a/.changeset/event-log-race-repro-local.md b/.changeset/event-log-race-repro-local.md new file mode 100644 index 0000000000..30c61fe13e --- /dev/null +++ b/.changeset/event-log-race-repro-local.md @@ -0,0 +1,4 @@ +--- +--- + +Shrink the event log race repro CI job and add a local world-postgres runner diff --git a/.changeset/local-empty-vercel-url.md b/.changeset/local-empty-vercel-url.md new file mode 100644 index 0000000000..14c2b58305 --- /dev/null +++ b/.changeset/local-empty-vercel-url.md @@ -0,0 +1,5 @@ +--- +"@workflow/core": patch +--- + +Treat an empty `VERCEL_URL` as not running on Vercel, so a `.env.local` left behind by `vercel env pull` no longer makes local runs fail with `Invalid URL` diff --git a/.github/workflows/event-log-race-repro.yml b/.github/workflows/event-log-race-repro.yml index 10eddf10d2..6f78986f2a 100644 --- a/.github/workflows/event-log-race-repro.yml +++ b/.github/workflows/event-log-race-repro.yml @@ -4,56 +4,54 @@ on: pull_request: branches: [main, stable] types: [opened, reopened, synchronize, labeled] + # Every input below is passed straight through to the harness, and a blank one + # lands on the harness' own default in + # `packages/core/e2e/event-log-race-repro.test.ts` — which is the only place + # these numbers are defined. Do not reintroduce `default:` values here: a second + # copy drifts, and the copy CI uses would stop matching the one a local run gets. + # + # The defaults are sized for a per-PR regression check. To soak for a *rate* + # instead, dispatch with the historical scale: step_storm_attempts 600, + # hook_storm_attempts 600, attempts 200, concurrency 40, budget_ms 4500000 — + # and raise `timeout-minutes` below, which is sized for the default scale. workflow_dispatch: inputs: step_storm_attempts: description: 'Number of step-storm runs (racing steps vs. watchdog + poke-hook pressure)' required: false - default: '600' hook_storm_attempts: description: 'Number of hook-storm runs (racing hook deliveries vs. watchdog — the production shape)' required: false - default: '600' attempts: description: 'Number of hook-sleep control runs (calibration baseline, ~0.1% historical corruption rate)' required: false - default: '200' concurrency: description: 'Number of concurrent workflow runs' required: false - default: '40' rounds: description: 'Rounds of racing branches per storm run (longer log = more room for a divergence to surface)' required: false - default: '6' width: description: 'Racing branches per round (each one is an independent wake source, so this is the main concurrency dial)' required: false - default: '8' watchdog_ms: description: 'Per-branch watchdog timeout. Branches that time out emit an extra step, which is what shifts the correlation-ID sequence' required: false - default: '2500' step_delay_ms: description: 'step-storm: raced step duration. Must straddle watchdog_ms once jitter is applied' required: false - default: '2200' hook_resume_stagger_ms: description: 'hook-storm: per-index delay inside a round resume burst. width * this should exceed watchdog_ms' required: false - default: '400' poke_interval_ms: description: 'step-storm: cadence of out-of-band resumes to the never-read poke hook' required: false - default: '750' run_timeout_ms: description: 'Per-run timeout before classifying as stuck' required: false - default: '240000' budget_ms: description: 'Wall-clock budget for launching attempts. Must stay far enough below the job timeout to leave room for rendering the summary' required: false - default: '4500000' concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -63,7 +61,12 @@ jobs: event-log-race-repro: name: Event Log Race Repro runs-on: ubuntu-latest - timeout-minutes: 120 + # Sized for the harness' default scale: its own test timeout is + # `budget_ms + run_timeout_ms + 60s` (~17 min at the defaults), and the rest is + # checkout, build, the deployment wait, and rendering the summary. A soak + # dispatch that raises `budget_ms` has to raise this too, or the runner kills + # the job before the summary is written. + timeout-minutes: 25 if: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'event-log-race-repro') }} permissions: contents: read @@ -115,18 +118,21 @@ jobs: WORKFLOW_VERCEL_PROJECT: "prj_yjkM7UdHliv8bfxZ1sMJQf1pMpdi" WORKFLOW_VERCEL_PROJECT_SLUG: example-nextjs-workflow-turbopack VERCEL_WORKFLOW_SERVER_URL: ${{ github.ref != 'refs/heads/main' && secrets.VERCEL_WORKFLOW_SERVER_URL || '' }} - EVENT_LOG_RACE_REPRO_STEP_STORM_ATTEMPTS: ${{ inputs.step_storm_attempts || '600' }} - EVENT_LOG_RACE_REPRO_HOOK_STORM_ATTEMPTS: ${{ inputs.hook_storm_attempts || '600' }} - EVENT_LOG_RACE_REPRO_ATTEMPTS: ${{ inputs.attempts || '200' }} - EVENT_LOG_RACE_REPRO_CONCURRENCY: ${{ inputs.concurrency || '40' }} - EVENT_LOG_RACE_REPRO_ROUNDS: ${{ inputs.rounds || '6' }} - EVENT_LOG_RACE_REPRO_WIDTH: ${{ inputs.width || '8' }} - EVENT_LOG_RACE_REPRO_WATCHDOG_MS: ${{ inputs.watchdog_ms || '2500' }} - EVENT_LOG_RACE_REPRO_STEP_DELAY_MS: ${{ inputs.step_delay_ms || '2200' }} - EVENT_LOG_RACE_REPRO_HOOK_RESUME_STAGGER_MS: ${{ inputs.hook_resume_stagger_ms || '400' }} - EVENT_LOG_RACE_REPRO_POKE_INTERVAL_MS: ${{ inputs.poke_interval_ms || '750' }} - EVENT_LOG_RACE_REPRO_RUN_TIMEOUT_MS: ${{ inputs.run_timeout_ms || '240000' }} - EVENT_LOG_RACE_REPRO_BUDGET_MS: ${{ inputs.budget_ms || '4500000' }} + # Deliberately unguarded pass-through: on a `pull_request` event every + # `inputs.*` is empty, and the harness reads an empty variable as unset + # and falls back to its own default. See the note on the inputs above. + EVENT_LOG_RACE_REPRO_STEP_STORM_ATTEMPTS: ${{ inputs.step_storm_attempts }} + EVENT_LOG_RACE_REPRO_HOOK_STORM_ATTEMPTS: ${{ inputs.hook_storm_attempts }} + EVENT_LOG_RACE_REPRO_ATTEMPTS: ${{ inputs.attempts }} + EVENT_LOG_RACE_REPRO_CONCURRENCY: ${{ inputs.concurrency }} + EVENT_LOG_RACE_REPRO_ROUNDS: ${{ inputs.rounds }} + EVENT_LOG_RACE_REPRO_WIDTH: ${{ inputs.width }} + EVENT_LOG_RACE_REPRO_WATCHDOG_MS: ${{ inputs.watchdog_ms }} + EVENT_LOG_RACE_REPRO_STEP_DELAY_MS: ${{ inputs.step_delay_ms }} + EVENT_LOG_RACE_REPRO_HOOK_RESUME_STAGGER_MS: ${{ inputs.hook_resume_stagger_ms }} + EVENT_LOG_RACE_REPRO_POKE_INTERVAL_MS: ${{ inputs.poke_interval_ms }} + EVENT_LOG_RACE_REPRO_RUN_TIMEOUT_MS: ${{ inputs.run_timeout_ms }} + EVENT_LOG_RACE_REPRO_BUDGET_MS: ${{ inputs.budget_ms }} - name: Fetch previous repro comment if: always() && github.event_name == 'pull_request' diff --git a/.gitignore b/.gitignore index 28b5575028..3241f9667c 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,10 @@ workbench/sveltekit/static/.well-known/workflow # Local e2e diagnostics dumps e2e-diagnostics-*.json + +# Event log race repro output (written to the repo root by the harness and by +# scripts/event-log-race-repro-local.sh) +event-log-race-repro-results.json +event-log-race-repro-summary.md +event-log-race-repro-previous-comment.md +event-log-race-repro-server.log diff --git a/AGENTS.md b/AGENTS.md index 78a02fb262..b17ffccad2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -127,6 +127,79 @@ VERCEL_OIDC_TOKEN="$(grep VERCEL_OIDC_TOKEN workbench/nextjs-turbopack/.env.loca pnpm run test:e2e ``` +### Event Log Race Repro + +`packages/core/e2e/event-log-race-repro.test.ts` is a dedicated harness for +`CORRUPTED_EVENT_LOG`. It drives three scenarios against one deployment: +`step-storm` and `hook-storm` (concurrent replays of a single run racing the +per-branch watchdog — `hook-storm` is the production shape), plus a `hook-sleep` +control that provides the calibration baseline. Any outcome other than +`completed` fails the run, except `infra`, which means the harness could not +reach the deployment. + +Run it locally against `@workflow/world-postgres` and a locally started +workbench app — no Vercel deployment, no credentials: + +```bash +pnpm run test:e2e:event-log-race-repro:local +``` + +The script (`scripts/event-log-race-repro-local.sh`, `--help` for flags) brings up +the world-postgres container, applies migrations, builds and starts +`workbench/nextjs-turbopack` with `WORKFLOW_TARGET_WORLD` and +`WORKFLOW_PUBLIC_MANIFEST=1` set **at build time** (both are build-time inputs; +missing either silently yields a world-local app or a 404 manifest), runs the +harness, prints the same summary table CI posts, and tears the server down. +Postgres is left running for the next iteration unless `--teardown` is passed. + +Scale is controlled entirely by `EVENT_LOG_RACE_REPRO_*` environment variables. +Their defaults live only in `event-log-race-repro.test.ts` — neither the CI +workflow nor the local script defines a second copy. The default scale (14 runs) +is a per-PR regression check, not a rate measurement; a clean run means "the +storms did not trip it", not "the rate is below X". To soak for a *rate*, use the +historical scale: + +```bash +EVENT_LOG_RACE_REPRO_STEP_STORM_ATTEMPTS=600 \ +EVENT_LOG_RACE_REPRO_HOOK_STORM_ATTEMPTS=600 \ +EVENT_LOG_RACE_REPRO_ATTEMPTS=200 \ +EVENT_LOG_RACE_REPRO_CONCURRENCY=40 \ +EVENT_LOG_RACE_REPRO_BUDGET_MS=4500000 \ + pnpm run test:e2e:event-log-race-repro:local --skip-build --skip-db-setup +``` + +Against world-postgres the storms bite much harder than the CI job's Vercel +preview does: on `main`, three 14-run passes failed 8 of their 18 `step-storm` +attempts with `CORRUPTED_EVENT_LOG` while `hook-storm` and `hook-sleep` stayed +clean, so the script exits non-zero. That is the harness working, not a broken +setup, and it is why the local runner is the fast signal while a fix is in +flight — at 14 runs a green CI job means "the storms did not trip it", nowhere +near "the rate is below X". + +One thing about a local run is unlike CI and is worth knowing before you read a +result: in CI each replay gets its own Fluid invocation, while here every replay +of every run shares one Next.js process. world-postgres gives that process (and +the harness process) 50 embedded Graphile Worker slots each, and ~100 replays in +one heap saturates GC — measured on a 12-core laptop, all 14 attempts came back +`stuck` with the server at 6.4 GB RSS and Postgres idle. The script therefore +sets `WORKFLOW_POSTGRES_WORKER_CONCURRENCY=10` (override by exporting it) and +raises the app's old-space limit (`--heap-mb`). If a local run reports `stuck` +rather than `CORRUPTED_EVENT_LOG`, suspect the machine before the SDK. + +In CI the same harness runs from `.github/workflows/event-log-race-repro.yml`, +triggered by adding the `event-log-race-repro` label to a PR (or by +`workflow_dispatch`, whose inputs are the soak dial — raise `timeout-minutes` in +that dispatch's branch if you raise `budget_ms`). Results land in a sticky PR +comment that keeps a history of previous runs and their configs. + +To poke at a run afterwards, the CLI reads the same world from the environment: + +```bash +WORKFLOW_TARGET_WORLD=@workflow/world-postgres \ +WORKFLOW_POSTGRES_URL=postgres://world:world@localhost:5432/world \ + pnpm wf inspect +``` + ### Example App Development ```bash diff --git a/package.json b/package.json index cc5a597989..aa05c91c81 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "typecheck": "turbo typecheck", "test:e2e": "vitest run packages/core/e2e/e2e.test.ts packages/core/e2e/e2e-agent.test.ts", "test:e2e:event-log-race-repro": "vitest run packages/core/e2e/event-log-race-repro.test.ts", + "test:e2e:event-log-race-repro:local": "bash scripts/event-log-race-repro-local.sh", "test:e2e:nextjs-webpack:staged": "node scripts/test-staged-nextjs-webpack.mjs", "test:docs": "pnpm --filter @workflow/docs-typecheck test:docs", "bench": "vitest run packages/core/e2e/benchmark.test.ts", diff --git a/packages/core/e2e/event-log-race-repro.test.ts b/packages/core/e2e/event-log-race-repro.test.ts index 0135b16747..c2895ff944 100644 --- a/packages/core/e2e/event-log-race-repro.test.ts +++ b/packages/core/e2e/event-log-race-repro.test.ts @@ -25,14 +25,23 @@ import { getWorkflowMetadata, setupWorld, trackRun } from './utils'; * * The `step-storm` and `hook-storm` scenarios supply all three by construction * (see `workflows/103_event_log_corruption_repro.ts`). `hook-sleep` is retained - * as a low-attempt calibration control: it is the shape that has historically - * produced a nonzero — but very low, ~0.1% — corruption rate, so its rate is the - * yardstick the storms are meant to beat. + * as a calibration control: it is the shape that has historically produced a + * nonzero — but very low, ~0.1% — corruption rate, so its rate is the yardstick + * the storms are meant to beat. * - * This job is a *measurement*, not a pass/fail gate on the SDK's happy path: - * every non-`completed`, non-`infra` outcome is reported and fails the test, so - * a corruption shows up loudly and the sticky PR comment can be diffed between a + * Every non-`completed`, non-`infra` outcome is reported and fails the test, so a + * corruption shows up loudly and the sticky PR comment can be diffed between a * baseline run and a fix run. + * + * At its default scale this is a *regression check*, not a rate measurement: a + * few tens of runs cannot resolve a per-run corruption rate, so a clean run means + * "the storms did not trip it", not "the rate is below X". Measuring a rate — or + * comparing one against the `hook-sleep` baseline — needs the soak scale, which + * is what the `workflow_dispatch` inputs on `event-log-race-repro.yml` exist for. + * The default is deliberately small because the storms are per-run amplifiers: + * each run's own `rounds` x `width` fan-out supplies the concurrency, so a + * regression that the shape can catch at all usually shows up in a handful of + * runs, and the attempt count buys resolution rather than sensitivity. */ const deploymentUrl = process.env.DEPLOYMENT_URL; @@ -170,15 +179,25 @@ function envBoolean(name: string, fallback: boolean) { return fallback; } +// These are the only copy of the harness' scale. The CI workflow passes its +// `workflow_dispatch` inputs straight through and `envNumber` treats an unset or +// empty variable as absent, so a blank input lands on the value below rather +// than on a second default maintained in YAML. const config: ReproConfig = { - stepStormAttempts: envNumber('EVENT_LOG_RACE_REPRO_STEP_STORM_ATTEMPTS', 600), - hookStormAttempts: envNumber('EVENT_LOG_RACE_REPRO_HOOK_STORM_ATTEMPTS', 600), - hookSleepAttempts: envNumber('EVENT_LOG_RACE_REPRO_ATTEMPTS', 200), - concurrency: envNumber('EVENT_LOG_RACE_REPRO_CONCURRENCY', 40), - // 75 min leaves ~20 min of the job's 120-min cap for checkout, build, the - // deployment wait, and rendering the summary, even at the worst case of one - // in-flight attempt draining its full `runTimeoutMs` after the budget ends. - budgetMs: envNumber('EVENT_LOG_RACE_REPRO_BUDGET_MS', 75 * 60_000), + stepStormAttempts: envNumber('EVENT_LOG_RACE_REPRO_STEP_STORM_ATTEMPTS', 6), + hookStormAttempts: envNumber('EVENT_LOG_RACE_REPRO_HOOK_STORM_ATTEMPTS', 6), + hookSleepAttempts: envNumber('EVENT_LOG_RACE_REPRO_ATTEMPTS', 2), + // Cross-run concurrency is throughput only — the race being reproduced is + // between concurrent replays *within* one run, driven by `rounds`/`width` and + // the watchdog timings. So this is set to clear the default attempt count in a + // couple of waves, not to maximize pressure. + concurrency: envNumber('EVENT_LOG_RACE_REPRO_CONCURRENCY', 8), + // Headroom, not a target: the default attempt count needs a few minutes, and a + // budget that fires before the job's `timeout-minutes` is what lets the + // summary be rendered at all. It has to stay far enough below that cap to + // absorb the worst case of one in-flight attempt draining its full + // `runTimeoutMs` after the budget ends (see `testTimeoutMs` below). + budgetMs: envNumber('EVENT_LOG_RACE_REPRO_BUDGET_MS', 12 * 60_000), runTimeoutMs: envNumber('EVENT_LOG_RACE_REPRO_RUN_TIMEOUT_MS', 240_000), hookTimeoutMs: envNumber('EVENT_LOG_RACE_REPRO_HOOK_TIMEOUT_MS', 60_000), rounds: envNumber('EVENT_LOG_RACE_REPRO_ROUNDS', 6), diff --git a/packages/core/src/runtime/step-executor.ts b/packages/core/src/runtime/step-executor.ts index bb68eab33b..63a32939eb 100644 --- a/packages/core/src/runtime/step-executor.ts +++ b/packages/core/src/runtime/step-executor.ts @@ -271,7 +271,11 @@ export async function executeStep( stepId, stepName, } = params; - const isVercel = process.env.VERCEL_URL !== undefined; + // Truthiness, not presence: `vercel env pull` writes `VERCEL_URL=""` into + // `.env.local`, and a framework that loads that file locally would otherwise + // put us on the Vercel branch with nothing to build a host from, making + // `https://` the base URL of every step. + const isVercel = Boolean(process.env.VERCEL_URL); // Unfenced when the caller passes no fence — every World that fences // ignores the field it does not understand, so this is the same create it // was before either mechanism existed. diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 76fd5e00dd..6583fd54aa 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -179,7 +179,11 @@ export async function runWorkflow( const fixedTimestamp = runIdCreatedAt(workflowRun.runId) ?? +workflowRun.createdAt; - const isVercel = process.env.VERCEL_URL !== undefined; + // Truthiness, not presence: `vercel env pull` writes `VERCEL_URL=""` into + // `.env.local`, and a framework that loads that file locally would otherwise + // put us on the Vercel branch with nothing to build a host from, making + // `https://` the base URL of every run. + const isVercel = Boolean(process.env.VERCEL_URL); // Load getPort lazily to prevent Turbopack from tracing get-port's // fs ops (readdir, readFile) into the flow route bundle. The resolved // port is cached per process (see get-port-lazy.ts), so this is cheap diff --git a/scripts/event-log-race-repro-local.sh b/scripts/event-log-race-repro-local.sh new file mode 100755 index 0000000000..ab12c22ed3 --- /dev/null +++ b/scripts/event-log-race-repro-local.sh @@ -0,0 +1,385 @@ +#!/usr/bin/env bash +# +# Run the event-log race repro harness locally, against @workflow/world-postgres +# and a workbench app started on this machine. No Vercel deployment, no GitHub +# Actions, no VERCEL_* credentials. +# +# This is the same harness `.github/workflows/event-log-race-repro.yml` runs +# (`packages/core/e2e/event-log-race-repro.test.ts`), pointed at a local backend. +# It exists because the ordering here is easy to get wrong in ways that fail +# silently rather than loudly: +# +# * WORKFLOW_PUBLIC_MANIFEST=1 must be set at *build* time — the Next.js builder +# emits the manifest as a static file during the build, and without it the +# harness cannot resolve workflow IDs. +# * WORKFLOW_TARGET_WORLD must be set at *build* time too. `withWorkflow()` +# defaults it to `local` when VERCEL_DEPLOYMENT_ID is absent, which bakes the +# filesystem backend into the app and leaves the harness talking to a +# different world than the app is. +# * The schema has to exist before the app boots, or Graphile Worker starts +# against an unmigrated database. +# * The queue has to be empty before the app boots. Postgres outlives the app +# here, so an interrupted run leaves its unfinished flow messages behind and +# the next boot resumes all of them — thousands of stale jobs starving the +# run you actually care about. See clear_queue below. +# +# Usage: scripts/event-log-race-repro-local.sh [options] +# Run with --help for options. + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +APP_NAME="nextjs-turbopack" +PORT="3000" +USE_DEV="0" +SKIP_BUILD="0" +SKIP_DB_SETUP="0" +USE_DOCKER="1" +TEARDOWN="0" +KEEP_QUEUE="0" +# In CI every replay of a run lands in its own Fluid invocation; here they all +# land in one Next.js process, and a storm run has tens of replays in flight at +# once, each holding a VM sandbox and its own copy of the event log. That is +# enough to exceed Node's default old-space limit (~4 GB) and take the server +# down mid-run with an OOM, which then shows up as unrelated `stuck` outcomes. +SERVER_HEAP_MB="8192" +# world-postgres runs an embedded Graphile Worker in every process that loads it, +# so the app and the harness each get this many slots. The package default is 50, +# i.e. ~100 replays in flight against one Next.js process. Measured on a 12-core +# laptop at the default: next-server pegged at ~120% CPU and 6.4 GB RSS against +# an 8 GB old space while Postgres sat idle (1 active connection), all 100 slots +# stayed locked, and every one of the 14 attempts came back `stuck` — GC +# saturation, not a database or a queue limit. Bounded, the same 14 attempts run +# to completion with no `stuck` at all. 10 is also world-postgres's pool size, so +# Graphile Worker stops warning that concurrency outruns maxPoolSize. Bounding +# replays per process does not weaken the repro: the race is between a handful of +# concurrent replays of one run, not a throughput effect. +LOCAL_WORKER_CONCURRENCY="10" + +COMPOSE_FILE="packages/world-postgres/docker-compose.yaml" +# Matches the compose file and the `e2e-local-postgres` job in tests.yml. +DEFAULT_POSTGRES_URL="postgres://world:world@localhost:5432/world" +RESULTS_FILE="event-log-race-repro-results.json" +SERVER_LOG="event-log-race-repro-server.log" +RENDERER=".github/scripts/render-event-log-race-repro-results.js" + +usage() { + cat <<'EOF' +Run the event-log race repro harness against world-postgres + a local workbench app. + +Options: + --app NAME Workbench app to drive (default: nextjs-turbopack). + The repro workflow fixtures (101_hook_sleep_repro.ts, + 103_event_log_corruption_repro.ts) only exist in + workbench/nextjs-turbopack; another app needs them copied + or symlinked in first. + --port N Port for the app (default: 3000). + --heap-mb N Old-space limit for the app process (default: 8192). The + server holds every concurrent replay of every run, so the + default heap is not enough; lower this only if the machine + cannot spare it, and expect OOM-induced `stuck` outcomes. + --dev Use `pnpm dev` instead of a production build + `pnpm start`. + Faster to iterate on workflow fixtures; less like CI. + --skip-build Skip `pnpm build` and the app build. Use when only the + harness or the driver changed. + --skip-db-setup Skip applying migrations (schema already set up). + --no-docker Do not manage Postgres. Point WORKFLOW_POSTGRES_URL at your + own instance; the schema still needs to exist. + --keep-queue Leave queued Graphile Worker jobs in place. By default they + are deleted before the app boots: an interrupted earlier run + leaves its flow messages queued, and resuming those abandoned + runs saturates the app so this run reports `stuck` for + reasons that have nothing to do with the event log. Only + useful if you are inspecting the leftovers themselves. + --teardown Stop and delete the Postgres container on exit. Off by + default so repeat runs skip container startup. + -h, --help Show this help. + +Scale knobs are read from the environment by the harness itself, so anything +already exported is passed through untouched: + + EVENT_LOG_RACE_REPRO_STEP_STORM_ATTEMPTS EVENT_LOG_RACE_REPRO_ROUNDS + EVENT_LOG_RACE_REPRO_HOOK_STORM_ATTEMPTS EVENT_LOG_RACE_REPRO_WIDTH + EVENT_LOG_RACE_REPRO_ATTEMPTS EVENT_LOG_RACE_REPRO_WATCHDOG_MS + EVENT_LOG_RACE_REPRO_CONCURRENCY EVENT_LOG_RACE_REPRO_STEP_DELAY_MS + EVENT_LOG_RACE_REPRO_BUDGET_MS EVENT_LOG_RACE_REPRO_POKE_INTERVAL_MS + EVENT_LOG_RACE_REPRO_RUN_TIMEOUT_MS EVENT_LOG_RACE_REPRO_HOOK_RESUME_STAGGER_MS + +This script deliberately sets none of them. Their defaults — and the full list — +live in packages/core/e2e/event-log-race-repro.test.ts, which is the single +source of truth the CI workflow also defers to. + +The one knob this script does set is WORKFLOW_POSTGRES_WORKER_CONCURRENCY, which +caps how many replays the app and the harness each run at once. world-postgres +defaults it to 50, i.e. ~100 replays sharing one Next.js process, which on a +12-core laptop saturates GC and reports all 14 attempts as `stuck`. It is set to +10 here instead, matching the pool size. Export your own value to override. + +Examples: + # Default scale (a regression check, a few minutes). + scripts/event-log-race-repro-local.sh + + # One run per scenario, to smoke the plumbing. + EVENT_LOG_RACE_REPRO_STEP_STORM_ATTEMPTS=1 \ + EVENT_LOG_RACE_REPRO_HOOK_STORM_ATTEMPTS=1 \ + EVENT_LOG_RACE_REPRO_ATTEMPTS=1 \ + scripts/event-log-race-repro-local.sh + + # Soak for a rate, reusing an already-built app and database. + EVENT_LOG_RACE_REPRO_STEP_STORM_ATTEMPTS=600 \ + EVENT_LOG_RACE_REPRO_HOOK_STORM_ATTEMPTS=600 \ + EVENT_LOG_RACE_REPRO_ATTEMPTS=200 \ + EVENT_LOG_RACE_REPRO_CONCURRENCY=40 \ + EVENT_LOG_RACE_REPRO_BUDGET_MS=4500000 \ + scripts/event-log-race-repro-local.sh --skip-build --skip-db-setup +EOF +} + +while [ $# -gt 0 ]; do + case "$1" in + --app) APP_NAME="${2:?--app needs a value}"; shift 2 ;; + --port) PORT="${2:?--port needs a value}"; shift 2 ;; + --heap-mb) SERVER_HEAP_MB="${2:?--heap-mb needs a value}"; shift 2 ;; + --dev) USE_DEV="1"; shift ;; + --skip-build) SKIP_BUILD="1"; shift ;; + --skip-db-setup) SKIP_DB_SETUP="1"; shift ;; + --no-docker) USE_DOCKER="0"; shift ;; + --keep-queue) KEEP_QUEUE="1"; shift ;; + --teardown) TEARDOWN="1"; shift ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; echo >&2; usage >&2; exit 2 ;; + esac +done + +log() { printf '\n\033[1m[repro]\033[0m %s\n' "$*"; } +die() { printf '\n\033[1;31m[repro]\033[0m %s\n' "$*" >&2; exit 1; } + +APP_DIR="workbench/$APP_NAME" +[ -d "$APP_DIR" ] || die "No such workbench app: $APP_DIR" + +# The harness resolves these two workflow files through the deployment's +# manifest, so an app without them fails one attempt at a time with opaque +# harness errors instead of up front. +for fixture in 101_hook_sleep_repro.ts 103_event_log_corruption_repro.ts; do + [ -e "$APP_DIR/workflows/$fixture" ] || + die "$APP_DIR/workflows/$fixture is missing — the repro fixtures live in workbench/nextjs-turbopack." +done + +export WORKFLOW_POSTGRES_URL="${WORKFLOW_POSTGRES_URL:-$DEFAULT_POSTGRES_URL}" +export WORKFLOW_TARGET_WORLD="@workflow/world-postgres" +export WORKFLOW_PUBLIC_MANIFEST="1" +export WORKFLOW_POSTGRES_WORKER_CONCURRENCY="${WORKFLOW_POSTGRES_WORKER_CONCURRENCY:-$LOCAL_WORKER_CONCURRENCY}" +export PORT="$PORT" +DEPLOYMENT_URL="http://localhost:$PORT" +MANIFEST_URL="$DEPLOYMENT_URL/.well-known/workflow/v1/manifest.json" + +# --- port --------------------------------------------------------------------- + +port_pids() { lsof -ti "tcp:$PORT" 2>/dev/null || true; } + +if [ -n "$(port_pids)" ]; then + die "Port $PORT is already in use (pids: $(port_pids | tr '\n' ' ')). Stop it or pass --port." +fi + +# --- cleanup ------------------------------------------------------------------ + +SERVER_PID="" + +cleanup() { + local status=$? + set +e + if [ -n "$SERVER_PID" ]; then + log "Stopping the app (pid $SERVER_PID)" + # `pnpm start` is a wrapper around `next start`, so the pnpm pid is not the + # listener. Kill the wrapper, then its children, then anything still holding + # the port — a process-group kill is not portable here (macOS has no setsid). + pkill -P "$SERVER_PID" >/dev/null 2>&1 + kill "$SERVER_PID" >/dev/null 2>&1 + # Then wait for the port to actually come free rather than just asking. The + # listener takes a moment to unbind, and the next run's up-front port check + # runs long before that — back-to-back runs otherwise die on "port in use". + local pids i + for i in 1 2 3 4 5 6 7 8 9 10; do + pids="$(port_pids)" + [ -n "$pids" ] || break + # SIGTERM first, SIGKILL from the third try on. + if [ "$i" -lt 3 ]; then + echo "$pids" | xargs kill >/dev/null 2>&1 + else + echo "$pids" | xargs kill -9 >/dev/null 2>&1 + fi + sleep 1 + done + [ -z "$(port_pids)" ] || + log "Port $PORT is still held by $(port_pids | tr '\n' ' ')— the next run needs --port or a manual kill." + fi + if [ "$TEARDOWN" = "1" ] && [ "$USE_DOCKER" = "1" ]; then + log "Removing the Postgres container" + docker compose -f "$COMPOSE_FILE" down -v >/dev/null 2>&1 + fi + set -e + return $status +} +trap cleanup EXIT + +# --- postgres ----------------------------------------------------------------- + +if [ "$USE_DOCKER" = "1" ]; then + command -v docker >/dev/null 2>&1 || die "docker not found. Install Docker, or use --no-docker with your own Postgres." + log "Starting Postgres ($COMPOSE_FILE)" + docker compose -f "$COMPOSE_FILE" up -d + + log "Waiting for Postgres to accept connections" + for _ in $(seq 1 60); do + if docker compose -f "$COMPOSE_FILE" exec -T postgres pg_isready -U world -d world >/dev/null 2>&1; then + break + fi + sleep 1 + done + docker compose -f "$COMPOSE_FILE" exec -T postgres pg_isready -U world -d world >/dev/null 2>&1 || + die "Postgres did not become ready. Check: docker compose -f $COMPOSE_FILE logs postgres" +else + log "Using the Postgres at WORKFLOW_POSTGRES_URL (not managed by this script)" +fi + +# --- queue --------------------------------------------------------------------- + +# Runs a single statement as the `world` user. Prefers the container's own psql so +# no local Postgres client is required. +pg_exec() { + if [ "$USE_DOCKER" = "1" ]; then + docker compose -f "$COMPOSE_FILE" exec -T postgres psql -U world -d world -At -c "$1" 2>/dev/null + else + psql "$WORKFLOW_POSTGRES_URL" -At -c "$1" 2>/dev/null + fi +} + +# The jobs table is private to graphile-worker and has been renamed across its +# versions, so resolve it instead of hardcoding one name, and no-op when the +# schema does not exist yet (a fresh database). +QUEUE_TABLE_SQL="select coalesce(to_regclass('graphile_worker._private_jobs')::text, to_regclass('graphile_worker.jobs')::text, '')" + +clear_queue() { + local table count + table="$(pg_exec "$QUEUE_TABLE_SQL" || true)" + [ -n "$table" ] || return 0 + count="$(pg_exec "select count(*) from $table" || echo "0")" + [ "${count:-0}" -gt 0 ] || return 0 + pg_exec "delete from $table" >/dev/null || + die "Could not clear $table. Pass --keep-queue to skip this, or reset the database." + # Loud on purpose: a backlog here means the previous run was interrupted, which + # is worth knowing when comparing results between runs. + log "Discarded $count queued job(s) left over from an earlier run ($table)" +} + +if [ "$KEEP_QUEUE" = "0" ]; then + if [ "$USE_DOCKER" = "0" ] && ! command -v psql >/dev/null 2>&1; then + log "psql not found — leaving the queue as it is. Stale jobs from an interrupted run will compete with this one." + else + clear_queue + fi +fi + +# --- build -------------------------------------------------------------------- + +if [ "$SKIP_BUILD" = "0" ]; then + # Turbo-cached, so this is cheap on repeat runs. Needed before the migration + # step, which loads packages/world-postgres/dist/cli.js. + log "Building packages" + pnpm build +fi + +if [ "$SKIP_DB_SETUP" = "0" ]; then + log "Applying the world-postgres schema" + ./packages/world-postgres/bin/setup.js +fi + +if [ "$SKIP_BUILD" = "0" ] && [ "$USE_DEV" = "0" ]; then + # WORKFLOW_PUBLIC_MANIFEST and WORKFLOW_TARGET_WORLD are exported above, so + # they apply to this build as well as to the server started below. Both are + # build-time inputs — see the header comment. + log "Building $APP_NAME" + pnpm --filter "$APP_NAME" build +fi + +# --- server ------------------------------------------------------------------- + +if [ "$USE_DEV" = "1" ]; then + SERVER_CMD="dev" +else + SERVER_CMD="start" +fi + +log "Starting $APP_NAME ($SERVER_CMD) on $DEPLOYMENT_URL — logging to $SERVER_LOG" +: > "$SERVER_LOG" +( + cd "$APP_DIR" && + NODE_OPTIONS="${NODE_OPTIONS:+$NODE_OPTIONS }--max-old-space-size=$SERVER_HEAP_MB" \ + pnpm "$SERVER_CMD" >> "$REPO_ROOT/$SERVER_LOG" 2>&1 +) & +SERVER_PID=$! + +# A 200 on the manifest proves both that the server is up and that the public +# manifest the harness depends on was actually emitted — a build without +# WORKFLOW_PUBLIC_MANIFEST=1 serves the app fine and 404s here. +log "Waiting for the workflow manifest at $MANIFEST_URL" +ready="0" +for _ in $(seq 1 180); do + if ! kill -0 "$SERVER_PID" 2>/dev/null; then + tail -40 "$SERVER_LOG" >&2 + die "The app exited before serving the manifest. Full log: $SERVER_LOG" + fi + if curl -fsS -o /dev/null "$MANIFEST_URL" 2>/dev/null; then + ready="1" + break + fi + sleep 1 +done +[ "$ready" = "1" ] || { + tail -40 "$SERVER_LOG" >&2 + die "Timed out waiting for $MANIFEST_URL. Full log: $SERVER_LOG" +} + +# --- harness ------------------------------------------------------------------ + +log "Running the repro harness" +rm -f "$RESULTS_FILE" +set +e +DEPLOYMENT_URL="$DEPLOYMENT_URL" \ +APP_NAME="$APP_NAME" \ +NODE_OPTIONS="--enable-source-maps" \ + pnpm run test:e2e:event-log-race-repro --reporter=default +harness_status=$? +set -e + +# A server that died mid-run makes every remaining attempt time out, so the +# harness reports a wall of `stuck` runs with no hint of why. Say so plainly — +# an OOM or a crash here is a local-environment problem, not a repro. +if ! kill -0 "$SERVER_PID" 2>/dev/null; then + log "The app is no longer running — it died during the harness. Last lines:" + tail -20 "$SERVER_LOG" >&2 + SERVER_PID="" + harness_status=1 +fi + +# --- summary ------------------------------------------------------------------ + +if [ -f "$RESULTS_FILE" ]; then + log "Summary" + # Same renderer the CI job uses for its sticky comment; with no --run-url it + # just prints the tables. + node "$RENDERER" "$RESULTS_FILE" + log "Full results: $RESULTS_FILE / app log: $SERVER_LOG" +else + log "No $RESULTS_FILE was written — the harness died before its first checkpoint." +fi + +if [ "$USE_DOCKER" = "1" ] && [ "$TEARDOWN" = "0" ]; then + log "Postgres is still running. Stop it with: docker compose -f $COMPOSE_FILE down -v" +fi + +# The harness' own exit code is the gate: it fails on any non-`completed`, +# non-`infra` outcome, which is the same rule as the renderer's --check. +exit "$harness_status" diff --git a/vitest.config.ts b/vitest.config.ts index 12e5918d6c..4addd1e955 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -1,7 +1,14 @@ -import { defineConfig } from 'vitest/config'; +import { configDefaults, defineConfig } from 'vitest/config'; export default defineConfig({ test: { testTimeout: 60_000, + // Positional file arguments are regex filters, not paths, so + // `vitest run packages/core/e2e/x.test.ts` also matches + // `.claude/worktrees//packages/core/e2e/x.test.ts` when agent + // worktrees live inside the repo (see .gitignore). Those copies belong to + // other branches: they would run their own version of the suite against the + // same backend and overwrite the same result files. + exclude: [...configDefaults.exclude, '**/.claude/**'], }, }); From 0bac69c5f97fb90a8f729c98c5d505c296fe3e31 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Sat, 1 Aug 2026 12:21:03 -0700 Subject: [PATCH 21/28] fix(core): take slot claims one at a time, tight against the log's tail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A slot claim is an assertion that nothing has been published since the view the writer decided from. Numbering a concurrent batch up front can only assert that for the first of them: the rest sit above slots their own siblings have yet to fill, so a foreign event landing in that space clears their fences too and the batch commits decisions taken without it. Claims are now drawn one at a time off a per-log write chain, so every write names the position immediately after the tail its writer saw. A rejection stops the whole batch rather than only its own write — the log's tail stops advancing while the backend's moves on, so the claims behind it fall inside the occupied range and are rejected in turn. That is the intent: the batch was decided from a log missing an event, so none of it should land. Rejected claims no longer re-address the same write to a free slot by default. Re-sending commits the stale decision anyway, and the missing event may be the one that would have taken the workflow down another branch; WORKFLOW_SLOT_RETRY_BUDGET takes the other side of that trade. world-local and world-postgres check the claim against the log's tail rather than the slot being free. Allocation is append-only, so a position left unwritten by an abandoned reservation stays empty for good, and a caller numbering from a stale snapshot aims straight at it — landing an event below events another replay already consumed. Measured on the world-postgres race repro: 52 runs, 0 CORRUPTED_EVENT_LOG, against 8 of 18 step-storm attempts corrupted on main. Co-Authored-By: Claude Opus 5 --- .../docs/v5/configuration/runtime-tuning.mdx | 9 +- packages/core/src/runtime/helpers.test.ts | 345 +++++++++++------- packages/core/src/runtime/helpers.ts | 207 ++++++++--- .../world-local/src/storage/events-storage.ts | 15 +- .../src/storage/slot-identity.test.ts | 34 +- .../world-local/src/storage/slots.test.ts | 26 +- packages/world-local/src/storage/slots.ts | 35 +- packages/world-postgres/src/storage.ts | 26 +- 8 files changed, 477 insertions(+), 220 deletions(-) diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index 254b0d4815..9c469084ec 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -52,11 +52,18 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL - Default: enabled - Numbers a new run's events by position instead of by ULID: `evnt_…001` is the run's first event, `evnt_…002` its second. Positions are allocated in order, so the log reads in the order it was written regardless of clock skew between writers. -- Contention becomes explicit rather than silent. Two writers proposing the same position cannot both win: the loser gets a 409 ([`SlotConflictError`](/docs/api-reference/workflow-errors/slot-conflict-error)) carrying the events it was missing, and the runtime merges them, replays, and re-proposes at the next free position. +- Contention becomes explicit rather than silent. Two writers proposing the same position cannot both win: the loser gets a 409 ([`SlotConflictError`](/docs/api-reference/workflow-errors/slot-conflict-error)) carrying the events it was missing, and the run replays from the top against a log that now includes them. - Applies only to runs created while it is enabled. A run keeps the identity scheme it was created with for life, so turning the flag on or off never affects runs already in flight. - Requires a World that supports it. A World that does not rejects the run outright rather than mis-numbering its events. - Set `0` or `false` to disable, which numbers new runs by ULID as before. +### `WORKFLOW_SLOT_RETRY_BUDGET` + +- Default: `0` +- How many times a run numbered by `WORKFLOW_SLOT_IDENTITY` may re-send an event creation that lost its position, before the rejection propagates and the run replays. +- Zero means never. A rejected position proves the replay decided from an event log missing at least one event, and re-sending the same write to a free position commits that decision anyway — the missing event may be the one that would have taken the workflow down another branch. +- Raise it for a run that would rather keep a batch of writes contiguous than replay, accepting that a write may land under a decision the replay would have revised. + ## Inline execution ### `WORKFLOW_V2_TIMEOUT_MS` diff --git a/packages/core/src/runtime/helpers.test.ts b/packages/core/src/runtime/helpers.test.ts index 9cf79dcdfc..410e0d0263 100644 --- a/packages/core/src/runtime/helpers.test.ts +++ b/packages/core/src/runtime/helpers.test.ts @@ -527,8 +527,9 @@ describe('slot bookkeeping', () => { const slotEvent = (slot: number) => makeEvent(slotEventId(slot)); it('reads maxSlot from the highest slot present, not the last element', () => { - // `appendUniqueEvents` appends without sorting, so a merged log's last - // element is not necessarily its newest event. + // Nothing forces a caller's array into slot order — a World is free to + // hand back a page in whatever order its index produced — so the highest + // slot is a scan, not a peek at the last element. const log = toMutableEventLog([slotEvent(3), slotEvent(1)], 'c0'); expect(log.maxSlot).toBe(3); expect(log.nextSlot).toBe(4); @@ -604,6 +605,33 @@ describe('slot bookkeeping', () => { ]); }); + it('restores slot order when a merge brings in a lower slot', () => { + // Arrival order is not log order: a slot is reserved when its event is + // issued and written when the issue resolves, so a lower slot can be + // learned after a higher one. The replay consumes this array positionally, + // so an event left sitting ahead of the one it followed decides races the + // wrong way. + const log = toMutableEventLog([slotEvent(1), slotEvent(4)], 'c0'); + mergeLoadedEvents(log, [slotEvent(3), slotEvent(2)]); + expect(log.events.map((e) => e.eventId)).toEqual([ + slotEventId(1), + slotEventId(2), + slotEventId(3), + slotEventId(4), + ]); + }); + + it('leaves a ULID log in the order the World returned it', () => { + // ULID ids are minted at write time, so arrival order *is* log order and + // the World's ordering is the authority. + const events = [makeUlidEvent(1_700_000_000_000)]; + const later = makeUlidEvent(1_700_000_001_000); + const earlier = makeUlidEvent(1_699_999_999_000); + const log = toMutableEventLog(events, 'c0'); + mergeLoadedEvents(log, [later, earlier]); + expect(log.events).toEqual([events[0], later, earlier]); + }); + it('hands out contiguous distinct slots for a synchronous burst', () => { // The suspension flush issues every operation synchronously and awaits // them together; without contiguous reservation they would all propose the @@ -704,105 +732,189 @@ describe('withSlotRetry', () => { expect(eventsListMock).not.toHaveBeenCalled(); }); - it('merges the conflict delta and reclaims past it, without reloading', async () => { - // The inline delta is the whole point of the 409 body: the client learns - // which events it was missing without a follow-up round-trip. + it('takes each claim only once the create ahead of it has committed', async () => { + // A claim fences out a concurrent writer only while it names the slot right + // after the tail the writer saw, so a second create cannot be numbered + // until the first has landed. const log = toMutableEventLog([slotEvent(1)], 'c0'); - const claimed: string[] = []; - const op = vi.fn(async ({ eventId }: { eventId?: string }) => { - claimed.push(eventId as string); - if (claimed.length === 1) { - throw new SlotConflictError('taken', { - eventId: eventId as string, - events: [slotEvent(2), slotEvent(3)], - cursor: 'c1', - }); - } - return 'done'; + const claimed: (string | undefined)[] = []; + let releaseFirst!: () => void; + const firstLanded = new Promise((resolve) => { + releaseFirst = resolve; }); - await expect(withSlotRetry('wrun_test', log, op)).resolves.toBe('done'); - expect(claimed).toEqual([slotEventId(2), slotEventId(4)]); - expect(log.events).toHaveLength(3); - expect(log.cursor).toBe('c1'); - expect(eventsListMock).not.toHaveBeenCalled(); + const first = withSlotRetry('wrun_test', log, async (fence) => { + claimed.push(fence.eventId); + await firstLanded; + return 'first'; + }); + const second = withSlotRetry('wrun_test', log, async (fence) => { + claimed.push(fence.eventId); + return 'second'; + }); + + await Promise.resolve(); + expect(claimed).toEqual([slotEventId(2)]); + + releaseFirst(); + await expect(first).resolves.toBe('first'); + await expect(second).resolves.toBe('second'); + expect(claimed).toEqual([slotEventId(2), slotEventId(3)]); }); - it('tops the delta up from the backend when it was truncated', async () => { + it('rejects a lost claim rather than re-addressing the write', async () => { + // A 409 says this replay decided from a log missing an event. Moving the + // same write to a free slot would commit that decision anyway, so the + // rejection propagates and the run replays. const log = toMutableEventLog([slotEvent(1)], 'c0'); - eventsListMock.mockResolvedValueOnce({ - data: [slotEvent(3)], - cursor: 'c2', - hasMore: false, - }); - let attempts = 0; - const op = vi.fn(async () => { - attempts++; - if (attempts === 1) { - throw new SlotConflictError('taken', { - eventId: slotEventId(2), - events: [slotEvent(2)], - cursor: 'c1', - hasMore: true, - }); - } - return 'done'; + const op = vi.fn(async ({ eventId }: { eventId?: string }) => { + throw new SlotConflictError('taken', { + eventId: eventId as string, + events: [slotEvent(2)], + cursor: 'c1', + }); }); - await expect(withSlotRetry('wrun_test', log, op)).resolves.toBe('done'); - expect(eventsListMock).toHaveBeenCalledTimes(1); - expect(log.maxSlot).toBe(3); - expect(op).toHaveBeenLastCalledWith({ - eventId: slotEventId(4), - maxSlot: 3, - }); + await expect(withSlotRetry('wrun_test', log, op)).rejects.toBeInstanceOf( + SlotConflictError + ); + expect(op).toHaveBeenCalledTimes(1); + expect(eventsListMock).not.toHaveBeenCalled(); }); - it('falls back to a full incremental load when the rejection carried no delta', async () => { + it('leaves the rest of the batch claiming into the occupied range', async () => { + // The tail stops advancing for this log while the backend's moves on, so + // the siblings behind a rejected claim propose slots the backend has + // already filled and are rejected with it. const log = toMutableEventLog([slotEvent(1)], 'c0'); - eventsListMock.mockResolvedValueOnce({ - data: [slotEvent(2)], - cursor: 'c1', - hasMore: false, + const loser = withSlotRetry('wrun_test', log, async ({ eventId }) => { + throw new SlotConflictError('taken', { eventId: eventId as string }); }); - let attempts = 0; - const op = vi.fn(async () => { - attempts++; - if (attempts === 1) { - throw new SlotConflictError('taken', { eventId: slotEventId(2) }); - } - return 'done'; + await expect(loser).rejects.toBeInstanceOf(SlotConflictError); + + const sibling = await withSlotRetry( + 'wrun_test', + log, + async (fence) => fence.eventId + ); + expect(sibling).toBe(slotEventId(2)); + }); + + describe('under a reclaim budget', () => { + // `WORKFLOW_SLOT_RETRY_BUDGET` takes the other side of the default trade: + // it keeps a batch contiguous by re-issuing the write past what it was + // missing, at the cost of committing a decision taken without it. + beforeEach(() => { + vi.stubEnv( + 'WORKFLOW_SLOT_RETRY_BUDGET', + String(PRECONDITION_MAX_RELOAD_RETRIES) + ); + }); + afterEach(() => { + vi.unstubAllEnvs(); }); - await expect(withSlotRetry('wrun_test', log, op)).resolves.toBe('done'); - expect(eventsListMock).toHaveBeenCalledTimes(1); - expect(op).toHaveBeenLastCalledWith({ - eventId: slotEventId(3), - maxSlot: 2, + it('merges the conflict delta and reclaims past it, without reloading', async () => { + // The inline delta is the whole point of the 409 body: the client learns + // which events it was missing without a follow-up round-trip. + const log = toMutableEventLog([slotEvent(1)], 'c0'); + const claimed: string[] = []; + const op = vi.fn(async ({ eventId }: { eventId?: string }) => { + claimed.push(eventId as string); + if (claimed.length === 1) { + throw new SlotConflictError('taken', { + eventId: eventId as string, + events: [slotEvent(2), slotEvent(3)], + cursor: 'c1', + }); + } + return 'done'; + }); + + await expect(withSlotRetry('wrun_test', log, op)).resolves.toBe('done'); + expect(claimed).toEqual([slotEventId(2), slotEventId(4)]); + expect(log.events).toHaveLength(3); + expect(log.cursor).toBe('c1'); + expect(eventsListMock).not.toHaveBeenCalled(); }); - }); - it('rethrows the conflict once the reclaim budget is spent', async () => { - // Escaping to a fresh replay is the correct fallback, not a failure mode: - // the merged events can change what the workflow body decides, and only a - // replay from the top can act on that. - const log = toMutableEventLog([slotEvent(1)], 'c0'); - eventsListMock.mockResolvedValue({ - data: [], - cursor: 'c1', - hasMore: false, + it('tops the delta up from the backend when it was truncated', async () => { + const log = toMutableEventLog([slotEvent(1)], 'c0'); + eventsListMock.mockResolvedValueOnce({ + data: [slotEvent(3)], + cursor: 'c2', + hasMore: false, + }); + let attempts = 0; + const op = vi.fn(async () => { + attempts++; + if (attempts === 1) { + throw new SlotConflictError('taken', { + eventId: slotEventId(2), + events: [slotEvent(2)], + cursor: 'c1', + hasMore: true, + }); + } + return 'done'; + }); + + await expect(withSlotRetry('wrun_test', log, op)).resolves.toBe('done'); + expect(eventsListMock).toHaveBeenCalledTimes(1); + // The reclaim saw slot 3 as the tail and then became it. + expect(log.maxSlot).toBe(4); + expect(op).toHaveBeenLastCalledWith({ + eventId: slotEventId(4), + maxSlot: 3, + }); }); - const op = vi.fn(async ({ eventId }: { eventId?: string }) => { - throw new SlotConflictError('taken', { eventId: eventId as string }); + + it('falls back to a full incremental load when the rejection carried no delta', async () => { + const log = toMutableEventLog([slotEvent(1)], 'c0'); + eventsListMock.mockResolvedValueOnce({ + data: [slotEvent(2)], + cursor: 'c1', + hasMore: false, + }); + let attempts = 0; + const op = vi.fn(async () => { + attempts++; + if (attempts === 1) { + throw new SlotConflictError('taken', { eventId: slotEventId(2) }); + } + return 'done'; + }); + + await expect(withSlotRetry('wrun_test', log, op)).resolves.toBe('done'); + expect(eventsListMock).toHaveBeenCalledTimes(1); + expect(op).toHaveBeenLastCalledWith({ + eventId: slotEventId(3), + maxSlot: 2, + }); }); - await expect(withSlotRetry('wrun_test', log, op)).rejects.toBeInstanceOf( - SlotConflictError - ); - expect(op).toHaveBeenCalledTimes(PRECONDITION_MAX_RELOAD_RETRIES + 1); - expect(eventsListMock).toHaveBeenCalledTimes( - PRECONDITION_MAX_RELOAD_RETRIES - ); + it('rethrows the conflict once the reclaim budget is spent', async () => { + // Escaping to a fresh replay is the correct fallback, not a failure mode: + // the merged events can change what the workflow body decides, and only a + // replay from the top can act on that. + const log = toMutableEventLog([slotEvent(1)], 'c0'); + eventsListMock.mockResolvedValue({ + data: [], + cursor: 'c1', + hasMore: false, + }); + const op = vi.fn(async ({ eventId }: { eventId?: string }) => { + throw new SlotConflictError('taken', { eventId: eventId as string }); + }); + + await expect(withSlotRetry('wrun_test', log, op)).rejects.toBeInstanceOf( + SlotConflictError + ); + expect(op).toHaveBeenCalledTimes(PRECONDITION_MAX_RELOAD_RETRIES + 1); + expect(eventsListMock).toHaveBeenCalledTimes( + PRECONDITION_MAX_RELOAD_RETRIES + ); + }); }); it('rethrows a non-conflict error immediately, without merging', async () => { @@ -824,28 +936,20 @@ describe('withEventCreateFence', () => { eventsListMock.mockReset(); }); - it('fences a slot-numbered run by event id and retries its 409s', async () => { + it('fences a slot-numbered run by event id and escapes its 409s', async () => { const log = toMutableEventLog([makeEvent(slotEventId(1))], 'c0'); - let attempts = 0; const op = vi.fn(async ({ eventId }: { eventId?: string }) => { - attempts++; - if (attempts === 1) { - throw new SlotConflictError('taken', { - eventId: eventId as string, - events: [makeEvent(slotEventId(2))], - cursor: 'c1', - }); - } - return 'done'; + throw new SlotConflictError('taken', { + eventId: eventId as string, + events: [makeEvent(slotEventId(2))], + cursor: 'c1', + }); }); await expect( withEventCreateFence('wrun_test', log, SPEC_VERSION_SLOT_IDENTITY, op) - ).resolves.toBe('done'); - expect(op).toHaveBeenLastCalledWith({ - eventId: slotEventId(3), - maxSlot: 2, - }); + ).rejects.toBeInstanceOf(SlotConflictError); + expect(op).toHaveBeenCalledWith({ eventId: slotEventId(2), maxSlot: 1 }); }); it('fences a ULID-numbered run by watermark and retries its 412s', async () => { @@ -880,10 +984,10 @@ describe('stepClaimFence', () => { eventsListMock.mockReset(); }); - it('numbers a batch in the order its claims were built, not the order they fire', async () => { - // The batch is built during replay and only starts racing afterwards, so - // the slot of each member has to be fixed at build time for the numbering - // to be replay-stable. + it('numbers a batch in the order its claims fire, each one above the last', async () => { + // Slots go out one at a time so every claim names the tail its writer saw. + // A step that publishes the `step_created` it deferred takes the two slots + // below its claim, and the next claim starts above both. const log = toMutableEventLog([slotEvent(1)], 'c0'); const first = stepClaimFence('wrun_test', log, SPEC_VERSION_SLOT_IDENTITY, { extraEvents: 1, @@ -895,19 +999,16 @@ describe('stepClaimFence', () => { claimed.push(fence?.eventId); return Promise.resolve('ok'); }; - // Fired in reverse: the numbering must not depend on it. - await second(record); await first(record); + await second(record); - expect(claimed).toEqual([slotEventId(4), slotEventId(3)]); + expect(claimed).toEqual([slotEventId(3), slotEventId(4)]); }); - it('reclaims a lost slot in place, keeping the batch adjacent', async () => { - // The server allocates an outside event from the same next-free pointer - // the client reserves from, so losing a claim is routine. Abandoning the - // batch on it would leave this step's events far later in the log than its - // siblings' — an order no single replay can consume — and leave the lost - // slot permanently empty. + it('abandons the batch when a claim is lost', async () => { + // The lost claim proves the batch was decided from a log missing an event, + // so the sibling behind it re-proposes the slot the backend has already + // filled instead of stepping over it. const log = toMutableEventLog([slotEvent(1)], 'c0'); const claim = stepClaimFence('wrun_test', log, SPEC_VERSION_SLOT_IDENTITY); const sibling = stepClaimFence( @@ -919,23 +1020,19 @@ describe('stepClaimFence', () => { const claimed: string[] = []; const op = vi.fn(async (fence?: { eventId?: string }) => { claimed.push(fence?.eventId as string); - if (claimed.length === 1) { - // An out-of-band hook took slot 2 while the batch was being built. - throw new SlotConflictError('taken', { - eventId: fence?.eventId as string, - events: [slotEvent(2)], - cursor: 'c1', - }); - } - return 'done'; + // An out-of-band hook took slot 2 while the batch was being built. + throw new SlotConflictError('taken', { + eventId: fence?.eventId as string, + events: [slotEvent(2)], + cursor: 'c1', + }); }); - await expect(claim(op)).resolves.toBe('done'); + await expect(claim(op)).rejects.toBeInstanceOf(SlotConflictError); await expect(sibling(async (f) => f?.eventId)).resolves.toBe( - slotEventId(3) + slotEventId(2) ); - // Reclaimed above the merged event rather than propagating the conflict. - expect(claimed).toEqual([slotEventId(2), slotEventId(4)]); + expect(claimed).toEqual([slotEventId(2)]); }); it('leaves a ULID-numbered batch on one shared watermark, unretried', async () => { diff --git a/packages/core/src/runtime/helpers.ts b/packages/core/src/runtime/helpers.ts index 586c0f1a68..951c0fd0d8 100644 --- a/packages/core/src/runtime/helpers.ts +++ b/packages/core/src/runtime/helpers.ts @@ -17,6 +17,7 @@ import type { import { getQueueTopicPrefix, HealthCheckPayloadSchema, + isSlotId, maxSlotOf, resolveQueueNamespace, SPEC_VERSION_CURRENT, @@ -465,7 +466,22 @@ function recordRequestedEventCursor( } /** - * Appends events whose IDs are not already present in `target`. + * Appends events whose IDs are not already present in `target`, keeping a + * slot-numbered log in slot order. + * + * Arrival order is not log order under slot identity. A slot is reserved when + * its event is issued and written when that issue resolves, so a lower slot can + * be committed after a higher one, and a merge that only appends leaves the + * array in the order the events were *learned*, not the order they occupy. + * That difference decides races: the replay consumes this array positionally + * — the delivery barriers in `pendingDeliveryBarriers` are keyed on the index — + * so a `step_completed` sitting ahead of a `wait_completed` it actually + * follows makes the replay take the branch the log does not record, and the + * next event it reads belongs to a step it never started. + * + * Sorting by event id *is* sorting by slot: ids are zero-padded to a fixed + * width, and a slot-numbered run's log carries no ULID ids to interleave with + * them. * * Pass the IDs currently present in `target` when appending repeatedly to the * same array. The set is updated alongside `target`. @@ -480,12 +496,18 @@ export function appendUniqueEvents( } const ids = targetIds ?? new Set(target.map((event) => event.eventId)); + let outOfOrder = false; for (const event of events) { if (!ids.has(event.eventId)) { ids.add(event.eventId); + outOfOrder ||= + target.length > 0 && event.eventId < target[target.length - 1].eventId; target.push(event); } } + if (outOfOrder && isSlotId(target[0].eventId)) { + target.sort((a, b) => (a.eventId < b.eventId ? -1 : 1)); + } } function assertEventPaginationProgress( @@ -665,11 +687,23 @@ export interface MutableEventLog { */ maxSlot: number; /** - * Next slot `reserveSlot` will hand out. Absolute, and it only ever moves - * forward: a merge can raise it past the events it brought in, but must never - * lower it onto a slot already handed to a writer that is still in flight. + * Next slot `reserveSlot` will hand out. Only a writer holding the log's + * write chain may draw from it, and it is rewound to `maxSlot + 1` when a + * claim is rejected so the rest of the batch claims into the occupied range + * and is rejected with it. */ nextSlot: number; + /** + * Tail of the chain of creates numbered off this log, or `undefined` when + * none is in flight. + * + * Slot claims are taken one at a time. A claim only fences out a concurrent + * writer if it names the slot right after the log's committed tail: numbering + * a whole concurrent batch up front hands its later writes slots far enough + * above the tail that a foreign event landing in between clears every fence + * they carry, and the batch commits decisions taken without it. + */ + writeChain?: Promise; } /** @@ -709,19 +743,11 @@ export function mergeLoadedEvents( } /** - * Claims the next free slot in `log`, synchronously at the moment an event is - * issued. + * Claims the next free slot in `log`. * - * Reservations are contiguous rather than all-`maxSlot + 1` because a - * suspension flushes its operations concurrently: without them every operation - * in the flush would propose the same slot and all but one would conflict, on - * every single flush. Operations are built in deterministic replay order, so - * the slot each one draws is replay-stable too. - * - * The pointer is never rewound by a merge, only pushed forward. A writer that - * loses its slot merges the delta and reserves again while its siblings still - * hold theirs; rewinding to `maxSlot + 1` would hand it a sibling's slot and - * turn one conflict into a chain of them. + * Only call this while holding the log's write chain: the claim is the fence, + * and it only fences anything while it names the slot immediately after the + * tail this writer has seen. */ export function reserveSlot(log: MutableEventLog): number { const slot = log.nextSlot; @@ -927,6 +953,57 @@ function reserveSlotFence( return { eventId: slotEventId(reserveSlot(log)), maxSlot }; } +/** + * Runs one slot-numbered create with the log's claim to itself, taking its slot + * only once every create ahead of it on the log has settled. + * + * A slot claim is an assertion about the tail: "nothing has been published + * since the view I decided from". Claims handed out up front to a concurrent + * batch can only assert that about the first of them — the rest sit above slots + * their own siblings have yet to fill, so a foreign event landing in that space + * satisfies their fences too and they commit on a view that is already missing + * it. Taking claims one at a time keeps every write's fence tight against the + * tail the writer actually saw. + * + * A rejection therefore stops the whole batch rather than only its own write: + * the log's tail stops advancing while the backend's moves on, so the claims + * behind it fall inside the occupied range and are rejected in turn. That is + * the intent — the batch was decided from a log missing an event, so none of it + * should land. + */ +async function withSerializedClaim( + log: MutableEventLog, + extraEvents: number, + op: (fence: EventCreateFence) => Promise +): Promise { + const ahead = log.writeChain; + let done!: () => void; + log.writeChain = new Promise((resolve) => { + done = resolve; + }); + if (ahead) { + await ahead; + } + try { + const fence = reserveSlotFence(log, extraEvents); + const result = await op(fence); + log.maxSlot = Math.max( + log.maxSlot, + maxSlotOf([{ eventId: fence.eventId ?? '' }]) + ); + log.nextSlot = Math.max(log.nextSlot, log.maxSlot + 1); + return result; + } catch (error) { + // The slots this attempt drew are not the writer's, and the tail is at + // least as high as the claim that lost. Rewinding onto the occupied range + // is what makes the rest of the batch fail with it. + log.nextSlot = log.maxSlot + 1; + throw error; + } finally { + done(); + } +} + /** * Runs one event create under whichever fence its run uses, handling a lost * claim however that run's scheme requires. @@ -938,23 +1015,22 @@ export type FencedCreate = ( /** * The fence for an inline step's `step_started` claim. * - * A slot-numbered run retries a lost claim in place; a watermark-guarded run - * does not, and lets the rejection abandon the batch for a fresh replay. The - * asymmetry is in what a rejection proves, and it is the difference between a - * batch that stays contiguous and one that splits: + * Both numbering schemes abandon the batch when the claim is rejected, and for + * the same reason: a rejection says this replay decided from a log it had not + * fully seen, and a decision made that way is not one to re-address to a free + * number. A 412 says the newest outside event is younger than the snapshot; a + * 409 says the number this replay counted to belongs to someone else. Either + * way the missing event can be the one that would have sent the workflow down + * another branch, and because correlation IDs are counted in branch order, + * every ID after that branch moves with it — so the re-issued write lands under + * an identity that now names a different step. * - * - A 412 compares the *time* of the newest outside event, so every claim in a - * batch carries the same fence value and a stale view fails all of them. The - * batch is abandoned as a unit, nothing is written, and the fresh replay - * reschedules from a complete view. - * - A 409 only proves another writer took this write's *number*. That happens - * routinely without any staleness: the server allocates outside events from - * the same next-free pointer the client reserves from, so any outside event - * landing mid-batch takes the slot the batch's next claim is holding. Fencing - * the batch on it splits it — the loser writes nothing while its siblings - * commit, and the loser's events land far later in the log (or never), leaving - * an order no single replay can consume. Taking another number instead keeps - * the batch's events adjacent and its slots dense. + * The cost is a batch that splits rather than staying contiguous: the loser + * writes nothing while its siblings commit. That is the intended trade. The + * fresh replay reschedules the whole batch from a complete view, and slots are + * allowed to have holes, so a split costs a round-trip rather than a + * correctness property. `WORKFLOW_SLOT_RETRY_BUDGET` reinstates the in-place + * retry for a run that would rather have the contiguity. */ export function stepClaimFence( runId: string, @@ -963,54 +1039,69 @@ export function stepClaimFence( options?: { extraEvents?: number } ): FencedCreate { if (usesSlotIdentity(specVersion)) { - // Reserved here, synchronously, rather than when the claim fires: a batch's - // claims have to be numbered in replay order, and they only start racing - // each other afterwards. Retries re-reserve at that point by necessity — - // the merged delta has moved the log — but by then this write is the only - // one of the batch still choosing a slot. - const initialFence = reserveSlotFence(log, options?.extraEvents ?? 0); - return (op) => withSlotRetry(runId, log, op, { ...options, initialFence }); + return (op) => withSlotRetry(runId, log, op, options); } const fence = eventCreateFenceFor(log, specVersion, options); return (op) => op(fence); } +/** + * How many times a lost claim may be re-issued in place before the rejection + * propagates and the run replays. + * + * Zero — the default — means never. A 409 proves this replay decided from a log + * missing at least one event, and re-sending the same write only moves a + * decision that may already be wrong to a free number: the merged event can be + * the very one that would have sent the workflow down another branch, and + * because correlation IDs are counted in branch order, every ID after that + * branch renumbers with it. Retrying in place keeps a batch contiguous at the + * cost of committing that stale decision, so the default trades contiguity for + * a fresh replay. `WORKFLOW_SLOT_RETRY_BUDGET` takes the other side. + */ +function slotRetryBudget(): number { + const raw = process.env.WORKFLOW_SLOT_RETRY_BUDGET; + const parsed = raw === undefined ? Number.NaN : Number(raw); + return Number.isInteger(parsed) && parsed >= 0 ? parsed : 0; +} + /** * Runs a replay-context event creation that claims its own event slot. * * The claim is the fence: the backend inserts the proposed `eventId` * conditionally, so a `SlotConflictError` (409) proves another writer got there - * first and that this replay therefore ran against an event log missing at least - * one event. Re-sending the same write is pointless — it would lose the same - * slot again — so each attempt merges the events it was missing (inline off the - * rejection, topped up from the backend when the delta was truncated) and claims - * a fresh slot past them. + * first and that this replay therefore ran against an event log missing at + * least one event. By default the rejection propagates immediately and the run + * is re-invoked from the queue for a fresh replay, because merged events can + * change what the workflow body decides and only a replay from the top can act + * on them. * - * Bounded by `PRECONDITION_MAX_RELOAD_RETRIES`, after which the error - * propagates and the run is re-invoked from the queue for a fresh replay. That - * fallback is not merely a giving-up path: merged events can change what the - * workflow body decides, and only a replay from the top can act on them. + * Under a non-zero {@link slotRetryBudget} the write is instead re-issued in + * place, merging what it was missing (inline off the rejection, topped up from + * the backend when the delta was truncated) and claiming a fresh slot past it. */ export async function withSlotRetry( runId: string, log: MutableEventLog, op: (fence: EventCreateFence) => Promise, - options?: { extraEvents?: number; initialFence?: EventCreateFence } + options?: { extraEvents?: number } ): Promise { for (let attempt = 0; ; attempt++) { - // Claimed per attempt, not once up front: a merged delta moves the log's - // high-water mark, so the previous claim is stale by definition. The first - // attempt can carry a slot the caller reserved earlier, for a caller whose - // numbering has to be assigned in a particular order (see stepClaimFence). - const fence = - (attempt === 0 ? options?.initialFence : undefined) ?? - reserveSlotFence(log, options?.extraEvents ?? 0); - const eventId = fence.eventId; + let eventId = ''; try { - return await op(fence); + // Claimed per attempt, not once up front: a merged delta moves the log's + // high-water mark, so the previous claim is stale by definition. + return await withSerializedClaim( + log, + options?.extraEvents ?? 0, + (fence) => { + eventId = fence.eventId ?? ''; + return op(fence); + } + ); } catch (error) { if ( !SlotConflictError.is(error) || + attempt >= slotRetryBudget() || attempt >= PRECONDITION_MAX_RELOAD_RETRIES ) { throw error; diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts index 1339a2e0bd..33ad225bb5 100644 --- a/packages/world-local/src/storage/events-storage.ts +++ b/packages/world-local/src/storage/events-storage.ts @@ -1114,15 +1114,18 @@ export function createEventsStorage( reserved.add(companionSlot); slots.claim(effectiveRunId, companionSlot); } - // Reject a doomed claim before the materialization below creates - // the step, hook or wait this event will now never accompany. A - // caller that re-proposes at the next slot would otherwise - // collide with its own orphan and read that as "my write already - // landed". See SlotBook.isWritten. + // A claim has to clear the log's tail, not merely be free: a + // position left unwritten by an abandoned reservation stays empty + // for good, and a caller numbering from a stale snapshot aims + // straight at it, landing an event below events another replay + // already consumed. Rejected before the materialization below + // creates the step, hook or wait this event will now never + // accompany. See SlotBook.highestWritten. + const tail = await slots.highestWritten(effectiveRunId); for (const slot of companionSlot === undefined ? [claimedSlot] : [companionSlot, claimedSlot]) { - if (await slots.isWritten(effectiveRunId, slot)) { + if (slot <= tail) { throw await slotConflict( effectiveRunId, slotEventId(slot), diff --git a/packages/world-local/src/storage/slot-identity.test.ts b/packages/world-local/src/storage/slot-identity.test.ts index 5b9eaf2626..f1ee5df07b 100644 --- a/packages/world-local/src/storage/slot-identity.test.ts +++ b/packages/world-local/src/storage/slot-identity.test.ts @@ -120,15 +120,35 @@ describe('numbering', () => { await expect(slotsOf(runId)).resolves.toEqual([1, 2]); }); - it('lists the log in slot order, not in the order writes started', async () => { - // A writer that loses its slot re-proposes above the winner while keeping - // the wall-clock stamp it started with, so `createdAt` order and slot order - // disagree. Replay consumes the log in list order, so list order has to be - // slot order — what the sort key gives the other backends for free. + it('rejects a claim on a free position below the log’s tail', async () => { + // The undercut that a "is the position free?" check cannot catch. A + // position claimed by a write that then failed is never filled, so a log + // carries holes below its tail — and a caller numbering from a snapshot + // that predates the events above one of those holes aims straight at it. + // Let it land and the event sits below events another replay has already + // consumed: the log stays internally consistent while its order silently + // changes, which is enough to flip a race between a step and a sleep from + // one replay to the next. A claim asserts a complete log, so a claim that + // does not clear the tail is a conflict, exactly as a taken one is. const runId = await newSlotRun(); await createStep(runId, 'step_late', slotEventId(3)); - await createStep(runId, 'step_early', slotEventId(2)); - await expect(slotsOf(runId)).resolves.toEqual([1, 2, 3]); + await expect( + createStep(runId, 'step_early', slotEventId(2)) + ).rejects.toThrow(SlotConflictError); + // The hole stays a hole, and the log stays in slot order. + await expect(slotsOf(runId)).resolves.toEqual([1, 3]); + }); + + it('accepts the claim immediately above a tail with a hole below it', async () => { + // The fence rejects at-or-below, so the first position above the tail has + // to stay writable — otherwise every write following a hole would conflict + // forever and the run could never make progress again. + const runId = await newSlotRun(); + await createStep(runId, 'step_late', slotEventId(3)); + expect(await createStep(runId, 'step_next', slotEventId(4))).toBe( + slotEventId(4) + ); + await expect(slotsOf(runId)).resolves.toEqual([1, 3, 4]); }); it('keeps a burst of concurrent writers dense', async () => { diff --git a/packages/world-local/src/storage/slots.test.ts b/packages/world-local/src/storage/slots.test.ts index 06f2986b71..318b95c204 100644 --- a/packages/world-local/src/storage/slots.test.ts +++ b/packages/world-local/src/storage/slots.test.ts @@ -2,6 +2,7 @@ import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; import { + FIRST_SLOT, SPEC_VERSION_CURRENT, SPEC_VERSION_SLOT_IDENTITY, slotEventId, @@ -186,20 +187,33 @@ describe('release', () => { }); }); -describe('isWritten', () => { +describe('highestWritten', () => { it('reads the log to answer for a run it has never seen', async () => { await writeEvents(1, 2); const book = createSlotBook(basedir); - await expect(book.isWritten(RUN_ID, 2)).resolves.toBe(true); - await expect(book.isWritten(RUN_ID, 3)).resolves.toBe(false); + await expect(book.highestWritten(RUN_ID)).resolves.toBe(2); }); - it('is false for a slot that is only reserved', async () => { + it('reports the tail of a log with a hole below it', async () => { + // The case the tail exists to catch: slot 3 was claimed by a write that + // never published, so it is free — but a claim on it would land below the + // event at 4 that a replay may already have consumed. + await writeEvents(1, 2, 4); + const book = createSlotBook(basedir); + await expect(book.highestWritten(RUN_ID)).resolves.toBe(4); + }); + + it('is below the first slot for a run with no events', async () => { + const book = createSlotBook(basedir); + await expect(book.highestWritten(RUN_ID)).resolves.toBe(FIRST_SLOT - 1); + }); + + it('ignores a slot that is only reserved', async () => { // A reservation is not a publish, so a caller claiming the slot has to be // allowed through to the write that actually decides it. const book = createSlotBook(basedir); - const slot = await book.reserve(RUN_ID); - await expect(book.isWritten(RUN_ID, slot)).resolves.toBe(false); + await book.reserve(RUN_ID); + await expect(book.highestWritten(RUN_ID)).resolves.toBe(FIRST_SLOT - 1); }); }); diff --git a/packages/world-local/src/storage/slots.ts b/packages/world-local/src/storage/slots.ts index 25b2b8e522..1b1822e704 100644 --- a/packages/world-local/src/storage/slots.ts +++ b/packages/world-local/src/storage/slots.ts @@ -88,18 +88,29 @@ export interface SlotBook { */ claim(runId: string, slot: number): void; /** - * Whether `slot` is already occupied by a published event, seeding from disk - * if this run has not been read yet. + * The highest position this run has published, seeding from disk if the run + * has not been read yet, or `FIRST_SLOT - 1` for a log with no events. * - * Lets a doomed claim be rejected *before* the create materializes its step, - * hook or wait: the entity mutation runs ahead of the event publish, so a - * claim that only fails at the publish leaves an entity behind with no event, - * and the caller's re-proposal at the next slot then collides with its own - * orphan. A `false` here is not a promise — the publish is still the - * authority — but it turns the case that actually happens (a caller numbering - * from a stale log) into a clean conflict. + * This is the tail a claim has to clear. "Free" is not the property a claim + * needs: allocation is append-only, so a position left unwritten by an + * abandoned reservation stays empty for good, and a caller numbering from a + * snapshot that predates the events above such a hole aims straight at it. Let + * that write land and the event sits *below* events another replay already + * consumed — the log stays internally consistent while its order silently + * changes, which is enough to flip a race between a step and a sleep from one + * replay to the next. + * + * Reading the tail before the write also lets a doomed claim be rejected + * *before* the create materializes its step, hook or wait: the entity mutation + * runs ahead of the event publish, so a claim that only fails at the publish + * leaves an entity behind with no event, and the caller's re-proposal at the + * next slot then collides with its own orphan. A tail read here is not a + * promise — the publish is still the authority, and another instance sharing + * the data directory may have written above it — but it turns the case that + * actually happens (a caller numbering from a stale log) into a clean + * conflict. */ - isWritten(runId: string, slot: number): Promise; + highestWritten(runId: string): Promise; /** * Forgets a reserved or claimed slot whose publish is never going to happen, * so nothing waits on it. The position itself is not handed out again: it may @@ -243,9 +254,9 @@ export function createSlotBook(basedir: string, tag?: string): SlotBook { } }, - async isWritten(runId, slot) { + async highestWritten(runId) { const book = await open(runId); - return book.written.has(slot); + return Math.max(FIRST_SLOT - 1, ...book.written); }, release(runId, slot) { diff --git a/packages/world-postgres/src/storage.ts b/packages/world-postgres/src/storage.ts index ccb70bb73c..9a13a17abb 100644 --- a/packages/world-postgres/src/storage.ts +++ b/packages/world-postgres/src/storage.ts @@ -71,8 +71,8 @@ import { type Drizzle, Schema } from './drizzle/index.js'; import type { SerializedContent } from './drizzle/schema.js'; import { type EventIds, - eventExists, highestEventId, + highestSlotOf, placeEvent, RUN_CREATED_SLOT, } from './slots.js'; @@ -787,11 +787,25 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { ); } eventId = params.eventId; - if (await eventExists(drizzle, effectiveRunId, eventId)) { - // Reject a doomed claim before the materialization below creates the - // step, hook or wait this event will now never accompany. A caller - // that re-proposes at the next slot would otherwise collide with its - // own orphan and read that as "my write already landed". + // A claim has to clear the log's tail, not merely be free. A position + // claimed by a write that then failed is never filled, so a log can + // carry holes below its tail; a caller numbering from a snapshot that + // predates the events above such a hole aims straight at it, and a + // free-position check would accept the write. The event then lands + // *below* events another replay has already consumed — the log stays + // internally consistent while its order silently changes, which is + // enough to flip a race between a step and a sleep from one replay to + // the next. + // + // Checked here, before the materialization below creates the step, hook + // or wait this event would now never accompany: a caller that + // re-proposes at the next position would otherwise collide with its own + // orphan and read that as "my write already landed". + seedHighestEventId = + seedHighestEventId ?? + (await highestEventId(drizzle, effectiveRunId)) ?? + undefined; + if (claimedSlot <= highestSlotOf(seedHighestEventId)) { throw await slotConflict(effectiveRunId, eventId, params); } } From bb761337d9cd9ef6cbbcf0431f5e07c4c993764c Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Sat, 1 Aug 2026 12:21:17 -0700 Subject: [PATCH 22/28] chore: changeset for tail-tight slot claims Co-Authored-By: Claude Opus 5 --- .changeset/slot-claims-tail-tight.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/slot-claims-tail-tight.md diff --git a/.changeset/slot-claims-tail-tight.md b/.changeset/slot-claims-tail-tight.md new file mode 100644 index 0000000000..346b2c92de --- /dev/null +++ b/.changeset/slot-claims-tail-tight.md @@ -0,0 +1,7 @@ +--- +'@workflow/core': patch +'@workflow/world-local': patch +'@workflow/world-postgres': patch +--- + +Take event slot claims one at a time against the event log's tail, so a replay that decided from a log missing an event is rejected instead of committing. From f6fa8ec2190357076f1221be8330f05df0c43d4d Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Sat, 1 Aug 2026 13:30:41 -0700 Subject: [PATCH 23/28] Sort imports --- packages/core/src/runtime/helpers.test.ts | 2 +- packages/core/src/runtime/suspension-handler.ts | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/core/src/runtime/helpers.test.ts b/packages/core/src/runtime/helpers.test.ts index b6227a4aa4..3ae748d1d0 100644 --- a/packages/core/src/runtime/helpers.test.ts +++ b/packages/core/src/runtime/helpers.test.ts @@ -22,6 +22,7 @@ import { } from '../serialization.js'; import { appendUniqueEvents, + claimFenceFor, eventCreateFenceFor, getWorkflowQueueName, handleHealthCheckMessage, @@ -31,7 +32,6 @@ import { loadWorkflowRunEvents, memoizeEncryptionKey, mergeLoadedEvents, - claimFenceFor, preconditionEventDelta, preconditionSnapshotParams, reserveSlot, diff --git a/packages/core/src/runtime/suspension-handler.ts b/packages/core/src/runtime/suspension-handler.ts index 729dbbf945..e167d83b05 100644 --- a/packages/core/src/runtime/suspension-handler.ts +++ b/packages/core/src/runtime/suspension-handler.ts @@ -30,8 +30,8 @@ import * as Attribute from '../telemetry/semantic-conventions.js'; import { getAbortStreamIdFromToken } from '../util.js'; import { getMaxInlineSteps } from './constants.js'; import { - type EventCreator, claimFenceFor, + type EventCreator, type MutableEventLog, } from './helpers.js'; import { ReplayRecoveryReporter } from './replay-recovery-reporter.js'; @@ -278,9 +278,10 @@ export async function handleSuspension({ // replay produces. const createGuarded: EventCreator = (data, params) => eventLog - ? claimFenceFor(eventLog, run.specVersion)((fence) => - createEvent(data, { ...params, ...fence }) - ) + ? claimFenceFor( + eventLog, + run.specVersion + )((fence) => createEvent(data, { ...params, ...fence })) : createEvent(data, params); // Separate queue items by type const stepItems = suspension.steps.filter( From c6f6c96caf58fbb466c3867582a659fc45fcc1b5 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Sat, 1 Aug 2026 14:20:26 -0700 Subject: [PATCH 24/28] Heal a slot-numbered log from its cursor after a stale-write rejection A restarted replay reloaded its whole event log because a hole under ULID identity is defined by time while a cursor filters lexicographically, so an incremental load can miss it. Neither half of that holds once a run numbers its events by slot: slot ids sort in write order, so everything the replay was missing is strictly above the cursor, and a dense log holds exactly maxSlot events, so the count proves afterwards that the page closed the gap. A short count falls back to the authoritative load. Measured on the step-storm repro against world-postgres, where a never-read poke hook rejects nearly every claim: two attempts went from ~400s and scoring stuck to 85.7s and 87.7s, with restart and re-invocation counts unchanged. --- .changeset/slot-restart-cursor-top-up.md | 6 + packages/core/src/runtime.ts | 64 ++++- .../runtime/precondition-guard-replay.test.ts | 219 +++++++++++++++--- 3 files changed, 260 insertions(+), 29 deletions(-) create mode 100644 .changeset/slot-restart-cursor-top-up.md diff --git a/.changeset/slot-restart-cursor-top-up.md b/.changeset/slot-restart-cursor-top-up.md new file mode 100644 index 0000000000..18752ca81f --- /dev/null +++ b/.changeset/slot-restart-cursor-top-up.md @@ -0,0 +1,6 @@ +--- +'@workflow/core': patch +'workflow': patch +--- + +Recover faster from a concurrent write on runs that number their events by slot, by topping the event log up from its cursor instead of reloading it in full diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index c611de7a68..d58cae37be 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -21,6 +21,7 @@ import { FIRST_SLOT, getQueueTopicPrefix, isLegacySpecVersion, + maxSlotOf, ROOT_RUN_ID_ATTRIBUTE, type RunInput, resolveQueueNamespace, @@ -718,6 +719,12 @@ export function workflowEntrypoint( let cachedEvents: Event[] | null = null; let eventsCursor: string | null = null; + // Set when a restarted replay is recovering by topping its + // cached log up from the cursor rather than reloading it + // whole. The next load verifies the result is dense before + // the replay trusts it; see the consume site below the load. + let slotTopUpPending = false; + // Inline-delta optimization: when an inline step's terminal // write returns the event-log delta since the pre-write // cursor (a supporting World only), we stash it here so the @@ -933,7 +940,7 @@ export function workflowEntrypoint( ids: Set; restart: number; reason: string; - source: 'inline-delta' | 'full-reload'; + source: 'inline-delta' | 'full-reload' | 'slot-top-up'; } | null = null; /** * Report what a stale-snapshot restart's reload found, once @@ -1032,6 +1039,25 @@ export function workflowEntrypoint( // to merge it into; with no base log the restart has to load // the whole thing anyway. const usedDelta = Boolean(delta && cachedEvents); + // Without a delta, a slot-numbered log still heals from its + // cursor rather than from a full reload. Slot ids sort in + // write order, so every event this replay was missing is + // strictly above the cursor and one incremental page brings + // it in — and density (a dense log from slot 1 holds + // exactly `maxSlot` events) proves afterwards that it did, + // so nothing is being trusted here that is not checked. + // Neither property holds under ULID ids, which is why those + // restarts reload whole. + const topsUpFromCursor = + !usedDelta && + usesSlotIdentity(workflowRun?.specVersion) && + cachedEvents !== null && + eventsCursor !== null; + const restartSource = usedDelta + ? 'inline-delta' + : topsUpFromCursor + ? 'slot-top-up' + : 'full-reload'; // Snapshot the set being discarded while it is still in // hand; the comparison happens once the next load resolves. preconditionRestartBaseline = cachedEvents @@ -1041,7 +1067,7 @@ export function workflowEntrypoint( ), restart: preconditionRestarts, reason, - source: usedDelta ? 'inline-delta' : 'full-reload', + source: restartSource, } : null; if (usedDelta) { @@ -1049,6 +1075,18 @@ export function workflowEntrypoint( // (`pendingInlineDelta && cachedEvents`) with no // events.list round trip at all. pendingInlineDelta = delta; + } else if (topsUpFromCursor) { + // Keep the cached log and its cursor: the loop's + // incremental branch fetches the page above the cursor + // and appends it, and the density check below the load + // sends the restart to a full reload if that page did not + // close the gap. Appends land above everything already + // scanned for payload prewarming, so no rescan is needed + // unless that fallback fires. + slotTopUpPending = true; + preloadedEvents = undefined; + preloadedEventsCursor = undefined; + pendingInlineDelta = null; } else { // MUST be a full, cursor-less reload. The cursor filters // by lexicographic event id while a hole is defined by @@ -1075,7 +1113,7 @@ export function workflowEntrypoint( reason, loopIteration, preconditionRestarts, - source: usedDelta ? 'inline-delta' : 'full-reload', + source: restartSource, } ); span?.setAttributes({ @@ -1818,6 +1856,26 @@ export function workflowEntrypoint( // the wait pass, which may swap in a freshly loaded array. cachedEvents = events; + if (slotTopUpPending) { + slotTopUpPending = false; + // A slot-numbered log is dense from slot 1, so a + // complete one holds exactly `maxSlot` events. A short + // count means the page above the cursor did not bring + // in everything the restart was missing — the only + // other reading, a permanent hole from a write that + // took a slot and then failed, is equally unrecoverable + // from here — so fall back to the authoritative load. + if (maxSlotOf(events) !== events.length) { + const loaded = await loadWorkflowRunEvents(runId); + events = loaded.events; + eventsCursor = loaded.cursor; + cachedEvents = events; + // The reload can insert events below the prefix + // already scanned for payload prewarming. + replayPayloadCache.resetScan(); + } + } + reportPreconditionRestartReload(events); // Detect concurrent completion via the event log: if diff --git a/packages/core/src/runtime/precondition-guard-replay.test.ts b/packages/core/src/runtime/precondition-guard-replay.test.ts index fefa5a0bcf..6239277a6d 100644 --- a/packages/core/src/runtime/precondition-guard-replay.test.ts +++ b/packages/core/src/runtime/precondition-guard-replay.test.ts @@ -10,7 +10,10 @@ * 2. The restart reloads the whole event log with no cursor, because a hole is * defined by ULID time while a cursor filters lexicographically — unless * the World attached the missing events to the 412, which the runtime - * consumes with no events.list round trip at all (first restart only). + * consumes with no events.list round trip at all (first restart only). A run + * numbering its events by slot instead heals from its cursor, since slot ids + * sort in write order and density proves afterwards that the page closed the + * gap; a short count sends it back to the full reload. * 3. Restarts are bounded; once the bound is spent the runtime schedules a * delayed re-invocation instead of failing the run — and that escalation is * itself counted on the queue message, so a run that can never observe its @@ -19,11 +22,19 @@ * Modeled on wait-completion-replay.test.ts, but with real ULID event IDs so * latestEventStateUpdatedAt() actually derives snapshot times. */ -import { PreconditionFailedError, RUN_ERROR_CODES } from '@workflow/errors'; +import { + PreconditionFailedError, + RUN_ERROR_CODES, + SlotConflictError, +} from '@workflow/errors'; import { type CreateEventRequest, type Event, + FIRST_SLOT, SPEC_VERSION_CURRENT, + SPEC_VERSION_SLOT_IDENTITY, + slotFromId, + slotIdBody, type WorkflowRun, type World, } from '@workflow/world'; @@ -88,6 +99,8 @@ interface SnapshotParams { stateUpdatedAt: number | undefined; stateEventCount: number | undefined; stateCursor: string | undefined; + /** The claimed position, on a run that numbers its events by slot. */ + eventId?: string | undefined; } async function runPreconditionScenario(options: { @@ -98,6 +111,17 @@ async function runPreconditionScenario(options: { * or a payload the runtime must refuse to narrow (`malformed`). */ attachDelta?: 'complete' | 'malformed'; + /** + * Number the run's events and correlation ids by slot rather than by ULID, + * and reject with the 409 a taken slot produces instead of the 412. + */ + slotIdentity?: boolean; + /** + * Slot mode only: land a second out-of-band event ahead of the hook and hide + * it from the page above the cursor, so a restart that tops up incrementally + * ends holding a log that is short of its own highest slot. + */ + hideFirstOutsideEventFromCursor?: boolean; }) { vi.spyOn(Date, 'now').mockReturnValue(+fixedNow); @@ -112,14 +136,24 @@ async function runPreconditionScenario(options: { undefined ); + const SPEC = options.slotIdentity + ? SPEC_VERSION_SLOT_IDENTITY + : SPEC_VERSION_CURRENT; + const { globalThis: vmGlobalThis } = createContext({ seed: `${runId}:${workflowName}:${deploymentId}`, fixedTimestamp: +startedAt, }); const vmUlid = monotonicFactory(() => vmGlobalThis.Math.random()); + // Hooks keep their ULID ids in both modes; steps and waits count per kind + // once the run is on slot identity. const hookCorrelationId = `hook_${vmUlid(+startedAt)}`; - const syncStep0CorrelationId = `step_${vmUlid(+startedAt)}`; - const waitCorrelationId = `wait_${vmUlid(+startedAt)}`; + const syncStep0CorrelationId = options.slotIdentity + ? `step_${slotIdBody(FIRST_SLOT)}` + : `step_${vmUlid(+startedAt)}`; + const waitCorrelationId = options.slotIdentity + ? `wait_${slotIdBody(FIRST_SLOT)}` + : `wait_${vmUlid(+startedAt)}`; const workflowRun: WorkflowRun = { runId, @@ -127,23 +161,35 @@ async function runPreconditionScenario(options: { status: 'running', input: workflowArgs, deploymentId, - specVersion: SPEC_VERSION_CURRENT, + specVersion: SPEC, startedAt, createdAt: startedAt, updatedAt: startedAt, }; // Real ULID event IDs at controlled times so latestEventStateUpdatedAt() - // resolves an actual epoch-ms snapshot from the loaded log. + // resolves an actual epoch-ms snapshot from the loaded log. Under slot + // identity the id is instead the next free slot, or the one the writer + // claimed, and the log stays dense from slot 1. const hostUlid = monotonicFactory(); let eventIndex = 0; - const event = (data: CreateEventRequest, atMs?: number): Event => { + let nextSlot = FIRST_SLOT; + const event = ( + data: CreateEventRequest, + atMs?: number, + claimedEventId?: string + ): Event => { const t = atMs ?? +startedAt + ++eventIndex * 100; + let eventId = `evnt_${hostUlid(t)}`; + if (options.slotIdentity) { + eventId = claimedEventId ?? `evnt_${slotIdBody(nextSlot)}`; + nextSlot = Math.max(nextSlot, slotFromId(eventId) ?? nextSlot) + 1; + } return { ...data, - specVersion: data.specVersion ?? SPEC_VERSION_CURRENT, + specVersion: data.specVersion ?? SPEC, runId, - eventId: `evnt_${hostUlid(t)}`, + eventId, createdAt: new Date(t), } as Event; }; @@ -151,19 +197,19 @@ async function runPreconditionScenario(options: { const staleEvents: Event[] = [ event({ eventType: 'run_created', - specVersion: SPEC_VERSION_CURRENT, + specVersion: SPEC, eventData: { deploymentId, workflowName, input: workflowArgs }, }), - event({ eventType: 'run_started', specVersion: SPEC_VERSION_CURRENT }), + event({ eventType: 'run_started', specVersion: SPEC }), event({ eventType: 'hook_created', - specVersion: SPEC_VERSION_CURRENT, + specVersion: SPEC, correlationId: hookCorrelationId, eventData: { token: hookToken }, }), event({ eventType: 'step_created', - specVersion: SPEC_VERSION_CURRENT, + specVersion: SPEC, correlationId: syncStep0CorrelationId, eventData: { stepName: 'syncStep', @@ -176,12 +222,12 @@ async function runPreconditionScenario(options: { }), event({ eventType: 'step_started', - specVersion: SPEC_VERSION_CURRENT, + specVersion: SPEC, correlationId: syncStep0CorrelationId, }), event({ eventType: 'step_completed', - specVersion: SPEC_VERSION_CURRENT, + specVersion: SPEC, correlationId: syncStep0CorrelationId, eventData: { result: await dehydrateStepReturnValue(undefined, runId, undefined), @@ -189,7 +235,7 @@ async function runPreconditionScenario(options: { }), event({ eventType: 'wait_created', - specVersion: SPEC_VERSION_CURRENT, + specVersion: SPEC, correlationId: waitCorrelationId, eventData: { resumeAt: new Date(+startedAt - 1_000) }, }), @@ -199,10 +245,30 @@ async function runPreconditionScenario(options: { const staleEventsCursor = 'cursor-after-stale-events'; const OUTSIDE_EVENT_MS = +startedAt + 5_000; + // Lands ahead of the winning delivery and is withheld from the page above + // the cursor, so an incremental top-up comes back holding fewer events than + // its own highest slot names. + const hiddenOutsideEvent = options.hideFirstOutsideEventFromCursor + ? event( + { + eventType: 'hook_received', + specVersion: SPEC, + correlationId: hookCorrelationId, + eventData: { + payload: await dehydrateStepReturnValue( + { value: 'hook-poke' }, + runId, + undefined + ), + }, + }, + OUTSIDE_EVENT_MS - 1 + ) + : undefined; const hookReceivedEvent = event( { eventType: 'hook_received', - specVersion: SPEC_VERSION_CURRENT, + specVersion: SPEC, correlationId: hookCorrelationId, eventData: { payload: await dehydrateStepReturnValue( @@ -233,7 +299,9 @@ async function runPreconditionScenario(options: { }) => { const data = params.pagination?.cursor === staleEventsCursor - ? durableEvents.slice(staleEvents.length) + ? durableEvents + .slice(staleEvents.length) + .filter((e) => e !== hiddenOutsideEvent) : [...durableEvents]; return { data, @@ -262,6 +330,7 @@ async function runPreconditionScenario(options: { stateUpdatedAt?: number; stateEventCount?: number; stateCursor?: string; + eventId?: string; } ) => { createParams.push({ @@ -269,6 +338,7 @@ async function runPreconditionScenario(options: { stateUpdatedAt: params?.stateUpdatedAt, stateEventCount: params?.stateEventCount, stateCursor: params?.stateCursor, + eventId: params?.eventId, }); if (request.eventType === 'run_started') { @@ -278,11 +348,32 @@ async function runPreconditionScenario(options: { if (request.eventType === 'wait_completed') { // The out-of-band hook payload becomes durable just before the // wait_completed commit — this is the exact race the guard closes. + if (hiddenOutsideEvent && !durableEvents.includes(hiddenOutsideEvent)) { + durableEvents.push(hiddenOutsideEvent); + } if (!durableEvents.includes(hookReceivedEvent)) { durableEvents.push(hookReceivedEvent); } if (waitCompletedRejections < (options.rejectWaitCompletedTimes ?? 0)) { waitCompletedRejections++; + // Both rejections say the same thing — "you decided from a log you + // had not fully seen" — and differ only in how the writer found out: + // a watermark comparison, or the slot it claimed already being + // occupied by the event it was missing. + if (options.slotIdentity) { + throw new SlotConflictError( + `Event ${params?.eventId} is already taken.`, + { + eventId: params?.eventId ?? '', + events: + options.attachDelta === 'complete' + ? [hookReceivedEvent] + : undefined, + cursor: + options.attachDelta === 'complete' ? 'cursor-409' : undefined, + } + ); + } throw new PreconditionFailedError( 'Run state is stale: the client event log is missing at least one event at or before its snapshot.', options.attachDelta === undefined @@ -303,17 +394,30 @@ async function runPreconditionScenario(options: { !!request.eventData && (request.eventData as { input?: unknown }).input !== undefined; let effectiveRequest = request; + // A claim covers the whole write, so a lazy start's synthesized + // step_created takes the claimed position and the step_started the one + // after it — the two extra slots the caller reserved for exactly this. + let claimedEventId = params?.eventId; + const takeClaim = () => { + const claim = claimedEventId; + claimedEventId = undefined; + return claim; + }; if (lazyStepStart) { const lazyData = request.eventData as { stepName?: string; input?: unknown; }; - const syntheticStepCreated = event({ - eventType: 'step_created', - specVersion: SPEC_VERSION_CURRENT, - correlationId: request.correlationId, - eventData: { stepName: lazyData.stepName, input: lazyData.input }, - } as CreateEventRequest); + const syntheticStepCreated = event( + { + eventType: 'step_created', + specVersion: SPEC, + correlationId: request.correlationId, + eventData: { stepName: lazyData.stepName, input: lazyData.input }, + } as CreateEventRequest, + undefined, + takeClaim() + ); durableEvents.push(syntheticStepCreated); createdEvents.push(syntheticStepCreated); const { input: _strippedInput, ...startEventData } = lazyData; @@ -323,7 +427,7 @@ async function runPreconditionScenario(options: { } as CreateEventRequest; } - const created = event(effectiveRequest); + const created = event(effectiveRequest, undefined, takeClaim()); durableEvents.push(created); createdEvents.push(created); if (effectiveRequest.eventType === 'step_started') { @@ -343,7 +447,7 @@ async function runPreconditionScenario(options: { const queue = vi.fn().mockResolvedValue({ messageId: 'msg_step' }); const fakeWorld = { - specVersion: SPEC_VERSION_CURRENT, + specVersion: SPEC, createQueueHandler: vi.fn((_prefix, handler) => { capturedHandler = handler; return vi.fn(); @@ -1051,6 +1155,69 @@ describe('precondition guard through the real replay loop', () => { ); }); + it('heals a slot-numbered log from its cursor instead of reloading it whole', async () => { + const result = await runPreconditionScenario({ + slotIdentity: true, + rejectWaitCompletedTimes: 1, + }); + await result.handlerInvocation; + + expect(result.waitCompletedRejectionCount()).toBe(1); + // Slot ids sort in write order, so the event the snapshot was missing is + // strictly above the cursor and one incremental page brings it in. The + // full reload the ULID path needs exists only because a hole defined by + // ULID time can sort below the cursor. + expect(cursorlessLoads(result.listEvents)).toBe(0); + + const waitCreates = result.createParams.filter( + (c) => c.eventType === 'wait_completed' + ); + expect(waitCreates).toHaveLength(2); + // No watermark is sent on a run that fences by position. + expect(waitCreates[0]?.stateUpdatedAt).toBeUndefined(); + // The rejected claim sat at the position the hook had just taken; the + // restarted replay claims the one after it. + expect(slotFromId(waitCreates[1]?.eventId ?? '')).toBe( + (slotFromId(waitCreates[0]?.eventId ?? '') ?? 0) + 1 + ); + + // Replay after the restart observed the hook and took the hook branch. + expect(result.createdEvents).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + eventType: 'step_created', + eventData: expect.objectContaining({ stepName: 'drainStep' }), + }), + ]) + ); + }); + + it('falls back to the full reload when a slot top-up leaves the log short of its own highest slot', async () => { + const result = await runPreconditionScenario({ + slotIdentity: true, + rejectWaitCompletedTimes: 1, + hideFirstOutsideEventFromCursor: true, + }); + await result.handlerInvocation; + + // A slot log is dense from slot 1, so a complete one holds exactly + // `maxSlot` events. The page above the cursor withheld one, the count came + // up short, and the restart fell through to the authoritative load — the + // check is what lets the cheap path be taken in the first place. + expect(cursorlessLoads(result.listEvents)).toBe(1); + expect( + result.createParams.filter((c) => c.eventType === 'wait_completed') + ).toHaveLength(2); + expect(result.createdEvents).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + eventType: 'step_created', + eventData: expect.objectContaining({ stepName: 'drainStep' }), + }), + ]) + ); + }); + it('falls back to the full reload when the 412 payload does not narrow to events', async () => { const result = await runPreconditionScenario({ rejectWaitCompletedTimes: 1, From 671dff7c9cb84a2acc99b0dec72e3c13f4332fa1 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Sat, 1 Aug 2026 14:30:22 -0700 Subject: [PATCH 25/28] Raise the in-process restart budget when restarts heal from the cursor A slot-numbered run recovers from a rejected write by reading the page after its cursor, not by reloading its log. The budget of 3 was priced for the reload: spending it buys a re-invocation, a queue hop plus the 2s re-invoke delay, in place of restarts that now cost a fraction of what the bound was protecting against. Measured on the step-storm repro against world-postgres at 6-way concurrency: the six runs went from 196-241s, two of them past the harness timeout and scored stuck, to 119-148s with none timing out. --- .changeset/slot-restart-budget.md | 6 +++++ .../docs/v5/configuration/runtime-tuning.mdx | 4 +++- packages/core/src/runtime.ts | 6 +++-- packages/core/src/runtime/constants.ts | 22 +++++++++++++++++-- .../runtime/precondition-guard-replay.test.ts | 21 ++++++++++++++++++ 5 files changed, 54 insertions(+), 5 deletions(-) create mode 100644 .changeset/slot-restart-budget.md diff --git a/.changeset/slot-restart-budget.md b/.changeset/slot-restart-budget.md new file mode 100644 index 0000000000..6a4c52c909 --- /dev/null +++ b/.changeset/slot-restart-budget.md @@ -0,0 +1,6 @@ +--- +'@workflow/core': patch +'workflow': patch +--- + +Let a run that numbers its events by position absorb more concurrent-write rejections in one invocation, instead of falling back to a delayed re-invocation diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index 1bba399962..b0e4ae8229 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -52,9 +52,11 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL ### `WORKFLOW_PRECONDITION_MAX_INPROCESS_RESTARTS` -- Default: `3` +- Default: `3`, or `12` on a run numbering its events by position (see [`WORKFLOW_SLOT_IDENTITY`](#workflow_slot_identity)) - How many times a single invocation restarts its replay in-process after a rejected event creation before it falls back to a re-invocation. - A restart reloads the event log and rebuilds the workflow from scratch, so it costs a replay but no queue round trip. A World may attach the missing events to its rejection, in which case the first restart needs no event-log request at all. +- A run numbering its events by position reads only the page after its cursor instead of reloading the log, since positions are allocated in order and every event it was missing sorts above what it already has. Restarts are cheap enough there that the higher default is worth taking before a re-invocation and its delay. +- Setting this overrides both defaults. ### `WORKFLOW_PRECONDITION_MAX_REINVOCATIONS` diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index d58cae37be..d801722553 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -1019,7 +1019,9 @@ export function workflowEntrypoint( ): boolean => { if ( preconditionRestarts >= - getPreconditionMaxInProcessRestarts() + getPreconditionMaxInProcessRestarts( + usesSlotIdentity(workflowRun?.specVersion) + ) ) { return false; } @@ -1159,7 +1161,7 @@ export function workflowEntrypoint( return { reinvoked: false, error: new WorkflowRuntimeError( - `Event creation was rejected as stale after ${maxReinvocations} re-invocations of ${getPreconditionMaxInProcessRestarts()} in-process replay restarts each: this run cannot observe its own event log completely enough to make progress. Last rejection (${reason}): ${error instanceof Error ? error.message : String(error)}`, + `Event creation was rejected as stale after ${maxReinvocations} re-invocations of ${getPreconditionMaxInProcessRestarts(usesSlotIdentity(workflowRun?.specVersion))} in-process replay restarts each: this run cannot observe its own event log completely enough to make progress. Last rejection (${reason}): ${error instanceof Error ? error.message : String(error)}`, { cause: error } ), }; diff --git a/packages/core/src/runtime/constants.ts b/packages/core/src/runtime/constants.ts index d5fea8383b..ab912651d7 100644 --- a/packages/core/src/runtime/constants.ts +++ b/packages/core/src/runtime/constants.ts @@ -397,14 +397,32 @@ export function getReplayDivergenceMaxRetries(): number { // run-level budget below then applies. export const PRECONDITION_MAX_INPROCESS_RESTARTS = 3; +// A run that numbers its events by slot recovers by topping its log up from +// its cursor, so a restart costs one incremental page instead of a full +// reload. The tight bound above is priced for the reload; spending it here +// buys a re-invocation — a queue hop plus `PRECONDITION_REINVOKE_DELAY_SECONDS` +// — in place of restarts that are orders of magnitude cheaper than the thing +// the bound was protecting against. Measured on the step-storm repro against +// world-postgres at 6-way concurrency, raising this to 12 took the six runs +// from 196–241s (two of them exceeding the harness timeout) to 119–148s with +// none timing out. +export const PRECONDITION_MAX_INPROCESS_RESTARTS_INCREMENTAL = 12; + /** * Effective in-process replay-restart budget for stale-snapshot rejections. * Override via `WORKFLOW_PRECONDITION_MAX_INPROCESS_RESTARTS`. + * + * @param incremental Whether a restart heals from the log's cursor rather than + * reloading it whole, which holds for runs on slot identity. */ -export function getPreconditionMaxInProcessRestarts(): number { +export function getPreconditionMaxInProcessRestarts( + incremental = false +): number { return envNumber( 'WORKFLOW_PRECONDITION_MAX_INPROCESS_RESTARTS', - PRECONDITION_MAX_INPROCESS_RESTARTS, + incremental + ? PRECONDITION_MAX_INPROCESS_RESTARTS_INCREMENTAL + : PRECONDITION_MAX_INPROCESS_RESTARTS, { integer: true } ); } diff --git a/packages/core/src/runtime/precondition-guard-replay.test.ts b/packages/core/src/runtime/precondition-guard-replay.test.ts index 6239277a6d..e9559008b9 100644 --- a/packages/core/src/runtime/precondition-guard-replay.test.ts +++ b/packages/core/src/runtime/precondition-guard-replay.test.ts @@ -53,6 +53,7 @@ import { getPreconditionMaxInProcessRestarts, getPreconditionMaxReinvocations, getPreconditionReinvokeDelaySeconds, + PRECONDITION_MAX_INPROCESS_RESTARTS, } from './constants.js'; import { setWorld } from './world.js'; @@ -1218,6 +1219,26 @@ describe('precondition guard through the real replay loop', () => { ); }); + it('spends a larger in-process restart budget when restarts heal from the cursor', async () => { + // One more rejection than a reloading run is allowed to absorb. That run + // would be out of budget and re-invoked; this one keeps recovering in + // process, because each of its restarts costs one incremental page rather + // than a full reload and so is not worth a queue hop to avoid. + const rejections = PRECONDITION_MAX_INPROCESS_RESTARTS + 1; + const result = await runPreconditionScenario({ + slotIdentity: true, + rejectWaitCompletedTimes: rejections, + }); + + await expect(result.handlerInvocation).resolves.toBeUndefined(); + expect(result.waitCompletedRejectionCount()).toBe(rejections); + expect( + result.createParams.filter((c) => c.eventType === 'wait_completed') + ).toHaveLength(rejections + 1); + // Every one of them healed from the cursor. + expect(cursorlessLoads(result.listEvents)).toBe(0); + }); + it('falls back to the full reload when the 412 payload does not narrow to events', async () => { const result = await runPreconditionScenario({ rejectWaitCompletedTimes: 1, From 351046fbdbee8e7229a71879da91b2ee01a62c35 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Sat, 1 Aug 2026 14:42:59 -0700 Subject: [PATCH 26/28] Lead the e2e run-diagnostics timeline with the event id On a run numbering its events by position the id is the position, so a gapped or out-of-order log is readable straight off the timeline. The correlation id already on the line cannot report it: attribute and hook correlation ids stay on ULIDs in both modes. --- packages/core/e2e/utils.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/core/e2e/utils.ts b/packages/core/e2e/utils.ts index 8c57ebed70..46f78b501e 100644 --- a/packages/core/e2e/utils.ts +++ b/packages/core/e2e/utils.ts @@ -659,7 +659,11 @@ async function getRunDiagnostics(tracked: TrackedRun): Promise { const elapsed = baseTime ? ((event.createdAt?.getTime?.() ?? 0) - baseTime) / 1000 : 0; - const prefix = ` +${elapsed.toFixed(1)}s`; + // The event's own id leads the line: on a run numbering its events + // by position it says where in the log the event sits, which is what + // a diagnosis of an out-of-order or gapped log needs and what the + // correlation id below cannot report. + const prefix = ` ${event.eventId} +${elapsed.toFixed(1)}s`; let detail = event.eventType; if ('eventData' in event) { const data = (event as any).eventData; From 117292c940e7e36d7575a82fff9cfe33691155b2 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Sat, 1 Aug 2026 15:42:54 -0700 Subject: [PATCH 27/28] Skip a step whose concurrent start another handler already wrote A slot rejection whose delta already contains a step_started for this exact step is the ordinary "another handler owns it" outcome wearing a slot rejection, so treat it the way an unfenced write's EntityConflictError is treated and skip, rather than restarting the replay to re-derive a claim that will lose again. The identity test is strict: correlation ids are positional under slot identity, so a diverged replay can reach the same step number naming a different call. Step name must match, and on the lazy path the inputs must be byte-identical. --- .changeset/slot-duplicate-start-skip.md | 6 + .../core/src/runtime/step-executor.test.ts | 151 ++++++++++++++++++ packages/core/src/runtime/step-executor.ts | 112 ++++++++++++- 3 files changed, 263 insertions(+), 6 deletions(-) create mode 100644 .changeset/slot-duplicate-start-skip.md diff --git a/.changeset/slot-duplicate-start-skip.md b/.changeset/slot-duplicate-start-skip.md new file mode 100644 index 0000000000..fc434c6abf --- /dev/null +++ b/.changeset/slot-duplicate-start-skip.md @@ -0,0 +1,6 @@ +--- +'@workflow/core': patch +'workflow': patch +--- + +Skip a step whose concurrent start another handler already wrote, instead of restarting the replay to rediscover it diff --git a/packages/core/src/runtime/step-executor.test.ts b/packages/core/src/runtime/step-executor.test.ts index 530b6a3ca0..a0e9e615a3 100644 --- a/packages/core/src/runtime/step-executor.test.ts +++ b/packages/core/src/runtime/step-executor.test.ts @@ -1,6 +1,7 @@ import { mkdtempSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { SlotConflictError } from '@workflow/errors'; import type { Event, World } from '@workflow/world'; import { SPEC_VERSION_CURRENT } from '@workflow/world'; import { createWorld } from '@workflow/world-local'; @@ -210,3 +211,153 @@ describe('executeStep — compute instance stamping', () => { expect(started[0]?.[2]).toMatchObject(preconditionSnapshot); }); }); + +// A run that numbers its events by position rejects a claim whose slot was +// taken, and the rejection carries the events that took it. When those events +// show the same step already started, the loser is in the ordinary "another +// handler owns this step" position and skips — the outcome an unfenced write +// reaches via EntityConflictError. Anything less than proof of the same call +// must propagate instead, because correlation ids are positional: a replay +// that diverged can reach the same step number naming a different call. +describe('executeStep — slot rejection carrying a duplicate start', () => { + afterEach(() => { + counter += 1; + }); + + function startedEvent(opts: { + stepId: string; + stepName: string; + input?: unknown; + }) { + return { + eventType: 'step_started', + correlationId: opts.stepId, + eventData: { + stepName: opts.stepName, + ...(opts.input !== undefined ? { input: opts.input } : {}), + }, + }; + } + + async function runAgainstRejection(opts: { + events: unknown[]; + lazyStepInput?: Uint8Array; + onBody?: () => void; + }) { + const world = makeWorld(); + const stepName = uniqueStepName(); + let bodyRuns = 0; + const { runId, stepId } = await setupRunningStep({ + world, + stepName, + onBody: () => { + bodyRuns += 1; + opts.onBody?.(); + }, + }); + + const rejection = new SlotConflictError('slot taken', { + eventId: 'evnt_00000000000000000000000007', + events: opts.events.map((build) => + typeof build === 'function' + ? (build as (ids: { stepId: string; stepName: string }) => unknown)({ + stepId, + stepName, + }) + : build + ), + }); + + const run = () => + executeStep({ + world, + workflowRunId: runId, + workflowName: 'wf', + workflowStartedAt: Date.now(), + stepId, + stepName, + lazyStepInput: opts.lazyStepInput, + // Take the awaited claim path so the rejection is translated before a + // body ever runs; the optimistic path reconciles the same way but + // would run the body first and muddy the assertion. + suppressOptimisticStart: true, + claimFence: () => Promise.reject(rejection), + }); + + return { run, bodyRuns: () => bodyRuns, stepName, stepId }; + } + + it('skips when the delta already started this step (no lazy input to compare)', async () => { + const { run, bodyRuns } = await runAgainstRejection({ + events: [ + (ids: { stepId: string; stepName: string }) => startedEvent(ids), + ], + }); + + await expect(run()).resolves.toEqual({ type: 'skipped' }); + expect(bodyRuns()).toBe(0); + }); + + it('skips when the delta started this step with byte-identical input', async () => { + const input = new Uint8Array([1, 2, 3]); + const { run, bodyRuns } = await runAgainstRejection({ + lazyStepInput: input, + events: [ + (ids: { stepId: string; stepName: string }) => + startedEvent({ ...ids, input: new Uint8Array([1, 2, 3]) }), + ], + }); + + await expect(run()).resolves.toEqual({ type: 'skipped' }); + expect(bodyRuns()).toBe(0); + }); + + it('propagates when the delta started the same slot under a different step name', async () => { + const { run } = await runAgainstRejection({ + events: [ + (ids: { stepId: string }) => + startedEvent({ ...ids, stepName: 'step//./other//someOtherStep' }), + ], + }); + + await expect(run()).rejects.toThrow(SlotConflictError); + }); + + it('propagates when the delta started this step with different input bytes', async () => { + const { run } = await runAgainstRejection({ + lazyStepInput: new Uint8Array([1, 2, 3]), + events: [ + (ids: { stepId: string; stepName: string }) => + startedEvent({ ...ids, input: new Uint8Array([1, 2, 4]) }), + ], + }); + + await expect(run()).rejects.toThrow(SlotConflictError); + }); + + it('propagates when the delta start carries a remote ref instead of inline bytes', async () => { + // A ref says nothing about the value behind it, so identity is unprovable + // and the replay restart is the only correct answer. + const { run } = await runAgainstRejection({ + lazyStepInput: new Uint8Array([1, 2, 3]), + events: [ + (ids: { stepId: string; stepName: string }) => + startedEvent({ ...ids, input: { ref: 'payload_abc' } }), + ], + }); + + await expect(run()).rejects.toThrow(SlotConflictError); + }); + + it('propagates when the delta holds no start for this step', async () => { + const { run } = await runAgainstRejection({ + events: [ + { eventType: 'step_completed', correlationId: 'step_other' }, + (ids: { stepName: string }) => + startedEvent({ ...ids, stepId: 'step_other' }), + ], + }); + + await expect(run()).rejects.toThrow(SlotConflictError); + }); +}); diff --git a/packages/core/src/runtime/step-executor.ts b/packages/core/src/runtime/step-executor.ts index b427c75d6d..8d1e7f61bf 100644 --- a/packages/core/src/runtime/step-executor.ts +++ b/packages/core/src/runtime/step-executor.ts @@ -4,6 +4,7 @@ import { FatalError, RetryableError, RunExpiredError, + SlotConflictError, ThrottleError, TooEarlyError, WorkflowRuntimeError, @@ -87,6 +88,73 @@ function extractInlineDelta( }; } +/** + * Byte-equality for two dehydrated step inputs, used to decide whether a + * conflicting `step_started` describes the same call as ours. Only inline + * bytes can answer that: a `SerializedData` that came back as a remote ref + * carries no payload, and two refs being unequal says nothing about the + * values behind them, so anything that is not a pair of `Uint8Array`s is + * reported as "cannot tell". + */ +function sameSerializedInput( + a: SerializedData | undefined, + b: SerializedData | undefined +): boolean { + if (!(a instanceof Uint8Array) || !(b instanceof Uint8Array)) return false; + if (a.byteLength !== b.byteLength) return false; + for (let i = 0; i < a.byteLength; i++) { + if (a[i] !== b[i]) return false; + } + return true; +} + +/** + * Decide whether a slot rejection carrying a delta is a *benign duplicate*: + * the events we lost the slot to already contain a `step_started` for this + * exact step, so a concurrent handler is running (or has run) the very call + * we were about to claim. The step is then someone else's to finish and we + * can skip, exactly as we do on the `EntityConflictError` the unfenced path + * would have raised instead. + * + * The identity test is deliberately strict. Correlation ids are positional + * under slot identity, so a replay that diverged — a `Promise.race` between + * a hook and a step resolving the other way, say — can arrive at the same + * `step_N` naming a different call. Matching on the correlation id alone + * would let such a replay adopt a foreign step's completion as its own. So + * the step name must match, and on the lazy path (where we hold the input) + * the inputs must be byte-identical. Anything we cannot prove identical + * returns false and takes the replay restart, which is always correct and + * merely slower. + */ +function isBenignDuplicateStart( + events: unknown[], + stepId: string, + stepName: string, + lazyStepInput: SerializedData | undefined +): boolean { + for (const candidate of events) { + if (!candidate || typeof candidate !== 'object') continue; + const event = candidate as { + eventType?: unknown; + correlationId?: unknown; + eventData?: { stepName?: unknown; input?: unknown } | null; + }; + if (event.eventType !== 'step_started') continue; + if (event.correlationId !== stepId) continue; + if (event.eventData?.stepName !== stepName) continue; + if (lazyStepInput === undefined) return true; + if ( + sameSerializedInput( + lazyStepInput, + event.eventData?.input as SerializedData + ) + ) { + return true; + } + } + return false; +} + export interface StepExecutorParams { world: World; workflowRunId: string; @@ -500,6 +568,37 @@ export async function executeStep( }); return { type: 'skipped' }; } + if ( + SlotConflictError.is(err) && + isBenignDuplicateStart( + err.events, + stepId, + stepName, + params.lazyStepInput + ) + ) { + // We lost the slot to a writer that had already started this same + // step, so this is the ordinary "another handler owns it" outcome + // wearing a slot rejection — the same thing an unfenced write reports + // as EntityConflictError. Skip rather than restart the replay: the + // restart would re-derive this identical claim and lose again. + // + // The delta on the error is deliberately NOT merged into the log. + // Adopting events this VM never replayed would let the next claim + // fence itself against a tail it cannot account for, which is the + // one thing the fence exists to prevent. Siblings in the batch fail + // with their own rejection and the handler defers as usual. + runtimeLogger.debug('Step already started by a concurrent writer', { + stepName, + stepId, + workflowRunId, + }); + span?.setAttributes({ + ...Attribute.StepSkipped(true), + ...Attribute.StepSkipReason('completed'), + }); + return { type: 'skipped' }; + } if (TooEarlyError.is(err)) { const timeoutSeconds = Math.max(1, err.retryAfter ?? 1); runtimeLogger.debug('Step retryAfter timestamp not yet reached', { @@ -590,9 +689,10 @@ export async function executeStep( // wait IS part of that stretch under turbo. stepStartPostSentAtMs = Date.now(); // Fence the claim — see StepExecutorParams.claimFence. A rejection - // the fence does not retry surfaces via reconcileOptimisticStart as a - // non-translatable error: the body result is discarded and the - // rejection propagates to the caller. + // the fence does not retry surfaces via reconcileOptimisticStart: + // the body result is discarded, and unless the rejection proves + // another writer already started this same step (a benign + // duplicate, skipped) it propagates to the caller. return runClaim((fence) => createEvent( { @@ -652,9 +752,9 @@ export async function executeStep( : {}; stepStartPostSentAtMs = Date.now(); // Fence the claim — see StepExecutorParams.claimFence. A rejection the - // fence does not retry is intentionally NOT translated by - // startErrorToResult below, so it propagates to the caller for a fresh - // replay. + // fence does not retry propagates to the caller for a fresh replay, + // except where startErrorToResult below can read the rejection's delta + // as "another writer already started this exact step" and skip. const startResult = await runClaim((fence) => createEvent( { From 2305e329c7f97b080c880518dece6794e96cff77 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Sat, 1 Aug 2026 15:43:03 -0700 Subject: [PATCH 28/28] Back off between in-process replay restarts Concurrent replays of one run contend for the same event slots, and the loser of a rejected write restarted immediately, putting every loser back in contention at once so none pulled ahead. Wait a full-jitter interval first, doubling per restart to a 400ms cap. Tunable with WORKFLOW_PRECONDITION_RESTART_BACKOFF_MS (0 disables) and WORKFLOW_PRECONDITION_RESTART_BACKOFF_MAX_MS. --- .changeset/precondition-restart-backoff.md | 6 ++ .../docs/v5/configuration/runtime-tuning.mdx | 12 ++++ packages/core/src/runtime.ts | 24 +++++++ packages/core/src/runtime/constants.test.ts | 60 ++++++++++++++++++ packages/core/src/runtime/constants.ts | 63 +++++++++++++++++++ 5 files changed, 165 insertions(+) create mode 100644 .changeset/precondition-restart-backoff.md diff --git a/.changeset/precondition-restart-backoff.md b/.changeset/precondition-restart-backoff.md new file mode 100644 index 0000000000..8e739cd8f0 --- /dev/null +++ b/.changeset/precondition-restart-backoff.md @@ -0,0 +1,6 @@ +--- +'@workflow/core': patch +'workflow': patch +--- + +Space out in-process replay restarts with a randomized backoff so concurrent replays of one run stop contending in lockstep diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index b0e4ae8229..7feac45f69 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -70,6 +70,18 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL - Delay before a re-invocation caused by a rejected event creation. - Unlike an in-process restart, which re-reads immediately, a re-invocation only happens once the in-process budget failed to catch up — so the delay gives the other writers a moment to quiesce. +### `WORKFLOW_PRECONDITION_RESTART_BACKOFF_MS` + +- Default: `15` +- Base for the randomized wait before an in-process replay restart re-derives, in milliseconds. The wait doubles with each restart the invocation has spent, up to [`WORKFLOW_PRECONDITION_RESTART_BACKOFF_MAX_MS`](#workflow_precondition_restart_backoff_max_ms), and is drawn uniformly from zero to that bound. +- The wait exists for runs with several replays in flight at once — a fan-out of steps completing together, or a burst of hooks. A rejected event creation means another replay got there first; if every loser re-derives immediately they all contend again, and none of them pulls far enough ahead to finish. Drawing each wait from the full range spreads the retries apart. +- Set to `0` to restart without waiting. + +### `WORKFLOW_PRECONDITION_RESTART_BACKOFF_MAX_MS` + +- Default: `400` +- Ceiling on the wait described above. + ### `WORKFLOW_SLOT_IDENTITY` - Default: enabled diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index d801722553..018ca5ad7a 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -55,6 +55,7 @@ import { getReplayDivergenceMaxRetries, isInlineOwnershipEnabled, isTurboEnabled, + preconditionRestartBackoffMs, } from './runtime/constants.js'; import { countStepStartedEvents } from './runtime/count-step-started-events.js'; import { @@ -925,6 +926,14 @@ export function workflowEntrypoint( // Precondition (412) recovery: how many times this invocation // has thrown away its replay and started over in-process. let preconditionRestarts = 0; + /** + * Wait the next loop iteration owes before re-deriving, set + * by `restartReplayInProcess`. Held here rather than awaited + * at the restart's call sites because the restart is decided + * from three of them, in synchronous code, while the loop head + * is the single point every restart passes through. + */ + let pendingRestartBackoffMs = 0; /** * Event ids the discarded replay held, kept until the next * load resolves so a restart can report what its reload @@ -1121,6 +1130,8 @@ export function workflowEntrypoint( span?.setAttributes({ 'workflow.precondition_restarts': preconditionRestarts, }); + pendingRestartBackoffMs = + preconditionRestartBackoffMs(preconditionRestarts); return true; }; @@ -1719,6 +1730,19 @@ export function workflowEntrypoint( while (true) { loopIteration++; + // A restart lost a slot to a concurrent replay of this same + // run. Pause before re-deriving so the winner gets a clear + // window to extend the log: re-deriving immediately puts + // every loser back in contention at once, and none of them + // pulls ahead. + if (pendingRestartBackoffMs > 0) { + const backoffMs = pendingRestartBackoffMs; + pendingRestartBackoffMs = 0; + await new Promise((resolve) => + setTimeout(resolve, backoffMs) + ); + } + // Replay-budget check: bail out (retry or fail) if // non-step time within this invocation has exceeded // the configured budget. Step bodies are excluded diff --git a/packages/core/src/runtime/constants.test.ts b/packages/core/src/runtime/constants.test.ts index 208250e8e6..919c4edd14 100644 --- a/packages/core/src/runtime/constants.test.ts +++ b/packages/core/src/runtime/constants.test.ts @@ -18,6 +18,9 @@ import { MAX_REPLAY_TIMEOUT_MS, MIN_MAX_INLINE_STEPS, MIN_REPLAY_TIMEOUT_MS, + PRECONDITION_RESTART_BACKOFF_BASE_MS, + PRECONDITION_RESTART_BACKOFF_MAX_MS, + preconditionRestartBackoffMs, REPLAY_TIMEOUT_MS, } from './constants.js'; @@ -393,3 +396,60 @@ describe('getInlineOwnershipLeaseSeconds', () => { expect(getInlineOwnershipLeaseSeconds()).toBe(1); }); }); + +describe('preconditionRestartBackoffMs', () => { + const BASE_ENV = 'WORKFLOW_PRECONDITION_RESTART_BACKOFF_MS'; + const MAX_ENV = 'WORKFLOW_PRECONDITION_RESTART_BACKOFF_MAX_MS'; + + beforeEach(() => { + delete process.env[BASE_ENV]; + delete process.env[MAX_ENV]; + }); + + afterEach(() => { + delete process.env[BASE_ENV]; + delete process.env[MAX_ENV]; + }); + + it('doubles the window on each successive restart', () => { + // Draw the top of the window so the window itself is observable. + const top = () => 0.999_999; + expect(preconditionRestartBackoffMs(1, top)).toBe( + PRECONDITION_RESTART_BACKOFF_BASE_MS - 1 + ); + expect(preconditionRestartBackoffMs(2, top)).toBe( + PRECONDITION_RESTART_BACKOFF_BASE_MS * 2 - 1 + ); + expect(preconditionRestartBackoffMs(3, top)).toBe( + PRECONDITION_RESTART_BACKOFF_BASE_MS * 4 - 1 + ); + }); + + it('draws over the whole window, not a fixed delay plus noise', () => { + // Full jitter is the property that decorrelates concurrent replays; a + // floor would keep them in lockstep however long the wait. + expect(preconditionRestartBackoffMs(5, () => 0)).toBe(0); + expect(preconditionRestartBackoffMs(5, () => 0.5)).toBeLessThan( + preconditionRestartBackoffMs(5, () => 0.999_999) + ); + }); + + it('caps the window', () => { + expect(preconditionRestartBackoffMs(40, () => 0.999_999)).toBe( + PRECONDITION_RESTART_BACKOFF_MAX_MS - 1 + ); + }); + + it('is disabled by a zero base', () => { + process.env[BASE_ENV] = '0'; + expect(preconditionRestartBackoffMs(1, () => 0.999_999)).toBe(0); + expect(preconditionRestartBackoffMs(9, () => 0.999_999)).toBe(0); + }); + + it('honours overrides of both the base and the cap', () => { + process.env[BASE_ENV] = '100'; + process.env[MAX_ENV] = '150'; + expect(preconditionRestartBackoffMs(1, () => 0.999_999)).toBe(99); + expect(preconditionRestartBackoffMs(2, () => 0.999_999)).toBe(149); + }); +}); diff --git a/packages/core/src/runtime/constants.ts b/packages/core/src/runtime/constants.ts index ab912651d7..9a42399899 100644 --- a/packages/core/src/runtime/constants.ts +++ b/packages/core/src/runtime/constants.ts @@ -465,3 +465,66 @@ export function getPreconditionReinvokeDelaySeconds(): number { { integer: true } ); } + +// Concurrent replays of one run contend for the same event slots, and a +// rejected write costs its loser a restart. With no pause the losers re-derive +// at full speed and collide again immediately, so a run under heavy fan-out can +// spend its whole invocation budget with no writer ever pulling far enough +// ahead to finish. A short randomized wait between restarts breaks the lockstep +// by spreading the retries. +export const PRECONDITION_RESTART_BACKOFF_BASE_MS = 15; + +// Ceiling on that wait. The restart is cheap (one incremental page plus a +// re-derive), so the backoff must stay well under the cost of the +// re-invocation it is competing with. +export const PRECONDITION_RESTART_BACKOFF_MAX_MS = 400; + +/** + * Effective base for the in-process restart backoff. Override via + * `WORKFLOW_PRECONDITION_RESTART_BACKOFF_MS`; `0` disables the wait entirely. + */ +export function getPreconditionRestartBackoffBaseMs(): number { + return envNumber( + 'WORKFLOW_PRECONDITION_RESTART_BACKOFF_MS', + PRECONDITION_RESTART_BACKOFF_BASE_MS, + { integer: true } + ); +} + +/** + * Effective ceiling for the in-process restart backoff. Override via + * `WORKFLOW_PRECONDITION_RESTART_BACKOFF_MAX_MS`. + */ +export function getPreconditionRestartBackoffMaxMs(): number { + return envNumber( + 'WORKFLOW_PRECONDITION_RESTART_BACKOFF_MAX_MS', + PRECONDITION_RESTART_BACKOFF_MAX_MS, + { integer: true } + ); +} + +/** + * Full-jitter backoff for the `restarts`-th in-process replay restart, in + * milliseconds. + * + * Full jitter (a uniform draw over the whole window, not a fixed delay plus + * noise) is what actually decorrelates the racers: equal-length waits would + * keep colliding replays in lockstep no matter how long they were. + * + * @param restarts 1-based count of restarts spent so far, including this one. + * @param random Injectable uniform source; defaults to `Math.random`. The delay + * never reaches the event log, so it does not affect replay determinism. + */ +export function preconditionRestartBackoffMs( + restarts: number, + random: () => number = Math.random +): number { + const base = getPreconditionRestartBackoffBaseMs(); + if (base <= 0) return 0; + const exponent = Math.max(0, restarts - 1); + const window = Math.min( + getPreconditionRestartBackoffMaxMs(), + base * 2 ** exponent + ); + return Math.floor(random() * window); +}