diff --git a/demos/objects_3d/README.md b/demos/objects_3d/README.md index af4a4f322..8463ea049 100644 --- a/demos/objects_3d/README.md +++ b/demos/objects_3d/README.md @@ -2,49 +2,115 @@ # 3D Object Boxes -Run the existing 2D object detector, sample the depth mesh inside each -detection's 2D box, and fit an oriented 3D bounding box around the -points. No new ML model — just a small PCA on the points the device's -own depth sensor already gives us. - -## Why not Objectron / Cube R-CNN / etc - -Most monocular 3D detectors exist because their target platform doesn't -have depth. xrblocks does (`xb.core.depth`), so we can skip the model -entirely and use real metric depth + the SDK's existing 2D detector. - -That gets us: - -- Categories = whatever the 2D detector recognises (lots, with the - Gemini backend), not just shoe / chair / cup / camera. -- Real metric scale from the headset's depth sensor, not estimated - relative depth. -- Real yaw orientation from PCA on the actual points. -- Zero model download. - -## How the box gets fit - -1. `xb.core.world.objects.runDetection()` returns 2D boxes + a - centre-point world position per object. -2. Sample an 18×18 grid of normalised UVs inside the 2D box and call - `xb.core.depth.getVertex(u, v)` for each to get world-space points. -3. Drop points more than ~1.2 m from the SDK's centre point — that - peels off background / foreground bleeding through the box. -4. PCA in the horizontal plane (XZ) gives the yaw of the dominant - axis. Y is left gravity-aligned. Min/max along the rotated axes - gives the footprint, min/max world-Y gives the height. -5. Render as `THREE.LineSegments(EdgesGeometry(BoxGeometry))` rotated - to the PCA yaw, with the label floating above. - -## Running - -Serve the repo root and open `/demos/objects_3d/`. Press **Detect** -(in the screen panel or the spatial panel). Works in the simulator and -on Android XR. - -## What's next - -If this lands well, the natural follow-up is a `box3d: true` option -on `world.objects.runDetection()` so apps can ask for oriented 3D -boxes alongside the existing 2D box + centre point — same primitive, -in the SDK rather than each demo redoing it. +Turns 2D object detections plus the depth mesh into oriented 3D bounding boxes, +using the `objects3d` addon (`Object3DDetector`). + +The integration is three lines: + +```js +const detector = new Object3DDetector({showDebugBoxes: true}); +xb.add(detector); +const objects = await detector.detect(); +``` + +Everything else on the page is the debug panel, which exists so the pipeline can +be diagnosed on a headset where there is no console. + +## Setup + +Needs a Gemini API key. Create `keys.json` in this directory (gitignored): + +```json +{"gemini": {"apiKey": "YOUR_KEY"}} +``` + +Then serve the repo (`npm run dev` from the repo root) and open +`http://localhost:8080/demos/objects_3d/`. + +When this page is embedded in the docs site the key comes from the iframe's +`?key=` parameter instead, so no `keys.json` is needed there. + +To skip the key entirely, switch the detector picker to `mediapipe` (fixed COCO +class set, no network). + +## The debug panel + +The same controls exist twice: as a DOM panel for desktop, and as a draggable +spatial panel for immersive XR, where the DOM is invisible. In XR the panel is +the only way to trigger a detection — pinch is deliberately not bound, so +grabbing and dragging the panel cannot fire one by accident. + +**Actions** — `detect`, `clear`, and `copy`, which puts the full diagnostics +record on the clipboard as JSON and logs it to the console. + +**Camera rotation offset** (`yaw` / `pitch` / `roll`, ±5° per press) applies +**live** to the next detection — no reload. This is the knob for nulling out a +constant calibration error between the SDK's estimated passthrough-camera +extrinsics and the actual hardware. Also available as `?camYawDeg=-30` etc. + +**Toggles that reload the page**, because they must be set before `xb.init()`: + +| Control | Query param | What it does | +| -------------- | ------------------- | --------------------------------------------------------------------------------------------- | +| `matchDepth` | `?matchDepthView=0` | **On by default.** Ask the platform for view-aligned depth instead of raw depth-sensor frames | +| `fullResDepth` | `?fullResDepth=1` | Rebuild the full-resolution depth mesh every frame (see below) | +| `detector` | `?backend=` | `gemini` / `mediapipe` / `both` | +| `mask` | `?mask=` | `slimsam` / `mediapipe` | + +**Diagnostics**, refreshed after each detection (4 Hz in XR): + +| Row | Why it matters | +| --------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `frame latency` | Age of the captured video frame. Large values mean the pixels predate the pose, which rotates every box by the head motion in between. Under ~150 ms is healthy. | +| `pose match` | How close the pose we paired with the frame was to the frame's capture time. | +| `camera model` | `device` means the SDK passthrough-camera model; `RENDER (fallback)` means the old, wrong-FOV path — a red flag on device. | +| `depth remap` | Whether the platform's view→depth-buffer UV remap is the identity. | +| `depth vs eye` | Angle between the depth camera's reported orientation and the left eye's. | +| `rejections` | Which sanity gate discarded detections (`farFromRoom` usually means the room-scale defaults need tuning). | +| `timings` | Per-stage wall clock, so you can see whether a slow detect is Gemini, SAM, or geometry. | + +## Diagnosing a coherent rotation error + +If every box from one detection lands rotated by roughly the same angle, run one +detection while holding your head **perfectly still for ~2 seconds**, then one +while turning your head: + +- **Rotation gone when still** → the captures were pairing fresh tracking poses + with stale video frames. The pipeline now waits for a fresh frame and pairs the + capture with the pose at the frame's `captureTime`; check `frame latency`. +- **Rotation identical in both** (same axis, same angle) → a constant + calibration error. Null it with the yaw/pitch/roll buttons, then bake the + value into `cameraRotationOffset`. +- **`depth vs eye` is large** → check `matchDepth` is still on (it is by + default). Turning it off measurably rotates the boxes on Galaxy XR, which is + how we learned the depth mesh — the surface every ray lands on — was the + rotated ingredient rather than the RGB camera model. + +## Frame rate + +The one setting that dominates is `fullResDepth`, off by default. Turning it on +sets `options.depth.depthMesh.updateFullResolutionGeometry = true`, so every +depth frame unprojects the full 154×154 grid — ~23.7k vertices, each a +`Matrix4` transform plus a divide — on the main thread, on top of the 40×40 +downsampled mesh the SDK always maintains. That is ~15× the per-frame vertex +work for no accuracy gain, because the detector calls +`depth.updateFullResolutionDepthMesh()` once inside `detect()` anyway, paying +the cost per detection rather than per frame. + +The spatial debug panel costs a little too, since it pulls in uikit/yoga layout +and MSDF text rendering. Neither setting affects correctness, only frame rate. + +## Tuning for a real room + +The fitter defaults are tuned for the simulator's cabin scene. On a headset, +pass bounds that match the real space: + +```js +new Object3DDetector({ + showDebugBoxes: true, + roomHalf: 4, // walls at x/z = ±4 m + sceneBounds: {maxXZ: 8, minY: -1, maxY: 5}, // reject boxes outside this + maxRayDistance: 12, + cameraRotationOffset: {yaw: 0, pitch: 0, roll: 0}, // radians, per-unit calibration +}); +``` diff --git a/demos/objects_3d/index.html b/demos/objects_3d/index.html index f5b612581..e3a4d902f 100644 --- a/demos/objects_3d/index.html +++ b/demos/objects_3d/index.html @@ -50,197 +50,142 @@ display: flex; flex-direction: column; gap: 10px; - background: rgba(20, 20, 20, 0.85); + background: rgba(20, 20, 20, 0.88); padding: 14px 18px; border-radius: 12px; font-family: system-ui, sans-serif; color: #eee; - max-width: 540px; - width: 92%; + max-width: 620px; + } + .panel .row { + display: flex; + gap: 10px; + align-items: center; + flex-wrap: wrap; } .panel button { - padding: 9px 14px; + padding: 8px 13px; border-radius: 8px; border: 1px solid #444; background: #2a2a2a; color: white; cursor: pointer; - font-size: 14px; + font-size: 13px; } - .panel button:hover:not(:disabled) { + .panel button:hover { background: #3a3a3a; } .panel button:disabled { opacity: 0.4; cursor: default; } - .panel .row { - display: flex; - gap: 8px; - align-items: center; - } - .panel select { - padding: 6px 8px; - border-radius: 6px; - border: 1px solid #444; - background: #2a2a2a; - color: white; - font-size: 12px; - cursor: pointer; - } .panel label { - font-size: 11px; - color: #aaa; - } - .panel .status { font-size: 12px; color: #aaa; - } - .key-overlay { - position: fixed; - inset: 0; - background: rgba(0, 0, 0, 0.85); display: flex; + gap: 6px; align-items: center; - justify-content: center; - z-index: 999; - font-family: system-ui, sans-serif; } - .key-card { - background: #1a1a1a; - color: #eee; - padding: 24px; - border-radius: 12px; - max-width: 420px; - width: 92%; + .panel .status { + font-size: 12px; + color: #9ad; } - .key-card h2 { - margin: 0 0 8px 0; + .panel .hint { + font-size: 11px; + color: #777; } - .key-card p { - font-size: 13px; - color: #bbb; + #diagnostics { + font-family: ui-monospace, Menlo, Consolas, monospace; + font-size: 11px; line-height: 1.5; - } - .key-card a { - color: #4cd964; - } - .key-card input { - width: 100%; - box-sizing: border-box; - padding: 10px; - font-size: 14px; - background: #2a2a2a; - border: 1px solid #444; - color: white; - border-radius: 8px; - margin: 10px 0; - } - .key-card button { - padding: 10px 16px; - background: #4cd964; - border: none; + color: #bbb; + background: rgba(0, 0, 0, 0.35); border-radius: 8px; - color: #000; - font-weight: 600; - cursor: pointer; + padding: 8px 10px; + margin: 0; + max-height: 220px; + overflow-y: auto; + white-space: pre; } - - - -
- - + + + + Ready. +
+
+ + + +
- - - - + + + + +
-
- Press detect to fit oriented 3D boxes around things in front of you. +
+ Rotation offsets and orientation mode apply live to the next detect. The + two checkboxes and the detector/mask pickers reload the page (they must + be set before xb.init).
+ 
No detection yet.
diff --git a/src/addons/objects3d/Object3DDetector.ts b/src/addons/objects3d/Object3DDetector.ts index 06bb09feb..1e5e3882e 100644 --- a/src/addons/objects3d/Object3DDetector.ts +++ b/src/addons/objects3d/Object3DDetector.ts @@ -1,21 +1,34 @@ /** * Reusable 3-D object-detection Script addon. * - * `Object3DDetector` wraps the full pipeline from the `objects_3d` demo into a - * reusable {@link Script}: snap camera + depth mesh, run 2-D detection, obtain + * `Object3DDetector` wraps the full pipeline in a reusable {@link Script}: + * snap camera + depth mesh, run 2-D detection, obtain * per-object segmentation masks, raycast depth samples into world space, fit an * oriented bounding box (OBB), fuse across views, and optionally show debug * wireframe boxes. */ import * as THREE from 'three'; -import {Script, core, enableAcceleratedRaycast} from 'xrblocks'; +import { + Script, + core, + enableAcceleratedRaycast, + getCameraParametersSnapshot, + getDeviceCameraWorldFromView, +} from 'xrblocks'; import {Detected3DObject} from './Detected3DObject'; import { anchorFromBboxCenter, sampleDepthInMaskAcrossFrames, } from './geometry/DepthSampling'; +import {buildFrozenCamera} from './geometry/FrozenCamera'; +import {PoseRing} from './geometry/PoseRing'; +import { + estimateRoomYawFromMesh, + RoomFrameAccumulator, + yawRelativeToRoom, +} from './geometry/RoomFrame'; import { fuseIntoBoxes, snapBoxToFloor, @@ -28,6 +41,7 @@ import { rejectByAnchorDepth, rejectByY, } from './geometry/ObbFitting'; +import type {OrientationMode, OrientationOptions} from './geometry/ObbFitting'; import {categorize, isSurfaceLabel, isTinyFlatLabel} from './labels/Categories'; import {samEncodeSnapshot, samMaskFromBbox, getSam} from './masks/SamMask'; import {segmenterMaskFromSnapshot} from './masks/SegmenterMask'; @@ -85,6 +99,120 @@ export interface Object3DDetectorOptions { * @defaultValue `false` */ showDebugBoxes?: boolean; + /** + * Maximum ray-hit distance in metres when sampling the depth mesh. + * @defaultValue `12` + */ + maxRayDistance?: number; + /** + * World-space sanity bounds; fitted boxes whose centre falls outside are + * rejected. Tuned for a room-scale scene around the session origin. + * @defaultValue `{maxXZ: 6, minY: -1, maxY: 5}` + */ + sceneBounds?: {maxXZ?: number; minY?: number; maxY?: number}; + /** + * Assumed distance in metres from the session origin to the cardinal + * walls, used by the tiny-flat fitter (switches, outlets) to snap onto a + * wall plane. The default matches the simulator's wood-cabin scene; tune + * it (or avoid tiny-flat labels) for real rooms. + * @defaultValue `3` + */ + roomHalf?: number; + /** + * Extra rotation applied to the device-camera pose at capture time, in + * radians (YXZ order, i.e. yaw about +Y first). Use this to null out a + * constant per-unit calibration error between the SDK's estimated + * passthrough-camera extrinsics and the actual hardware: if detections + * land rotated clockwise (viewed from above) by θ, pass `{yaw: θ}`. + * @defaultValue `{yaw: 0, pitch: 0, roll: 0}` + */ + cameraRotationOffset?: {yaw?: number; pitch?: number; roll?: number}; + /** + * How fitted yaws are reconciled with the room. Defaults to + * `{mode: 'roomFrame'}`, which estimates the room's own wall direction from + * the depth mesh and falls back to it only when an object's own orientation + * is ill-determined. Pass `{mode: 'cardinal'}` for the legacy behaviour of + * snapping every box to the session origin's axes. + */ + orientation?: OrientationOptions; +} + +/** + * Machine-readable record of what one {@link Object3DDetector.detect} call + * observed about its inputs. Chiefly useful for diagnosing on-device + * misalignment, where the interesting quantities (how stale the captured + * video frame was, whether the depth mesh is rotated relative to the render + * view) are invisible from the fitted boxes alone. + */ +export interface Object3DDetectorDiagnostics { + /** `performance.now()` when the detect call started. */ + startedAtMs: number; + /** Age of the captured video frame at snapshot time, or `null` when the + * browser exposes no `captureTime` for the stream. Large values mean the + * pixels predate the pose, which rotates every box by the head motion in + * between. */ + frameLatencyMs: number | null; + /** Gap between the frame's capture time and the timestamp of the recorded + * pose used for it. `null` when no historical pose was applied. */ + poseMatchErrorMs: number | null; + /** Poses currently held in the history ring. */ + poseRingSize: number; + /** Whether the frozen camera came from the SDK's device-camera model + * (`true`) or fell back to a clone of the XR render camera (`false`). */ + usedDeviceCameraModel: boolean; + /** Vertical FOV and aspect of the frozen camera actually raycast through. */ + cameraFovDeg: number; + cameraAspect: number; + /** Extra rotation applied on top of the SDK extrinsics, in degrees. */ + cameraRotationOffsetDeg: {yaw: number; pitch: number; roll: number}; + snapshotWidth: number; + snapshotHeight: number; + /** Whether the platform's view→depth-buffer UV remap is the identity. */ + depthRemapIsIdentity: boolean | null; + /** Angle between the depth camera's reported orientation and the left eye's, + * in degrees. A large value with `matchDepthView: false` means the depth + * mesh every ray lands on is itself rotated. */ + depthVsEyeRotationDeg: number | null; + /** Vertices in the frozen depth mesh snapshot. */ + depthMeshVertices: number | null; + /** 2-D detections returned by the detector backend. */ + detections2d: number; + /** Detections that survived fitting and the sanity gates. */ + fitted3d: number; + /** Count of each rejection reason across all detections. */ + rejections: Record; + /** Wall-clock milliseconds per stage. */ + timings: { + freshFrameWait: number; + snapshot: number; + depthMeshSnapshot: number; + detect2d: number; + masksAndFit: number; + total: number; + }; + /** Populated when the call bailed out early. */ + error: string | null; + /** Orientation policy in force for this call. */ + orientationMode: OrientationMode; + /** Estimated room yaw in degrees, or `null` when no frame was available. */ + roomYawDeg: number | null; + /** Confidence of the room frame, in `[0, 1]`. */ + roomYawConfidence: number | null; + /** Vertical surface area that voted for the room frame, in m². */ + roomFrameSupportM2: number | null; + /** + * Per-object yaw outcome. `roomRelativeYawDeg` is the useful one on device: + * if wall-aligned furniture reads ≈0 here but the boxes still look wrong, + * the fault is upstream in the camera model rather than in fitting. + */ + yawStats: Array<{ + label: string; + category: string; + yawDeg: number; + roomRelativeYawDeg: number; + confidence: number; + method: string; + }>; } // Kick off the three-mesh-bvh dynamic import at module load so it is ready @@ -94,8 +222,9 @@ const _bvhReady: Promise = enableAcceleratedRaycast().catch( ); /** - * Extracts the 3-D object-detection pipeline from the `objects_3d` demo into a - * reusable {@link Script}. Attach it to the scene before `xb.init()`, then + * The 3-D object-detection pipeline as a reusable {@link Script}. See the + * `objects_3d` demo for a worked integration. Attach it to the scene before + * `xb.init()`, then * call `await detector.detect()` to populate `detector.results`. * * ```ts @@ -108,9 +237,25 @@ const _bvhReady: Promise = enableAcceleratedRaycast().catch( * ``` */ export class Object3DDetector extends Script { - private readonly _opts: Required; + private readonly _opts: Required< + Omit< + Object3DDetectorOptions, + 'sceneBounds' | 'cameraRotationOffset' | 'orientation' + > + > & { + sceneBounds: {maxXZ: number; minY: number; maxY: number}; + cameraRotationOffset: {yaw: number; pitch: number; roll: number}; + orientation: Required> & { + roomYaw: number | null; + }; + }; private _results: Detected3DObject[] = []; private _detectInFlight = false; + // ~1.3 s of pose history at 90 fps, enough to cover passthrough video + // pipeline latency when pairing a capture with its capture-time pose. + private readonly _poseRing = new PoseRing(120); + private readonly _roomFrame = new RoomFrameAccumulator(); + private _diagnostics: Object3DDetectorDiagnostics | null = null; /** * @param options - Configuration options. @@ -122,15 +267,121 @@ export class Object3DDetector extends Script { maskBackend: options.maskBackend ?? 'slimsam', fuseAcrossViews: options.fuseAcrossViews ?? true, showDebugBoxes: options.showDebugBoxes ?? false, + maxRayDistance: options.maxRayDistance ?? 12, + sceneBounds: { + maxXZ: options.sceneBounds?.maxXZ ?? MAX_CENTER_HORIZONTAL_DISTANCE_M, + minY: options.sceneBounds?.minY ?? MIN_CENTER_HEIGHT_M, + maxY: options.sceneBounds?.maxY ?? MAX_CENTER_HEIGHT_M, + }, + roomHalf: options.roomHalf ?? 3, + cameraRotationOffset: { + yaw: options.cameraRotationOffset?.yaw ?? 0, + pitch: options.cameraRotationOffset?.pitch ?? 0, + roll: options.cameraRotationOffset?.roll ?? 0, + }, + orientation: { + mode: options.orientation?.mode ?? 'roomFrame', + roomYaw: options.orientation?.roomYaw ?? null, + roomYawConfidence: options.orientation?.roomYawConfidence ?? 0, + snapToleranceRad: + options.orientation?.snapToleranceRad ?? (12 * Math.PI) / 180, + minYawConfidence: options.orientation?.minYawConfidence ?? 0.35, + }, }; } + /** + * Record the device-camera pose every frame so {@link detect} can pair a + * captured video frame with the pose at the frame's `captureTime` — the + * passthrough video lags head tracking, so the pose at snapshot time is + * newer than the snapshot's pixels. + */ + override update(): void { + const deviceCamera = core.deviceCamera; + if (!deviceCamera || deviceCamera.simulatorCamera) return; + const xrCameras = core.renderer?.xr?.getCamera?.(); + if (!xrCameras?.cameras?.length) return; + try { + this._poseRing.push( + performance.now(), + getDeviceCameraWorldFromView( + core.camera, + xrCameras, + deviceCamera, + this._targetDevice() + ) + ); + } catch (_e) { + // Pose momentarily unavailable; skip this frame. + } + } + /** Currently fitted {@link Detected3DObject} instances from the last * (or accumulated) detect run. */ get results(): Detected3DObject[] { return this._results; } + /** + * Diagnostics from the most recent {@link detect} call, or `null` before + * the first one. See {@link Object3DDetectorDiagnostics}. + */ + get diagnostics(): Object3DDetectorDiagnostics | null { + return this._diagnostics; + } + + /** Poses currently held in the capture-time pose history ring. */ + get poseRingSize(): number { + return this._poseRing.size; + } + + /** Extra rotation applied to the device-camera pose, in radians. */ + get cameraRotationOffset(): {yaw: number; pitch: number; roll: number} { + return {...this._opts.cameraRotationOffset}; + } + + /** + * Adjust the camera rotation offset between detections, so a calibration + * error can be nulled out interactively instead of by reloading. Omitted + * components are left unchanged. + */ + setCameraRotationOffset(offset: { + yaw?: number; + pitch?: number; + roll?: number; + }): void { + const current = this._opts.cameraRotationOffset; + current.yaw = offset.yaw ?? current.yaw; + current.pitch = offset.pitch ?? current.pitch; + current.roll = offset.roll ?? current.roll; + } + + /** The orientation policy currently in force. */ + get orientationMode(): OrientationMode { + return this._opts.orientation.mode; + } + + /** + * Switch orientation policy between detections, so the modes can be + * A/B compared on device without reloading. + */ + setOrientationMode(mode: OrientationMode): void { + this._opts.orientation.mode = mode; + } + + /** The room frame accumulated so far, or `null` before any usable estimate. */ + get roomFrame(): ReturnType { + return this._roomFrame.current; + } + + /** + * Discard the accumulated room frame. Call this after the user recenters or + * moves to a different space; {@link clearDetections} does it too. + */ + resetRoomFrame(): void { + this._roomFrame.reset(); + } + /** * Remove all existing results and their debug visuals from the scene. * Call this to reset the detector before a new area scan. @@ -152,6 +403,7 @@ export class Object3DDetector extends Script { this.remove(obj); } this._results = []; + this._roomFrame.reset(); } /** @@ -188,6 +440,77 @@ export class Object3DDetector extends Script { } this._detectInFlight = true; + const t0 = performance.now(); + const diag: Object3DDetectorDiagnostics = { + startedAtMs: t0, + frameLatencyMs: null, + poseMatchErrorMs: null, + poseRingSize: this._poseRing.size, + usedDeviceCameraModel: false, + cameraFovDeg: 0, + cameraAspect: 0, + cameraRotationOffsetDeg: { + yaw: THREE.MathUtils.radToDeg(this._opts.cameraRotationOffset.yaw), + pitch: THREE.MathUtils.radToDeg(this._opts.cameraRotationOffset.pitch), + roll: THREE.MathUtils.radToDeg(this._opts.cameraRotationOffset.roll), + }, + snapshotWidth: 0, + snapshotHeight: 0, + depthRemapIsIdentity: null, + depthVsEyeRotationDeg: null, + depthMeshVertices: null, + detections2d: 0, + fitted3d: 0, + rejections: {}, + timings: { + freshFrameWait: 0, + snapshot: 0, + depthMeshSnapshot: 0, + detect2d: 0, + masksAndFit: 0, + total: 0, + }, + error: null, + orientationMode: this._opts.orientation.mode, + roomYawDeg: null, + roomYawConfidence: null, + roomFrameSupportM2: null, + yawStats: [], + }; + const bail = (message: string): Detected3DObject[] => { + console.warn(`[Object3DDetector] ${message}`); + diag.error = message; + diag.timings.total = performance.now() - t0; + this._diagnostics = diag; + this._detectInFlight = false; + return this._results; + }; + + if (!deviceCamera) { + return bail('device camera not available'); + } + + // Wait for a fresh video frame before snapshotting: inside an immersive + // session the hidden