Skip to content

Commit aa3f3b9

Browse files
committed
fix(voice): preserve manual edits during dictation
1 parent 770aaa7 commit aa3f3b9

4 files changed

Lines changed: 217 additions & 2 deletions

File tree

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
/** @vitest-environment jsdom */
2+
import { act } from 'react'
3+
import { createRoot, type Root } from 'react-dom/client'
4+
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
5+
import type { useSpeechToText } from '@/hooks/use-speech-to-text'
6+
7+
const mocks = vi.hoisted(() => ({ speech: vi.fn<typeof useSpeechToText>() }))
8+
vi.mock('@/hooks/use-speech-to-text', () => ({ useSpeechToText: mocks.speech }))
9+
10+
import { useVoiceInput } from '@/hooks/use-voice-input'
11+
12+
let root: Root
13+
let container: HTMLDivElement
14+
let value: string
15+
let voice: ReturnType<typeof useVoiceInput>
16+
17+
function Harness() {
18+
voice = useVoiceInput({
19+
organizationId: 'organization-a',
20+
getValue: () => value,
21+
onChange: (next) => {
22+
value = next
23+
},
24+
})
25+
return null
26+
}
27+
28+
function transcript(text: string) {
29+
act(() => mocks.speech.mock.calls.at(-1)![0].onTranscript(text))
30+
}
31+
32+
beforeEach(() => {
33+
vi.clearAllMocks()
34+
vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true)
35+
mocks.speech.mockReturnValue({
36+
isListening: false,
37+
isSupported: true,
38+
audioLevelsRef: { current: new Float32Array(5) },
39+
toggleListening: vi.fn(),
40+
resetTranscript: vi.fn(),
41+
})
42+
value = 'Find'
43+
container = document.createElement('div')
44+
document.body.appendChild(container)
45+
root = createRoot(container)
46+
act(() => root.render(<Harness />))
47+
act(() => voice.toggleListening())
48+
})
49+
50+
afterEach(() => {
51+
act(() => root.unmount())
52+
container.remove()
53+
vi.unstubAllGlobals()
54+
})
55+
56+
describe('voice input manual edits', () => {
57+
it('preserves edits made before the first transcript arrives', () => {
58+
value = 'Summarize'
59+
transcript('the report')
60+
expect(value).toBe('Summarize the report')
61+
})
62+
63+
it('preserves a changed prefix while updating partial speech', () => {
64+
transcript('the repor')
65+
value = 'Summarize the repor'
66+
transcript('the report')
67+
expect(value).toBe('Summarize the report')
68+
transcript('the report from June')
69+
expect(value).toBe('Summarize the report from June')
70+
})
71+
72+
it.each(['Replacement draft', ''])(
73+
'preserves a replaced or cleared draft (%s)',
74+
(replacement) => {
75+
transcript('the report')
76+
value = replacement
77+
transcript('the report')
78+
expect(value).toBe(replacement)
79+
transcript('the report from June')
80+
expect(value).toBe(replacement ? `${replacement} from June` : 'from June')
81+
}
82+
)
83+
84+
it('preserves corrections inside dictated text across later partial updates', () => {
85+
transcript('the red report')
86+
value = 'Find the blue report'
87+
transcript('the red report from June')
88+
expect(value).toBe('Find the blue report from June')
89+
transcript('the red report from July')
90+
expect(value).toBe('Find the blue report from July')
91+
})
92+
93+
it('keeps manual suffixes after speech revisions and before new speech', () => {
94+
transcript('the red report')
95+
value += ' and notes'
96+
transcript('the blue report')
97+
expect(value).toBe('Find the blue report and notes')
98+
transcript('the blue report from June')
99+
expect(value).toBe('Find the blue report and notes from June')
100+
})
101+
102+
it('prefers the manual correction when speech revises the same word', () => {
103+
value = ''
104+
act(() => voice.toggleListening())
105+
transcript('hel')
106+
value = 'help'
107+
transcript('hello')
108+
expect(value).toBe('help')
109+
transcript('hello again')
110+
expect(value).toBe('help again')
111+
})
112+
113+
it('keeps appended speech when another revision conflicts with a manual correction', () => {
114+
transcript('the red report')
115+
value = 'Find the blue report'
116+
transcript('the green report from June')
117+
expect(value).toBe('Find the blue report from June')
118+
transcript('the green report from June and July')
119+
expect(value).toBe('Find the blue report from June and July')
120+
})
121+
122+
it('preserves separate manual edits around an independently revised word', () => {
123+
transcript('the red report')
124+
value = 'Summarize the red report and notes'
125+
transcript('the green report from June')
126+
expect(value).toBe('Summarize the green report and notes from June')
127+
})
128+
129+
it('starts fresh after the draft and transcript are cleared on submit', () => {
130+
transcript('the report')
131+
value = ''
132+
act(() => voice.resetTranscript())
133+
transcript('Next question')
134+
expect(value).toBe('Next question')
135+
})
136+
137+
it('continues dictation after a large draft is manually replaced', () => {
138+
const longTranscript = Array.from({ length: 300 }, (_, index) => `word${index}`).join(' ')
139+
transcript(longTranscript)
140+
value = 'New draft'
141+
transcript(`${longTranscript} next question`)
142+
expect(value).toBe('New draft next question')
143+
})
144+
})

‎apps/sim/hooks/use-voice-input.ts‎

Lines changed: 71 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import { useCallback, useRef, useState } from 'react'
44
import { toast } from '@sim/emcn'
5+
import { diffWordsWithSpace } from 'diff'
56
import { getDesktopBridge } from '@/lib/desktop'
67
import { type SpeechToTextError, useSpeechToText } from '@/hooks/use-speech-to-text'
78

@@ -13,6 +14,64 @@ interface UseVoiceInputProps {
1314
onUsageLimitExceeded?: (message?: string, isMemberLimit?: boolean) => void
1415
}
1516

17+
interface TextEdit {
18+
from: number
19+
to: number
20+
insert: string
21+
}
22+
23+
function textChanges(before: string, after: string): TextEdit[] {
24+
const parts = diffWordsWithSpace(before, after, { maxEditLength: 256 })
25+
if (!parts) return [{ from: 0, to: before.length, insert: after }]
26+
const edits: TextEdit[] = []
27+
let position = 0
28+
let edit: TextEdit | undefined
29+
for (const part of parts) {
30+
if (part.added || part.removed) {
31+
edit ??= { from: position, to: position, insert: '' }
32+
if (part.added) edit.insert += part.value
33+
if (part.removed) {
34+
position += part.value.length
35+
edit.to = position
36+
}
37+
} else {
38+
if (edit) edits.push(edit)
39+
edit = undefined
40+
position += part.value.length
41+
}
42+
}
43+
if (edit) edits.push(edit)
44+
return edits
45+
}
46+
47+
/** Rebase independent speech updates around manual edits; the user's text wins conflicts. */
48+
function mergeTranscript(previous: string, next: string, current: string): string {
49+
if (current === previous) return next
50+
if (next === previous) return current
51+
52+
const speechEdits = textChanges(previous, next)
53+
const manualEdits = textChanges(previous, current)
54+
let result = current
55+
for (const speech of speechEdits.reverse()) {
56+
let offset = 0
57+
let conflict = false
58+
for (const edit of manualEdits) {
59+
if (speech.to <= edit.from && speech.from < edit.from) continue
60+
if (speech.from >= edit.to && (speech.from > edit.from || /^\s/.test(speech.insert))) {
61+
offset += edit.insert.length - (edit.to - edit.from)
62+
} else {
63+
conflict = true
64+
break
65+
}
66+
}
67+
if (!conflict) {
68+
result =
69+
result.slice(0, speech.from + offset) + speech.insert + result.slice(speech.to + offset)
70+
}
71+
}
72+
return current ? result : result.trimStart()
73+
}
74+
1675
/** Shares draft appending and microphone recovery across chat and search inputs. */
1776
export function useVoiceInput({
1877
workspaceId,
@@ -22,6 +81,7 @@ export function useVoiceInput({
2281
onUsageLimitExceeded = (message) => toast.error(message || 'You are out of credits.'),
2382
}: UseVoiceInputProps) {
2483
const prefixRef = useRef('')
84+
const previousTranscriptValueRef = useRef('')
2585
const getValueRef = useRef(getValue)
2686
getValueRef.current = getValue
2787
const [permissionHelpOpen, setPermissionHelpOpen] = useState(false)
@@ -66,18 +126,27 @@ export function useVoiceInput({
66126
} = useSpeechToText({
67127
workspaceId,
68128
organizationId,
69-
onTranscript: (text) => onChange(prefixRef.current ? `${prefixRef.current} ${text}` : text),
129+
onTranscript: (text) => {
130+
const next = prefixRef.current ? `${prefixRef.current} ${text}` : text
131+
const value = mergeTranscript(previousTranscriptValueRef.current, next, getValueRef.current())
132+
previousTranscriptValueRef.current = next
133+
onChange(value)
134+
},
70135
onUsageLimitExceeded,
71136
onError: handleSpeechError,
72137
})
73138

74139
const toggleListening = useCallback(() => {
75-
if (!speech.isListening) prefixRef.current = getValueRef.current()
140+
if (!speech.isListening) {
141+
prefixRef.current = getValueRef.current()
142+
previousTranscriptValueRef.current = prefixRef.current
143+
}
76144
rawToggle()
77145
}, [speech.isListening, rawToggle])
78146

79147
const resetTranscript = useCallback(() => {
80148
prefixRef.current = ''
149+
previousTranscriptValueRef.current = ''
81150
rawReset()
82151
}, [rawReset])
83152

‎apps/sim/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,7 @@
175175
"csv-parse": "7.0.2",
176176
"date-fns": "4.1.0",
177177
"decimal.js": "10.6.0",
178+
"diff": "8.0.4",
178179
"docx-preview": "^0.3.7",
179180
"docx": "^9.6.1",
180181
"drizzle-orm": "^0.45.2",

‎bun.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)