chenbhao commited on
Commit
94f44a2
·
1 Parent(s): 23780d5

feat: voice mode asr

Browse files
.gitignore CHANGED
@@ -4,6 +4,9 @@ cli
4
  .codex
5
  VersperClaw
6
 
 
 
 
7
  # Desktop (Tauri) build artifacts
8
  desktop/src-tauri/target/
9
  desktop/src-tauri/binaries/
 
4
  .codex
5
  VersperClaw
6
 
7
+ # Prebuilt native binaries (tracked in claude-code/ reference)
8
+ vendor/audio-capture/
9
+
10
  # Desktop (Tauri) build artifacts
11
  desktop/src-tauri/target/
12
  desktop/src-tauri/binaries/
bun.lock CHANGED
The diff for this file is too large to render. See raw diff
 
package.json CHANGED
@@ -2,6 +2,9 @@
2
  "name": "claude-code-source-snapshot",
3
  "version": "2.1.87",
4
  "private": true,
 
 
 
5
  "description": "Reconstructed Bun CLI workspace for the Claude Code source snapshot.",
6
  "type": "module",
7
  "packageManager": "bun@1.3.11",
@@ -57,10 +60,13 @@
57
  "@opentelemetry/semantic-conventions": "^1.40.0",
58
  "@smithy/core": "^3.23.13",
59
  "@smithy/node-http-handler": "^4.5.1",
 
60
  "@yukiakai/tls-fetch": "^2.0.2",
61
  "ajv": "^8.18.0",
62
  "asciichart": "^1.5.25",
 
63
  "auto-bind": "^5.0.1",
 
64
  "bidi-js": "^1.0.3",
65
  "cacache": "^20.0.4",
66
  "chalk": "^5.6.2",
@@ -69,6 +75,7 @@
69
  "cli-highlight": "^2.1.11",
70
  "code-excerpt": "^4.0.0",
71
  "diff": "^8.0.4",
 
72
  "emoji-regex": "^10.6.0",
73
  "env-paths": "^4.0.0",
74
  "execa": "^9.6.1",
@@ -112,7 +119,6 @@
112
  "ws": "^8.20.0",
113
  "xss": "^1.0.15",
114
  "yaml": "^2.8.3",
115
- "@tavily/core": "^0.6.1",
116
  "zod": "^4.3.6"
117
  },
118
  "devDependencies": {
 
2
  "name": "claude-code-source-snapshot",
3
  "version": "2.1.87",
4
  "private": true,
5
+ "workspaces": [
6
+ "packages/*"
7
+ ],
8
  "description": "Reconstructed Bun CLI workspace for the Claude Code source snapshot.",
9
  "type": "module",
10
  "packageManager": "bun@1.3.11",
 
60
  "@opentelemetry/semantic-conventions": "^1.40.0",
61
  "@smithy/core": "^3.23.13",
62
  "@smithy/node-http-handler": "^4.5.1",
63
+ "@tavily/core": "^0.6.1",
64
  "@yukiakai/tls-fetch": "^2.0.2",
65
  "ajv": "^8.18.0",
66
  "asciichart": "^1.5.25",
67
+ "audio-capture-napi": "workspace:*",
68
  "auto-bind": "^5.0.1",
69
+ "axios": "^1.15.2",
70
  "bidi-js": "^1.0.3",
71
  "cacache": "^20.0.4",
72
  "chalk": "^5.6.2",
 
75
  "cli-highlight": "^2.1.11",
76
  "code-excerpt": "^4.0.0",
77
  "diff": "^8.0.4",
78
+ "doubaoime-asr": "^0.1.0",
79
  "emoji-regex": "^10.6.0",
80
  "env-paths": "^4.0.0",
81
  "execa": "^9.6.1",
 
119
  "ws": "^8.20.0",
120
  "xss": "^1.0.15",
121
  "yaml": "^2.8.3",
 
122
  "zod": "^4.3.6"
123
  },
124
  "devDependencies": {
packages/audio-capture-napi/package.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "audio-capture-napi",
3
+ "version": "1.0.0",
4
+ "private": true,
5
+ "type": "module",
6
+ "main": "./src/index.ts",
7
+ "types": "./src/index.ts"
8
+ }
packages/audio-capture-napi/src/index.ts ADDED
@@ -0,0 +1,181 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { createRequire } from 'node:module'
2
+ import { dirname, resolve, sep } from 'node:path'
3
+ import { fileURLToPath } from 'node:url'
4
+ // createRequire works in both Bun and Node.js ESM contexts.
5
+ // Needed because this package is "type": "module" but uses require() for
6
+ // loading native .node addons — bare require is not available in Node.js ESM.
7
+ const nodeRequire = createRequire(import.meta.url)
8
+
9
+ /**
10
+ * Resolve the "vendor root" directory where native .node binaries live.
11
+ *
12
+ * - Dev mode: import.meta.url → packages/audio-capture-napi/src/index.ts
13
+ * → vendor root = <project>/vendor/
14
+ * - Bun build: import.meta.url → dist/chunk-xxx.js
15
+ * → vendor root = <project>/dist/vendor/
16
+ * - Vite build: import.meta.url → dist/chunks/chunk-xxx.js
17
+ * → vendor root = <project>/dist/vendor/
18
+ */
19
+ function getVendorRoot(): string {
20
+ const filePath = fileURLToPath(import.meta.url)
21
+ const dir = dirname(filePath)
22
+ const parts = dir.split(sep)
23
+ const distIdx = parts.lastIndexOf('dist')
24
+ if (distIdx !== -1) {
25
+ return parts.slice(0, distIdx + 1).join(sep) + sep + 'vendor'
26
+ }
27
+ // Dev mode — go up from packages/audio-capture-napi/src/ to project root
28
+ return resolve(dir, '..', '..', '..', 'vendor')
29
+ }
30
+
31
+ type AudioCaptureNapi = {
32
+ startRecording(onData: (data: Buffer) => void, onEnd: () => void): boolean
33
+ stopRecording(): void
34
+ isRecording(): boolean
35
+ startPlayback(sampleRate: number, channels: number): boolean
36
+ writePlaybackData(data: Buffer): void
37
+ stopPlayback(): void
38
+ isPlaying(): boolean
39
+ // TCC microphone authorization status (macOS only):
40
+ // 0 = notDetermined, 1 = restricted, 2 = denied, 3 = authorized.
41
+ // Linux: always returns 3 (authorized) — no system-level microphone permission API.
42
+ // Windows: returns 3 (authorized) if registry key absent or allowed,
43
+ // 2 (denied) if microphone access is explicitly denied.
44
+ microphoneAuthorizationStatus?(): number
45
+ }
46
+
47
+ let cachedModule: AudioCaptureNapi | null = null
48
+ let loadAttempted = false
49
+
50
+ function loadModule(): AudioCaptureNapi | null {
51
+ if (loadAttempted) {
52
+ return cachedModule
53
+ }
54
+ loadAttempted = true
55
+
56
+ // Supported platforms: macOS (darwin), Linux, Windows (win32)
57
+ const platform = process.platform
58
+ if (platform !== 'darwin' && platform !== 'linux' && platform !== 'win32') {
59
+ return null
60
+ }
61
+
62
+ // Candidate 1: native-embed path (bun compile). AUDIO_CAPTURE_NODE_PATH is
63
+ // defined at build time in build-with-plugins.ts for native builds only — the
64
+ // define resolves it to the static literal "../../audio-capture.node" so bun
65
+ // compile can rewrite it to /$bunfs/root/audio-capture.node. MUST stay a
66
+ // direct require(env var) — bun cannot analyze require(variable) from a loop.
67
+ if (process.env.AUDIO_CAPTURE_NODE_PATH) {
68
+ try {
69
+ // eslint-disable-next-line @typescript-eslint/no-require-imports
70
+ cachedModule = nodeRequire(
71
+ process.env.AUDIO_CAPTURE_NODE_PATH,
72
+ ) as AudioCaptureNapi
73
+ return cachedModule
74
+ } catch {
75
+ // fall through to runtime fallbacks below
76
+ }
77
+ }
78
+
79
+ // Candidates 2-5: resolved vendor path + relative fallbacks.
80
+ // The primary candidate uses getVendorRoot() to find the correct dist root
81
+ // regardless of chunk nesting depth. Relative fallbacks cover edge cases.
82
+ const platformDir = `${process.arch}-${platform}`
83
+ const binaryRel = `audio-capture/${platformDir}/audio-capture.node`
84
+ const vendorRoot = getVendorRoot()
85
+ const fallbacks = [
86
+ resolve(vendorRoot, binaryRel),
87
+ `./vendor/${binaryRel}`,
88
+ `../vendor/${binaryRel}`,
89
+ `../../vendor/${binaryRel}`,
90
+ `${process.cwd()}/vendor/${binaryRel}`,
91
+ ]
92
+ for (const p of fallbacks) {
93
+ try {
94
+ // eslint-disable-next-line @typescript-eslint/no-require-imports
95
+ cachedModule = nodeRequire(p) as AudioCaptureNapi
96
+ return cachedModule
97
+ } catch {
98
+ // try next
99
+ }
100
+ }
101
+ return null
102
+ }
103
+
104
+ export function isNativeAudioAvailable(): boolean {
105
+ return loadModule() !== null
106
+ }
107
+
108
+ export function startNativeRecording(
109
+ onData: (data: Buffer) => void,
110
+ onEnd: () => void,
111
+ ): boolean {
112
+ const mod = loadModule()
113
+ if (!mod) {
114
+ return false
115
+ }
116
+ return mod.startRecording(onData, onEnd)
117
+ }
118
+
119
+ export function stopNativeRecording(): void {
120
+ const mod = loadModule()
121
+ if (!mod) {
122
+ return
123
+ }
124
+ mod.stopRecording()
125
+ }
126
+
127
+ export function isNativeRecordingActive(): boolean {
128
+ const mod = loadModule()
129
+ if (!mod) {
130
+ return false
131
+ }
132
+ return mod.isRecording()
133
+ }
134
+
135
+ export function startNativePlayback(
136
+ sampleRate: number,
137
+ channels: number,
138
+ ): boolean {
139
+ const mod = loadModule()
140
+ if (!mod) {
141
+ return false
142
+ }
143
+ return mod.startPlayback(sampleRate, channels)
144
+ }
145
+
146
+ export function writeNativePlaybackData(data: Buffer): void {
147
+ const mod = loadModule()
148
+ if (!mod) {
149
+ return
150
+ }
151
+ mod.writePlaybackData(data)
152
+ }
153
+
154
+ export function stopNativePlayback(): void {
155
+ const mod = loadModule()
156
+ if (!mod) {
157
+ return
158
+ }
159
+ mod.stopPlayback()
160
+ }
161
+
162
+ export function isNativePlaying(): boolean {
163
+ const mod = loadModule()
164
+ if (!mod) {
165
+ return false
166
+ }
167
+ return mod.isPlaying()
168
+ }
169
+
170
+ // Returns the microphone authorization status.
171
+ // On macOS, returns the TCC status: 0=notDetermined, 1=restricted, 2=denied, 3=authorized.
172
+ // On Linux, always returns 3 (authorized) — no system-level mic permission API.
173
+ // On Windows, returns 3 (authorized) if registry key absent or allowed, 2 (denied) if explicitly denied.
174
+ // Returns 0 (notDetermined) if the native module is unavailable.
175
+ export function microphoneAuthorizationStatus(): number {
176
+ const mod = loadModule()
177
+ if (!mod || !mod.microphoneAuthorizationStatus) {
178
+ return 0
179
+ }
180
+ return mod.microphoneAuthorizationStatus()
181
+ }
packages/audio-capture-napi/tsconfig.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "extends": "../../tsconfig.base.json",
3
+ "include": ["src/**/*.ts"],
4
+ "exclude": ["node_modules", "dist"]
5
+ }
scripts/build.ts CHANGED
@@ -1,5 +1,5 @@
1
- import { chmodSync, existsSync, mkdirSync } from 'fs'
2
- import { dirname } from 'path'
3
 
4
  const pkg = await Bun.file(new URL('../package.json', import.meta.url)).json() as {
5
  name: string
@@ -203,5 +203,15 @@ if (existsSync(outfile)) {
203
  chmodSync(outfile, 0o755)
204
  }
205
 
 
 
 
 
 
 
 
 
 
 
206
  console.log(`Built ${outfile}`)
207
 
 
1
+ import { chmodSync, cpSync, existsSync, mkdirSync } from 'fs'
2
+ import { dirname, join } from 'path'
3
 
4
  const pkg = await Bun.file(new URL('../package.json', import.meta.url)).json() as {
5
  name: string
 
203
  chmodSync(outfile, 0o755)
204
  }
205
 
206
+ // Copy vendor audio-capture binaries to dist/ for runtime resolution
207
+ if (!compile) {
208
+ const distDir = dirname(outfile)
209
+ const vendorDir = join(distDir, 'vendor')
210
+ if (!existsSync(vendorDir)) {
211
+ cpSync('vendor', vendorDir, { recursive: true })
212
+ console.log(`Copied vendor/ → ${vendorDir}/`)
213
+ }
214
+ }
215
+
216
  console.log(`Built ${outfile}`)
217
 
src/commands/voice/index.ts CHANGED
@@ -1,17 +1,16 @@
1
  import type { Command } from '../../commands.js'
2
  import {
 
3
  isVoiceGrowthBookEnabled,
4
- isVoiceModeEnabled,
5
  } from '../../voice/voiceModeEnabled.js'
6
 
7
  const voice = {
8
  type: 'local',
9
  name: 'voice',
10
  description: 'Toggle voice mode',
11
- availability: ['claude-ai'],
12
  isEnabled: () => isVoiceGrowthBookEnabled(),
13
  get isHidden() {
14
- return !isVoiceModeEnabled()
15
  },
16
  supportsNonInteractive: false,
17
  load: () => import('./voice.js'),
 
1
  import type { Command } from '../../commands.js'
2
  import {
3
+ isVoiceAvailable,
4
  isVoiceGrowthBookEnabled,
 
5
  } from '../../voice/voiceModeEnabled.js'
6
 
7
  const voice = {
8
  type: 'local',
9
  name: 'voice',
10
  description: 'Toggle voice mode',
 
11
  isEnabled: () => isVoiceGrowthBookEnabled(),
12
  get isHidden() {
13
+ return !isVoiceAvailable()
14
  },
15
  supportsNonInteractive: false,
16
  load: () => import('./voice.js'),
src/commands/voice/voice.ts CHANGED
@@ -9,22 +9,13 @@ import {
9
  getInitialSettings,
10
  updateSettingsForSource,
11
  } from '../../utils/settings/settings.js'
12
- import { isVoiceModeEnabled } from '../../voice/voiceModeEnabled.js'
13
 
14
  const LANG_HINT_MAX_SHOWS = 2
15
 
16
  export const call: LocalCommandCall = async () => {
17
- // Check auth and kill-switch before allowing voice mode
18
- if (!isVoiceModeEnabled()) {
19
- // Differentiate: OAuth-less users get an auth hint, everyone else
20
- // gets nothing (command shouldn't be reachable when the kill-switch is on).
21
- if (!isAnthropicAuthEnabled()) {
22
- return {
23
- type: 'text' as const,
24
- value:
25
- 'Voice mode requires a Claude.ai account. Please run /login to sign in.',
26
- }
27
- }
28
  return {
29
  type: 'text' as const,
30
  value: 'Voice mode is not available.',
@@ -60,6 +51,28 @@ export const call: LocalCommandCall = async () => {
60
  )
61
  const { checkRecordingAvailability } = await import('../../services/voice.js')
62
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
63
  // Check recording availability (microphone access)
64
  const recording = await checkRecordingAvailability()
65
  if (!recording.available) {
@@ -70,15 +83,6 @@ export const call: LocalCommandCall = async () => {
70
  }
71
  }
72
 
73
- // Check for API key
74
- if (!isVoiceStreamAvailable()) {
75
- return {
76
- type: 'text' as const,
77
- value:
78
- 'Voice mode requires a Claude.ai account. Please run /login to sign in.',
79
- }
80
- }
81
-
82
  // Check for recording tools
83
  const { checkVoiceDependencies, requestMicrophonePermission } = await import(
84
  '../../services/voice.js'
@@ -111,8 +115,11 @@ export const call: LocalCommandCall = async () => {
111
  }
112
  }
113
 
114
- // All checks passed — enable voice
115
- const result = updateSettingsForSource('userSettings', { voiceEnabled: true })
 
 
 
116
  if (result.error) {
117
  return {
118
  type: 'text' as const,
@@ -143,8 +150,9 @@ export const call: LocalCommandCall = async () => {
143
  voiceLangHintLastLanguage: stt.code,
144
  }))
145
  }
 
146
  return {
147
  type: 'text' as const,
148
- value: `Voice mode enabled. Hold ${key} to record.${langNote}`,
149
  }
150
  }
 
9
  getInitialSettings,
10
  updateSettingsForSource,
11
  } from '../../utils/settings/settings.js'
12
+ import { isVoiceAvailable } from '../../voice/voiceModeEnabled.js'
13
 
14
  const LANG_HINT_MAX_SHOWS = 2
15
 
16
  export const call: LocalCommandCall = async () => {
17
+ // Check only the GrowthBook kill-switch voice might use Doubao (no auth)
18
+ if (!isVoiceAvailable()) {
 
 
 
 
 
 
 
 
 
19
  return {
20
  type: 'text' as const,
21
  value: 'Voice mode is not available.',
 
51
  )
52
  const { checkRecordingAvailability } = await import('../../services/voice.js')
53
 
54
+ // Determine available STT backend: prefer Anthropic, fall back to Doubao
55
+ const hasAnthropicAuth = isAnthropicAuthEnabled() && isVoiceStreamAvailable()
56
+ let useDoubao = false
57
+ if (!hasAnthropicAuth) {
58
+ const { isDoubaoAvailable } = await import('../../services/doubaoSTT.js')
59
+ if (!(await isDoubaoAvailable())) {
60
+ // Neither backend is available
61
+ if (!isAnthropicAuthEnabled()) {
62
+ return {
63
+ type: 'text' as const,
64
+ value:
65
+ 'Voice mode requires a Claude.ai account or the Doubao ASR backend. Please run /login to sign in, or install Doubao ASR.',
66
+ }
67
+ }
68
+ return {
69
+ type: 'text' as const,
70
+ value: 'Voice mode is not available.',
71
+ }
72
+ }
73
+ useDoubao = true
74
+ }
75
+
76
  // Check recording availability (microphone access)
77
  const recording = await checkRecordingAvailability()
78
  if (!recording.available) {
 
83
  }
84
  }
85
 
 
 
 
 
 
 
 
 
 
86
  // Check for recording tools
87
  const { checkVoiceDependencies, requestMicrophonePermission } = await import(
88
  '../../services/voice.js'
 
115
  }
116
  }
117
 
118
+ // All checks passed — enable voice and store backend choice
119
+ const result = updateSettingsForSource('userSettings', {
120
+ voiceEnabled: true,
121
+ ...(useDoubao ? { voiceProvider: 'doubao' as const } : { voiceProvider: 'anthropic' as const }),
122
+ })
123
  if (result.error) {
124
  return {
125
  type: 'text' as const,
 
150
  voiceLangHintLastLanguage: stt.code,
151
  }))
152
  }
153
+ const backendNote = useDoubao ? ' [Doubao ASR]' : ''
154
  return {
155
  type: 'text' as const,
156
+ value: `Voice mode enabled. Hold ${key} to record.${langNote}${backendNote}`,
157
  }
158
  }
src/hooks/useVoice.ts CHANGED
@@ -9,6 +9,7 @@
9
  import { useCallback, useEffect, useRef, useState } from 'react'
10
  import { useSetVoiceState } from '../context/voice.js'
11
  import { useTerminalFocus } from '../ink/hooks/use-terminal-focus.js'
 
12
  import {
13
  type AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
14
  logEvent,
@@ -20,6 +21,7 @@ import {
20
  isVoiceStreamAvailable,
21
  type VoiceStreamConnection,
22
  } from '../services/voiceStreamSTT.js'
 
23
  import { logForDebugging } from '../utils/debug.js'
24
  import { toError } from '../utils/errors.js'
25
  import { getSystemLocaleLanguage } from '../utils/intl.js'
@@ -133,6 +135,10 @@ export function normalizeLanguageForSTT(language: string | undefined): {
133
  return { code: DEFAULT_STT_LANGUAGE, fellBackFrom: language }
134
  }
135
 
 
 
 
 
136
  // Lazy-loaded voice module. We defer importing voice.ts (and its native
137
  // audio-capture-napi dependency) until voice input is actually activated.
138
  // On macOS, loading the native audio module can trigger a TCC microphone
@@ -574,7 +580,7 @@ export function useVoice({
574
  // stop when it loses focus. This enables a "multi-clauding army"
575
  // workflow where voice input follows window focus.
576
  useEffect(() => {
577
- if (!enabled || !focusMode) {
578
  // Focus mode was disabled while a focus-driven recording was active —
579
  // stop the recording so it doesn't linger until the silence timer fires.
580
  if (focusTriggeredRef.current && stateRef.current === 'recording') {
@@ -778,7 +784,17 @@ export function useVoice({
778
 
779
  const attemptConnect = (keyterms: string[]): void => {
780
  const myAttemptGen = attemptGenRef.current
781
- void connectVoiceStream(
 
 
 
 
 
 
 
 
 
 
782
  {
783
  onTranscript: (text: string, isFinal: boolean) => {
784
  if (isStale()) return
@@ -1007,7 +1023,12 @@ export function useVoice({
1007
  })
1008
  }
1009
 
1010
- void getVoiceKeyterms().then(attemptConnect)
 
 
 
 
 
1011
  }
1012
 
1013
  // ── Hold-to-talk handler ────────────────────────────────────────────
@@ -1021,7 +1042,10 @@ export function useVoice({
1021
  // delay of ~500ms on macOS).
1022
  const handleKeyEvent = useCallback(
1023
  (fallbackMs = REPEAT_FALLBACK_MS): void => {
1024
- if (!enabled || !isVoiceStreamAvailable()) {
 
 
 
1025
  return
1026
  }
1027
 
 
9
  import { useCallback, useEffect, useRef, useState } from 'react'
10
  import { useSetVoiceState } from '../context/voice.js'
11
  import { useTerminalFocus } from '../ink/hooks/use-terminal-focus.js'
12
+ import { isDoubaoAvailableSync } from '../services/doubaoSTT.js'
13
  import {
14
  type AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
15
  logEvent,
 
21
  isVoiceStreamAvailable,
22
  type VoiceStreamConnection,
23
  } from '../services/voiceStreamSTT.js'
24
+ import { connectDoubaoStream } from '../services/doubaoSTT.js'
25
  import { logForDebugging } from '../utils/debug.js'
26
  import { toError } from '../utils/errors.js'
27
  import { getSystemLocaleLanguage } from '../utils/intl.js'
 
135
  return { code: DEFAULT_STT_LANGUAGE, fellBackFrom: language }
136
  }
137
 
138
+ function isDoubaoProvider(): boolean {
139
+ return getInitialSettings().voiceProvider === 'doubao'
140
+ }
141
+
142
  // Lazy-loaded voice module. We defer importing voice.ts (and its native
143
  // audio-capture-napi dependency) until voice input is actually activated.
144
  // On macOS, loading the native audio module can trigger a TCC microphone
 
580
  // stop when it loses focus. This enables a "multi-clauding army"
581
  // workflow where voice input follows window focus.
582
  useEffect(() => {
583
+ if (!enabled || !focusMode || isDoubaoProvider()) {
584
  // Focus mode was disabled while a focus-driven recording was active —
585
  // stop the recording so it doesn't linger until the silence timer fires.
586
  if (focusTriggeredRef.current && stateRef.current === 'recording') {
 
784
 
785
  const attemptConnect = (keyterms: string[]): void => {
786
  const myAttemptGen = attemptGenRef.current
787
+ // Select STT backend based on settings.voiceProvider
788
+ const connectFn = isDoubaoProvider()
789
+ ? (
790
+ cbs: Parameters<typeof connectDoubaoStream>[0],
791
+ opts: Parameters<typeof connectDoubaoStream>[1],
792
+ ) => connectDoubaoStream(cbs, opts)
793
+ : (
794
+ cbs: Parameters<typeof connectVoiceStream>[0],
795
+ opts: Parameters<typeof connectVoiceStream>[1],
796
+ ) => connectVoiceStream(cbs, opts)
797
+ void connectFn(
798
  {
799
  onTranscript: (text: string, isFinal: boolean) => {
800
  if (isStale()) return
 
1023
  })
1024
  }
1025
 
1026
+ // Doubao backend doesn't use keyterms — skip the async fetch
1027
+ if (isDoubaoProvider()) {
1028
+ attemptConnect([])
1029
+ } else {
1030
+ void getVoiceKeyterms().then(attemptConnect)
1031
+ }
1032
  }
1033
 
1034
  // ── Hold-to-talk handler ────────────────────────────────────────────
 
1042
  // delay of ~500ms on macOS).
1043
  const handleKeyEvent = useCallback(
1044
  (fallbackMs = REPEAT_FALLBACK_MS): void => {
1045
+ const sttAvailable = isDoubaoProvider()
1046
+ ? isDoubaoAvailableSync()
1047
+ : isVoiceStreamAvailable()
1048
+ if (!enabled || !sttAvailable) {
1049
  return
1050
  }
1051
 
src/services/doubaoSTT.ts ADDED
@@ -0,0 +1,258 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // Doubao (豆包) ASR speech-to-text adapter for voice mode.
2
+ //
3
+ // Wraps the doubaoime-asr npm package to expose the same interface as
4
+ // voiceStreamSTT.ts. The doubao backend uses an AsyncGenerator-based
5
+ // streaming protocol internally; this adapter bridges it to the
6
+ // send/finalize/close pattern used by useVoice.ts.
7
+
8
+ import { homedir } from 'node:os'
9
+ import type { ASRResponse } from 'doubaoime-asr'
10
+ import type {
11
+ FinalizeSource,
12
+ VoiceStreamCallbacks,
13
+ VoiceStreamConnection,
14
+ } from './voiceStreamSTT.js'
15
+ import { logForDebugging } from '../utils/debug.js'
16
+ import { logError } from '../utils/log.js'
17
+
18
+ // Re-export FinalizeSource so useVoice can import from either module
19
+ export type { FinalizeSource } from './voiceStreamSTT.js'
20
+
21
+ // ─── AsyncIterable audio queue ─────────────────────────────────────────
22
+
23
+ // A push-based queue that implements AsyncIterable<Uint8Array>.
24
+ // send() pushes chunks; push(null) signals end-of-stream.
25
+ class AudioChunkQueue {
26
+ private chunks: (Uint8Array | null)[] = []
27
+ private waiting: ((result: IteratorResult<Uint8Array>) => void) | null = null
28
+ private done = false
29
+
30
+ push(chunk: Uint8Array | null): void {
31
+ if (this.done) return
32
+ if (chunk === null) {
33
+ this.done = true
34
+ if (this.waiting) {
35
+ const resolve = this.waiting
36
+ this.waiting = null
37
+ resolve({ value: undefined, done: true })
38
+ }
39
+ return
40
+ }
41
+ if (this.waiting) {
42
+ const resolve = this.waiting
43
+ this.waiting = null
44
+ resolve({ value: chunk, done: false })
45
+ } else {
46
+ this.chunks.push(chunk)
47
+ }
48
+ }
49
+
50
+ abort(): void {
51
+ this.done = true
52
+ this.chunks.length = 0
53
+ if (this.waiting) {
54
+ const resolve = this.waiting
55
+ this.waiting = null
56
+ resolve({ value: undefined, done: true })
57
+ }
58
+ }
59
+
60
+ [Symbol.asyncIterator](): AsyncIterator<Uint8Array> {
61
+ return {
62
+ next: async (): Promise<IteratorResult<Uint8Array>> => {
63
+ if (this.chunks.length > 0) {
64
+ const chunk = this.chunks.shift()!
65
+ return { value: chunk, done: false }
66
+ }
67
+ if (this.done) {
68
+ return { value: undefined, done: true }
69
+ }
70
+ return new Promise<IteratorResult<Uint8Array>>(resolve => {
71
+ this.waiting = resolve
72
+ })
73
+ },
74
+ }
75
+ }
76
+ }
77
+
78
+ // ─── Availability ────────────────────────────────────────────────────────
79
+
80
+ let doubaoAvailable: boolean | null = null
81
+
82
+ export async function isDoubaoAvailable(): Promise<boolean> {
83
+ if (doubaoAvailable !== null) return doubaoAvailable
84
+ try {
85
+ await import('doubaoime-asr')
86
+ doubaoAvailable = true
87
+ } catch {
88
+ doubaoAvailable = false
89
+ }
90
+ return doubaoAvailable
91
+ }
92
+
93
+ // Synchronous check — returns cached result or optimistic true when
94
+ // VOICE_PROVIDER=doubao is set and no cached result exists yet.
95
+ // The actual import happens in connectDoubaoStream which reports errors.
96
+ export function isDoubaoAvailableSync(): boolean {
97
+ if (doubaoAvailable !== null) return doubaoAvailable
98
+ return true
99
+ }
100
+
101
+ // ─── Connection ──────────────────────────────────────────────────────────
102
+
103
+ export async function connectDoubaoStream(
104
+ callbacks: VoiceStreamCallbacks,
105
+ _options?: { language?: string },
106
+ ): Promise<VoiceStreamConnection | null> {
107
+ let doubaoAsr: typeof import('doubaoime-asr')
108
+ try {
109
+ doubaoAsr = await import('doubaoime-asr')
110
+ } catch (err) {
111
+ logError(
112
+ new Error(
113
+ `[doubao-asr] Failed to import doubaoime-asr package: ${String(err)}`,
114
+ ),
115
+ )
116
+ callbacks.onError(`doubaoime-asr package import failed: ${String(err)}`, {
117
+ fatal: true,
118
+ })
119
+ return null
120
+ }
121
+
122
+ const { transcribeRealtime, ASRConfig, ResponseType } = doubaoAsr
123
+
124
+ const queue = new AudioChunkQueue()
125
+ let finalized = false
126
+
127
+ // Resolve handle for finalize() promise — wrapped in an object to avoid
128
+ // TypeScript closure-scope type narrowing issues (TS2349 "not callable").
129
+ const finalizeHandle: { resolve: ((source: FinalizeSource) => void) | null } =
130
+ { resolve: null }
131
+
132
+ const connection: VoiceStreamConnection = {
133
+ send(audioChunk: Buffer): void {
134
+ if (finalized) return
135
+ queue.push(
136
+ new Uint8Array(
137
+ audioChunk.buffer,
138
+ audioChunk.byteOffset,
139
+ audioChunk.byteLength,
140
+ ),
141
+ )
142
+ },
143
+ finalize(): Promise<FinalizeSource> {
144
+ if (finalized) return Promise.resolve<FinalizeSource>('ws_already_closed')
145
+ finalized = true
146
+ queue.push(null) // signal end-of-stream to the generator
147
+ // Doubao returns FINAL_RESULT during recording — by the time the user
148
+ // releases the key, all transcripts are already in accumulatedRef.
149
+ // Resolve immediately so the UI skips the 'processing' state and goes
150
+ // straight to displaying the result.
151
+ logForDebugging('[doubao-asr] Finalize — resolving immediately')
152
+ return Promise.resolve<FinalizeSource>('post_closestream_endpoint')
153
+ },
154
+ close(): void {
155
+ finalized = true
156
+ queue.abort()
157
+ const r = finalizeHandle.resolve
158
+ finalizeHandle.resolve = null
159
+ if (r) r('ws_close')
160
+ callbacks.onClose()
161
+ },
162
+ isConnected(): boolean {
163
+ return true
164
+ },
165
+ }
166
+
167
+ // Start the ASR session in the background
168
+ const config = new ASRConfig({
169
+ credentialPath: `${homedir()}/.claude/tts/doubao/credentials.json`,
170
+ })
171
+
172
+ // Ensure credentials are initialized (may auto-generate)
173
+ try {
174
+ await config.ensureCredentials()
175
+ } catch (err) {
176
+ logError(
177
+ new Error(
178
+ `[doubao-asr] Credential initialization failed: ${String(err)}`,
179
+ ),
180
+ )
181
+ callbacks.onError(`Doubao ASR 凭证初始化失败: ${String(err)}`, {
182
+ fatal: true,
183
+ })
184
+ return null
185
+ }
186
+
187
+ // Fire onReady immediately — unlike the Anthropic WebSocket which needs to
188
+ // wait for a handshake, the doubao backend accepts audio through the queue
189
+ // and handles connection internally. The caller (useVoice.ts) needs onReady
190
+ // to fire before it will route audio chunks via connection.send().
191
+ logForDebugging('[doubao-asr] Firing onReady immediately')
192
+ callbacks.onReady(connection)
193
+
194
+ // Consume the AsyncGenerator in the background
195
+ void (async () => {
196
+ try {
197
+ const audioSource: AsyncIterable<Uint8Array> = queue
198
+ const gen: AsyncGenerator<ASRResponse> = transcribeRealtime(audioSource, {
199
+ config,
200
+ })
201
+
202
+ for await (const resp of gen) {
203
+ if (
204
+ finalized &&
205
+ resp.type !== ResponseType.FINAL_RESULT &&
206
+ resp.type !== ResponseType.SESSION_FINISHED
207
+ ) {
208
+ continue
209
+ }
210
+
211
+ switch (resp.type) {
212
+ case ResponseType.SESSION_STARTED:
213
+ logForDebugging('[doubao-asr] Session started')
214
+ break
215
+ case ResponseType.VAD_START:
216
+ logForDebugging('[doubao-asr] VAD detected speech start')
217
+ break
218
+ case ResponseType.INTERIM_RESULT:
219
+ if (resp.text) {
220
+ callbacks.onTranscript(resp.text, false)
221
+ }
222
+ break
223
+ case ResponseType.FINAL_RESULT:
224
+ if (resp.text) {
225
+ callbacks.onTranscript(resp.text, true)
226
+ }
227
+ break
228
+ case ResponseType.ERROR:
229
+ logError(new Error(`[doubao-asr] Error: ${resp.errorMsg}`))
230
+ if (!finalized) {
231
+ callbacks.onError(resp.errorMsg || 'Doubao ASR 识别错误')
232
+ }
233
+ break
234
+ case ResponseType.SESSION_FINISHED:
235
+ logForDebugging('[doubao-asr] Session finished')
236
+ break
237
+ default:
238
+ break
239
+ }
240
+ }
241
+
242
+ // Generator exhausted naturally
243
+ const r = finalizeHandle.resolve
244
+ finalizeHandle.resolve = null
245
+ if (r) r('post_closestream_endpoint')
246
+ } catch (err) {
247
+ logError(new Error(`[doubao-asr] Stream error: ${String(err)}`))
248
+ if (!finalized) {
249
+ callbacks.onError(`Doubao ASR 连接错误: ${String(err)}`)
250
+ }
251
+ const r2 = finalizeHandle.resolve
252
+ finalizeHandle.resolve = null
253
+ if (r2) r2('ws_close')
254
+ }
255
+ })()
256
+
257
+ return connection
258
+ }
src/utils/settings/types.ts CHANGED
@@ -867,6 +867,12 @@ export const SettingsSchema = lazySchema(() =>
867
  .boolean()
868
  .optional()
869
  .describe('Enable voice mode (hold-to-talk dictation)'),
 
 
 
 
 
 
870
  }
871
  : {}),
872
  ...(feature('KAIROS')
 
867
  .boolean()
868
  .optional()
869
  .describe('Enable voice mode (hold-to-talk dictation)'),
870
+ voiceProvider: z
871
+ .enum(['anthropic', 'doubao'])
872
+ .optional()
873
+ .describe(
874
+ 'Voice STT backend: "anthropic" (default) or "doubao" (Doubao ASR)',
875
+ ),
876
  }
877
  : {}),
878
  ...(feature('KAIROS')
src/voice/voiceModeEnabled.ts CHANGED
@@ -44,11 +44,18 @@ export function hasVoiceAuth(): boolean {
44
  }
45
 
46
  /**
47
- * Full runtime check: auth + GrowthBook kill-switch. Callers: `/voice`
48
- * (voice.ts, voice/index.ts), ConfigTool, VoiceModeNotice command-time
49
- * paths where a fresh keychain read is acceptable. For React render
50
- * paths use useVoiceEnabled() instead (memoizes the auth half).
51
  */
52
  export function isVoiceModeEnabled(): boolean {
53
  return hasVoiceAuth() && isVoiceGrowthBookEnabled()
54
  }
 
 
 
 
 
 
 
 
 
 
44
  }
45
 
46
  /**
47
+ * Full runtime check for Anthropic voice_stream backend.
48
+ * Returns true when both auth + GrowthBook kill-switch pass.
 
 
49
  */
50
  export function isVoiceModeEnabled(): boolean {
51
  return hasVoiceAuth() && isVoiceGrowthBookEnabled()
52
  }
53
+
54
+ /**
55
+ * Check if voice mode can be activated with any STT backend.
56
+ * Always returns true when VOICE_MODE feature flag is on and GrowthBook
57
+ * kill-switch is off — the Doubao backend does not require Anthropic auth.
58
+ */
59
+ export function isVoiceAvailable(): boolean {
60
+ return feature('VOICE_MODE')
61
+ }
vendor/audio-capture-src/index.ts ADDED
@@ -0,0 +1,147 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ type AudioCaptureNapi = {
2
+ startRecording(onData: (data: Buffer) => void, onEnd: () => void): boolean
3
+ stopRecording(): void
4
+ isRecording(): boolean
5
+ startPlayback(sampleRate: number, channels: number): boolean
6
+ writePlaybackData(data: Buffer): void
7
+ stopPlayback(): void
8
+ isPlaying(): boolean
9
+ // TCC microphone authorization status (macOS only):
10
+ // 0 = notDetermined, 1 = restricted, 2 = denied, 3 = authorized.
11
+ // Linux: always returns 3 (authorized) — no system-level microphone permission API.
12
+ // Windows: returns 3 (authorized) if registry key absent or allowed,
13
+ // 2 (denied) if microphone access is explicitly denied.
14
+ microphoneAuthorizationStatus?(): number
15
+ }
16
+
17
+ let cachedModule: AudioCaptureNapi | null = null
18
+ let loadAttempted = false
19
+
20
+ function loadModule(): AudioCaptureNapi | null {
21
+ if (loadAttempted) {
22
+ return cachedModule
23
+ }
24
+ loadAttempted = true
25
+
26
+ // Supported platforms: macOS (darwin), Linux, Windows (win32)
27
+ const platform = process.platform
28
+ if (platform !== 'darwin' && platform !== 'linux' && platform !== 'win32') {
29
+ return null
30
+ }
31
+
32
+ // Candidate 1: native-embed path (bun compile). AUDIO_CAPTURE_NODE_PATH is
33
+ // defined at build time in build-with-plugins.ts for native builds only — the
34
+ // define resolves it to the static literal "../../audio-capture.node" so bun
35
+ // compile can rewrite it to /$bunfs/root/audio-capture.node. MUST stay a
36
+ // direct require(env var) — bun cannot analyze require(variable) from a loop.
37
+ if (process.env.AUDIO_CAPTURE_NODE_PATH) {
38
+ try {
39
+ // eslint-disable-next-line @typescript-eslint/no-require-imports
40
+ cachedModule = require(
41
+ process.env.AUDIO_CAPTURE_NODE_PATH,
42
+ ) as AudioCaptureNapi
43
+ return cachedModule
44
+ } catch {
45
+ // fall through to runtime fallbacks below
46
+ }
47
+ }
48
+
49
+ // Candidates 2/3: npm-install and dev/source layouts. Dynamic require is
50
+ // fine here — in bundled output (node --target build) require() resolves at
51
+ // runtime relative to cli.js at the package root; in dev it resolves
52
+ // relative to this file (vendor/audio-capture-src/index.ts).
53
+ const platformDir = `${process.arch}-${platform}`
54
+ const fallbacks = [
55
+ `./vendor/audio-capture/${platformDir}/audio-capture.node`,
56
+ `../audio-capture/${platformDir}/audio-capture.node`,
57
+ ]
58
+ for (const p of fallbacks) {
59
+ try {
60
+ // eslint-disable-next-line @typescript-eslint/no-require-imports
61
+ cachedModule = require(p) as AudioCaptureNapi
62
+ return cachedModule
63
+ } catch {
64
+ // try next
65
+ }
66
+ }
67
+ return null
68
+ }
69
+
70
+ export function isNativeAudioAvailable(): boolean {
71
+ return loadModule() !== null
72
+ }
73
+
74
+ export function startNativeRecording(
75
+ onData: (data: Buffer) => void,
76
+ onEnd: () => void,
77
+ ): boolean {
78
+ const mod = loadModule()
79
+ if (!mod) {
80
+ return false
81
+ }
82
+ return mod.startRecording(onData, onEnd)
83
+ }
84
+
85
+ export function stopNativeRecording(): void {
86
+ const mod = loadModule()
87
+ if (!mod) {
88
+ return
89
+ }
90
+ mod.stopRecording()
91
+ }
92
+
93
+ export function isNativeRecordingActive(): boolean {
94
+ const mod = loadModule()
95
+ if (!mod) {
96
+ return false
97
+ }
98
+ return mod.isRecording()
99
+ }
100
+
101
+ export function startNativePlayback(
102
+ sampleRate: number,
103
+ channels: number,
104
+ ): boolean {
105
+ const mod = loadModule()
106
+ if (!mod) {
107
+ return false
108
+ }
109
+ return mod.startPlayback(sampleRate, channels)
110
+ }
111
+
112
+ export function writeNativePlaybackData(data: Buffer): void {
113
+ const mod = loadModule()
114
+ if (!mod) {
115
+ return
116
+ }
117
+ mod.writePlaybackData(data)
118
+ }
119
+
120
+ export function stopNativePlayback(): void {
121
+ const mod = loadModule()
122
+ if (!mod) {
123
+ return
124
+ }
125
+ mod.stopPlayback()
126
+ }
127
+
128
+ export function isNativePlaying(): boolean {
129
+ const mod = loadModule()
130
+ if (!mod) {
131
+ return false
132
+ }
133
+ return mod.isPlaying()
134
+ }
135
+
136
+ // Returns the microphone authorization status.
137
+ // On macOS, returns the TCC status: 0=notDetermined, 1=restricted, 2=denied, 3=authorized.
138
+ // On Linux, always returns 3 (authorized) — no system-level mic permission API.
139
+ // On Windows, returns 3 (authorized) if registry key absent or allowed, 2 (denied) if explicitly denied.
140
+ // Returns 0 (notDetermined) if the native module is unavailable.
141
+ export function microphoneAuthorizationStatus(): number {
142
+ const mod = loadModule()
143
+ if (!mod || !mod.microphoneAuthorizationStatus) {
144
+ return 0
145
+ }
146
+ return mod.microphoneAuthorizationStatus()
147
+ }