feat: voice mode asr
Browse files- .gitignore +3 -0
- bun.lock +0 -0
- package.json +7 -1
- packages/audio-capture-napi/package.json +8 -0
- packages/audio-capture-napi/src/index.ts +181 -0
- packages/audio-capture-napi/tsconfig.json +5 -0
- scripts/build.ts +12 -2
- src/commands/voice/index.ts +2 -3
- src/commands/voice/voice.ts +32 -24
- src/hooks/useVoice.ts +28 -4
- src/services/doubaoSTT.ts +258 -0
- src/utils/settings/types.ts +6 -0
- src/voice/voiceModeEnabled.ts +11 -4
- vendor/audio-capture-src/index.ts +147 -0
.gitignore
CHANGED
|
@@ -4,6 +4,9 @@ cli
|
|
| 4 |
.codex
|
| 5 |
VersperClaw
|
| 6 |
|
|
|
|
|
|
|
|
|
|
| 7 |
# Desktop (Tauri) build artifacts
|
| 8 |
desktop/src-tauri/target/
|
| 9 |
desktop/src-tauri/binaries/
|
|
|
|
| 4 |
.codex
|
| 5 |
VersperClaw
|
| 6 |
|
| 7 |
+
# Prebuilt native binaries (tracked in claude-code/ reference)
|
| 8 |
+
vendor/audio-capture/
|
| 9 |
+
|
| 10 |
# Desktop (Tauri) build artifacts
|
| 11 |
desktop/src-tauri/target/
|
| 12 |
desktop/src-tauri/binaries/
|
bun.lock
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
package.json
CHANGED
|
@@ -2,6 +2,9 @@
|
|
| 2 |
"name": "claude-code-source-snapshot",
|
| 3 |
"version": "2.1.87",
|
| 4 |
"private": true,
|
|
|
|
|
|
|
|
|
|
| 5 |
"description": "Reconstructed Bun CLI workspace for the Claude Code source snapshot.",
|
| 6 |
"type": "module",
|
| 7 |
"packageManager": "bun@1.3.11",
|
|
@@ -57,10 +60,13 @@
|
|
| 57 |
"@opentelemetry/semantic-conventions": "^1.40.0",
|
| 58 |
"@smithy/core": "^3.23.13",
|
| 59 |
"@smithy/node-http-handler": "^4.5.1",
|
|
|
|
| 60 |
"@yukiakai/tls-fetch": "^2.0.2",
|
| 61 |
"ajv": "^8.18.0",
|
| 62 |
"asciichart": "^1.5.25",
|
|
|
|
| 63 |
"auto-bind": "^5.0.1",
|
|
|
|
| 64 |
"bidi-js": "^1.0.3",
|
| 65 |
"cacache": "^20.0.4",
|
| 66 |
"chalk": "^5.6.2",
|
|
@@ -69,6 +75,7 @@
|
|
| 69 |
"cli-highlight": "^2.1.11",
|
| 70 |
"code-excerpt": "^4.0.0",
|
| 71 |
"diff": "^8.0.4",
|
|
|
|
| 72 |
"emoji-regex": "^10.6.0",
|
| 73 |
"env-paths": "^4.0.0",
|
| 74 |
"execa": "^9.6.1",
|
|
@@ -112,7 +119,6 @@
|
|
| 112 |
"ws": "^8.20.0",
|
| 113 |
"xss": "^1.0.15",
|
| 114 |
"yaml": "^2.8.3",
|
| 115 |
-
"@tavily/core": "^0.6.1",
|
| 116 |
"zod": "^4.3.6"
|
| 117 |
},
|
| 118 |
"devDependencies": {
|
|
|
|
| 2 |
"name": "claude-code-source-snapshot",
|
| 3 |
"version": "2.1.87",
|
| 4 |
"private": true,
|
| 5 |
+
"workspaces": [
|
| 6 |
+
"packages/*"
|
| 7 |
+
],
|
| 8 |
"description": "Reconstructed Bun CLI workspace for the Claude Code source snapshot.",
|
| 9 |
"type": "module",
|
| 10 |
"packageManager": "bun@1.3.11",
|
|
|
|
| 60 |
"@opentelemetry/semantic-conventions": "^1.40.0",
|
| 61 |
"@smithy/core": "^3.23.13",
|
| 62 |
"@smithy/node-http-handler": "^4.5.1",
|
| 63 |
+
"@tavily/core": "^0.6.1",
|
| 64 |
"@yukiakai/tls-fetch": "^2.0.2",
|
| 65 |
"ajv": "^8.18.0",
|
| 66 |
"asciichart": "^1.5.25",
|
| 67 |
+
"audio-capture-napi": "workspace:*",
|
| 68 |
"auto-bind": "^5.0.1",
|
| 69 |
+
"axios": "^1.15.2",
|
| 70 |
"bidi-js": "^1.0.3",
|
| 71 |
"cacache": "^20.0.4",
|
| 72 |
"chalk": "^5.6.2",
|
|
|
|
| 75 |
"cli-highlight": "^2.1.11",
|
| 76 |
"code-excerpt": "^4.0.0",
|
| 77 |
"diff": "^8.0.4",
|
| 78 |
+
"doubaoime-asr": "^0.1.0",
|
| 79 |
"emoji-regex": "^10.6.0",
|
| 80 |
"env-paths": "^4.0.0",
|
| 81 |
"execa": "^9.6.1",
|
|
|
|
| 119 |
"ws": "^8.20.0",
|
| 120 |
"xss": "^1.0.15",
|
| 121 |
"yaml": "^2.8.3",
|
|
|
|
| 122 |
"zod": "^4.3.6"
|
| 123 |
},
|
| 124 |
"devDependencies": {
|
packages/audio-capture-napi/package.json
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "audio-capture-napi",
|
| 3 |
+
"version": "1.0.0",
|
| 4 |
+
"private": true,
|
| 5 |
+
"type": "module",
|
| 6 |
+
"main": "./src/index.ts",
|
| 7 |
+
"types": "./src/index.ts"
|
| 8 |
+
}
|
packages/audio-capture-napi/src/index.ts
ADDED
|
@@ -0,0 +1,181 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { createRequire } from 'node:module'
|
| 2 |
+
import { dirname, resolve, sep } from 'node:path'
|
| 3 |
+
import { fileURLToPath } from 'node:url'
|
| 4 |
+
// createRequire works in both Bun and Node.js ESM contexts.
|
| 5 |
+
// Needed because this package is "type": "module" but uses require() for
|
| 6 |
+
// loading native .node addons — bare require is not available in Node.js ESM.
|
| 7 |
+
const nodeRequire = createRequire(import.meta.url)
|
| 8 |
+
|
| 9 |
+
/**
|
| 10 |
+
* Resolve the "vendor root" directory where native .node binaries live.
|
| 11 |
+
*
|
| 12 |
+
* - Dev mode: import.meta.url → packages/audio-capture-napi/src/index.ts
|
| 13 |
+
* → vendor root = <project>/vendor/
|
| 14 |
+
* - Bun build: import.meta.url → dist/chunk-xxx.js
|
| 15 |
+
* → vendor root = <project>/dist/vendor/
|
| 16 |
+
* - Vite build: import.meta.url → dist/chunks/chunk-xxx.js
|
| 17 |
+
* → vendor root = <project>/dist/vendor/
|
| 18 |
+
*/
|
| 19 |
+
function getVendorRoot(): string {
|
| 20 |
+
const filePath = fileURLToPath(import.meta.url)
|
| 21 |
+
const dir = dirname(filePath)
|
| 22 |
+
const parts = dir.split(sep)
|
| 23 |
+
const distIdx = parts.lastIndexOf('dist')
|
| 24 |
+
if (distIdx !== -1) {
|
| 25 |
+
return parts.slice(0, distIdx + 1).join(sep) + sep + 'vendor'
|
| 26 |
+
}
|
| 27 |
+
// Dev mode — go up from packages/audio-capture-napi/src/ to project root
|
| 28 |
+
return resolve(dir, '..', '..', '..', 'vendor')
|
| 29 |
+
}
|
| 30 |
+
|
| 31 |
+
type AudioCaptureNapi = {
|
| 32 |
+
startRecording(onData: (data: Buffer) => void, onEnd: () => void): boolean
|
| 33 |
+
stopRecording(): void
|
| 34 |
+
isRecording(): boolean
|
| 35 |
+
startPlayback(sampleRate: number, channels: number): boolean
|
| 36 |
+
writePlaybackData(data: Buffer): void
|
| 37 |
+
stopPlayback(): void
|
| 38 |
+
isPlaying(): boolean
|
| 39 |
+
// TCC microphone authorization status (macOS only):
|
| 40 |
+
// 0 = notDetermined, 1 = restricted, 2 = denied, 3 = authorized.
|
| 41 |
+
// Linux: always returns 3 (authorized) — no system-level microphone permission API.
|
| 42 |
+
// Windows: returns 3 (authorized) if registry key absent or allowed,
|
| 43 |
+
// 2 (denied) if microphone access is explicitly denied.
|
| 44 |
+
microphoneAuthorizationStatus?(): number
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
let cachedModule: AudioCaptureNapi | null = null
|
| 48 |
+
let loadAttempted = false
|
| 49 |
+
|
| 50 |
+
function loadModule(): AudioCaptureNapi | null {
|
| 51 |
+
if (loadAttempted) {
|
| 52 |
+
return cachedModule
|
| 53 |
+
}
|
| 54 |
+
loadAttempted = true
|
| 55 |
+
|
| 56 |
+
// Supported platforms: macOS (darwin), Linux, Windows (win32)
|
| 57 |
+
const platform = process.platform
|
| 58 |
+
if (platform !== 'darwin' && platform !== 'linux' && platform !== 'win32') {
|
| 59 |
+
return null
|
| 60 |
+
}
|
| 61 |
+
|
| 62 |
+
// Candidate 1: native-embed path (bun compile). AUDIO_CAPTURE_NODE_PATH is
|
| 63 |
+
// defined at build time in build-with-plugins.ts for native builds only — the
|
| 64 |
+
// define resolves it to the static literal "../../audio-capture.node" so bun
|
| 65 |
+
// compile can rewrite it to /$bunfs/root/audio-capture.node. MUST stay a
|
| 66 |
+
// direct require(env var) — bun cannot analyze require(variable) from a loop.
|
| 67 |
+
if (process.env.AUDIO_CAPTURE_NODE_PATH) {
|
| 68 |
+
try {
|
| 69 |
+
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
| 70 |
+
cachedModule = nodeRequire(
|
| 71 |
+
process.env.AUDIO_CAPTURE_NODE_PATH,
|
| 72 |
+
) as AudioCaptureNapi
|
| 73 |
+
return cachedModule
|
| 74 |
+
} catch {
|
| 75 |
+
// fall through to runtime fallbacks below
|
| 76 |
+
}
|
| 77 |
+
}
|
| 78 |
+
|
| 79 |
+
// Candidates 2-5: resolved vendor path + relative fallbacks.
|
| 80 |
+
// The primary candidate uses getVendorRoot() to find the correct dist root
|
| 81 |
+
// regardless of chunk nesting depth. Relative fallbacks cover edge cases.
|
| 82 |
+
const platformDir = `${process.arch}-${platform}`
|
| 83 |
+
const binaryRel = `audio-capture/${platformDir}/audio-capture.node`
|
| 84 |
+
const vendorRoot = getVendorRoot()
|
| 85 |
+
const fallbacks = [
|
| 86 |
+
resolve(vendorRoot, binaryRel),
|
| 87 |
+
`./vendor/${binaryRel}`,
|
| 88 |
+
`../vendor/${binaryRel}`,
|
| 89 |
+
`../../vendor/${binaryRel}`,
|
| 90 |
+
`${process.cwd()}/vendor/${binaryRel}`,
|
| 91 |
+
]
|
| 92 |
+
for (const p of fallbacks) {
|
| 93 |
+
try {
|
| 94 |
+
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
| 95 |
+
cachedModule = nodeRequire(p) as AudioCaptureNapi
|
| 96 |
+
return cachedModule
|
| 97 |
+
} catch {
|
| 98 |
+
// try next
|
| 99 |
+
}
|
| 100 |
+
}
|
| 101 |
+
return null
|
| 102 |
+
}
|
| 103 |
+
|
| 104 |
+
export function isNativeAudioAvailable(): boolean {
|
| 105 |
+
return loadModule() !== null
|
| 106 |
+
}
|
| 107 |
+
|
| 108 |
+
export function startNativeRecording(
|
| 109 |
+
onData: (data: Buffer) => void,
|
| 110 |
+
onEnd: () => void,
|
| 111 |
+
): boolean {
|
| 112 |
+
const mod = loadModule()
|
| 113 |
+
if (!mod) {
|
| 114 |
+
return false
|
| 115 |
+
}
|
| 116 |
+
return mod.startRecording(onData, onEnd)
|
| 117 |
+
}
|
| 118 |
+
|
| 119 |
+
export function stopNativeRecording(): void {
|
| 120 |
+
const mod = loadModule()
|
| 121 |
+
if (!mod) {
|
| 122 |
+
return
|
| 123 |
+
}
|
| 124 |
+
mod.stopRecording()
|
| 125 |
+
}
|
| 126 |
+
|
| 127 |
+
export function isNativeRecordingActive(): boolean {
|
| 128 |
+
const mod = loadModule()
|
| 129 |
+
if (!mod) {
|
| 130 |
+
return false
|
| 131 |
+
}
|
| 132 |
+
return mod.isRecording()
|
| 133 |
+
}
|
| 134 |
+
|
| 135 |
+
export function startNativePlayback(
|
| 136 |
+
sampleRate: number,
|
| 137 |
+
channels: number,
|
| 138 |
+
): boolean {
|
| 139 |
+
const mod = loadModule()
|
| 140 |
+
if (!mod) {
|
| 141 |
+
return false
|
| 142 |
+
}
|
| 143 |
+
return mod.startPlayback(sampleRate, channels)
|
| 144 |
+
}
|
| 145 |
+
|
| 146 |
+
export function writeNativePlaybackData(data: Buffer): void {
|
| 147 |
+
const mod = loadModule()
|
| 148 |
+
if (!mod) {
|
| 149 |
+
return
|
| 150 |
+
}
|
| 151 |
+
mod.writePlaybackData(data)
|
| 152 |
+
}
|
| 153 |
+
|
| 154 |
+
export function stopNativePlayback(): void {
|
| 155 |
+
const mod = loadModule()
|
| 156 |
+
if (!mod) {
|
| 157 |
+
return
|
| 158 |
+
}
|
| 159 |
+
mod.stopPlayback()
|
| 160 |
+
}
|
| 161 |
+
|
| 162 |
+
export function isNativePlaying(): boolean {
|
| 163 |
+
const mod = loadModule()
|
| 164 |
+
if (!mod) {
|
| 165 |
+
return false
|
| 166 |
+
}
|
| 167 |
+
return mod.isPlaying()
|
| 168 |
+
}
|
| 169 |
+
|
| 170 |
+
// Returns the microphone authorization status.
|
| 171 |
+
// On macOS, returns the TCC status: 0=notDetermined, 1=restricted, 2=denied, 3=authorized.
|
| 172 |
+
// On Linux, always returns 3 (authorized) — no system-level mic permission API.
|
| 173 |
+
// On Windows, returns 3 (authorized) if registry key absent or allowed, 2 (denied) if explicitly denied.
|
| 174 |
+
// Returns 0 (notDetermined) if the native module is unavailable.
|
| 175 |
+
export function microphoneAuthorizationStatus(): number {
|
| 176 |
+
const mod = loadModule()
|
| 177 |
+
if (!mod || !mod.microphoneAuthorizationStatus) {
|
| 178 |
+
return 0
|
| 179 |
+
}
|
| 180 |
+
return mod.microphoneAuthorizationStatus()
|
| 181 |
+
}
|
packages/audio-capture-napi/tsconfig.json
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"extends": "../../tsconfig.base.json",
|
| 3 |
+
"include": ["src/**/*.ts"],
|
| 4 |
+
"exclude": ["node_modules", "dist"]
|
| 5 |
+
}
|
scripts/build.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
-
import { chmodSync, existsSync, mkdirSync } from 'fs'
|
| 2 |
-
import { dirname } from 'path'
|
| 3 |
|
| 4 |
const pkg = await Bun.file(new URL('../package.json', import.meta.url)).json() as {
|
| 5 |
name: string
|
|
@@ -203,5 +203,15 @@ if (existsSync(outfile)) {
|
|
| 203 |
chmodSync(outfile, 0o755)
|
| 204 |
}
|
| 205 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 206 |
console.log(`Built ${outfile}`)
|
| 207 |
|
|
|
|
| 1 |
+
import { chmodSync, cpSync, existsSync, mkdirSync } from 'fs'
|
| 2 |
+
import { dirname, join } from 'path'
|
| 3 |
|
| 4 |
const pkg = await Bun.file(new URL('../package.json', import.meta.url)).json() as {
|
| 5 |
name: string
|
|
|
|
| 203 |
chmodSync(outfile, 0o755)
|
| 204 |
}
|
| 205 |
|
| 206 |
+
// Copy vendor audio-capture binaries to dist/ for runtime resolution
|
| 207 |
+
if (!compile) {
|
| 208 |
+
const distDir = dirname(outfile)
|
| 209 |
+
const vendorDir = join(distDir, 'vendor')
|
| 210 |
+
if (!existsSync(vendorDir)) {
|
| 211 |
+
cpSync('vendor', vendorDir, { recursive: true })
|
| 212 |
+
console.log(`Copied vendor/ → ${vendorDir}/`)
|
| 213 |
+
}
|
| 214 |
+
}
|
| 215 |
+
|
| 216 |
console.log(`Built ${outfile}`)
|
| 217 |
|
src/commands/voice/index.ts
CHANGED
|
@@ -1,17 +1,16 @@
|
|
| 1 |
import type { Command } from '../../commands.js'
|
| 2 |
import {
|
|
|
|
| 3 |
isVoiceGrowthBookEnabled,
|
| 4 |
-
isVoiceModeEnabled,
|
| 5 |
} from '../../voice/voiceModeEnabled.js'
|
| 6 |
|
| 7 |
const voice = {
|
| 8 |
type: 'local',
|
| 9 |
name: 'voice',
|
| 10 |
description: 'Toggle voice mode',
|
| 11 |
-
availability: ['claude-ai'],
|
| 12 |
isEnabled: () => isVoiceGrowthBookEnabled(),
|
| 13 |
get isHidden() {
|
| 14 |
-
return !
|
| 15 |
},
|
| 16 |
supportsNonInteractive: false,
|
| 17 |
load: () => import('./voice.js'),
|
|
|
|
| 1 |
import type { Command } from '../../commands.js'
|
| 2 |
import {
|
| 3 |
+
isVoiceAvailable,
|
| 4 |
isVoiceGrowthBookEnabled,
|
|
|
|
| 5 |
} from '../../voice/voiceModeEnabled.js'
|
| 6 |
|
| 7 |
const voice = {
|
| 8 |
type: 'local',
|
| 9 |
name: 'voice',
|
| 10 |
description: 'Toggle voice mode',
|
|
|
|
| 11 |
isEnabled: () => isVoiceGrowthBookEnabled(),
|
| 12 |
get isHidden() {
|
| 13 |
+
return !isVoiceAvailable()
|
| 14 |
},
|
| 15 |
supportsNonInteractive: false,
|
| 16 |
load: () => import('./voice.js'),
|
src/commands/voice/voice.ts
CHANGED
|
@@ -9,22 +9,13 @@ import {
|
|
| 9 |
getInitialSettings,
|
| 10 |
updateSettingsForSource,
|
| 11 |
} from '../../utils/settings/settings.js'
|
| 12 |
-
import {
|
| 13 |
|
| 14 |
const LANG_HINT_MAX_SHOWS = 2
|
| 15 |
|
| 16 |
export const call: LocalCommandCall = async () => {
|
| 17 |
-
// Check
|
| 18 |
-
if (!
|
| 19 |
-
// Differentiate: OAuth-less users get an auth hint, everyone else
|
| 20 |
-
// gets nothing (command shouldn't be reachable when the kill-switch is on).
|
| 21 |
-
if (!isAnthropicAuthEnabled()) {
|
| 22 |
-
return {
|
| 23 |
-
type: 'text' as const,
|
| 24 |
-
value:
|
| 25 |
-
'Voice mode requires a Claude.ai account. Please run /login to sign in.',
|
| 26 |
-
}
|
| 27 |
-
}
|
| 28 |
return {
|
| 29 |
type: 'text' as const,
|
| 30 |
value: 'Voice mode is not available.',
|
|
@@ -60,6 +51,28 @@ export const call: LocalCommandCall = async () => {
|
|
| 60 |
)
|
| 61 |
const { checkRecordingAvailability } = await import('../../services/voice.js')
|
| 62 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
// Check recording availability (microphone access)
|
| 64 |
const recording = await checkRecordingAvailability()
|
| 65 |
if (!recording.available) {
|
|
@@ -70,15 +83,6 @@ export const call: LocalCommandCall = async () => {
|
|
| 70 |
}
|
| 71 |
}
|
| 72 |
|
| 73 |
-
// Check for API key
|
| 74 |
-
if (!isVoiceStreamAvailable()) {
|
| 75 |
-
return {
|
| 76 |
-
type: 'text' as const,
|
| 77 |
-
value:
|
| 78 |
-
'Voice mode requires a Claude.ai account. Please run /login to sign in.',
|
| 79 |
-
}
|
| 80 |
-
}
|
| 81 |
-
|
| 82 |
// Check for recording tools
|
| 83 |
const { checkVoiceDependencies, requestMicrophonePermission } = await import(
|
| 84 |
'../../services/voice.js'
|
|
@@ -111,8 +115,11 @@ export const call: LocalCommandCall = async () => {
|
|
| 111 |
}
|
| 112 |
}
|
| 113 |
|
| 114 |
-
// All checks passed — enable voice
|
| 115 |
-
const result = updateSettingsForSource('userSettings', {
|
|
|
|
|
|
|
|
|
|
| 116 |
if (result.error) {
|
| 117 |
return {
|
| 118 |
type: 'text' as const,
|
|
@@ -143,8 +150,9 @@ export const call: LocalCommandCall = async () => {
|
|
| 143 |
voiceLangHintLastLanguage: stt.code,
|
| 144 |
}))
|
| 145 |
}
|
|
|
|
| 146 |
return {
|
| 147 |
type: 'text' as const,
|
| 148 |
-
value: `Voice mode enabled. Hold ${key} to record.${langNote}`,
|
| 149 |
}
|
| 150 |
}
|
|
|
|
| 9 |
getInitialSettings,
|
| 10 |
updateSettingsForSource,
|
| 11 |
} from '../../utils/settings/settings.js'
|
| 12 |
+
import { isVoiceAvailable } from '../../voice/voiceModeEnabled.js'
|
| 13 |
|
| 14 |
const LANG_HINT_MAX_SHOWS = 2
|
| 15 |
|
| 16 |
export const call: LocalCommandCall = async () => {
|
| 17 |
+
// Check only the GrowthBook kill-switch — voice might use Doubao (no auth)
|
| 18 |
+
if (!isVoiceAvailable()) {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
return {
|
| 20 |
type: 'text' as const,
|
| 21 |
value: 'Voice mode is not available.',
|
|
|
|
| 51 |
)
|
| 52 |
const { checkRecordingAvailability } = await import('../../services/voice.js')
|
| 53 |
|
| 54 |
+
// Determine available STT backend: prefer Anthropic, fall back to Doubao
|
| 55 |
+
const hasAnthropicAuth = isAnthropicAuthEnabled() && isVoiceStreamAvailable()
|
| 56 |
+
let useDoubao = false
|
| 57 |
+
if (!hasAnthropicAuth) {
|
| 58 |
+
const { isDoubaoAvailable } = await import('../../services/doubaoSTT.js')
|
| 59 |
+
if (!(await isDoubaoAvailable())) {
|
| 60 |
+
// Neither backend is available
|
| 61 |
+
if (!isAnthropicAuthEnabled()) {
|
| 62 |
+
return {
|
| 63 |
+
type: 'text' as const,
|
| 64 |
+
value:
|
| 65 |
+
'Voice mode requires a Claude.ai account or the Doubao ASR backend. Please run /login to sign in, or install Doubao ASR.',
|
| 66 |
+
}
|
| 67 |
+
}
|
| 68 |
+
return {
|
| 69 |
+
type: 'text' as const,
|
| 70 |
+
value: 'Voice mode is not available.',
|
| 71 |
+
}
|
| 72 |
+
}
|
| 73 |
+
useDoubao = true
|
| 74 |
+
}
|
| 75 |
+
|
| 76 |
// Check recording availability (microphone access)
|
| 77 |
const recording = await checkRecordingAvailability()
|
| 78 |
if (!recording.available) {
|
|
|
|
| 83 |
}
|
| 84 |
}
|
| 85 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 86 |
// Check for recording tools
|
| 87 |
const { checkVoiceDependencies, requestMicrophonePermission } = await import(
|
| 88 |
'../../services/voice.js'
|
|
|
|
| 115 |
}
|
| 116 |
}
|
| 117 |
|
| 118 |
+
// All checks passed — enable voice and store backend choice
|
| 119 |
+
const result = updateSettingsForSource('userSettings', {
|
| 120 |
+
voiceEnabled: true,
|
| 121 |
+
...(useDoubao ? { voiceProvider: 'doubao' as const } : { voiceProvider: 'anthropic' as const }),
|
| 122 |
+
})
|
| 123 |
if (result.error) {
|
| 124 |
return {
|
| 125 |
type: 'text' as const,
|
|
|
|
| 150 |
voiceLangHintLastLanguage: stt.code,
|
| 151 |
}))
|
| 152 |
}
|
| 153 |
+
const backendNote = useDoubao ? ' [Doubao ASR]' : ''
|
| 154 |
return {
|
| 155 |
type: 'text' as const,
|
| 156 |
+
value: `Voice mode enabled. Hold ${key} to record.${langNote}${backendNote}`,
|
| 157 |
}
|
| 158 |
}
|
src/hooks/useVoice.ts
CHANGED
|
@@ -9,6 +9,7 @@
|
|
| 9 |
import { useCallback, useEffect, useRef, useState } from 'react'
|
| 10 |
import { useSetVoiceState } from '../context/voice.js'
|
| 11 |
import { useTerminalFocus } from '../ink/hooks/use-terminal-focus.js'
|
|
|
|
| 12 |
import {
|
| 13 |
type AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
|
| 14 |
logEvent,
|
|
@@ -20,6 +21,7 @@ import {
|
|
| 20 |
isVoiceStreamAvailable,
|
| 21 |
type VoiceStreamConnection,
|
| 22 |
} from '../services/voiceStreamSTT.js'
|
|
|
|
| 23 |
import { logForDebugging } from '../utils/debug.js'
|
| 24 |
import { toError } from '../utils/errors.js'
|
| 25 |
import { getSystemLocaleLanguage } from '../utils/intl.js'
|
|
@@ -133,6 +135,10 @@ export function normalizeLanguageForSTT(language: string | undefined): {
|
|
| 133 |
return { code: DEFAULT_STT_LANGUAGE, fellBackFrom: language }
|
| 134 |
}
|
| 135 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 136 |
// Lazy-loaded voice module. We defer importing voice.ts (and its native
|
| 137 |
// audio-capture-napi dependency) until voice input is actually activated.
|
| 138 |
// On macOS, loading the native audio module can trigger a TCC microphone
|
|
@@ -574,7 +580,7 @@ export function useVoice({
|
|
| 574 |
// stop when it loses focus. This enables a "multi-clauding army"
|
| 575 |
// workflow where voice input follows window focus.
|
| 576 |
useEffect(() => {
|
| 577 |
-
if (!enabled || !focusMode) {
|
| 578 |
// Focus mode was disabled while a focus-driven recording was active —
|
| 579 |
// stop the recording so it doesn't linger until the silence timer fires.
|
| 580 |
if (focusTriggeredRef.current && stateRef.current === 'recording') {
|
|
@@ -778,7 +784,17 @@ export function useVoice({
|
|
| 778 |
|
| 779 |
const attemptConnect = (keyterms: string[]): void => {
|
| 780 |
const myAttemptGen = attemptGenRef.current
|
| 781 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 782 |
{
|
| 783 |
onTranscript: (text: string, isFinal: boolean) => {
|
| 784 |
if (isStale()) return
|
|
@@ -1007,7 +1023,12 @@ export function useVoice({
|
|
| 1007 |
})
|
| 1008 |
}
|
| 1009 |
|
| 1010 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1011 |
}
|
| 1012 |
|
| 1013 |
// ── Hold-to-talk handler ────────────────────────────────────────────
|
|
@@ -1021,7 +1042,10 @@ export function useVoice({
|
|
| 1021 |
// delay of ~500ms on macOS).
|
| 1022 |
const handleKeyEvent = useCallback(
|
| 1023 |
(fallbackMs = REPEAT_FALLBACK_MS): void => {
|
| 1024 |
-
|
|
|
|
|
|
|
|
|
|
| 1025 |
return
|
| 1026 |
}
|
| 1027 |
|
|
|
|
| 9 |
import { useCallback, useEffect, useRef, useState } from 'react'
|
| 10 |
import { useSetVoiceState } from '../context/voice.js'
|
| 11 |
import { useTerminalFocus } from '../ink/hooks/use-terminal-focus.js'
|
| 12 |
+
import { isDoubaoAvailableSync } from '../services/doubaoSTT.js'
|
| 13 |
import {
|
| 14 |
type AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
|
| 15 |
logEvent,
|
|
|
|
| 21 |
isVoiceStreamAvailable,
|
| 22 |
type VoiceStreamConnection,
|
| 23 |
} from '../services/voiceStreamSTT.js'
|
| 24 |
+
import { connectDoubaoStream } from '../services/doubaoSTT.js'
|
| 25 |
import { logForDebugging } from '../utils/debug.js'
|
| 26 |
import { toError } from '../utils/errors.js'
|
| 27 |
import { getSystemLocaleLanguage } from '../utils/intl.js'
|
|
|
|
| 135 |
return { code: DEFAULT_STT_LANGUAGE, fellBackFrom: language }
|
| 136 |
}
|
| 137 |
|
| 138 |
+
function isDoubaoProvider(): boolean {
|
| 139 |
+
return getInitialSettings().voiceProvider === 'doubao'
|
| 140 |
+
}
|
| 141 |
+
|
| 142 |
// Lazy-loaded voice module. We defer importing voice.ts (and its native
|
| 143 |
// audio-capture-napi dependency) until voice input is actually activated.
|
| 144 |
// On macOS, loading the native audio module can trigger a TCC microphone
|
|
|
|
| 580 |
// stop when it loses focus. This enables a "multi-clauding army"
|
| 581 |
// workflow where voice input follows window focus.
|
| 582 |
useEffect(() => {
|
| 583 |
+
if (!enabled || !focusMode || isDoubaoProvider()) {
|
| 584 |
// Focus mode was disabled while a focus-driven recording was active —
|
| 585 |
// stop the recording so it doesn't linger until the silence timer fires.
|
| 586 |
if (focusTriggeredRef.current && stateRef.current === 'recording') {
|
|
|
|
| 784 |
|
| 785 |
const attemptConnect = (keyterms: string[]): void => {
|
| 786 |
const myAttemptGen = attemptGenRef.current
|
| 787 |
+
// Select STT backend based on settings.voiceProvider
|
| 788 |
+
const connectFn = isDoubaoProvider()
|
| 789 |
+
? (
|
| 790 |
+
cbs: Parameters<typeof connectDoubaoStream>[0],
|
| 791 |
+
opts: Parameters<typeof connectDoubaoStream>[1],
|
| 792 |
+
) => connectDoubaoStream(cbs, opts)
|
| 793 |
+
: (
|
| 794 |
+
cbs: Parameters<typeof connectVoiceStream>[0],
|
| 795 |
+
opts: Parameters<typeof connectVoiceStream>[1],
|
| 796 |
+
) => connectVoiceStream(cbs, opts)
|
| 797 |
+
void connectFn(
|
| 798 |
{
|
| 799 |
onTranscript: (text: string, isFinal: boolean) => {
|
| 800 |
if (isStale()) return
|
|
|
|
| 1023 |
})
|
| 1024 |
}
|
| 1025 |
|
| 1026 |
+
// Doubao backend doesn't use keyterms — skip the async fetch
|
| 1027 |
+
if (isDoubaoProvider()) {
|
| 1028 |
+
attemptConnect([])
|
| 1029 |
+
} else {
|
| 1030 |
+
void getVoiceKeyterms().then(attemptConnect)
|
| 1031 |
+
}
|
| 1032 |
}
|
| 1033 |
|
| 1034 |
// ── Hold-to-talk handler ────────────────────────────────────────────
|
|
|
|
| 1042 |
// delay of ~500ms on macOS).
|
| 1043 |
const handleKeyEvent = useCallback(
|
| 1044 |
(fallbackMs = REPEAT_FALLBACK_MS): void => {
|
| 1045 |
+
const sttAvailable = isDoubaoProvider()
|
| 1046 |
+
? isDoubaoAvailableSync()
|
| 1047 |
+
: isVoiceStreamAvailable()
|
| 1048 |
+
if (!enabled || !sttAvailable) {
|
| 1049 |
return
|
| 1050 |
}
|
| 1051 |
|
src/services/doubaoSTT.ts
ADDED
|
@@ -0,0 +1,258 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Doubao (豆包) ASR speech-to-text adapter for voice mode.
|
| 2 |
+
//
|
| 3 |
+
// Wraps the doubaoime-asr npm package to expose the same interface as
|
| 4 |
+
// voiceStreamSTT.ts. The doubao backend uses an AsyncGenerator-based
|
| 5 |
+
// streaming protocol internally; this adapter bridges it to the
|
| 6 |
+
// send/finalize/close pattern used by useVoice.ts.
|
| 7 |
+
|
| 8 |
+
import { homedir } from 'node:os'
|
| 9 |
+
import type { ASRResponse } from 'doubaoime-asr'
|
| 10 |
+
import type {
|
| 11 |
+
FinalizeSource,
|
| 12 |
+
VoiceStreamCallbacks,
|
| 13 |
+
VoiceStreamConnection,
|
| 14 |
+
} from './voiceStreamSTT.js'
|
| 15 |
+
import { logForDebugging } from '../utils/debug.js'
|
| 16 |
+
import { logError } from '../utils/log.js'
|
| 17 |
+
|
| 18 |
+
// Re-export FinalizeSource so useVoice can import from either module
|
| 19 |
+
export type { FinalizeSource } from './voiceStreamSTT.js'
|
| 20 |
+
|
| 21 |
+
// ─── AsyncIterable audio queue ─────────────────────────────────────────
|
| 22 |
+
|
| 23 |
+
// A push-based queue that implements AsyncIterable<Uint8Array>.
|
| 24 |
+
// send() pushes chunks; push(null) signals end-of-stream.
|
| 25 |
+
class AudioChunkQueue {
|
| 26 |
+
private chunks: (Uint8Array | null)[] = []
|
| 27 |
+
private waiting: ((result: IteratorResult<Uint8Array>) => void) | null = null
|
| 28 |
+
private done = false
|
| 29 |
+
|
| 30 |
+
push(chunk: Uint8Array | null): void {
|
| 31 |
+
if (this.done) return
|
| 32 |
+
if (chunk === null) {
|
| 33 |
+
this.done = true
|
| 34 |
+
if (this.waiting) {
|
| 35 |
+
const resolve = this.waiting
|
| 36 |
+
this.waiting = null
|
| 37 |
+
resolve({ value: undefined, done: true })
|
| 38 |
+
}
|
| 39 |
+
return
|
| 40 |
+
}
|
| 41 |
+
if (this.waiting) {
|
| 42 |
+
const resolve = this.waiting
|
| 43 |
+
this.waiting = null
|
| 44 |
+
resolve({ value: chunk, done: false })
|
| 45 |
+
} else {
|
| 46 |
+
this.chunks.push(chunk)
|
| 47 |
+
}
|
| 48 |
+
}
|
| 49 |
+
|
| 50 |
+
abort(): void {
|
| 51 |
+
this.done = true
|
| 52 |
+
this.chunks.length = 0
|
| 53 |
+
if (this.waiting) {
|
| 54 |
+
const resolve = this.waiting
|
| 55 |
+
this.waiting = null
|
| 56 |
+
resolve({ value: undefined, done: true })
|
| 57 |
+
}
|
| 58 |
+
}
|
| 59 |
+
|
| 60 |
+
[Symbol.asyncIterator](): AsyncIterator<Uint8Array> {
|
| 61 |
+
return {
|
| 62 |
+
next: async (): Promise<IteratorResult<Uint8Array>> => {
|
| 63 |
+
if (this.chunks.length > 0) {
|
| 64 |
+
const chunk = this.chunks.shift()!
|
| 65 |
+
return { value: chunk, done: false }
|
| 66 |
+
}
|
| 67 |
+
if (this.done) {
|
| 68 |
+
return { value: undefined, done: true }
|
| 69 |
+
}
|
| 70 |
+
return new Promise<IteratorResult<Uint8Array>>(resolve => {
|
| 71 |
+
this.waiting = resolve
|
| 72 |
+
})
|
| 73 |
+
},
|
| 74 |
+
}
|
| 75 |
+
}
|
| 76 |
+
}
|
| 77 |
+
|
| 78 |
+
// ─── Availability ────────────────────────────────────────────────────────
|
| 79 |
+
|
| 80 |
+
let doubaoAvailable: boolean | null = null
|
| 81 |
+
|
| 82 |
+
export async function isDoubaoAvailable(): Promise<boolean> {
|
| 83 |
+
if (doubaoAvailable !== null) return doubaoAvailable
|
| 84 |
+
try {
|
| 85 |
+
await import('doubaoime-asr')
|
| 86 |
+
doubaoAvailable = true
|
| 87 |
+
} catch {
|
| 88 |
+
doubaoAvailable = false
|
| 89 |
+
}
|
| 90 |
+
return doubaoAvailable
|
| 91 |
+
}
|
| 92 |
+
|
| 93 |
+
// Synchronous check — returns cached result or optimistic true when
|
| 94 |
+
// VOICE_PROVIDER=doubao is set and no cached result exists yet.
|
| 95 |
+
// The actual import happens in connectDoubaoStream which reports errors.
|
| 96 |
+
export function isDoubaoAvailableSync(): boolean {
|
| 97 |
+
if (doubaoAvailable !== null) return doubaoAvailable
|
| 98 |
+
return true
|
| 99 |
+
}
|
| 100 |
+
|
| 101 |
+
// ─── Connection ──────────────────────────────────────────────────────────
|
| 102 |
+
|
| 103 |
+
export async function connectDoubaoStream(
|
| 104 |
+
callbacks: VoiceStreamCallbacks,
|
| 105 |
+
_options?: { language?: string },
|
| 106 |
+
): Promise<VoiceStreamConnection | null> {
|
| 107 |
+
let doubaoAsr: typeof import('doubaoime-asr')
|
| 108 |
+
try {
|
| 109 |
+
doubaoAsr = await import('doubaoime-asr')
|
| 110 |
+
} catch (err) {
|
| 111 |
+
logError(
|
| 112 |
+
new Error(
|
| 113 |
+
`[doubao-asr] Failed to import doubaoime-asr package: ${String(err)}`,
|
| 114 |
+
),
|
| 115 |
+
)
|
| 116 |
+
callbacks.onError(`doubaoime-asr package import failed: ${String(err)}`, {
|
| 117 |
+
fatal: true,
|
| 118 |
+
})
|
| 119 |
+
return null
|
| 120 |
+
}
|
| 121 |
+
|
| 122 |
+
const { transcribeRealtime, ASRConfig, ResponseType } = doubaoAsr
|
| 123 |
+
|
| 124 |
+
const queue = new AudioChunkQueue()
|
| 125 |
+
let finalized = false
|
| 126 |
+
|
| 127 |
+
// Resolve handle for finalize() promise — wrapped in an object to avoid
|
| 128 |
+
// TypeScript closure-scope type narrowing issues (TS2349 "not callable").
|
| 129 |
+
const finalizeHandle: { resolve: ((source: FinalizeSource) => void) | null } =
|
| 130 |
+
{ resolve: null }
|
| 131 |
+
|
| 132 |
+
const connection: VoiceStreamConnection = {
|
| 133 |
+
send(audioChunk: Buffer): void {
|
| 134 |
+
if (finalized) return
|
| 135 |
+
queue.push(
|
| 136 |
+
new Uint8Array(
|
| 137 |
+
audioChunk.buffer,
|
| 138 |
+
audioChunk.byteOffset,
|
| 139 |
+
audioChunk.byteLength,
|
| 140 |
+
),
|
| 141 |
+
)
|
| 142 |
+
},
|
| 143 |
+
finalize(): Promise<FinalizeSource> {
|
| 144 |
+
if (finalized) return Promise.resolve<FinalizeSource>('ws_already_closed')
|
| 145 |
+
finalized = true
|
| 146 |
+
queue.push(null) // signal end-of-stream to the generator
|
| 147 |
+
// Doubao returns FINAL_RESULT during recording — by the time the user
|
| 148 |
+
// releases the key, all transcripts are already in accumulatedRef.
|
| 149 |
+
// Resolve immediately so the UI skips the 'processing' state and goes
|
| 150 |
+
// straight to displaying the result.
|
| 151 |
+
logForDebugging('[doubao-asr] Finalize — resolving immediately')
|
| 152 |
+
return Promise.resolve<FinalizeSource>('post_closestream_endpoint')
|
| 153 |
+
},
|
| 154 |
+
close(): void {
|
| 155 |
+
finalized = true
|
| 156 |
+
queue.abort()
|
| 157 |
+
const r = finalizeHandle.resolve
|
| 158 |
+
finalizeHandle.resolve = null
|
| 159 |
+
if (r) r('ws_close')
|
| 160 |
+
callbacks.onClose()
|
| 161 |
+
},
|
| 162 |
+
isConnected(): boolean {
|
| 163 |
+
return true
|
| 164 |
+
},
|
| 165 |
+
}
|
| 166 |
+
|
| 167 |
+
// Start the ASR session in the background
|
| 168 |
+
const config = new ASRConfig({
|
| 169 |
+
credentialPath: `${homedir()}/.claude/tts/doubao/credentials.json`,
|
| 170 |
+
})
|
| 171 |
+
|
| 172 |
+
// Ensure credentials are initialized (may auto-generate)
|
| 173 |
+
try {
|
| 174 |
+
await config.ensureCredentials()
|
| 175 |
+
} catch (err) {
|
| 176 |
+
logError(
|
| 177 |
+
new Error(
|
| 178 |
+
`[doubao-asr] Credential initialization failed: ${String(err)}`,
|
| 179 |
+
),
|
| 180 |
+
)
|
| 181 |
+
callbacks.onError(`Doubao ASR 凭证初始化失败: ${String(err)}`, {
|
| 182 |
+
fatal: true,
|
| 183 |
+
})
|
| 184 |
+
return null
|
| 185 |
+
}
|
| 186 |
+
|
| 187 |
+
// Fire onReady immediately — unlike the Anthropic WebSocket which needs to
|
| 188 |
+
// wait for a handshake, the doubao backend accepts audio through the queue
|
| 189 |
+
// and handles connection internally. The caller (useVoice.ts) needs onReady
|
| 190 |
+
// to fire before it will route audio chunks via connection.send().
|
| 191 |
+
logForDebugging('[doubao-asr] Firing onReady immediately')
|
| 192 |
+
callbacks.onReady(connection)
|
| 193 |
+
|
| 194 |
+
// Consume the AsyncGenerator in the background
|
| 195 |
+
void (async () => {
|
| 196 |
+
try {
|
| 197 |
+
const audioSource: AsyncIterable<Uint8Array> = queue
|
| 198 |
+
const gen: AsyncGenerator<ASRResponse> = transcribeRealtime(audioSource, {
|
| 199 |
+
config,
|
| 200 |
+
})
|
| 201 |
+
|
| 202 |
+
for await (const resp of gen) {
|
| 203 |
+
if (
|
| 204 |
+
finalized &&
|
| 205 |
+
resp.type !== ResponseType.FINAL_RESULT &&
|
| 206 |
+
resp.type !== ResponseType.SESSION_FINISHED
|
| 207 |
+
) {
|
| 208 |
+
continue
|
| 209 |
+
}
|
| 210 |
+
|
| 211 |
+
switch (resp.type) {
|
| 212 |
+
case ResponseType.SESSION_STARTED:
|
| 213 |
+
logForDebugging('[doubao-asr] Session started')
|
| 214 |
+
break
|
| 215 |
+
case ResponseType.VAD_START:
|
| 216 |
+
logForDebugging('[doubao-asr] VAD detected speech start')
|
| 217 |
+
break
|
| 218 |
+
case ResponseType.INTERIM_RESULT:
|
| 219 |
+
if (resp.text) {
|
| 220 |
+
callbacks.onTranscript(resp.text, false)
|
| 221 |
+
}
|
| 222 |
+
break
|
| 223 |
+
case ResponseType.FINAL_RESULT:
|
| 224 |
+
if (resp.text) {
|
| 225 |
+
callbacks.onTranscript(resp.text, true)
|
| 226 |
+
}
|
| 227 |
+
break
|
| 228 |
+
case ResponseType.ERROR:
|
| 229 |
+
logError(new Error(`[doubao-asr] Error: ${resp.errorMsg}`))
|
| 230 |
+
if (!finalized) {
|
| 231 |
+
callbacks.onError(resp.errorMsg || 'Doubao ASR 识别错误')
|
| 232 |
+
}
|
| 233 |
+
break
|
| 234 |
+
case ResponseType.SESSION_FINISHED:
|
| 235 |
+
logForDebugging('[doubao-asr] Session finished')
|
| 236 |
+
break
|
| 237 |
+
default:
|
| 238 |
+
break
|
| 239 |
+
}
|
| 240 |
+
}
|
| 241 |
+
|
| 242 |
+
// Generator exhausted naturally
|
| 243 |
+
const r = finalizeHandle.resolve
|
| 244 |
+
finalizeHandle.resolve = null
|
| 245 |
+
if (r) r('post_closestream_endpoint')
|
| 246 |
+
} catch (err) {
|
| 247 |
+
logError(new Error(`[doubao-asr] Stream error: ${String(err)}`))
|
| 248 |
+
if (!finalized) {
|
| 249 |
+
callbacks.onError(`Doubao ASR 连接错误: ${String(err)}`)
|
| 250 |
+
}
|
| 251 |
+
const r2 = finalizeHandle.resolve
|
| 252 |
+
finalizeHandle.resolve = null
|
| 253 |
+
if (r2) r2('ws_close')
|
| 254 |
+
}
|
| 255 |
+
})()
|
| 256 |
+
|
| 257 |
+
return connection
|
| 258 |
+
}
|
src/utils/settings/types.ts
CHANGED
|
@@ -867,6 +867,12 @@ export const SettingsSchema = lazySchema(() =>
|
|
| 867 |
.boolean()
|
| 868 |
.optional()
|
| 869 |
.describe('Enable voice mode (hold-to-talk dictation)'),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 870 |
}
|
| 871 |
: {}),
|
| 872 |
...(feature('KAIROS')
|
|
|
|
| 867 |
.boolean()
|
| 868 |
.optional()
|
| 869 |
.describe('Enable voice mode (hold-to-talk dictation)'),
|
| 870 |
+
voiceProvider: z
|
| 871 |
+
.enum(['anthropic', 'doubao'])
|
| 872 |
+
.optional()
|
| 873 |
+
.describe(
|
| 874 |
+
'Voice STT backend: "anthropic" (default) or "doubao" (Doubao ASR)',
|
| 875 |
+
),
|
| 876 |
}
|
| 877 |
: {}),
|
| 878 |
...(feature('KAIROS')
|
src/voice/voiceModeEnabled.ts
CHANGED
|
@@ -44,11 +44,18 @@ export function hasVoiceAuth(): boolean {
|
|
| 44 |
}
|
| 45 |
|
| 46 |
/**
|
| 47 |
-
* Full runtime check
|
| 48 |
-
*
|
| 49 |
-
* paths where a fresh keychain read is acceptable. For React render
|
| 50 |
-
* paths use useVoiceEnabled() instead (memoizes the auth half).
|
| 51 |
*/
|
| 52 |
export function isVoiceModeEnabled(): boolean {
|
| 53 |
return hasVoiceAuth() && isVoiceGrowthBookEnabled()
|
| 54 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 44 |
}
|
| 45 |
|
| 46 |
/**
|
| 47 |
+
* Full runtime check for Anthropic voice_stream backend.
|
| 48 |
+
* Returns true when both auth + GrowthBook kill-switch pass.
|
|
|
|
|
|
|
| 49 |
*/
|
| 50 |
export function isVoiceModeEnabled(): boolean {
|
| 51 |
return hasVoiceAuth() && isVoiceGrowthBookEnabled()
|
| 52 |
}
|
| 53 |
+
|
| 54 |
+
/**
|
| 55 |
+
* Check if voice mode can be activated with any STT backend.
|
| 56 |
+
* Always returns true when VOICE_MODE feature flag is on and GrowthBook
|
| 57 |
+
* kill-switch is off — the Doubao backend does not require Anthropic auth.
|
| 58 |
+
*/
|
| 59 |
+
export function isVoiceAvailable(): boolean {
|
| 60 |
+
return feature('VOICE_MODE')
|
| 61 |
+
}
|
vendor/audio-capture-src/index.ts
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
type AudioCaptureNapi = {
|
| 2 |
+
startRecording(onData: (data: Buffer) => void, onEnd: () => void): boolean
|
| 3 |
+
stopRecording(): void
|
| 4 |
+
isRecording(): boolean
|
| 5 |
+
startPlayback(sampleRate: number, channels: number): boolean
|
| 6 |
+
writePlaybackData(data: Buffer): void
|
| 7 |
+
stopPlayback(): void
|
| 8 |
+
isPlaying(): boolean
|
| 9 |
+
// TCC microphone authorization status (macOS only):
|
| 10 |
+
// 0 = notDetermined, 1 = restricted, 2 = denied, 3 = authorized.
|
| 11 |
+
// Linux: always returns 3 (authorized) — no system-level microphone permission API.
|
| 12 |
+
// Windows: returns 3 (authorized) if registry key absent or allowed,
|
| 13 |
+
// 2 (denied) if microphone access is explicitly denied.
|
| 14 |
+
microphoneAuthorizationStatus?(): number
|
| 15 |
+
}
|
| 16 |
+
|
| 17 |
+
let cachedModule: AudioCaptureNapi | null = null
|
| 18 |
+
let loadAttempted = false
|
| 19 |
+
|
| 20 |
+
function loadModule(): AudioCaptureNapi | null {
|
| 21 |
+
if (loadAttempted) {
|
| 22 |
+
return cachedModule
|
| 23 |
+
}
|
| 24 |
+
loadAttempted = true
|
| 25 |
+
|
| 26 |
+
// Supported platforms: macOS (darwin), Linux, Windows (win32)
|
| 27 |
+
const platform = process.platform
|
| 28 |
+
if (platform !== 'darwin' && platform !== 'linux' && platform !== 'win32') {
|
| 29 |
+
return null
|
| 30 |
+
}
|
| 31 |
+
|
| 32 |
+
// Candidate 1: native-embed path (bun compile). AUDIO_CAPTURE_NODE_PATH is
|
| 33 |
+
// defined at build time in build-with-plugins.ts for native builds only — the
|
| 34 |
+
// define resolves it to the static literal "../../audio-capture.node" so bun
|
| 35 |
+
// compile can rewrite it to /$bunfs/root/audio-capture.node. MUST stay a
|
| 36 |
+
// direct require(env var) — bun cannot analyze require(variable) from a loop.
|
| 37 |
+
if (process.env.AUDIO_CAPTURE_NODE_PATH) {
|
| 38 |
+
try {
|
| 39 |
+
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
| 40 |
+
cachedModule = require(
|
| 41 |
+
process.env.AUDIO_CAPTURE_NODE_PATH,
|
| 42 |
+
) as AudioCaptureNapi
|
| 43 |
+
return cachedModule
|
| 44 |
+
} catch {
|
| 45 |
+
// fall through to runtime fallbacks below
|
| 46 |
+
}
|
| 47 |
+
}
|
| 48 |
+
|
| 49 |
+
// Candidates 2/3: npm-install and dev/source layouts. Dynamic require is
|
| 50 |
+
// fine here — in bundled output (node --target build) require() resolves at
|
| 51 |
+
// runtime relative to cli.js at the package root; in dev it resolves
|
| 52 |
+
// relative to this file (vendor/audio-capture-src/index.ts).
|
| 53 |
+
const platformDir = `${process.arch}-${platform}`
|
| 54 |
+
const fallbacks = [
|
| 55 |
+
`./vendor/audio-capture/${platformDir}/audio-capture.node`,
|
| 56 |
+
`../audio-capture/${platformDir}/audio-capture.node`,
|
| 57 |
+
]
|
| 58 |
+
for (const p of fallbacks) {
|
| 59 |
+
try {
|
| 60 |
+
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
| 61 |
+
cachedModule = require(p) as AudioCaptureNapi
|
| 62 |
+
return cachedModule
|
| 63 |
+
} catch {
|
| 64 |
+
// try next
|
| 65 |
+
}
|
| 66 |
+
}
|
| 67 |
+
return null
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
export function isNativeAudioAvailable(): boolean {
|
| 71 |
+
return loadModule() !== null
|
| 72 |
+
}
|
| 73 |
+
|
| 74 |
+
export function startNativeRecording(
|
| 75 |
+
onData: (data: Buffer) => void,
|
| 76 |
+
onEnd: () => void,
|
| 77 |
+
): boolean {
|
| 78 |
+
const mod = loadModule()
|
| 79 |
+
if (!mod) {
|
| 80 |
+
return false
|
| 81 |
+
}
|
| 82 |
+
return mod.startRecording(onData, onEnd)
|
| 83 |
+
}
|
| 84 |
+
|
| 85 |
+
export function stopNativeRecording(): void {
|
| 86 |
+
const mod = loadModule()
|
| 87 |
+
if (!mod) {
|
| 88 |
+
return
|
| 89 |
+
}
|
| 90 |
+
mod.stopRecording()
|
| 91 |
+
}
|
| 92 |
+
|
| 93 |
+
export function isNativeRecordingActive(): boolean {
|
| 94 |
+
const mod = loadModule()
|
| 95 |
+
if (!mod) {
|
| 96 |
+
return false
|
| 97 |
+
}
|
| 98 |
+
return mod.isRecording()
|
| 99 |
+
}
|
| 100 |
+
|
| 101 |
+
export function startNativePlayback(
|
| 102 |
+
sampleRate: number,
|
| 103 |
+
channels: number,
|
| 104 |
+
): boolean {
|
| 105 |
+
const mod = loadModule()
|
| 106 |
+
if (!mod) {
|
| 107 |
+
return false
|
| 108 |
+
}
|
| 109 |
+
return mod.startPlayback(sampleRate, channels)
|
| 110 |
+
}
|
| 111 |
+
|
| 112 |
+
export function writeNativePlaybackData(data: Buffer): void {
|
| 113 |
+
const mod = loadModule()
|
| 114 |
+
if (!mod) {
|
| 115 |
+
return
|
| 116 |
+
}
|
| 117 |
+
mod.writePlaybackData(data)
|
| 118 |
+
}
|
| 119 |
+
|
| 120 |
+
export function stopNativePlayback(): void {
|
| 121 |
+
const mod = loadModule()
|
| 122 |
+
if (!mod) {
|
| 123 |
+
return
|
| 124 |
+
}
|
| 125 |
+
mod.stopPlayback()
|
| 126 |
+
}
|
| 127 |
+
|
| 128 |
+
export function isNativePlaying(): boolean {
|
| 129 |
+
const mod = loadModule()
|
| 130 |
+
if (!mod) {
|
| 131 |
+
return false
|
| 132 |
+
}
|
| 133 |
+
return mod.isPlaying()
|
| 134 |
+
}
|
| 135 |
+
|
| 136 |
+
// Returns the microphone authorization status.
|
| 137 |
+
// On macOS, returns the TCC status: 0=notDetermined, 1=restricted, 2=denied, 3=authorized.
|
| 138 |
+
// On Linux, always returns 3 (authorized) — no system-level mic permission API.
|
| 139 |
+
// On Windows, returns 3 (authorized) if registry key absent or allowed, 2 (denied) if explicitly denied.
|
| 140 |
+
// Returns 0 (notDetermined) if the native module is unavailable.
|
| 141 |
+
export function microphoneAuthorizationStatus(): number {
|
| 142 |
+
const mod = loadModule()
|
| 143 |
+
if (!mod || !mod.microphoneAuthorizationStatus) {
|
| 144 |
+
return 0
|
| 145 |
+
}
|
| 146 |
+
return mod.microphoneAuthorizationStatus()
|
| 147 |
+
}
|