A-Mahla's picture
A-Mahla HF Staff
Add SPEECH_TO_SPEECH_URL: deploy-pinned direct URL that overrides the LB
1cfad64
Raw
History Blame Contribute Delete
57.2 kB
// @ts-check
/**
* Minimal voice conversation app, talking to a Hugging Face speech-to-speech
* backend over **WebSocket** (drop-in alternative to the WebRTC variant).
*
* Click the orb -> we ask for the mic, POST a session on the LB, open a
* WebSocket on the routed compute endpoint, push session.update + mic
* audio, play back the TTS audio. The orb visually reflects the live
* state (idle, connecting, listening, user-speaking, processing,
* ai-speaking).
*
* The only meaningful difference vs. the WebRTC main.js is that the
* client owns its own AudioContext (no `attachOutputTrack`), so we hand
* it the MediaStream directly.
*
* @typedef {"idle" | "connecting" | "queued" | "your-turn" | "listening" | "user-speaking" | "processing" | "ai-speaking" | "error"} AppState
*/
import { S2sWsRealtimeClient } from "./ws/s2s-ws-client.js";
import { $, truncateError, DEBUG } from "./ui/dom.js";
import { ChatView } from "./ui/chat.js";
import { Account } from "./ui/account.js";
const DEFAULT_VOICE = "Aiden";
const DEFAULT_INSTRUCTIONS =
"You are a friendly voice assistant. " +
"Keep replies short, warm, and spoken. Avoid long monologues.";
// Appended to the user's instructions whenever at least one tool is enabled.
// Stops the model from announcing capabilities ("Yes, I can search") and then
// idling for the next turn — it should act immediately in the same response.
const TOOL_USE_HINT =
" When the user's request calls for one of your tools, do not describe your " +
"capabilities or say you can do it and wait for another turn. Instead, say " +
'a brief acknowledgement like "Let me search for that..." and call the tool ' +
"right away in the same response.";
const STORAGE_KEYS = {
// Direct s2s server URL, used only when the deploy has no LOAD_BALANCER_URL
// (in LB mode the browser never learns the LB address — it POSTs /api/session).
directUrl: "s2s.ws.directUrl",
voice: "s2s.ws.voice",
instructions: "s2s.ws.instructions",
tools: "s2s.ws.tools",
searchKey: "s2s.ws.searchKey",
noiseGate: "s2s.ws.noiseGate",
};
// ── Noise gate ──────────────────────────────────────────────────────────────
// The Settings cursor sets the gate's open threshold in dBFS. Its leftmost
// position is an OFF detent (gate disabled, pure passthrough); the rest of the
// travel is the active threshold. The cursor shares the meter's dB axis, so the
// handle sits on the level bar — raise it until room noise stops lighting it up.
// The slider range IS the shared axis: the live meter fill and the threshold
// thumb both map across [GATE_OFF_DB, GATE_MAX_DB], so the thumb sits exactly
// where the gate cuts on the same scale as the level bar.
const GATE_OFF_DB = -66; // slider minimum = off / bottom of the meter axis
const GATE_MAX_DB = -3; // slider maximum = most aggressive / top of the meter axis
const GATE_DEFAULT_DB = -50; // first-run default: a gentle gate, enabled
/** @param {number} thresholdDb @returns {import("./ws/s2s-ws-client.js").NoiseGate} */
function gateParams(thresholdDb) {
return { enabled: thresholdDb > GATE_OFF_DB, thresholdDb };
}
// ── Tools ─────────────────────────────────────────────────────────────────
// Function tools we declare to the backend. The model decides when to call
// one; the executor below runs it and returns the result (see runTool).
/** @type {Record<string, import("./ws/s2s-ws-client.js").ToolDef>} */
const TOOL_DEFS = {
web_search: {
type: "function",
name: "web_search",
description:
"Search the web for current or factual information you don't already know " +
"(news, prices, facts, documentation). Returns the top results with titles, " +
"snippets and URLs.",
parameters: {
type: "object",
properties: { query: { type: "string", description: "The search query." } },
required: ["query"],
},
},
camera_snapshot: {
type: "function",
name: "camera_snapshot",
description:
"Capture the current frame from the user's webcam so you can see what they " +
"are showing you. Use it whenever the user refers to something visual or " +
"asks you to look.",
parameters: { type: "object", properties: {}, required: [] },
},
};
/** Longest edge of the snapshot sent to the VLM, in px (keeps payload sane). */
const SNAPSHOT_MAX_EDGE = 768;
const SNAPSHOT_QUALITY = 0.7;
function loadSettings() {
return {
directUrl: localStorage.getItem(STORAGE_KEYS.directUrl) || "",
voice: localStorage.getItem(STORAGE_KEYS.voice) || DEFAULT_VOICE,
instructions: localStorage.getItem(STORAGE_KEYS.instructions) || DEFAULT_INSTRUCTIONS,
noiseGate: loadGateThreshold(),
};
}
/** Stored gate threshold (dBFS), clamped to the slider range. Defaults to a
* gentle enabled gate (GATE_DEFAULT_DB) when the user hasn't set one yet. */
function loadGateThreshold() {
const stored = localStorage.getItem(STORAGE_KEYS.noiseGate);
// getItem returns null when unset, and Number(null) === 0 (finite!), so guard
// the missing/empty case explicitly before coercing — otherwise the default
// never fires and 0 clamps to the slider max.
if (stored === null || stored === "") return GATE_DEFAULT_DB;
const raw = Number(stored);
if (!Number.isFinite(raw)) return GATE_DEFAULT_DB;
return Math.min(GATE_MAX_DB, Math.max(GATE_OFF_DB, Math.round(raw)));
}
/** @param {ReturnType<typeof loadSettings>} s */
function saveSettings(s) {
localStorage.setItem(STORAGE_KEYS.directUrl, s.directUrl);
localStorage.setItem(STORAGE_KEYS.voice, s.voice);
localStorage.setItem(STORAGE_KEYS.instructions, s.instructions);
localStorage.setItem(STORAGE_KEYS.noiseGate, String(s.noiseGate));
}
/** @returns {{ web_search: boolean, camera_snapshot: boolean }} */
function loadTools() {
try {
const raw = JSON.parse(localStorage.getItem(STORAGE_KEYS.tools) || "{}");
// Both tools default ON (web search still only activates when a key exists).
// We never call getUserMedia on page load — the camera only actually starts
// on a user gesture (conversation start), so a default-on flag doesn't
// silently resume the webcam; an explicit saved `false` is respected.
return {
web_search: raw.web_search ?? true,
camera_snapshot: raw.camera_snapshot ?? true,
};
} catch {
return { web_search: true, camera_snapshot: true };
}
}
function saveTools() {
localStorage.setItem(STORAGE_KEYS.tools, JSON.stringify(toolsEnabled));
}
/** @type {Record<AppState, { caption: string; disabled: boolean }>} */
const STATE_VIEWS = {
idle: { caption: "Tap to start", disabled: false },
connecting: { caption: "Connecting", disabled: true },
queued: { caption: "Finding you a spot…", disabled: true },
"your-turn": { caption: "You're up! 🎉", disabled: true },
listening: { caption: "", disabled: false },
"user-speaking": { caption: "", disabled: false },
processing: { caption: "", disabled: false },
"ai-speaking": { caption: "", disabled: false },
error: { caption: "Tap to retry", disabled: false },
};
/** @type {Record<AppState, string>} */
const STATE_CLASS = {
idle: "state-idle",
connecting: "state-connecting",
queued: "state-queued",
"your-turn": "state-your-turn",
listening: "state-listening",
"user-speaking": "state-user-speaking",
processing: "state-processing",
"ai-speaking": "state-ai-speaking",
error: "state-error",
};
/** @type {ReadonlySet<AppState>} */
const LIVE_STATES = new Set(["listening", "user-speaking", "processing", "ai-speaking"]);
/** @type {HTMLButtonElement} */
const circleBtn = $("#main-circle");
/** @type {HTMLParagraphElement} */
const circleCaption = $("#circle-caption");
/** @type {HTMLParagraphElement} */
const circleSubcaption = $("#circle-subcaption");
/** @type {HTMLElement} */
const orbWrap = $(".orb-wrap");
/** @type {HTMLButtonElement} */
const micBtn = $("#mic-btn");
/** @type {HTMLButtonElement} */
const stopBtn = $("#stop-btn");
/** @type {HTMLElement} */
const queueActions = $("#queue-actions");
/** @type {HTMLButtonElement} */
const joinQueueBtn = $("#join-queue-btn");
/** @type {HTMLButtonElement} */
const leaveQueueBtn = $("#leave-queue-btn");
/** @type {HTMLButtonElement} */
const settingsBtn = $("#settings-btn");
/** @type {HTMLDialogElement} */
const settingsModal = $("#settings-modal");
/** @type {HTMLButtonElement} */
const aboutBtn = $("#about-btn");
/** @type {HTMLDialogElement} */
const aboutModal = $("#about-modal");
/** @type {HTMLButtonElement} */
const aboutClose = $("#about-close");
/** @type {HTMLButtonElement} */
const toolsBtn = $("#tools-btn");
/** @type {HTMLDialogElement} */
const toolsModal = $("#tools-modal");
/** @type {HTMLButtonElement} */
const toolsClose = $("#tools-close");
/** @type {HTMLInputElement} */
const toolWebSwitch = $("#tool-web");
/** @type {HTMLInputElement} */
const toolCamSwitch = $("#tool-cam");
/** @type {HTMLElement} */
const toolWebRow = $("#tool-web-row");
/** @type {HTMLElement} */
const toolWebHint = $("#tool-web-hint");
/** @type {HTMLElement} */
const toolCamHint = $("#tool-cam-hint");
/** @type {HTMLInputElement} */
const searchKeyInput = $("#search-key");
/** @type {HTMLElement} */
const camPip = $("#cam-pip");
/** @type {HTMLVideoElement} */
const camVideo = $("#cam-video");
/** @type {HTMLInputElement} */
const inputLbUrl = $("#lb-url");
/** @type {HTMLElement} */
const connField = $("#conn-field");
/** @type {HTMLElement} */
const connHint = $("#conn-hint");
/** @type {HTMLSelectElement} */
const inputVoice = $("#voice");
/** @type {HTMLTextAreaElement} */
const inputInstructions = $("#instructions");
/** @type {HTMLInputElement} */
const inputNoiseGate = $("#noise-gate");
/** @type {HTMLElement} */
const gateValue = $("#gate-value");
/** @type {HTMLElement} */
const gateMeterFill = $("#gate-meter-fill");
/** @type {HTMLElement} */
const micGate = $("#mic-gate");
const mgaArc = /** @type {SVGSVGElement} */ (document.querySelector("#mic-gate-arc"));
const mgaTrack = /** @type {SVGPathElement} */ (document.querySelector("#mga-track"));
const mgaFill = /** @type {SVGPathElement} */ (document.querySelector("#mga-fill"));
const mgaHit = /** @type {SVGPathElement} */ (document.querySelector("#mga-hit"));
const mgaHandle = /** @type {SVGCircleElement} */ (document.querySelector("#mga-handle"));
/** @type {HTMLButtonElement} */
const restartBtn = $("#restart-conversation");
/** @type {HTMLElement} */
const restartHint = $("#restart-hint");
const settingsForm = /** @type {HTMLFormElement} */ (settingsModal.querySelector("form"));
/** @type {AppState} */
let currentState = "idle";
let settings = loadSettings();
// ── Connection target ────────────────────────────────────────────────────────
// Three modes, decided by the deploy via /api/config:
// • SPEECH_TO_SPEECH_URL set -> direct mode pinned by the deploy: the browser
// connects straight to that URL, shown read-only in Settings. Overrides the
// load balancer entirely.
// • LOAD_BALANCER_URL set -> original flow: POST the same-origin /api/session
// proxy (the server forwards to the LB; the LB address is never sent here).
// • neither (allowDirect) -> the user sets a speech-to-speech server URL and
// the browser connects to it directly (no load balancer, no /session).
let lbMode = false;
// Fail open: direct entry is allowed unless /api/config reports an LB URL. This
// way a missing/unreachable config (e.g. static hosting) leaves the field
// usable rather than locked.
let allowDirect = true;
// Deploy-pinned s2s URL (SPEECH_TO_SPEECH_URL). Non-empty -> locked direct
// mode: the field displays it read-only and the saved user URL is untouched.
let pinnedUrl = "";
// ── Tool state ──────────────────────────────────────────────────────────────
let toolsEnabled = loadTools();
// Whether the server holds a Serper key (learned from /api/config on load).
let serverSearchKey = false;
// A user-supplied key (fallback when the deploy has none). localStorage only.
let userSearchKey = localStorage.getItem(STORAGE_KEYS.searchKey) || "";
/** @type {MediaStream | null} */
let cameraStream = null;
/** Search is usable if the server has a key or the user supplied one. */
function searchAvailable() {
return serverSearchKey || !!userSearchKey;
}
/** Tool definitions for the currently-enabled (and usable) tools. */
function activeToolDefs() {
const defs = [];
if (toolsEnabled.web_search && searchAvailable()) defs.push(TOOL_DEFS.web_search);
if (toolsEnabled.camera_snapshot) defs.push(TOOL_DEFS.camera_snapshot);
return defs;
}
/** Instructions plus the hidden tool-use hint when any tool is active. */
function effectiveInstructions() {
const base = settings.instructions;
return activeToolDefs().length ? base + TOOL_USE_HINT : base;
}
/** Push the active tool set to a live session so toggles apply mid-call. */
function pushToolsToSession() {
if (!client || !LIVE_STATES.has(currentState)) return;
client.setTools(activeToolDefs());
// The hidden tool-use hint depends on whether any tool is active, so refresh
// instructions alongside the tool set.
client.updateSession({ instructions: effectiveInstructions() });
}
// ── Chat view ───────────────────────────────────────────────────────────────
// Owns the history panel, the ephemeral bubbles, and all transcript/tool
// streaming state. The client's events are forwarded to its on* methods.
const chat = new ChatView();
// ── Account / limiter ─────────────────────────────────────────────────────
// Login chip + daily-limit modal (inert unless the deploy is in LB mode). The
// server meters conversation time; the client just heartbeats a live session
// and tears down when the server reports the budget is spent.
const account = new Account();
let limiterOn = false;
let heartbeatTimer = 0;
let trackedSessionId = "";
let trackedTier = "";
// The waiting-queue ticket id while we're in line (else ""). Used to leave the
// queue on teardown / tab-close so we don't hold a phantom place.
let queuedTicketId = "";
/** @type {S2sWsRealtimeClient | null} */
let client = null;
/** @type {MediaStream | null} */
let micStream = null;
let micMuted = false;
/** @param {AppState} next */
function setState(next) {
currentState = next;
const view = STATE_VIEWS[next];
circleBtn.disabled = view.disabled;
circleBtn.className = `circle ${STATE_CLASS[next]}`;
if (next !== "error") setCaption(view.caption);
const live = LIVE_STATES.has(next);
orbWrap.classList.toggle("live", live);
micBtn.setAttribute("aria-hidden", live ? "false" : "true");
stopBtn.setAttribute("aria-hidden", live ? "false" : "true");
micBtn.tabIndex = live ? 0 : -1;
stopBtn.tabIndex = live ? 0 : -1;
// Queue affordances: "Leave queue" whenever we're in line; "Join now" only once
// it's our turn (a slot is held for us). Both live under #queue-actions.
const yourTurn = next === "your-turn";
const inLine = next === "queued" || yourTurn;
queueActions.hidden = !inLine;
joinQueueBtn.hidden = !yourTurn;
joinQueueBtn.tabIndex = yourTurn ? 0 : -1;
leaveQueueBtn.hidden = !inLine;
leaveQueueBtn.tabIndex = inLine ? 0 : -1;
if (!yourTurn) stopJoinCountdown();
// Warm reassurance under the terse position, only while waiting in line.
if (next === "queued") {
circleSubcaption.textContent =
"Sorry, we overhugged! 🤗 Every slot is busy, so we saved you a spot. Hang tight, you're moving up.";
circleSubcaption.hidden = false;
} else {
circleSubcaption.hidden = true;
}
updateRestartAvailability();
}
function updateRestartAvailability() {
// Restart works from any settled state — it tears down a live call (if any)
// and reconnects with the current settings. Only block while mid-connect or
// while waiting in the queue (restarting from there would just re-queue).
restartBtn.disabled =
currentState === "connecting" || currentState === "queued" || currentState === "your-turn";
restartHint.hidden = false;
restartHint.textContent = LIVE_STATES.has(currentState)
? "Reconnects now with the settings above."
: "Starts a conversation with the settings above.";
}
/**
* @param {string} text
* @param {"" | "error" | "muted"} [kind]
*/
function setCaption(text, kind = "") {
const trimmed = text.trim();
circleCaption.textContent = trimmed;
circleCaption.className = `circle-caption${kind ? ` ${kind}` : ""}${trimmed ? "" : " empty"}`;
}
function openSettings() {
syncConnectionUi();
inputVoice.value = settings.voice;
inputInstructions.value = settings.instructions;
syncGateUi();
updateRestartAvailability();
settingsModal.showModal();
}
/** dB position (clamped to the slider axis) as a 0..1 fraction of the track.
* @param {number} db */
function dbToFraction(db) {
const clamped = Math.min(GATE_MAX_DB, Math.max(GATE_OFF_DB, db));
return (clamped - GATE_OFF_DB) / (GATE_MAX_DB - GATE_OFF_DB);
}
/** @param {number} f @returns {number} dB at a 0..1 position on the gate axis. */
function fractionToDb(f) {
const clamped = Math.min(1, Math.max(0, f));
return Math.round(GATE_OFF_DB + clamped * (GATE_MAX_DB - GATE_OFF_DB));
}
// ── Radial gate arc (around the mic button, live during a call) ─────────────
// A 270° arc with the gap facing the orb (right). Fraction 0 (=Off) sits at the
// bottom-ish start; 1 (=max) at the top-ish end. The level fill and the
// threshold handle ride this same axis, mirroring the Settings widget.
const ARC_R = 40;
// A ~200° arc centred on the left (180°) so the wide gap faces the orb (right).
const ARC_SPAN_DEG = 200;
const ARC_START_DEG = 180 - ARC_SPAN_DEG / 2; // lower-left start; Off end
/** Point at fraction f (0..1) and radius r, in the 0..100 viewBox.
* @param {number} f @param {number} [r] */
function arcPoint(f, r = ARC_R) {
const deg = ARC_START_DEG + f * ARC_SPAN_DEG;
const rad = (deg * Math.PI) / 180;
return { x: 50 + r * Math.cos(rad), y: 50 + r * Math.sin(rad) };
}
/** SVG path `d` for the full 0..1 arc (clockwise). */
function fullArcD() {
const a = arcPoint(0);
const b = arcPoint(1);
const largeArc = ARC_SPAN_DEG > 180 ? 1 : 0;
return `M ${a.x} ${a.y} A ${ARC_R} ${ARC_R} 0 ${largeArc} 1 ${b.x} ${b.y}`;
}
/** One-time geometry: track, fill (dash-revealed) and the transparent hit band. */
function initGateArc() {
const d = fullArcD();
mgaTrack.setAttribute("d", d);
mgaFill.setAttribute("d", d);
mgaHit.setAttribute("d", d);
// pathLength 100 lets us reveal the fill by fraction via dashoffset.
mgaFill.setAttribute("pathLength", "100");
mgaFill.style.strokeDasharray = "100 100";
mgaFill.style.strokeDashoffset = "100"; // empty until levels arrive
renderGateHandle();
}
/** Place the threshold bead on the arc at the stored threshold; flag off state. */
function renderGateHandle() {
const off = settings.noiseGate <= GATE_OFF_DB;
const p = arcPoint(dbToFraction(settings.noiseGate));
mgaHandle.setAttribute("cx", String(p.x));
mgaHandle.setAttribute("cy", String(p.y));
micGate.classList.toggle("gate-off", off);
}
/** Paint a 0..1 live level onto the arc fill (and the Settings meter if open).
* Brightens the tick when the level crosses the threshold — i.e. the gate is
* actually open — but only when gating is enabled.
* @param {number} rms */
function paintInputLevel(rms) {
const db = rms > 0 ? 20 * Math.log10(rms) : GATE_OFF_DB;
const f = dbToFraction(db);
mgaFill.style.strokeDashoffset = String(100 * (1 - f));
if (settingsModal.open) gateMeterFill.style.width = `${f * 100}%`;
const enabled = settings.noiseGate > GATE_OFF_DB;
micGate.classList.toggle("gate-open", enabled && f >= dbToFraction(settings.noiseGate));
}
/** The single place that commits a new gate threshold: updates both controls,
* persists, and applies live to the running session.
* @param {number} db */
function setGateThreshold(db) {
settings.noiseGate = Math.min(GATE_MAX_DB, Math.max(GATE_OFF_DB, Math.round(db)));
const off = settings.noiseGate <= GATE_OFF_DB;
inputNoiseGate.value = String(settings.noiseGate);
gateValue.textContent = off ? "Off" : `${settings.noiseGate} dB`;
renderGateHandle();
localStorage.setItem(STORAGE_KEYS.noiseGate, String(settings.noiseGate));
if (client && LIVE_STATES.has(currentState)) {
client.setNoiseGate(gateParams(settings.noiseGate));
}
}
/** Reflect the stored gate threshold into the slider, label and arc handle. */
function syncGateUi() {
inputNoiseGate.value = String(settings.noiseGate);
const off = settings.noiseGate <= GATE_OFF_DB;
gateValue.textContent = off ? "Off" : `${settings.noiseGate} dB`;
renderGateHandle();
}
// Drag along the arc band to set the threshold (a tap on the glyph still mutes).
let gateDragging = false;
/** @param {PointerEvent} e */
function gatePointerToDb(e) {
const rect = mgaArc.getBoundingClientRect();
const cx = rect.left + rect.width / 2;
const cy = rect.top + rect.height / 2;
let deg = (Math.atan2(e.clientY - cy, e.clientX - cx) * 180) / Math.PI;
if (deg < 0) deg += 360;
// Map the on-arc angle to a fraction; angles in the right-side gap fall
// outside [0,1] and fractionToDb clamps them to the nearest end (just-below
// start -> Off, just-past end -> max).
const f = (deg - ARC_START_DEG) / ARC_SPAN_DEG;
return fractionToDb(f);
}
mgaHit.addEventListener("pointerdown", (e) => {
gateDragging = true;
mgaHit.setPointerCapture(e.pointerId);
setGateThreshold(gatePointerToDb(e));
});
mgaHit.addEventListener("pointermove", (e) => {
if (gateDragging) setGateThreshold(gatePointerToDb(e));
});
const endGateDrag = (/** @type {PointerEvent} */ e) => {
if (!gateDragging) return;
gateDragging = false;
try { mgaHit.releasePointerCapture(e.pointerId); } catch {}
};
mgaHit.addEventListener("pointerup", endGateDrag);
mgaHit.addEventListener("pointercancel", endGateDrag);
settingsBtn.addEventListener("click", openSettings);
// About panel: native <dialog>, Esc closes for free; also close on the X and
// on a click in the backdrop (a click whose target is the dialog itself).
aboutBtn.addEventListener("click", () => aboutModal.showModal());
// Mobile twin of the (i), living in the right-hand control cluster.
$("#about-btn-m").addEventListener("click", () => aboutModal.showModal());
aboutClose.addEventListener("click", () => aboutModal.close());
aboutModal.addEventListener("click", (e) => {
if (e.target === aboutModal) aboutModal.close();
});
// ── Tools panel ───────────────────────────────────────────────────────────
/** Reflect the current tool state into the panel controls. */
function syncToolsUi() {
const avail = searchAvailable();
toolWebSwitch.checked = toolsEnabled.web_search && avail;
toolWebSwitch.disabled = !avail;
toolWebRow.classList.toggle("disabled", !avail);
toolCamSwitch.checked = toolsEnabled.camera_snapshot;
if (serverSearchKey) {
// Key lives server-side: show it as configured, never expose it.
searchKeyInput.value = "";
searchKeyInput.placeholder = "•••••••• · provided by the server";
searchKeyInput.disabled = true;
toolWebHint.textContent = "Ready. The search key is held server-side and never sent to your browser.";
} else {
searchKeyInput.disabled = false;
searchKeyInput.value = userSearchKey;
searchKeyInput.placeholder = "Paste a Serper key to enable web search";
toolWebHint.textContent = userSearchKey
? "Using your key — stored in this browser only."
: "No server key configured. Add your own Serper key to enable web search.";
}
}
toolsBtn.addEventListener("click", () => { syncToolsUi(); toolsModal.showModal(); });
toolsClose.addEventListener("click", () => toolsModal.close());
toolsModal.addEventListener("click", (e) => {
if (e.target === toolsModal) toolsModal.close();
});
toolWebSwitch.addEventListener("change", () => {
if (toolWebSwitch.checked && !searchAvailable()) {
toolWebSwitch.checked = false; // guard: can't enable without a key
return;
}
toolsEnabled.web_search = toolWebSwitch.checked;
saveTools();
pushToolsToSession();
});
toolCamSwitch.addEventListener("change", async () => {
if (toolCamSwitch.checked) {
try {
// Flipping the switch always re-requests the camera, so a permission that
// was only dismissed earlier is asked again here.
await enableCamera();
} catch (err) {
toolCamSwitch.checked = false;
const denied = err instanceof Error && (err.name === "NotAllowedError" || err.name === "SecurityError");
toolCamHint.textContent = denied
? "Camera blocked. Allow it from the camera icon in your browser's address bar — it switches on automatically."
: `Camera unavailable${err instanceof Error ? `: ${err.message}` : ""}`;
return;
}
toolsEnabled.camera_snapshot = true;
toolCamHint.textContent = "Camera on. The assistant can take a snapshot when it needs to see.";
} else {
disableCamera();
toolsEnabled.camera_snapshot = false;
toolCamHint.textContent = "Let the assistant see through your webcam.";
}
saveTools();
pushToolsToSession();
});
searchKeyInput.addEventListener("input", () => {
if (serverSearchKey) return;
userSearchKey = searchKeyInput.value.trim();
if (userSearchKey) localStorage.setItem(STORAGE_KEYS.searchKey, userSearchKey);
else localStorage.removeItem(STORAGE_KEYS.searchKey);
const avail = searchAvailable();
toolWebSwitch.disabled = !avail;
toolWebRow.classList.toggle("disabled", !avail);
// Losing the key disables a previously-enabled tool.
if (!avail && toolsEnabled.web_search) {
toolsEnabled.web_search = false;
toolWebSwitch.checked = false;
saveTools();
pushToolsToSession();
}
toolWebHint.textContent = userSearchKey
? "Using your key — stored in this browser only."
: "No server key configured. Add your own Serper key to enable web search.";
});
// ── Camera ──────────────────────────────────────────────────────────────────
async function enableCamera() {
if (cameraStream) return;
cameraStream = await navigator.mediaDevices.getUserMedia({
video: { facingMode: "user" },
audio: false,
});
camVideo.srcObject = cameraStream;
try { await camVideo.play(); } catch { /* autoplay quirks; muted video is fine */ }
camPip.classList.add("visible");
camPip.setAttribute("aria-hidden", "false");
// Lets the footer reflow to the bottom-right (and hide on mobile) while the
// webcam preview occupies the bottom of the stage.
document.body.classList.add("cam-on");
}
function disableCamera() {
if (cameraStream) {
for (const t of cameraStream.getTracks()) t.stop();
cameraStream = null;
}
camVideo.srcObject = null;
camPip.classList.remove("visible");
camPip.setAttribute("aria-hidden", "true");
document.body.classList.remove("cam-on");
}
/** Auto-start the webcam on arrival (the camera tool is on by default). If the
* user declines the permission, switch the tool off and reflect it in the UI
* rather than nagging. */
async function autoStartCamera() {
if (!toolsEnabled.camera_snapshot || cameraStream) return;
try {
await enableCamera();
} catch (err) {
console.warn("[main] camera auto-start declined/failed:", err);
toolsEnabled.camera_snapshot = false;
saveTools();
syncToolsUi();
}
}
/** Track the browser's camera permission so a later re-grant (e.g. the user
* unblocks it from the address bar after a denial) turns the camera back on
* without another toggle, and a revoke turns it off. Best-effort: the
* Permissions API doesn't support "camera" everywhere (e.g. Safari). */
async function watchCameraPermission() {
try {
const status = await navigator.permissions?.query?.({ name: /** @type {any} */ ("camera") });
if (!status) return;
status.addEventListener("change", () => {
if (status.state === "granted") {
if (!toolsEnabled.camera_snapshot) { toolsEnabled.camera_snapshot = true; saveTools(); }
void autoStartCamera();
syncToolsUi();
} else if (status.state === "denied") {
disableCamera();
if (toolsEnabled.camera_snapshot) { toolsEnabled.camera_snapshot = false; saveTools(); }
syncToolsUi();
}
});
} catch {
// Permissions API unavailable for "camera" — the toggle still re-asks.
}
}
/**
* Grab the current webcam frame as a downscaled JPEG data URL. The preview is
* mirrored in CSS for a natural self-view, but we draw the raw (un-mirrored)
* video here so the model sees the scene in its true orientation.
* @returns {string | null}
*/
function captureSnapshot() {
if (!cameraStream || !camVideo.videoWidth) return null;
const vw = camVideo.videoWidth;
const vh = camVideo.videoHeight;
const scale = Math.min(1, SNAPSHOT_MAX_EDGE / Math.max(vw, vh));
const w = Math.max(1, Math.round(vw * scale));
const h = Math.max(1, Math.round(vh * scale));
const canvas = document.createElement("canvas");
canvas.width = w;
canvas.height = h;
const ctx = canvas.getContext("2d");
if (!ctx) return null;
ctx.drawImage(camVideo, 0, 0, w, h);
return canvas.toDataURL("image/jpeg", SNAPSHOT_QUALITY);
}
/** Brief shutter flash on the preview so the user sees a snapshot was taken. */
function flashPreview() {
camPip.classList.remove("flash");
void camPip.offsetWidth; // reflow so the animation restarts
camPip.classList.add("flash");
}
// ── Tool executor ─────────────────────────────────────────────────────────
// Runs the function the model called, returns the result, and asks for a
// response so the model speaks it. Errors come back as the tool output too, so
// the model can recover gracefully instead of the turn stalling.
/**
* Run the function the model called, return its result to the backend, and ask
* for a follow-up response. We also hand the result back to the caller so it
* can be shown in the conversation once the tool has actually run.
* @param {string} name @param {string} argsJson @param {string} callId
* @returns {Promise<{ output: string, image?: string }>}
*/
async function runTool(name, argsJson, callId) {
if (!client) return { output: "" };
let args = /** @type {Record<string, unknown>} */ ({});
try { args = JSON.parse(argsJson || "{}"); } catch { /* keep {} */ }
if (DEBUG) console.debug(`[tool] run name=${name} callId=${JSON.stringify(callId)} args=${argsJson}`);
if (!callId) console.warn("[tool] empty call_id — the backend didn't tag the call, can't return a function_call_output");
/** @type {{ output: string, image?: string }} */
let result = { output: "" };
try {
if (name === "web_search") {
const query = typeof args.query === "string" ? args.query : "";
result.output = await execWebSearch(query);
// Return the result and let the bare response.create (below) trigger the
// spoken answer.
client.sendToolOutput(callId, result.output);
} else if (name === "camera_snapshot") {
const dataUrl = captureSnapshot();
if (dataUrl) {
if (DEBUG) console.debug(`[tool] camera_snapshot captured frame (${dataUrl.length} chars), sending image + output`);
result = { output: "Snapshot captured from the webcam and attached as an image.", image: dataUrl };
// Return the tool output; the frame itself rides along with the
// response.create below (sent right before it), so the model sees the
// snapshot in the very response it's about to speak.
client.sendToolOutput(callId, result.output);
flashPreview();
} else {
console.warn("[tool] camera_snapshot: no frame — camera off or not ready");
result.output = "The camera is not available right now.";
client.sendToolOutput(callId, result.output);
}
} else {
result.output = `Unknown tool: ${name}`;
client.sendToolOutput(callId, result.output);
}
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
result.output = `Tool failed: ${msg}`;
client.sendToolOutput(callId, result.output);
}
if (DEBUG) console.debug(`[tool] requesting model response after ${name}`);
// Camera: the captured frame rides with the response.create (sent just before
// it) so it's in context for the reply. Other tools: a bare create.
client.requestResponse(result.image ? { image: result.image } : undefined);
return result;
}
/** @param {string} query @returns {Promise<string>} */
async function execWebSearch(query) {
if (!query) return "No query provided.";
/** @type {Record<string, string>} */
const body = { query };
// Only send a user key when there's no server key (server prefers its own).
if (!serverSearchKey && userSearchKey) body.key = userSearchKey;
const res = await fetch("api/search", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(body),
});
if (!res.ok) {
let detail = String(res.status);
try { const j = await res.json(); if (j.detail) detail = j.detail; } catch {}
throw new Error(`search error (${detail})`);
}
const json = await res.json();
// Date-stamp the header so the model treats these as fresh realtime facts
// rather than its (older) training knowledge.
const today = new Date().toISOString().slice(0, 10);
/** @type {string[]} */
const lines = [`Google search result from ${today}:`];
if (json.answer) lines.push(`Answer: ${json.answer}`);
for (const r of json.results || []) {
lines.push(`- ${r.title}: ${r.snippet} (${r.url})`);
}
return lines.length > 1 ? lines.join("\n") : `${lines[0]}\nNo results found.`;
}
/** Learn server config (search key + connection target), then refresh the UI. */
async function fetchConfig() {
try {
const res = await fetch("api/config");
if (res.ok) {
const json = await res.json();
serverSearchKey = !!json.search;
lbMode = !!json.lb;
// Lock to LB mode only when the deploy reports a load balancer.
allowDirect = json.allowDirect ?? !lbMode;
// Deploy-pinned direct URL (overrides the LB server-side already).
pinnedUrl = (json.s2sUrl || "").trim();
// The conversation-time limiter rides on the LB being present.
limiterOn = lbMode;
}
// Non-OK response: leave the fail-open default (allowDirect = true).
} catch {
// Config endpoint unreachable (e.g. static hosting): keep direct entry.
}
if (DEBUG) console.debug(`[ui] config: allowDirect=${allowDirect} lbMode=${lbMode}`);
// Login chip + remaining-budget (no-op / hidden when the limiter is off).
void account.refresh();
syncToolsUi();
syncConnectionUi();
}
/**
* Resolve where to connect, per the deploy's mode:
* • LB mode -> `{ sessionUrl }`, the client POSTs the same-origin /api/session
* proxy and the server forwards to the LB (its address stays server-side).
* • direct -> `{ directUrl }`, connect straight to the s2s WebSocket.
* Throws a user-facing error if direct mode is on but no URL was entered.
* @returns {{ sessionUrl: string } | { directUrl: string }}
*/
function connectionTarget() {
if (!allowDirect) {
return { sessionUrl: "api/session" };
}
const directUrl = buildDirectWsUrl(pinnedUrl || settings.directUrl);
if (!directUrl) {
throw new Error("Enter a speech-to-speech server URL in Settings.");
}
return { directUrl };
}
/**
* Normalise a user-typed server address into a realtime WebSocket URL.
* Accepts bare hosts (`localhost:8080`), http(s) URLs, or ws(s) URLs, and adds
* the `/v1/realtime` path when none is given. A full connect URL (with path
* and/or query) is preserved as-is.
* @param {string} raw @returns {string}
*/
function buildDirectWsUrl(raw) {
let s = (raw || "").trim();
if (!s) return "";
if (!/^wss?:\/\//i.test(s)) {
if (/^https?:\/\//i.test(s)) {
s = s.replace(/^http/i, "ws"); // http→ws, https→wss
} else {
const isLocal = /^(localhost|127\.0\.0\.1|\[::1\])(:|\/|$)/i.test(s);
s = (isLocal ? "ws://" : "wss://") + s;
}
}
try {
const u = new URL(s);
if (u.pathname === "" || u.pathname === "/") u.pathname = "/v1/realtime";
return u.toString();
} catch {
return s;
}
}
/** Create + resume an AudioContext synchronously (must run inside the user
* gesture so iOS lets it start). Returns null if construction fails. */
function createResumedAudioContext() {
try {
const Ctx = window.AudioContext || /** @type {any} */ (window).webkitAudioContext;
const ctx = new Ctx({ latencyHint: "interactive" });
if (ctx.state === "suspended") void ctx.resume().catch(() => {});
return /** @type {AudioContext} */ (ctx);
} catch (err) {
console.warn("[main] AudioContext init failed:", err);
return null;
}
}
/** Read the editable settings out of the form. The URL field is only honoured
* in free direct mode — in LB mode it's hidden, and when the deploy pins a
* URL it's read-only, so the user's saved URL survives either way. */
function readSettingsFromForm() {
return {
directUrl: allowDirect && !pinnedUrl ? inputLbUrl.value.trim() : settings.directUrl,
voice: inputVoice.value || DEFAULT_VOICE,
instructions: inputInstructions.value.trim() || DEFAULT_INSTRUCTIONS,
noiseGate: readGateThreshold(),
};
}
/** Gate threshold (dBFS) currently shown on the slider, clamped to range. */
function readGateThreshold() {
const v = Math.round(Number(inputNoiseGate.value));
if (!Number.isFinite(v)) return GATE_OFF_DB;
return Math.min(GATE_MAX_DB, Math.max(GATE_OFF_DB, v));
}
/** Adapt the connection field to the mode learned from /api/config. */
function syncConnectionUi() {
if (pinnedUrl) {
// Deploy-pinned URL: show it, but locked — the deployment owns it.
connField.hidden = false;
inputLbUrl.value = pinnedUrl;
inputLbUrl.readOnly = true;
connHint.classList.remove("error");
connHint.textContent = "Speech-to-speech server URL pinned by this deployment.";
} else if (allowDirect) {
// Direct mode: the user sets their own s2s server URL.
connField.hidden = false;
inputLbUrl.value = settings.directUrl;
inputLbUrl.readOnly = false;
inputLbUrl.placeholder = "http://localhost:port";
connHint.classList.remove("error");
connHint.textContent =
"URL of your speech-to-speech server, e.g. http://localhost:8080 (the app adds /v1/realtime).";
} else {
// LB mode: the load balancer URL is deployment-owned — hide it entirely so
// its address is never exposed in Settings.
connField.hidden = true;
}
}
/** True when the user must supply a server URL before connecting (direct mode
* with nothing set). */
function missingServerUrl() {
return allowDirect && !pinnedUrl && !buildDirectWsUrl(settings.directUrl);
}
/** Open Settings and point the user at the empty server-URL field. */
function promptServerUrl() {
if (settingsModal.open) syncConnectionUi();
else openSettings();
connHint.textContent = "Set the speech-to-speech server URL to start.";
connHint.classList.add("error");
inputLbUrl.focus();
}
settingsForm.addEventListener("submit", (event) => {
const submitter = /** @type {HTMLButtonElement | null} */ ((/** @type {SubmitEvent} */ (event)).submitter);
if (submitter?.value !== "save") return;
settings = readSettingsFromForm();
saveSettings(settings);
// Voice + instructions can apply to a live session without reconnecting; a
// changed connection URL only takes effect on the next restart.
if (client && LIVE_STATES.has(currentState)) {
client.updateSession({ voice: settings.voice, instructions: effectiveInstructions() });
}
});
// The noise gate applies live (worklet param), so tune it without a restart:
// update the label/marker, persist, and push straight to the running client.
inputNoiseGate.addEventListener("input", () => {
setGateThreshold(readGateThreshold());
});
restartBtn.addEventListener("click", async () => {
if (currentState === "connecting") return; // a connect is already underway
settings = readSettingsFromForm();
saveSettings(settings);
if (missingServerUrl()) { promptServerUrl(); return; } // keep settings open
settingsModal.close();
// Grab the AudioContext NOW, inside the click gesture — teardown() awaits, and
// creating it afterwards would fall outside the gesture (silent on iOS).
const audioContext = createResumedAudioContext();
try {
if (client) await teardown();
await doStart(audioContext);
} catch (err) {
await handleStartError(err);
}
});
circleBtn.addEventListener("click", async () => {
try {
if (currentState === "idle" || currentState === "error") {
if (missingServerUrl()) { promptServerUrl(); return; }
await doStart();
}
} catch (err) {
await handleStartError(err);
}
});
/** A failed start is either the daily limit (show the modal, return to idle) or
* a real fault (surface it). doStart already closed any orphan AudioContext.
* @param {any} err */
async function handleStartError(err) {
if (err && err.code === "limit") {
await teardown();
account.showLimit(err.tier);
return;
}
// The user left the queue (close() aborted the wait): teardown already reset
// the UI to idle, so there's nothing to report.
if (err && err.code === "aborted") return;
// The whole waiting line is full: a warm, reassuring modal rather than an error.
if (err && err.code === "queue-full") {
await teardown();
account.showBusy();
return;
}
// Our place lapsed (ticket reaped, or the join window ran out). Recoverable, not
// a fault: land on the retry state with a kind, plain-language reason.
if (err && (err.code === "queue-expired" || err.code === "join-expired")) {
await teardown();
setState("error");
setCaption(
err.code === "join-expired"
? "Your spot expired. Tap to rejoin."
: "That took a while. Tap to rejoin.",
"error",
);
return;
}
onFatalError(err);
}
micBtn.addEventListener("click", () => {
if (!micStream || !client) return;
micMuted = !micMuted;
for (const track of micStream.getAudioTracks()) {
track.enabled = !micMuted;
}
client.setMuted(micMuted);
micBtn.classList.toggle("muted", micMuted);
micBtn.setAttribute("aria-label", micMuted ? "Unmute" : "Mute");
micBtn.title = micMuted ? "Unmute" : "Mute";
});
stopBtn.addEventListener("click", async () => {
await teardown();
});
// "Leave queue": tear down the pending connect (aborts the poll wait) and drop
// our place in line. Same teardown path as stopping a live call.
leaveQueueBtn.addEventListener("click", async () => {
await teardown();
});
// "Join now": accept the held slot. The click is a user gesture, so the client
// re-resumes the AudioContext here (iOS) before dialing.
joinQueueBtn.addEventListener("click", () => {
stopJoinCountdown();
if (client) client.join();
});
const MIC_CONSTRAINTS = {
audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true },
};
/** Prompt for mic permission up front, then immediately release the tracks so no
* recording indicator lingers during a queue wait. Throws a friendly error if the
* user denies. */
async function primeMicPermission() {
try {
const s = await navigator.mediaDevices.getUserMedia(MIC_CONSTRAINTS);
for (const track of s.getTracks()) track.stop();
} catch (err) {
throw new Error(
`Microphone access denied${err instanceof Error ? `: ${err.message}` : ""}`,
);
}
}
/** Acquire the live capture stream once a slot is granted. Permission was primed
* in the tap gesture, so this is silent. Stored module-side for mute + teardown. */
async function acquireMicStream() {
micStream = await navigator.mediaDevices.getUserMedia(MIC_CONSTRAINTS);
return micStream;
}
/** @param {number} position Update the queued caption ("You're #N in line"). */
function onQueuePosition(position) {
const n = Number(position) || 0;
setCaption(n > 0 ? `You're #${n} in line` : "Finding you a spot…", "muted");
}
// ── "Your turn" join countdown ──────────────────────────────────────────────
// While a slot is held for us, show how long is left to accept it. The client's
// join gate expires just before the load balancer reclaims the slot.
let joinCountdownTimer = 0;
/** @param {number} sec */
function startJoinCountdown(sec) {
stopJoinCountdown();
let left = Math.max(0, Math.floor(sec));
const paint = () => {
joinQueueBtn.textContent = left > 0 ? `Join now (${left}s)` : "Join now";
};
paint();
joinCountdownTimer = window.setInterval(() => {
left -= 1;
if (left <= 0) {
stopJoinCountdown();
joinQueueBtn.textContent = "Join now";
return;
}
paint();
}, 1000);
}
function stopJoinCountdown() {
if (joinCountdownTimer) {
clearInterval(joinCountdownTimer);
joinCountdownTimer = 0;
}
}
/**
* Start a conversation. Pass a pre-created AudioContext when the caller already
* made one inside the tap/click gesture (required on iOS); otherwise one is
* created here, which is still inside the gesture for a direct orb tap.
* @param {AudioContext | null} [audioContext]
*/
async function doStart(audioContext = null) {
// Resolve the target before touching mic/audio so a misconfiguration (e.g.
// direct mode with no URL) fails fast with a clear message.
const target = connectionTarget();
chat.clear();
chat.reset();
setState("connecting");
setCaption("Asking for mic…", "muted");
// Create + resume the AudioContext SYNCHRONOUSLY, still inside the gesture.
// iOS Safari only starts an AudioContext from a user gesture; if we waited
// until after the getUserMedia / session-creation awaits below, it would stay
// suspended and the whole pipeline would be silent.
if (!audioContext) audioContext = createResumedAudioContext();
// Prime the mic permission now (get the prompt out of the way up front), then
// release it. The real capture stream is acquired only once a slot is granted
// (see acquireMicStream), so the mic 'in use' indicator never lights while we
// sit in the queue. Permission persists, so the later acquire is silent.
try {
await primeMicPermission();
} catch (err) {
if (audioContext) void audioContext.close().catch(() => {});
throw err;
}
// The webcam is started on arrival (autoStartCamera), so nothing to do here;
// a still-pending grant just means the snapshot tool isn't ready yet.
const c = new S2sWsRealtimeClient({
...target,
voice: settings.voice,
instructions: effectiveInstructions(),
acquireMic: acquireMicStream,
tools: activeToolDefs(),
noiseGate: gateParams(settings.noiseGate),
...(audioContext ? { audioContext } : {}),
});
client = c;
c.addEventListener("queue", (e) => {
const { position, queueId } = /** @type {CustomEvent<{ position: number; queueId: string }>} */ (e).detail;
if (queueId) queuedTicketId = queueId;
onQueuePosition(position);
});
c.addEventListener("ready-to-join", (e) => {
const { info, expiresSec } = /** @type {CustomEvent<{ info: import("./ws/s2s-ws-client.js").WsSessionInfo; expiresSec: number }>} */ (e).detail;
// A slot is held for us. We're out of the queue now, so drop the ticket ref.
// Track the granted session id already so that leaving (or letting the timer
// lapse) refunds the budget the server reserved at claim, even before we dial.
queuedTicketId = "";
if (info?.sessionId) {
trackedSessionId = info.sessionId;
trackedTier = info.tier || "anon";
}
startJoinCountdown(expiresSec);
});
c.addEventListener("status", (e) => {
const detail = /** @type {CustomEvent<{ status: string }>} */ (e).detail;
onClientStatus(detail.status);
});
c.addEventListener("transcript", (e) => {
const d = /** @type {CustomEvent<{ role: "user" | "assistant"; text: string; partial: boolean; itemId?: string; responseId?: string }>} */ (e).detail;
chat.onTranscript(d);
});
c.addEventListener("response-finished", (e) => {
const detail = /** @type {CustomEvent<{ responseId: string; status: string; audible?: boolean; transcript?: string }>} */ (e).detail;
chat.onResponseFinished(detail);
});
c.addEventListener("toolcall", (e) => {
const { name, arguments: args, callId } = /** @type {CustomEvent<{ name: string; arguments: string; callId: string }>} */ (e).detail;
chat.onToolCall(name);
// Execute the tool, then push it to the conversation once the result is in,
// so the toggle shows both the call input and its output together.
void runTool(name, args, callId).then(({ output, image }) => {
chat.onToolResult(name, args, output, image);
});
});
c.addEventListener("error", (e) => {
const detail = /** @type {CustomEvent<{ error: unknown }>} */ (e).detail;
onFatalError(detail.error);
});
c.addEventListener("server-error", (e) => {
// Non-fatal: the backend reported an error mid-session. Log it, keep the
// socket and the conversation alive (the model can recover on its own).
const detail = /** @type {CustomEvent<{ error: unknown }>} */ (e).detail;
const msg = detail.error instanceof Error ? detail.error.message : String(detail.error);
console.warn("[main] server error (non-fatal):", msg);
});
c.addEventListener("session", (e) => {
const info = /** @type {CustomEvent<{ info: import("./ws/s2s-ws-client.js").WsSessionInfo }>} */ (e).detail.info;
console.log("[ws] session created:", info.sessionId);
// A slot was granted — we're out of the queue; drop the ticket reference so
// teardown doesn't try to leave a line we already left.
queuedTicketId = "";
// A metered tier (anon / free): heartbeat so the server can extend the
// reservation and tell us when the daily budget runs out. PRO isn't limited.
if (info.limited && info.sessionId) {
trackedSessionId = info.sessionId;
trackedTier = info.tier || "anon";
startHeartbeat(info.heartbeatSec || 5);
}
});
c.addEventListener("input-level", (e) => {
const { rms } = /** @type {CustomEvent<{ rms: number }>} */ (e).detail;
paintInputLevel(rms);
});
try {
await c.connect();
} catch (err) {
// The grant can be refused (402 → limit) or the dial can fail. In LB mode
// the AudioContext hasn't been adopted by the client yet (the session POST
// runs first), so close the one we created here to avoid leaking it.
if (audioContext) void audioContext.close().catch(() => {});
throw err;
}
}
// ── Conversation-time heartbeat ─────────────────────────────────────────────
/** Ping the server every `sec` seconds so it can meter the live session; when
* it reports the daily budget is spent, cut the call and show the limit modal.
* @param {number} sec */
function startHeartbeat(sec) {
stopHeartbeat();
heartbeatTimer = window.setInterval(async () => {
if (!trackedSessionId) return;
try {
const res = await fetch("api/session/heartbeat", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ sessionId: trackedSessionId }),
keepalive: true,
});
const json = await res.json().catch(() => ({}));
if (json.expired) await onLimitReached();
} catch (err) {
// A transient network blip shouldn't kill the call; the next tick retries.
if (DEBUG) console.debug("[ui] heartbeat failed:", err);
}
}, Math.max(1, sec) * 1000);
}
function stopHeartbeat() {
if (heartbeatTimer) {
clearInterval(heartbeatTimer);
heartbeatTimer = 0;
}
}
/** The server cut the live session: tear down and explain why. */
async function onLimitReached() {
const tier = trackedTier;
stopHeartbeat();
await teardown();
account.showLimit(tier);
}
/** Tell the server a session ended so it reconciles + refunds the unused chunk.
* Uses sendBeacon so it still fires when the tab is closing. */
function endTrackedSession() {
if (!trackedSessionId) return;
const body = JSON.stringify({ sessionId: trackedSessionId });
try {
const blob = new Blob([body], { type: "application/json" });
if (!navigator.sendBeacon("api/session/end", blob)) {
void fetch("api/session/end", {
method: "POST", headers: { "Content-Type": "application/json" }, body, keepalive: true,
}).catch(() => {});
}
} catch {
// Best-effort; the server sweep reaps the session anyway.
}
trackedSessionId = "";
trackedTier = "";
}
/** Leave the waiting queue so the LB frees our place. sendBeacon so it still
* fires on tab close; the LB also reaps the ticket on TTL as a backstop. */
function endQueueTicket() {
if (!queuedTicketId) return;
const body = JSON.stringify({ queueId: queuedTicketId });
try {
const blob = new Blob([body], { type: "application/json" });
if (!navigator.sendBeacon("api/queue/end", blob)) {
void fetch("api/queue/end", {
method: "POST", headers: { "Content-Type": "application/json" }, body, keepalive: true,
}).catch(() => {});
}
} catch {
// Best-effort; the LB reaps the ticket on TTL anyway.
}
queuedTicketId = "";
}
/** @param {string} status */
function onClientStatus(status) {
switch (status) {
case "creating-session":
case "connecting":
setState("connecting");
break;
case "queued":
setState("queued");
break;
case "your-turn":
setState("your-turn");
break;
case "connected":
setState("listening");
break;
case "user-speaking":
setState("user-speaking");
break;
case "processing":
setState("processing");
break;
case "ai-speaking":
setState("ai-speaking");
break;
case "closed":
// teardown() will move us to idle
break;
case "error":
setState("error");
break;
}
}
async function teardown() {
stopHeartbeat();
stopJoinCountdown();
endTrackedSession();
endQueueTicket();
chat.reset({ dismiss: true });
if (client) {
try {
await client.close();
} catch (err) {
console.warn("[main] error closing client:", err);
}
client = null;
}
if (micStream) {
for (const track of micStream.getTracks()) track.stop();
micStream = null;
}
// The webcam is independent of the call lifecycle (it runs while the user is
// on the page), so we leave it on here — only the camera toggle stops it.
micMuted = false;
micBtn.classList.remove("muted");
setState("idle");
// Refresh the chip's remaining-today after the budget moved.
if (limiterOn) void account.refresh();
}
/** @param {unknown} err */
function onFatalError(err) {
console.error("[main] fatal:", err);
setState("error");
const message = err instanceof Error ? err.message : String(err);
setCaption(truncateError(message), "error");
void teardown().catch(() => {
setState("error");
setCaption(truncateError(message), "error");
});
}
setState("idle");
chat.renderEmptyState();
initGateArc();
void fetchConfig();
// Start the webcam as soon as the user lands (camera tool defaults on), and
// react to later permission changes (re-grant after a denial re-enables it).
void autoStartCamera();
void watchCameraPermission();
// Reconcile a live session if the tab is closed/hidden mid-call (no teardown).
window.addEventListener("pagehide", () => { endTrackedSession(); endQueueTicket(); });
requestAnimationFrame(() => {
document.body.classList.remove("booting");
});