From 134c95bd3b11f30b55c3f330a796bfbf995f5129 Mon Sep 17 00:00:00 2001 From: Brent Rager Date: Fri, 17 Jul 2026 23:36:17 -0400 Subject: [PATCH] SMOODEV-2674: Speaker toggle (STT-only voice) + voice-first thread adoption A speaker button beside the mic flips agent speech per-session: off sends tts:false in the start frame so the server skips TTS entirely and the visitor speaks-and-reads. voice.tts config sets the default. startVoice now connects the text session first when no conversation exists, so voice-first visitors land in a real thread that text turns continue. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Tz7j9eqPBBL36CEFZpjoWz --- .changeset/stt-only-voice.md | 5 +++ src/config.ts | 11 +++++-- src/element.test.ts | 64 +++++++++++++++++++++++++++++++++++- src/element.ts | 50 ++++++++++++++++++++++++---- src/styles.ts | 12 ++++--- src/voice-session.test.ts | 14 +++++++- src/voice-session.ts | 7 ++++ 7 files changed, 147 insertions(+), 16 deletions(-) create mode 100644 .changeset/stt-only-voice.md diff --git a/.changeset/stt-only-voice.md b/.changeset/stt-only-voice.md new file mode 100644 index 0000000..50b7e27 --- /dev/null +++ b/.changeset/stt-only-voice.md @@ -0,0 +1,5 @@ +--- +'@smooai/chat-widget': minor +--- + +Voice: speak-and-read mode + seamless voice→text continuity (SMOODEV-2674). A speaker toggle beside the mic lets visitors turn agent speech off — sessions start STT-only (`tts:false` on the browser-voice start frame; the server skips TTS entirely) while replies still arrive as chat bubbles. New `voice.tts` config sets the default. Starting voice now connects the text session first, so a voice-first visitor lands in a real conversation that continues seamlessly when they switch back to typing. diff --git a/src/config.ts b/src/config.ts index 77c4575..c94081e 100644 --- a/src/config.ts +++ b/src/config.ts @@ -18,6 +18,13 @@ export interface ChatWidgetVoiceConfig { enabled?: boolean; /** Browser-voice WS endpoint. Defaults to the hosted SmooAI voice service. */ url?: string; + /** + * Whether the agent SPEAKS its replies (SMOODEV-2674). `false` starts voice + * sessions in STT-only mode: the visitor talks, the replies arrive as text + * only. The visitor can flip this per-session with the speaker toggle. + * Default `true`. + */ + tts?: boolean; } export interface ChatWidgetTheme { @@ -175,7 +182,7 @@ export type ResolvedTheme = Required> & { theme: ResolvedTheme; - voice: { enabled: boolean; url: string }; + voice: { enabled: boolean; url: string; tts: boolean }; userName?: string; userEmail?: string; userPhone?: string; @@ -221,7 +228,7 @@ export function resolveConfig(config: ChatWidgetConfig): ResolvedConfig { allowChatRestore: config.allowChatRestore ?? true, allowAnonymous: config.allowAnonymous ?? false, showToolActivity: config.showToolActivity ?? false, - voice: { enabled: config.voice?.enabled ?? false, url: config.voice?.url ?? DEFAULT_VOICE_URL }, + voice: { enabled: config.voice?.enabled ?? false, url: config.voice?.url ?? DEFAULT_VOICE_URL, tts: config.voice?.tts ?? true }, theme: { text: theme.text ?? '#f8fafc', background: theme.background ?? '#040d30', diff --git a/src/element.test.ts b/src/element.test.ts index ac31c21..72f00b6 100644 --- a/src/element.test.ts +++ b/src/element.test.ts @@ -554,17 +554,79 @@ describe('voice config gating (SMOODEV-2534)', () => { disconnect: () => {}, }; await priv.startVoice(); - const session = priv.voiceSession as unknown as { opts: { url?: string; agentId: string; conversationId?: string } } | null; + const session = priv.voiceSession as unknown as { opts: { url?: string; agentId: string; conversationId?: string; tts?: boolean } } | null; expect(session).not.toBeNull(); expect(session!.opts.url).toBe('wss://voice.test/ws'); expect(session!.opts.agentId).toBe('a1'); // The text thread's conversation id rides along so voice resumes it. expect(session!.opts.conversationId).toBe('conv-42'); + // Agent speech defaults ON. + expect(session!.opts.tts).toBe(true); const mic = sr.querySelector('.composer .mic') as HTMLButtonElement; expect(mic.classList.contains('active')).toBe(true); expect(mic.getAttribute('aria-pressed')).toBe('true'); expect(mic.getAttribute('aria-label')).toBe('Stop voice'); } + + it('renders the agent-speech toggle; flipping it starts STT-only sessions (SMOODEV-2674)', async () => { + const origStart = VoiceSession.prototype.start; + VoiceSession.prototype.start = () => Promise.resolve(); + try { + const el = mountCfg({ voice: { enabled: true, url: 'wss://voice.test/ws' } }); + const sr = el.shadowRoot!; + const speech = sr.querySelector('.composer .speech') as HTMLButtonElement; + expect(speech).not.toBeNull(); + expect(speech.getAttribute('aria-pressed')).toBe('true'); // speaks by default + speech.click(); + expect(speech.getAttribute('aria-pressed')).toBe('false'); + expect(speech.classList.contains('off')).toBe(true); + const priv = el as unknown as { startVoice: () => Promise; voiceSession: { opts: { tts?: boolean } } | null; controller: unknown }; + priv.controller = { currentConversationId: 'conv-42', appendLocalMessage: () => {}, connect: () => Promise.resolve(), disconnect: () => {} }; + await priv.startVoice(); + expect(priv.voiceSession!.opts.tts).toBe(false); + } finally { + VoiceSession.prototype.start = origStart; + } + }); + + it('voice.tts:false config starts sessions STT-only by default', () => { + const sr = mountCfg({ voice: { enabled: true, tts: false } }).shadowRoot!; + const speech = sr.querySelector('.composer .speech') as HTMLButtonElement; + expect(speech.getAttribute('aria-pressed')).toBe('false'); + expect(speech.classList.contains('off')).toBe(true); + }); + + it('voice-first: startVoice connects the text session so voice joins a real thread (SMOODEV-2674)', async () => { + const origStart = VoiceSession.prototype.start; + VoiceSession.prototype.start = () => Promise.resolve(); + try { + const el = mountCfg({ voice: { enabled: true, url: 'wss://voice.test/ws' } }); + const priv = el as unknown as { + startVoice: () => Promise; + voiceSession: { opts: { conversationId?: string } } | null; + controller: unknown; + }; + // No conversation yet; connect() establishes one (the voice-first path). + let connected = false; + const stub = { + get currentConversationId() { + return connected ? 'conv-fresh' : null; + }, + appendLocalMessage: () => {}, + connect: () => { + connected = true; + return Promise.resolve(); + }, + disconnect: () => {}, + }; + priv.controller = stub; + await priv.startVoice(); + expect(connected).toBe(true); + expect(priv.voiceSession!.opts.conversationId).toBe('conv-fresh'); + } finally { + VoiceSession.prototype.start = origStart; + } + }); }); describe('Rich Interactions card registry', () => { diff --git a/src/element.ts b/src/element.ts index 3ce271a..dd17ac3 100644 --- a/src/element.ts +++ b/src/element.ts @@ -89,6 +89,10 @@ const ICON = { tool: ``, /** Voice toggle — a microphone. */ mic: ``, + /** Agent-speech toggle ON — a speaker with waves. */ + speaker: ``, + /** Agent-speech toggle OFF — a muted speaker. */ + speakerOff: ``, } as const; /** @@ -281,7 +285,10 @@ export class SmoothAgentChatElement extends HTMLElement { /** True while the pre-chat identity gate is showing (blocks premature connect). */ private gating = false; /** Voice config (SMOODEV-2534) — enabled=false renders zero voice UI. */ - private voiceCfg: { enabled: boolean; url: string } = { enabled: false, url: '' }; + private voiceCfg: { enabled: boolean; url: string; tts: boolean } = { enabled: false, url: '', tts: true }; + /** Agent speech on/off (SMOODEV-2674) — applies at the next voice session start. */ + private voiceTtsOn = true; + private speechBtn: HTMLButtonElement | null = null; /** Live voice session, or null when voice is off. */ private voiceSession: VoiceSession | null = null; @@ -538,8 +545,9 @@ export class SmoothAgentChatElement extends HTMLElement { const footerHtml = footerInner ? `
${footerInner}
` : ''; // Voice (SMOODEV-2534): mic toggle in the composer, only when enabled. this.voiceCfg = resolved.voice; + this.voiceTtsOn = resolved.voice.tts; const micHtml = resolved.voice.enabled - ? `` + ? `` : ''; const chatHtml = `
@@ -583,12 +591,14 @@ export class SmoothAgentChatElement extends HTMLElement { this.inputEl = container.querySelector('textarea'); this.sendBtn = container.querySelector('.send'); this.micBtn = container.querySelector('.mic'); + this.speechBtn = container.querySelector('.speech'); this.interruptEl = container.querySelector('.interrupt'); this.launcherEl?.addEventListener('click', () => this.openChat()); container.querySelector('.close')?.addEventListener('click', () => this.closeChat()); this.sendBtn?.addEventListener('click', () => this.submit()); this.micBtn?.addEventListener('click', () => this.toggleVoice()); + this.speechBtn?.addEventListener('click', () => this.toggleAgentSpeech()); this.inputEl?.addEventListener('input', () => this.autosize()); this.inputEl?.addEventListener('keydown', (ev) => { if (ev.key === 'Enter' && !ev.shiftKey) { @@ -1573,18 +1583,44 @@ export class SmoothAgentChatElement extends HTMLElement { void this.startVoice(); } + /** + * Flip agent speech on/off (SMOODEV-2674). Session-scoped: the choice is + * sent in the voice start frame (`tts:false` = STT-only, the server skips + * TTS entirely), so flipping mid-session takes effect on the NEXT session — + * the titles say so. Mid-session, turning speech off also cuts any audio + * that is currently playing. + */ + private toggleAgentSpeech(): void { + this.voiceTtsOn = !this.voiceTtsOn; + if (!this.voiceTtsOn && this.voiceSession?.isSpeaking) this.voiceSession.interrupt(); + const btn = this.speechBtn; + if (!btn) return; + btn.classList.toggle('off', !this.voiceTtsOn); + btn.setAttribute('aria-pressed', String(this.voiceTtsOn)); + btn.title = this.voiceTtsOn + ? 'Agent speaks replies — click for speak-and-read (no agent audio)' + : this.voiceSession + ? 'Agent voice off for the next session — replies stay text only' + : 'Agent voice off — replies are text only. Click to hear the agent'; + btn.innerHTML = this.voiceTtsOn ? ICON.speaker : ICON.speakerOff; // static, trusted + } + private async startVoice(): Promise { if (!this.controller || this.voiceSession || !this.voiceCfg.enabled) return; const config = this.readConfig(); if (!config) return; - // Best-effort: join the text thread when one exists. If the text session - // hasn't connected yet, voice starts a fresh thread (the frozen protocol - // never returns the voice-created conversation id, so it can't be adopted - // for later text turns — tracked as a follow-up on the server protocol). + // Voice and text share ONE conversation. If the text session hasn't + // connected yet (voice-first visitor), connect it now — voice turns then + // land in a REAL conversation the text thread already owns, so leaving + // voice continues seamlessly in text (SMOODEV-2674). connect() is + // idempotent; on failure voice still starts (server mints a thread). + if (!this.controller.currentConversationId) { + await this.controller.connect().catch(() => {}); + } const conversationId = this.controller.currentConversationId ?? undefined; const controller = this.controller; const session = new VoiceSession( - { url: this.voiceCfg.url, agentId: config.agentId, conversationId }, + { url: this.voiceCfg.url, agentId: config.agentId, conversationId, tts: this.voiceTtsOn }, { onTranscriptPartial: (text) => { // Live partial transcript in the input area while listening. diff --git a/src/styles.ts b/src/styles.ts index b539687..bed5b56 100644 --- a/src/styles.ts +++ b/src/styles.ts @@ -555,8 +555,9 @@ ${ .send:active { transform: scale(.94); } .send:disabled { opacity: .4; cursor: default; transform: none; box-shadow: none; } -/* Voice mic toggle (SMOODEV-2534) — ghost twin of .send; lights up while live. */ -.mic { +/* Voice mic toggle (SMOODEV-2534) — ghost twin of .send; lights up while live. + .speech (SMOODEV-2674) is the agent-speech on/off twin next to it. */ +.mic, .speech { width: 38px; height: 38px; flex: none; border: 1px solid color-mix(in srgb, var(--sac-border) 80%, transparent); @@ -569,9 +570,10 @@ ${ color: color-mix(in srgb, var(--sac-text) 65%, transparent); transition: transform .2s var(--sac-ease), color .2s ease, background .25s ease, box-shadow .25s ease; } -.mic svg { width: 18px; height: 18px; } -.mic:hover { color: var(--sac-text); transform: translateY(-1px) scale(1.05); } -.mic:active { transform: scale(.94); } +.mic svg, .speech svg { width: 18px; height: 18px; } +.mic:hover, .speech:hover { color: var(--sac-text); transform: translateY(-1px) scale(1.05); } +.mic:active, .speech:active { transform: scale(.94); } +.speech.off { opacity: .55; } .mic.active { border-color: transparent; background: linear-gradient(150deg, var(--sac-primary), var(--sac-primary-2)); diff --git a/src/voice-session.test.ts b/src/voice-session.test.ts index 25ad831..220b34b 100644 --- a/src/voice-session.test.ts +++ b/src/voice-session.test.ts @@ -94,7 +94,7 @@ class MockPlayer implements VoicePlayer { /** Build a session wired to mocks. `harness.mic(samples, rate)` injects a mic frame. */ function makeSession( - opts: { conversationId?: string; token?: string; bargeInThreshold?: number } = {}, + opts: { conversationId?: string; token?: string; bargeInThreshold?: number; tts?: boolean } = {}, events: VoiceSessionEvents = {}, ): { session: VoiceSession; ws: MockVoiceSocket; player: MockPlayer; mic: (samples: Float32Array, rate: number) => void; captureStopped: () => boolean } { const ws = new MockVoiceSocket(); @@ -148,6 +148,18 @@ describe('VoiceSession protocol framing', () => { expect(ws.jsonFrames()[0]).toEqual({ type: 'start', agent_id: 'agent-123', conversation_id: 'conv-9', token: 'jwt-abc' }); }); + it('sends tts:false in the start frame for STT-only sessions (SMOODEV-2674)', async () => { + const { session, ws } = makeSession({ tts: false }); + await session.start(); + ws.open(); + expect(ws.jsonFrames()[0]).toEqual({ type: 'start', agent_id: 'agent-123', tts: false }); + // Default (tts unset/true) omits the field — frozen-protocol compatible. + const { session: s2, ws: ws2 } = makeSession({ tts: true }); + await s2.start(); + ws2.open(); + expect(ws2.jsonFrames()[0]).toEqual({ type: 'start', agent_id: 'agent-123' }); + }); + it('streams mic frames as 16 kHz Int16 binary after the start frame', async () => { const { session, ws, mic } = makeSession(); await session.start(); diff --git a/src/voice-session.ts b/src/voice-session.ts index 5de85de..73f53b3 100644 --- a/src/voice-session.ts +++ b/src/voice-session.ts @@ -334,6 +334,12 @@ export interface VoiceSessionOptions { conversationId?: string; /** Optional JWT for authenticated contexts (public agents auth by Origin). */ token?: string; + /** + * Agent speech (SMOODEV-2674). `false` = STT-only: the visitor speaks and + * READS the replies — the server skips TTS entirely (no audio frames, no + * speaking events) and only streams `reply_text`. Default `true`. + */ + tts?: boolean; /** * RMS level (0..1) above which a mic frame counts as speech for barge-in * while the agent is speaking. Default 0.02 — comfortably above room noise @@ -392,6 +398,7 @@ export class VoiceSession { const start: Record = { type: 'start', agent_id: this.opts.agentId }; if (this.opts.conversationId) start.conversation_id = this.opts.conversationId; if (this.opts.token) start.token = this.opts.token; + if (this.opts.tts === false) start.tts = false; ws.send(JSON.stringify(start)); this.state = 'active'; });