diff --git a/components/speech/audioCapture.ts b/components/speech/audioCapture.ts new file mode 100644 index 00000000..76312115 --- /dev/null +++ b/components/speech/audioCapture.ts @@ -0,0 +1,210 @@ +import { + MicrophoneError, + type MicrophoneAudioFrame, + type MicrophoneAudioFrameSource, + type MicrophoneSession, +} from "./microphone"; + +export const TRANSCRIPTION_PCM_SAMPLE_RATE = 16000 as const; + +export interface PcmAudioChunk { + readonly samples: Int16Array; + readonly sampleRate: typeof TRANSCRIPTION_PCM_SAMPLE_RATE; + readonly capturedAtMs: number; +} + +export type PcmAudioChunkListener = (chunk: PcmAudioChunk) => void; + +export interface PcmMicrophoneCaptureOptions { + readonly outputChunkDurationMs?: number; +} + +const DEFAULT_OUTPUT_CHUNK_DURATION_MS = 100; + +export const float32ToPcm16 = (samples: Float32Array): Int16Array => { + const output = new Int16Array(samples.length); + for (let index = 0; index < samples.length; index += 1) { + const sample = Math.max(-1, Math.min(1, samples[index])); + output[index] = + sample < 0 ? Math.round(sample * 32768) : Math.round(sample * 32767); + } + return output; +}; + +export class StreamingLinearResampler { + private readonly inputSampleRate: number; + private readonly outputSampleRate: number; + private readonly step: number; + private sourcePosition = 0; + private previousSample?: number; + + constructor(inputSampleRate: number, outputSampleRate: number) { + if ( + !Number.isFinite(inputSampleRate) || + inputSampleRate <= 0 || + !Number.isFinite(outputSampleRate) || + outputSampleRate <= 0 + ) { + throw new MicrophoneError( + "invalidConfiguration", + "Audio sample rates must be finite positive numbers.", + ); + } + this.inputSampleRate = inputSampleRate; + this.outputSampleRate = outputSampleRate; + this.step = inputSampleRate / outputSampleRate; + } + + reset(): void { + this.sourcePosition = 0; + this.previousSample = undefined; + } + + process(input: Float32Array): Float32Array { + if (input.length === 0) return new Float32Array(); + if (this.inputSampleRate === this.outputSampleRate) { + return new Float32Array(input); + } + + const samples = + this.previousSample === undefined + ? input + : Float32Array.from([this.previousSample, ...input]); + const output: number[] = []; + while (this.sourcePosition + 1 < samples.length) { + const left = Math.floor(this.sourcePosition); + const fraction = this.sourcePosition - left; + output.push( + samples[left] + (samples[left + 1] - samples[left]) * fraction, + ); + this.sourcePosition += this.step; + } + + this.sourcePosition -= samples.length - 1; + this.previousSample = samples[samples.length - 1]; + return Float32Array.from(output); + } +} + +/** + * Converts a microphone session into gated, provider-ready PCM chunks. The + * microphone remains open while capture is stopped, so a task can prepare its + * resources before a timing-sensitive stimulus onset. + */ +export class PcmMicrophoneCapture { + private readonly microphone: MicrophoneSession; + private readonly outputChunkSamples: number; + private readonly listeners = new Set(); + private resampler?: StreamingLinearResampler; + private pendingSamples: number[] = []; + private frameSource?: MicrophoneAudioFrameSource; + private initializePromise?: Promise; + private active = false; + private closed = false; + + constructor( + microphone: MicrophoneSession, + options: PcmMicrophoneCaptureOptions = {}, + ) { + this.microphone = microphone; + const durationMs = + options.outputChunkDurationMs ?? DEFAULT_OUTPUT_CHUNK_DURATION_MS; + if (!Number.isFinite(durationMs) || durationMs <= 0) { + throw new MicrophoneError( + "invalidConfiguration", + "outputChunkDurationMs must be a finite positive number.", + ); + } + this.outputChunkSamples = Math.max( + 1, + Math.round((TRANSCRIPTION_PCM_SAMPLE_RATE * durationMs) / 1000), + ); + } + + initialize(): Promise { + if (this.closed) { + return Promise.reject( + new MicrophoneError( + "sessionClosed", + "Cannot initialize a closed PCM microphone capture.", + ), + ); + } + this.initializePromise ??= this.microphone + .subscribeToAudioFrames(this.handleAudioFrame) + .then((frameSource) => { + this.frameSource = frameSource; + }); + return this.initializePromise; + } + + start(): void { + if (this.closed || !this.frameSource) { + throw new MicrophoneError( + this.closed ? "sessionClosed" : "invalidConfiguration", + "PCM microphone capture must be initialized before it starts.", + ); + } + this.pendingSamples = []; + this.resampler?.reset(); + this.active = true; + this.frameSource.start(); + } + + stop(): void { + if (!this.active) return; + this.frameSource?.stop(); + this.active = false; + this.emitPending(true); + } + + subscribe(listener: PcmAudioChunkListener): () => void { + this.listeners.add(listener); + return () => this.listeners.delete(listener); + } + + async close(): Promise { + if (this.closed) return; + this.stop(); + this.closed = true; + try { + await this.initializePromise; + } catch { + // Initialization already reported its failure to the caller. + } + this.frameSource?.close(); + this.frameSource = undefined; + this.listeners.clear(); + } + + private readonly handleAudioFrame = (frame: MicrophoneAudioFrame): void => { + if (!this.active || this.closed) return; + if (!this.resampler) { + this.resampler = new StreamingLinearResampler( + frame.sampleRate, + TRANSCRIPTION_PCM_SAMPLE_RATE, + ); + } + const resampled = this.resampler.process(frame.samples); + for (const sample of resampled) this.pendingSamples.push(sample); + this.emitPending(false, frame.capturedAtMs); + }; + + private emitPending(flush: boolean, capturedAtMs = performance.now()): void { + while ( + this.pendingSamples.length >= this.outputChunkSamples || + (flush && this.pendingSamples.length > 0) + ) { + const count = flush + ? Math.min(this.outputChunkSamples, this.pendingSamples.length) + : this.outputChunkSamples; + const samples = Float32Array.from(this.pendingSamples.splice(0, count)); + const chunk: PcmAudioChunk = { + samples: float32ToPcm16(samples), + sampleRate: TRANSCRIPTION_PCM_SAMPLE_RATE, + capturedAtMs, + }; + for (const listener of this.listeners) listener(chunk); + } + } +} diff --git a/components/speech/deepgramRealtimeTranscriber.ts b/components/speech/deepgramRealtimeTranscriber.ts new file mode 100644 index 00000000..e459c88c --- /dev/null +++ b/components/speech/deepgramRealtimeTranscriber.ts @@ -0,0 +1,522 @@ +import { + TRANSCRIPTION_PCM_SAMPLE_RATE, + TranscriberError, + type PcmAudioChunk, + type StreamingTranscriber, + type TranscriberConnectionInfo, + type TranscriberEvent, + type TranscriberListener, + type TranscriberState, +} from "./transcriber"; +import { + createSpeechTokenProvider, + type SpeechTokenProviderOptions, +} from "./speechToken"; + +const PROVIDER = "deepgram"; +const DEFAULT_MODEL = "nova-3"; +const DEFAULT_BASE_URL = "wss://api.deepgram.com/v1/listen"; +const DEFAULT_CONNECT_TIMEOUT_MS = 8000; +const WEBSOCKET_OPEN = 1; +const WEBSOCKET_CLOSED = 3; + +export type DeepgramTokenProvider = () => Promise; + +interface DeepgramAlternative { + readonly transcript?: unknown; +} + +interface DeepgramMessage { + readonly type?: unknown; + readonly is_final?: unknown; + readonly speech_final?: unknown; + readonly from_finalize?: unknown; + readonly request_id?: unknown; + readonly channel?: { readonly alternatives?: readonly DeepgramAlternative[] }; + readonly err_code?: unknown; + readonly err_msg?: unknown; +} + +export interface DeepgramWebSocketLike { + readonly readyState: number; + addEventListener(type: "open", listener: () => void): void; + addEventListener( + type: "message", + listener: (event: MessageEvent) => void, + ): void; + addEventListener(type: "error", listener: (event: unknown) => void): void; + addEventListener(type: "close", listener: (event: CloseEvent) => void): void; + send(data: string | ArrayBuffer): void; + close(code?: number, reason?: string): void; +} + +export type DeepgramWebSocketFactory = ( + url: string, + protocols: readonly string[], +) => DeepgramWebSocketLike; + +export interface DeepgramRealtimeTranscriberOptions { + readonly tokenProvider: DeepgramTokenProvider; + readonly languageCode: string; + readonly keyterms?: readonly string[]; + readonly targetKeytermBiasEnabled: boolean; + readonly endpointingMs: number; + readonly model?: string; + readonly baseUrl?: string; + readonly connectTimeoutMs?: number; + readonly webSocketFactory?: DeepgramWebSocketFactory; + readonly now?: () => number; +} + +export type DeepgramTokenProviderOptions = SpeechTokenProviderOptions; + +const validateLanguageCode = (languageCode: string): string => { + const value = languageCode.trim(); + if (!/^[A-Za-z]{2,3}(?:-[A-Za-z0-9]{2,8})*$/.test(value)) { + throw new TranscriberError( + "invalidConfiguration", + "Deepgram languageCode must be a valid language or locale code.", + ); + } + return value; +}; + +const normalizeKeyterms = (keyterms: readonly string[] = []): string[] => { + const values = [...new Set(keyterms.map((value) => value.trim()))]; + if (values.some((value) => !value)) { + throw new TranscriberError( + "invalidConfiguration", + "Deepgram keyterms cannot be empty.", + ); + } + return values; +}; + +export const buildDeepgramRealtimeUrl = ( + options: Omit, +): string => { + if (!Number.isFinite(options.endpointingMs) || options.endpointingMs < 10) { + throw new TranscriberError( + "invalidConfiguration", + "Deepgram endpointingMs must be at least 10 milliseconds.", + ); + } + const url = new URL(options.baseUrl ?? DEFAULT_BASE_URL); + url.searchParams.set("model", options.model?.trim() || DEFAULT_MODEL); + url.searchParams.set("language", validateLanguageCode(options.languageCode)); + url.searchParams.set("encoding", "linear16"); + url.searchParams.set("sample_rate", String(TRANSCRIPTION_PCM_SAMPLE_RATE)); + url.searchParams.set("channels", "1"); + url.searchParams.set("interim_results", "true"); + url.searchParams.set("endpointing", String(options.endpointingMs)); + url.searchParams.set("mip_opt_out", "true"); + if (options.targetKeytermBiasEnabled) { + for (const keyterm of normalizeKeyterms(options.keyterms)) { + url.searchParams.append("keyterm", keyterm); + } + } + return url.toString(); +}; + +export const createDeepgramTokenProvider = ( + options: DeepgramTokenProviderOptions, +): DeepgramTokenProvider => createSpeechTokenProvider(PROVIDER, options); + +const defaultWebSocketFactory: DeepgramWebSocketFactory = (url, protocols) => { + if (typeof WebSocket === "undefined") { + throw new TranscriberError( + "unsupported", + "Realtime transcription is unavailable in this browser.", + ); + } + return new WebSocket(url, [...protocols]); +}; + +const pcm16LittleEndianBuffer = (samples: Int16Array): ArrayBuffer => { + const buffer = new ArrayBuffer(samples.length * 2); + const view = new DataView(buffer); + for (let index = 0; index < samples.length; index += 1) { + view.setInt16(index * 2, samples[index], true); + } + return buffer; +}; + +export class DeepgramRealtimeTranscriber implements StreamingTranscriber { + private readonly options: DeepgramRealtimeTranscriberOptions; + private readonly webSocketFactory: DeepgramWebSocketFactory; + private readonly now: () => number; + private readonly listeners = new Set(); + private stateValue: TranscriberState = "idle"; + private socket?: DeepgramWebSocketLike; + private info?: TranscriberConnectionInfo; + private activeUtteranceId?: string; + private committedParts: string[] = []; + private connectPromise?: Promise; + private resolveConnect?: (info: TranscriberConnectionInfo) => void; + private rejectConnect?: (error: TranscriberError) => void; + private connectTimeoutId?: ReturnType; + private closeExpected = false; + private closedEventSent = false; + + constructor(options: DeepgramRealtimeTranscriberOptions) { + if (!options || typeof options.tokenProvider !== "function") { + throw new TranscriberError( + "invalidConfiguration", + "A Deepgram token provider is required.", + ); + } + validateLanguageCode(options.languageCode); + buildDeepgramRealtimeUrl(options); + const timeout = options.connectTimeoutMs ?? DEFAULT_CONNECT_TIMEOUT_MS; + if (!Number.isFinite(timeout) || timeout <= 0) { + throw new TranscriberError( + "invalidConfiguration", + "Connection timeout must be a finite positive number.", + ); + } + this.options = { ...options, connectTimeoutMs: timeout }; + this.webSocketFactory = options.webSocketFactory ?? defaultWebSocketFactory; + this.now = options.now ?? (() => performance.now()); + } + + get state(): TranscriberState { + return this.stateValue; + } + + get connectionInfo(): TranscriberConnectionInfo | undefined { + return this.info; + } + + connect(): Promise { + if (this.connectPromise) return this.connectPromise; + if (this.stateValue !== "idle") { + return Promise.reject( + new TranscriberError( + this.stateValue === "closed" ? "closed" : "connectionFailure", + "The Deepgram connection cannot be started again.", + ), + ); + } + this.stateValue = "connecting"; + this.connectPromise = this.performConnect(); + return this.connectPromise; + } + + beginUtterance(utteranceId: string): void { + this.assertReady(); + const id = utteranceId.trim(); + if (!id) { + throw new TranscriberError( + "invalidConfiguration", + "A non-empty utterance identifier is required.", + ); + } + if (this.activeUtteranceId) { + throw new TranscriberError( + "utteranceConflict", + "A transcription utterance is already active.", + ); + } + this.activeUtteranceId = id; + this.committedParts = []; + } + + sendAudio(utteranceId: string, chunk: PcmAudioChunk): void { + this.assertActiveUtterance(utteranceId); + if ( + chunk.sampleRate !== TRANSCRIPTION_PCM_SAMPLE_RATE || + !(chunk.samples instanceof Int16Array) || + chunk.samples.length === 0 + ) { + throw new TranscriberError( + "invalidAudio", + "Realtime audio must be non-empty 16-bit PCM at 16 kHz.", + ); + } + this.socket?.send(pcm16LittleEndianBuffer(chunk.samples)); + } + + requestCommit(utteranceId: string): void { + this.assertActiveUtterance(utteranceId); + this.socket?.send(JSON.stringify({ type: "Finalize" })); + } + + endUtterance(utteranceId: string): void { + this.assertActiveUtterance(utteranceId); + this.activeUtteranceId = undefined; + this.committedParts = []; + } + + cancelUtterance(utteranceId: string): void { + this.assertActiveUtterance(utteranceId); + this.activeUtteranceId = undefined; + this.committedParts = []; + this.closeExpected = true; + this.socket?.close(1000, "utterance-cancelled"); + this.stateValue = "closed"; + } + + subscribe(listener: TranscriberListener): () => void { + this.listeners.add(listener); + return () => this.listeners.delete(listener); + } + + async close(): Promise { + if (this.stateValue === "closed") return; + this.closeExpected = true; + this.activeUtteranceId = undefined; + this.clearConnectTimeout(); + this.rejectConnect?.( + new TranscriberError( + "closed", + "The Deepgram transcription connection was closed.", + ), + ); + if (this.socket?.readyState === WEBSOCKET_OPEN) { + this.socket.send(JSON.stringify({ type: "CloseStream" })); + } + if (this.socket && this.socket.readyState !== WEBSOCKET_CLOSED) { + this.socket.close(1000, "client-close"); + } + this.stateValue = "closed"; + this.emitClosed(true); + this.listeners.clear(); + } + + private async performConnect(): Promise { + let token: string; + try { + token = (await this.options.tokenProvider()).trim(); + if (!token) throw new Error("Empty token"); + } catch (error) { + const mapped = + error instanceof TranscriberError + ? error + : new TranscriberError( + "credentialFailure", + "The Deepgram credential request failed.", + { retryable: true, originalError: error }, + ); + this.fail(mapped); + throw mapped; + } + + if (this.stateValue !== "connecting") { + throw new TranscriberError( + "closed", + "The Deepgram transcription connection was cancelled.", + ); + } + + const url = buildDeepgramRealtimeUrl(this.options); + try { + this.socket = this.webSocketFactory(url, ["bearer", token]); + } catch (error) { + const mapped = new TranscriberError( + "connectionFailure", + "The Deepgram realtime connection could not be created.", + { retryable: true, originalError: error }, + ); + this.fail(mapped); + throw mapped; + } + this.socket.addEventListener("open", this.handleOpen); + this.socket.addEventListener("message", this.handleMessage); + this.socket.addEventListener("error", this.handleSocketError); + this.socket.addEventListener("close", this.handleSocketClose); + + return new Promise((resolve, reject) => { + this.resolveConnect = resolve; + this.rejectConnect = reject; + this.connectTimeoutId = globalThis.setTimeout(() => { + const error = new TranscriberError( + "connectionTimeout", + "The Deepgram realtime connection did not open in time.", + { retryable: true }, + ); + this.fail(error); + this.socket?.close(1000, "connection-timeout"); + }, this.options.connectTimeoutMs); + }); + } + + private readonly handleOpen = (): void => { + if (this.stateValue !== "connecting") return; + this.info = { + provider: PROVIDER, + model: this.options.model?.trim() || DEFAULT_MODEL, + sessionId: "pending-metadata", + languageCode: validateLanguageCode(this.options.languageCode), + sampleRate: TRANSCRIPTION_PCM_SAMPLE_RATE, + }; + this.stateValue = "ready"; + this.clearConnectTimeout(); + this.resolveConnect?.(this.info); + this.resolveConnect = undefined; + this.rejectConnect = undefined; + }; + + private readonly handleMessage = (event: MessageEvent): void => { + if (typeof event.data !== "string") return; + let message: DeepgramMessage; + try { + message = JSON.parse(event.data) as DeepgramMessage; + } catch (error) { + this.handleFailure( + new TranscriberError( + "providerUnavailable", + "Deepgram returned malformed realtime data.", + { retryable: true, originalError: error }, + ), + ); + return; + } + + if (message.type === "Metadata" && typeof message.request_id === "string") { + if (this.info) + this.info = { ...this.info, sessionId: message.request_id }; + return; + } + if (message.type === "Error") { + this.handleFailure( + new TranscriberError( + message.err_code === "INVALID_AUTH" + ? "authenticationFailure" + : "providerUnavailable", + "Deepgram reported a transcription error.", + { retryable: message.err_code !== "INVALID_AUTH" }, + ), + ); + return; + } + if (message.type !== "Results" || !this.activeUtteranceId) return; + + const text = message.channel?.alternatives?.[0]?.transcript; + if (typeof text !== "string") return; + const receivedAtMs = this.now(); + if (message.is_final === true && text.trim()) { + this.committedParts.push(text.trim()); + } + const accumulated = [...this.committedParts]; + if (message.is_final !== true && text.trim()) accumulated.push(text.trim()); + const transcript = accumulated.join(" ").trim(); + + if (message.speech_final === true || message.from_finalize === true) { + this.committedParts = []; + this.emit({ + type: "commit", + utteranceId: this.activeUtteranceId, + text: transcript, + receivedAtMs, + }); + } else { + this.emit({ + type: "partial", + utteranceId: this.activeUtteranceId, + text: transcript, + settled: message.is_final === true, + receivedAtMs, + }); + } + }; + + private readonly handleSocketError = (event: unknown): void => { + this.handleFailure( + new TranscriberError( + "connectionFailure", + "The Deepgram realtime connection failed.", + { retryable: true, originalError: event }, + ), + ); + }; + + private readonly handleSocketClose = (event: CloseEvent): void => { + const expected = this.closeExpected || this.stateValue === "closed"; + if (!expected && this.stateValue !== "failed") { + this.handleFailure( + new TranscriberError( + "connectionFailure", + `The Deepgram connection closed unexpectedly${ + event.code ? ` (code ${event.code})` : "" + }.`, + { retryable: true }, + ), + ); + } + this.emitClosed(expected); + }; + + private assertReady(): void { + if ( + this.stateValue !== "ready" || + !this.socket || + this.socket.readyState !== WEBSOCKET_OPEN + ) { + throw new TranscriberError( + this.stateValue === "closed" ? "closed" : "notConnected", + "The Deepgram realtime transcriber is not ready.", + ); + } + } + + private assertActiveUtterance(utteranceId: string): void { + this.assertReady(); + if (!this.activeUtteranceId) { + throw new TranscriberError( + "noActiveUtterance", + "There is no active transcription utterance.", + ); + } + if (this.activeUtteranceId !== utteranceId) { + throw new TranscriberError( + "utteranceConflict", + "Audio was sent for a stale transcription utterance.", + ); + } + } + + private handleFailure(error: TranscriberError): void { + if (this.stateValue === "failed" || this.stateValue === "closed") return; + this.fail(error); + this.socket?.close(1011, "provider-error"); + } + + private fail(error: TranscriberError): void { + this.stateValue = "failed"; + this.clearConnectTimeout(); + this.rejectConnect?.(error); + this.resolveConnect = undefined; + this.rejectConnect = undefined; + this.emit({ + type: "error", + utteranceId: this.activeUtteranceId, + error, + receivedAtMs: this.now(), + }); + this.activeUtteranceId = undefined; + this.committedParts = []; + } + + private clearConnectTimeout(): void { + if (this.connectTimeoutId !== undefined) { + globalThis.clearTimeout(this.connectTimeoutId); + this.connectTimeoutId = undefined; + } + } + + private emit(event: TranscriberEvent): void { + for (const listener of this.listeners) { + try { + listener(event); + } catch { + // A consumer exception must not interrupt the provider connection. + } + } + } + + private emitClosed(expected: boolean): void { + if (this.closedEventSent) return; + this.closedEventSent = true; + this.emit({ type: "closed", expected, receivedAtMs: this.now() }); + } +} diff --git a/components/speech/elevenLabsRealtimeTranscriber.ts b/components/speech/elevenLabsRealtimeTranscriber.ts new file mode 100644 index 00000000..64e07787 --- /dev/null +++ b/components/speech/elevenLabsRealtimeTranscriber.ts @@ -0,0 +1,755 @@ +import { + TRANSCRIPTION_PCM_SAMPLE_RATE, + TranscriberError, + type PcmAudioChunk, + type StreamingTranscriber, + type TranscriberConnectionInfo, + type TranscriberEvent, + type TranscriberListener, + type TranscriberState, +} from "./transcriber"; +import { + createSpeechTokenProvider, + type SpeechTokenProviderOptions, +} from "./speechToken"; + +const ELEVENLABS_PROVIDER = "elevenlabs"; +const DEFAULT_MODEL = "scribe_v2_realtime"; +const DEFAULT_BASE_URL = "wss://api.elevenlabs.io/v1/speech-to-text/realtime"; +const DEFAULT_CONNECT_TIMEOUT_MS = 8000; +const MAX_REALTIME_KEYTERMS = 50; +const MAX_REALTIME_KEYTERM_LENGTH = 20; +const DEFAULT_NO_VERBATIM = true; +const WEBSOCKET_OPEN = 1; +const WEBSOCKET_CLOSED = 3; + +export interface ElevenLabsVadConfig { + readonly silenceThresholdSecs: number; + readonly threshold: number; + readonly minimumSpeechDurationMs: number; + readonly minimumSilenceDurationMs: number; +} + +export const DEFAULT_ELEVENLABS_VAD_CONFIG: ElevenLabsVadConfig = { + silenceThresholdSecs: 0.5, + threshold: 0.4, + minimumSpeechDurationMs: 100, + minimumSilenceDurationMs: 100, +}; + +export type ElevenLabsTokenProvider = () => Promise; + +export interface WebSocketMessageEventLike { + readonly data: unknown; +} + +export interface WebSocketCloseEventLike { + readonly code?: number; + readonly reason?: string; +} + +export interface WebSocketLike { + readonly readyState: number; + addEventListener( + type: "message", + listener: (event: WebSocketMessageEventLike) => void, + ): void; + addEventListener(type: "error", listener: (event: unknown) => void): void; + addEventListener( + type: "close", + listener: (event: WebSocketCloseEventLike) => void, + ): void; + send(data: string): void; + close(code?: number, reason?: string): void; +} + +export type WebSocketFactory = (url: string) => WebSocketLike; + +export interface ElevenLabsRealtimeTranscriberOptions { + readonly tokenProvider: ElevenLabsTokenProvider; + readonly languageCode: string; + readonly keyterms?: readonly string[]; + readonly targetKeytermBiasEnabled: boolean; + readonly noVerbatim?: boolean; + readonly model?: string; + readonly baseUrl?: string; + readonly vad?: Partial; + readonly connectTimeoutMs?: number; + readonly filterBackgroundAudio?: boolean; + readonly webSocketFactory?: WebSocketFactory; + readonly now?: () => number; +} + +export type TokenProviderOptions = SpeechTokenProviderOptions; + +interface ProviderMessage { + readonly message_type?: unknown; + readonly session_id?: unknown; + readonly text?: unknown; + readonly error?: unknown; +} + +const defaultWebSocketFactory: WebSocketFactory = (url) => { + if (typeof WebSocket === "undefined") { + throw new TranscriberError( + "unsupported", + "Realtime transcription is unavailable in this browser.", + ); + } + return new WebSocket(url); +}; + +const finiteInRange = ( + value: number, + minimum: number, + maximum: number, + name: string, +): void => { + if (!Number.isFinite(value) || value < minimum || value > maximum) { + throw new TranscriberError( + "invalidConfiguration", + `${name} must be between ${minimum} and ${maximum}.`, + ); + } +}; + +const validateLanguageCode = (languageCode: string): string => { + const trimmed = languageCode.trim().toLowerCase(); + if (!/^[a-z]{2,3}$/.test(trimmed)) { + throw new TranscriberError( + "invalidConfiguration", + "ElevenLabs languageCode must be an ISO-639-1 or ISO-639-3 code.", + ); + } + return trimmed; +}; + +export const normalizeElevenLabsKeyterms = ( + keyterms: readonly string[] = [], +): string[] => { + const unique: string[] = []; + const seen = new Set(); + + for (const keyterm of keyterms) { + const value = keyterm.trim(); + if (!value) { + throw new TranscriberError( + "invalidConfiguration", + "ElevenLabs keyterms cannot be empty.", + ); + } + if ([...value].length > MAX_REALTIME_KEYTERM_LENGTH) { + throw new TranscriberError( + "invalidConfiguration", + `Realtime keyterms cannot exceed ${MAX_REALTIME_KEYTERM_LENGTH} characters.`, + ); + } + if (!seen.has(value)) { + seen.add(value); + unique.push(value); + } + } + + if (unique.length > MAX_REALTIME_KEYTERMS) { + throw new TranscriberError( + "invalidConfiguration", + `Realtime transcription accepts at most ${MAX_REALTIME_KEYTERMS} keyterms.`, + ); + } + return unique; +}; + +const resolveVadConfig = ( + overrides: Partial = {}, +): ElevenLabsVadConfig => { + const vad = { ...DEFAULT_ELEVENLABS_VAD_CONFIG, ...overrides }; + finiteInRange(vad.silenceThresholdSecs, 0.3, 3, "silenceThresholdSecs"); + finiteInRange(vad.threshold, 0.1, 0.9, "vadThreshold"); + finiteInRange( + vad.minimumSpeechDurationMs, + 50, + 2000, + "minimumSpeechDurationMs", + ); + finiteInRange( + vad.minimumSilenceDurationMs, + 50, + 2000, + "minimumSilenceDurationMs", + ); + return vad; +}; + +export const buildElevenLabsRealtimeUrl = ( + token: string, + options: Omit, +): string => { + const trimmedToken = token.trim(); + if (!trimmedToken) { + throw new TranscriberError( + "credentialFailure", + "The realtime transcription credential was empty.", + { retryable: true }, + ); + } + + const model = options.model?.trim() || DEFAULT_MODEL; + const languageCode = validateLanguageCode(options.languageCode); + if (typeof options.targetKeytermBiasEnabled !== "boolean") { + throw new TranscriberError( + "invalidConfiguration", + "The ElevenLabs keyterm policy must be explicit.", + ); + } + if ( + options.noVerbatim !== undefined && + typeof options.noVerbatim !== "boolean" + ) { + throw new TranscriberError( + "invalidConfiguration", + "The ElevenLabs verbatim policy must be a boolean when provided.", + ); + } + const keyterms = !options.targetKeytermBiasEnabled + ? [] + : normalizeElevenLabsKeyterms(options.keyterms); + const vad = resolveVadConfig(options.vad); + const url = new URL(options.baseUrl ?? DEFAULT_BASE_URL); + + url.searchParams.set("token", trimmedToken); + url.searchParams.set("model_id", model); + url.searchParams.set("audio_format", "pcm_16000"); + url.searchParams.set("language_code", languageCode); + url.searchParams.set("commit_strategy", "vad"); + url.searchParams.set( + "vad_silence_threshold_secs", + String(vad.silenceThresholdSecs), + ); + url.searchParams.set("vad_threshold", String(vad.threshold)); + url.searchParams.set( + "min_speech_duration_ms", + String(vad.minimumSpeechDurationMs), + ); + url.searchParams.set( + "min_silence_duration_ms", + String(vad.minimumSilenceDurationMs), + ); + url.searchParams.set( + "no_verbatim", + String(options.noVerbatim ?? DEFAULT_NO_VERBATIM), + ); + url.searchParams.set("include_timestamps", "false"); + url.searchParams.set("enable_logging", "false"); + if (options.filterBackgroundAudio !== undefined) { + url.searchParams.set( + "filter_background_audio", + String(options.filterBackgroundAudio), + ); + } + for (const keyterm of keyterms) url.searchParams.append("keyterms", keyterm); + + return url.toString(); +}; + +export const createElevenLabsTokenProvider = ( + options: TokenProviderOptions, +): ElevenLabsTokenProvider => + createSpeechTokenProvider(ELEVENLABS_PROVIDER, options); + +const pcm16ToBase64 = (samples: Int16Array): string => { + const bytes = new Uint8Array(samples.length * 2); + const view = new DataView(bytes.buffer); + for (let index = 0; index < samples.length; index += 1) { + view.setInt16(index * 2, samples[index], true); + } + + let binary = ""; + const batchSize = 0x8000; + for (let offset = 0; offset < bytes.length; offset += batchSize) { + binary += String.fromCharCode( + ...bytes.subarray(offset, offset + batchSize), + ); + } + return globalThis.btoa(binary); +}; + +const providerError = (messageType: string): TranscriberError => { + switch (messageType) { + case "auth_error": + return new TranscriberError( + "authenticationFailure", + "ElevenLabs rejected the realtime credential.", + ); + case "quota_exceeded": + return new TranscriberError( + "quotaExceeded", + "The ElevenLabs transcription quota was exceeded.", + ); + case "rate_limited": + case "commit_throttled": + return new TranscriberError( + "rateLimited", + "ElevenLabs rate-limited the transcription session.", + { retryable: true }, + ); + case "unaccepted_terms": + return new TranscriberError( + "termsNotAccepted", + "The ElevenLabs Scribe terms have not been accepted.", + ); + case "session_time_limit_exceeded": + return new TranscriberError( + "sessionLimit", + "The ElevenLabs realtime session reached its time limit.", + { retryable: true }, + ); + case "input_error": + case "invalid_request": + case "chunk_size_exceeded": + return new TranscriberError( + "inputRejected", + "ElevenLabs rejected the realtime audio input.", + ); + case "queue_overflow": + case "resource_exhausted": + case "insufficient_audio_activity": + return new TranscriberError( + "providerUnavailable", + "ElevenLabs could not continue the realtime session.", + { retryable: true }, + ); + default: + return new TranscriberError( + "providerUnavailable", + "ElevenLabs reported a transcription error.", + { retryable: true }, + ); + } +}; + +export class ElevenLabsRealtimeTranscriber implements StreamingTranscriber { + private readonly options: ElevenLabsRealtimeTranscriberOptions; + private readonly webSocketFactory: WebSocketFactory; + private readonly now: () => number; + private readonly listeners = new Set(); + private stateValue: TranscriberState = "idle"; + private socket?: WebSocketLike; + private activeUtteranceId?: string; + private info?: TranscriberConnectionInfo; + private connectPromise?: Promise; + private closeExpected = false; + private closedEventSent = false; + private connectTimeoutId?: ReturnType; + private resolveConnect?: (info: TranscriberConnectionInfo) => void; + private rejectConnect?: (error: TranscriberError) => void; + + constructor(options: ElevenLabsRealtimeTranscriberOptions) { + if (!options || typeof options.tokenProvider !== "function") { + throw new TranscriberError( + "invalidConfiguration", + "An ElevenLabs token provider is required.", + ); + } + const connectTimeoutMs = + options.connectTimeoutMs ?? DEFAULT_CONNECT_TIMEOUT_MS; + if (!Number.isFinite(connectTimeoutMs) || connectTimeoutMs <= 0) { + throw new TranscriberError( + "invalidConfiguration", + "Connection timeout must be a finite positive number.", + ); + } + + validateLanguageCode(options.languageCode); + if (options.targetKeytermBiasEnabled) { + normalizeElevenLabsKeyterms(options.keyterms); + } + resolveVadConfig(options.vad); + this.options = { ...options, connectTimeoutMs }; + this.webSocketFactory = options.webSocketFactory ?? defaultWebSocketFactory; + this.now = options.now ?? (() => performance.now()); + } + + get state(): TranscriberState { + return this.stateValue; + } + + get connectionInfo(): TranscriberConnectionInfo | undefined { + return this.info; + } + + connect(): Promise { + if (this.connectPromise) return this.connectPromise; + if (this.stateValue !== "idle") { + return Promise.reject( + new TranscriberError( + this.stateValue === "closed" ? "closed" : "connectionFailure", + "The realtime transcription connection cannot be started again.", + ), + ); + } + + this.stateValue = "connecting"; + this.connectPromise = this.performConnect(); + return this.connectPromise; + } + + beginUtterance(utteranceId: string): void { + this.assertReady(); + const normalizedId = utteranceId.trim(); + if (!normalizedId) { + throw new TranscriberError( + "invalidConfiguration", + "A non-empty utterance identifier is required.", + ); + } + if (this.activeUtteranceId) { + throw new TranscriberError( + "utteranceConflict", + "A transcription utterance is already active.", + ); + } + this.activeUtteranceId = normalizedId; + } + + sendAudio(utteranceId: string, chunk: PcmAudioChunk): void { + this.assertActiveUtterance(utteranceId); + if ( + chunk.sampleRate !== TRANSCRIPTION_PCM_SAMPLE_RATE || + !(chunk.samples instanceof Int16Array) || + chunk.samples.length === 0 + ) { + throw new TranscriberError( + "invalidAudio", + "Realtime audio must be non-empty 16-bit PCM at 16 kHz.", + ); + } + + this.send({ + message_type: "input_audio_chunk", + audio_base_64: pcm16ToBase64(chunk.samples), + sample_rate: TRANSCRIPTION_PCM_SAMPLE_RATE, + }); + } + + requestCommit(utteranceId: string): void { + this.assertActiveUtterance(utteranceId); + this.send({ + message_type: "input_audio_chunk", + audio_base_64: "", + commit: true, + sample_rate: TRANSCRIPTION_PCM_SAMPLE_RATE, + }); + } + + endUtterance(utteranceId: string): void { + this.assertActiveUtterance(utteranceId); + this.activeUtteranceId = undefined; + } + + cancelUtterance(utteranceId: string): void { + this.assertActiveUtterance(utteranceId); + this.activeUtteranceId = undefined; + this.closeExpected = true; + this.socket?.close(1000, "utterance-cancelled"); + this.stateValue = "closed"; + } + + subscribe(listener: TranscriberListener): () => void { + this.listeners.add(listener); + return () => this.listeners.delete(listener); + } + + async close(): Promise { + if (this.stateValue === "closed") return; + this.closeExpected = true; + this.activeUtteranceId = undefined; + this.clearConnectTimeout(); + const closeError = new TranscriberError( + "closed", + "The realtime transcription connection was closed.", + ); + this.rejectConnect?.(closeError); + this.resolveConnect = undefined; + this.rejectConnect = undefined; + + if (this.socket && this.socket.readyState !== WEBSOCKET_CLOSED) { + this.socket.close(1000, "client-close"); + } + this.stateValue = "closed"; + this.emitClosed(true); + this.listeners.clear(); + } + + private async performConnect(): Promise { + let token: string; + try { + token = await this.options.tokenProvider(); + } catch (error) { + const mapped = + error instanceof TranscriberError + ? error + : new TranscriberError( + "credentialFailure", + "The realtime transcription credential request failed.", + { retryable: true, originalError: error }, + ); + this.fail(mapped); + throw mapped; + } + + if (this.stateValue !== "connecting") { + throw new TranscriberError( + "closed", + "The realtime transcription connection was cancelled.", + ); + } + + const url = buildElevenLabsRealtimeUrl(token, this.options); + try { + this.socket = this.webSocketFactory(url); + } catch (error) { + const mapped = + error instanceof TranscriberError + ? error + : new TranscriberError( + "connectionFailure", + "The ElevenLabs realtime connection could not be created.", + { retryable: true, originalError: error }, + ); + this.fail(mapped); + throw mapped; + } + + this.socket.addEventListener("message", this.handleMessage); + this.socket.addEventListener("error", this.handleSocketError); + this.socket.addEventListener("close", this.handleSocketClose); + + return new Promise((resolve, reject) => { + this.resolveConnect = resolve; + this.rejectConnect = reject; + this.connectTimeoutId = globalThis.setTimeout(() => { + const error = new TranscriberError( + "connectionTimeout", + "The ElevenLabs realtime connection did not become ready in time.", + { retryable: true }, + ); + this.fail(error); + this.socket?.close(1000, "connection-timeout"); + }, this.options.connectTimeoutMs); + }); + } + + private readonly handleMessage = (event: WebSocketMessageEventLike): void => { + if (typeof event.data !== "string") { + this.handleProviderFailure( + new TranscriberError( + "inputRejected", + "ElevenLabs returned an unsupported realtime message.", + ), + ); + return; + } + + let message: ProviderMessage; + try { + message = JSON.parse(event.data) as ProviderMessage; + } catch (error) { + this.handleProviderFailure( + new TranscriberError( + "providerUnavailable", + "ElevenLabs returned malformed realtime data.", + { retryable: true, originalError: error }, + ), + ); + return; + } + + if (typeof message.message_type !== "string") return; + const receivedAtMs = this.now(); + + switch (message.message_type) { + case "session_started": { + if ( + this.stateValue !== "connecting" || + typeof message.session_id !== "string" + ) { + return; + } + this.info = { + provider: ELEVENLABS_PROVIDER, + model: this.options.model?.trim() || DEFAULT_MODEL, + sessionId: message.session_id, + languageCode: validateLanguageCode(this.options.languageCode), + sampleRate: TRANSCRIPTION_PCM_SAMPLE_RATE, + }; + this.stateValue = "ready"; + this.clearConnectTimeout(); + this.resolveConnect?.(this.info); + this.resolveConnect = undefined; + this.rejectConnect = undefined; + break; + } + case "partial_transcript": + case "final_transcript": { + if (!this.activeUtteranceId || typeof message.text !== "string") return; + this.emit({ + type: "partial", + utteranceId: this.activeUtteranceId, + text: message.text, + settled: message.message_type === "final_transcript", + receivedAtMs, + }); + break; + } + case "committed_transcript": { + if (!this.activeUtteranceId || typeof message.text !== "string") return; + this.emit({ + type: "commit", + utteranceId: this.activeUtteranceId, + text: message.text, + receivedAtMs, + }); + break; + } + case "committed_transcript_with_timestamps": + case "final_transcript_with_timestamps": + case "committed_transcript_entities": + // These optional delayed messages do not define the utterance boundary. + break; + default: + if (message.message_type.endsWith("error")) { + this.handleProviderFailure(providerError(message.message_type)); + } else if ( + [ + "rate_limited", + "commit_throttled", + "quota_exceeded", + "unaccepted_terms", + "queue_overflow", + "resource_exhausted", + "session_time_limit_exceeded", + "chunk_size_exceeded", + "insufficient_audio_activity", + "invalid_request", + ].includes(message.message_type) + ) { + this.handleProviderFailure(providerError(message.message_type)); + } + } + }; + + private readonly handleSocketError = (event: unknown): void => { + this.handleProviderFailure( + new TranscriberError( + "connectionFailure", + "The ElevenLabs realtime connection failed.", + { retryable: true, originalError: event }, + ), + ); + }; + + private readonly handleSocketClose = ( + event: WebSocketCloseEventLike, + ): void => { + const expected = this.closeExpected || this.stateValue === "closed"; + if (!expected && this.stateValue !== "failed") { + this.handleProviderFailure( + new TranscriberError( + "connectionFailure", + `The ElevenLabs realtime connection closed unexpectedly${ + event.code ? ` (code ${event.code})` : "" + }.`, + { retryable: true }, + ), + ); + } + this.stateValue = expected ? "closed" : this.stateValue; + this.emitClosed(expected); + }; + + private assertReady(): void { + if ( + this.stateValue !== "ready" || + !this.socket || + this.socket.readyState !== WEBSOCKET_OPEN + ) { + throw new TranscriberError( + this.stateValue === "closed" ? "closed" : "notConnected", + "The realtime transcriber is not ready.", + ); + } + } + + private assertActiveUtterance(utteranceId: string): void { + this.assertReady(); + if (!this.activeUtteranceId) { + throw new TranscriberError( + "noActiveUtterance", + "There is no active transcription utterance.", + ); + } + if (utteranceId !== this.activeUtteranceId) { + throw new TranscriberError( + "utteranceConflict", + "Audio was sent for a stale transcription utterance.", + ); + } + } + + private send(payload: Record): void { + this.assertReady(); + try { + this.socket?.send(JSON.stringify(payload)); + } catch (error) { + throw new TranscriberError( + "connectionFailure", + "Realtime audio could not be sent to ElevenLabs.", + { retryable: true, originalError: error }, + ); + } + } + + private handleProviderFailure(error: TranscriberError): void { + if (this.stateValue === "closed" || this.stateValue === "failed") return; + this.fail(error); + this.socket?.close(1011, "provider-error"); + } + + private fail(error: TranscriberError): void { + this.stateValue = "failed"; + this.clearConnectTimeout(); + this.rejectConnect?.(error); + this.resolveConnect = undefined; + this.rejectConnect = undefined; + this.emit({ + type: "error", + utteranceId: this.activeUtteranceId, + error, + receivedAtMs: this.now(), + }); + this.activeUtteranceId = undefined; + } + + private clearConnectTimeout(): void { + if (this.connectTimeoutId !== undefined) { + globalThis.clearTimeout(this.connectTimeoutId); + this.connectTimeoutId = undefined; + } + } + + private emit(event: TranscriberEvent): void { + for (const listener of this.listeners) { + try { + listener(event); + } catch { + // A consumer exception must not interrupt the provider connection. + } + } + } + + private emitClosed(expected: boolean): void { + if (this.closedEventSent) return; + this.closedEventSent = true; + this.emit({ type: "closed", expected, receivedAtMs: this.now() }); + } +} diff --git a/components/speech/microphone.ts b/components/speech/microphone.ts index 26d1bc28..1fbbdf7e 100644 --- a/components/speech/microphone.ts +++ b/components/speech/microphone.ts @@ -1,3 +1,5 @@ +import { MICROPHONE_CAPTURE_WORKLET_SOURCE } from "./microphoneCaptureWorkletSource"; + export type MicrophoneErrorCode = | "unsupported" | "permissionDenied" @@ -44,6 +46,22 @@ export interface MicrophoneHealth { export type MicrophoneHealthListener = (health: MicrophoneHealth) => void; +export interface MicrophoneAudioFrame { + readonly samples: Float32Array; + readonly sampleRate: number; + readonly capturedAtMs: number; +} + +export type MicrophoneAudioFrameListener = ( + frame: MicrophoneAudioFrame, +) => void; + +export interface MicrophoneAudioFrameSource { + start(): void; + stop(): void; + close(): void; +} + export interface MicrophoneSession { readonly stream: MediaStream; readonly track: MediaStreamTrack; @@ -53,6 +71,9 @@ export interface MicrophoneSession { getHealth(): MicrophoneHealth; getTrackSettings(): MediaTrackSettings; readFrame(target: Float32Array): void; + subscribeToAudioFrames( + listener: MicrophoneAudioFrameListener, + ): Promise; subscribeToHealth(listener: MicrophoneHealthListener): () => void; close(): Promise; } @@ -68,6 +89,7 @@ export interface OpenMicrophoneOptions { const DEFAULT_ANALYSER_FFT_SIZE = 2048; const MIN_ANALYSER_FFT_SIZE = 32; const MAX_ANALYSER_FFT_SIZE = 32768; +const CAPTURE_FRAME_SIZE = 2048; const createFloatTimeDomainBuffer = (length: number) => new Float32Array(length); @@ -219,6 +241,7 @@ class BrowserMicrophoneSession implements MicrophoneSession { private readonly healthListeners = new Set(); private closePromise?: Promise; private closed = false; + private audioFrameSubscription?: Promise; constructor( stream: MediaStream, @@ -295,6 +318,31 @@ class BrowserMicrophoneSession implements MicrophoneSession { } } + subscribeToAudioFrames( + listener: MicrophoneAudioFrameListener, + ): Promise { + if (this.closed) { + return Promise.reject( + new MicrophoneError( + "sessionClosed", + "Cannot capture audio from a closed microphone session.", + ), + ); + } + if (this.audioFrameSubscription) { + return Promise.reject( + new MicrophoneError( + "invalidConfiguration", + "This microphone session already has an audio frame subscriber.", + ), + ); + } + + const subscription = this.createAudioFrameSubscription(listener); + this.audioFrameSubscription = subscription; + return subscription; + } + subscribeToHealth(listener: MicrophoneHealthListener): () => void { if (this.closed) { listener(this.getHealth()); @@ -337,6 +385,15 @@ class BrowserMicrophoneSession implements MicrophoneSession { this.track.removeEventListener("unmute", this.handleTrackHealthChange); this.track.removeEventListener("ended", this.handleTrackHealthChange); + if (this.audioFrameSubscription) { + try { + const frameSource = await this.audioFrameSubscription; + frameSource.close(); + } catch { + // Audio-frame setup failures do not change the remaining cleanup steps. + } + } + stopAllTracks(this.stream); disconnectAudioNode(this.sourceNode); @@ -346,6 +403,137 @@ class BrowserMicrophoneSession implements MicrophoneSession { this.notifyHealthListeners(); this.healthListeners.clear(); } + + private async createAudioFrameSubscription( + listener: MicrophoneAudioFrameListener, + ): Promise { + const emit = (samples: Float32Array): void => { + if (this.closed || samples.length === 0) return; + listener({ + samples, + sampleRate: this.audioContext.sampleRate, + capturedAtMs: performance.now(), + }); + }; + + if ( + this.audioContext.audioWorklet && + typeof AudioWorkletNode !== "undefined" && + typeof URL.createObjectURL === "function" + ) { + const moduleUrl = URL.createObjectURL( + new Blob([MICROPHONE_CAPTURE_WORKLET_SOURCE], { + type: "text/javascript", + }), + ); + try { + await this.audioContext.audioWorklet.addModule(moduleUrl); + } catch { + URL.revokeObjectURL(moduleUrl); + return this.createScriptProcessorSubscription(listener); + } + URL.revokeObjectURL(moduleUrl); + if (this.closed) { + throw new MicrophoneError( + "sessionClosed", + "The microphone session closed while audio capture was starting.", + ); + } + + const captureNode = new AudioWorkletNode( + this.audioContext, + "easyeyes-microphone-capture", + { + numberOfInputs: 1, + numberOfOutputs: 1, + outputChannelCount: [1], + processorOptions: { frameSize: CAPTURE_FRAME_SIZE }, + }, + ); + const silentOutput = this.audioContext.createGain(); + silentOutput.gain.value = 0; + const handleMessage = (event: MessageEvent): void => { + if (event.data instanceof Float32Array) emit(event.data); + }; + captureNode.port.addEventListener("message", handleMessage); + captureNode.port.start(); + this.sourceNode.connect(captureNode); + captureNode.connect(silentOutput); + silentOutput.connect(this.audioContext.destination); + + let closed = false; + return { + start: () => { + if (!closed) captureNode.port.postMessage({ type: "start" }); + }, + stop: () => { + if (!closed) captureNode.port.postMessage({ type: "stop" }); + }, + close: () => { + if (closed) return; + closed = true; + captureNode.port.removeEventListener("message", handleMessage); + captureNode.port.close(); + disconnectAudioNode(captureNode); + disconnectAudioNode(silentOutput); + }, + }; + } + + return this.createScriptProcessorSubscription(listener); + } + + private createScriptProcessorSubscription( + listener: MicrophoneAudioFrameListener, + ): MicrophoneAudioFrameSource { + if (typeof this.audioContext.createScriptProcessor === "function") { + const processor = this.audioContext.createScriptProcessor( + CAPTURE_FRAME_SIZE, + 1, + 1, + ); + const silentOutput = this.audioContext.createGain(); + silentOutput.gain.value = 0; + let capturing = false; + processor.onaudioprocess = (event): void => { + if (!capturing) return; + const input = event.inputBuffer.getChannelData(0); + if (!this.closed && input.length > 0) { + listener({ + samples: new Float32Array(input), + sampleRate: this.audioContext.sampleRate, + capturedAtMs: performance.now(), + }); + } + }; + this.sourceNode.connect(processor); + processor.connect(silentOutput); + silentOutput.connect(this.audioContext.destination); + + let closed = false; + return { + start: () => { + if (!closed) capturing = true; + }, + stop: () => { + capturing = false; + }, + close: () => { + if (closed) return; + closed = true; + capturing = false; + processor.onaudioprocess = null; + disconnectAudioNode(processor); + disconnectAudioNode(silentOutput); + }, + }; + } + + throw new MicrophoneError( + "audioGraphUnavailable", + "Continuous microphone audio capture is unavailable in this browser.", + ); + } } export const openMicrophone = async ( diff --git a/components/speech/microphoneCaptureWorkletSource.ts b/components/speech/microphoneCaptureWorkletSource.ts new file mode 100644 index 00000000..f5874bfd --- /dev/null +++ b/components/speech/microphoneCaptureWorkletSource.ts @@ -0,0 +1,56 @@ +export const MICROPHONE_CAPTURE_WORKLET_SOURCE = ` +class EasyEyesMicrophoneCaptureProcessor extends AudioWorkletProcessor { + constructor(options) { + super(); + const configuredFrameSize = options.processorOptions?.frameSize; + this.frameSize = + Number.isInteger(configuredFrameSize) && configuredFrameSize > 0 + ? configuredFrameSize + : 2048; + this.frame = new Float32Array(this.frameSize); + this.offset = 0; + this.capturing = false; + this.port.onmessage = (event) => { + if (event.data?.type === "start") { + this.frame = new Float32Array(this.frameSize); + this.offset = 0; + this.capturing = true; + } else if (event.data?.type === "stop") { + this.capturing = false; + this.offset = 0; + } + }; + } + + process(inputs, outputs) { + const input = inputs[0]?.[0]; + const output = outputs[0]?.[0]; + if (output) output.fill(0); + if (!input || !this.capturing) return true; + + let inputOffset = 0; + while (inputOffset < input.length) { + const count = Math.min( + input.length - inputOffset, + this.frameSize - this.offset, + ); + this.frame.set(input.subarray(inputOffset, inputOffset + count), this.offset); + this.offset += count; + inputOffset += count; + + if (this.offset === this.frameSize) { + const completedFrame = this.frame; + this.port.postMessage(completedFrame, [completedFrame.buffer]); + this.frame = new Float32Array(this.frameSize); + this.offset = 0; + } + } + return true; + } +} + +registerProcessor( + "easyeyes-microphone-capture", + EasyEyesMicrophoneCaptureProcessor, +); +`; diff --git a/components/speech/speechPreflight.ts b/components/speech/speechPreflight.ts index 3ffe293c..13250bca 100644 --- a/components/speech/speechPreflight.ts +++ b/components/speech/speechPreflight.ts @@ -50,6 +50,7 @@ interface SpeechPreflightRuntimeOptions { voiceTimeoutMs: number; permissionTimeoutMs: number; pollIntervalMs: number; + muteRecoveryGraceMs: number; minimumVoiceDurationMs: number; speechThresholdDbAboveNoise: number; absoluteSpeechFloorAcRms: number; @@ -82,6 +83,7 @@ const DEFAULT_RUNTIME_OPTIONS: SpeechPreflightRuntimeOptions = { voiceTimeoutMs: 5000, permissionTimeoutMs: 20000, pollIntervalMs: 50, + muteRecoveryGraceMs: 1000, minimumVoiceDurationMs: 100, speechThresholdDbAboveNoise: 9, absoluteSpeechFloorAcRms: 0.0025, @@ -133,13 +135,32 @@ const percentile = (values: number[], fraction: number): number => { return sorted[index]; }; -const assertUsableTrack = (session: MicrophoneSession): void => { +const waitForUsableTrack = async ( + session: MicrophoneSession, + config: SpeechPreflightRuntimeOptions, + now: () => number, + wait: (durationMs: number, signal?: AbortSignal) => Promise, + signal?: AbortSignal, +): Promise => { + const recoveryStartedAt = now(); + while (session.getHealth().state === "muted") { + throwIfAborted(signal); + if (now() - recoveryStartedAt >= config.muteRecoveryGraceMs) { + throw new MicrophoneError( + "deviceUnavailable", + "The microphone remained muted during preflight.", + ); + } + await wait(config.pollIntervalMs, signal); + } + if (session.getHealth().state !== "ready") { throw new MicrophoneError( "deviceUnavailable", "The microphone stopped providing usable input during preflight.", ); } + return now() - recoveryStartedAt; }; const waitForMicrophone = async ( @@ -267,7 +288,7 @@ export const runSpeechPreflight = async ({ permissionTimeoutMs: config.permissionTimeoutMs, }); session = await waitForMicrophone(request, signal); - assertUsableTrack(session); + await waitForUsableTrack(session, config, now, wait, signal); const frame = new Float32Array(session.frameSize); const ambientLevels: number[] = []; @@ -275,17 +296,23 @@ export const runSpeechPreflight = async ({ let ambientFrameCount = 0; onPhaseChange?.("measuringAmbient"); - const ambientStartedAt = now(); + let ambientDeadline = now() + config.ambientDurationMs; do { throwIfAborted(signal); - assertUsableTrack(session); + ambientDeadline += await waitForUsableTrack( + session, + config, + now, + wait, + signal, + ); session.readFrame(frame); const metrics = calculateAudioSignalMetrics(frame); ambientLevels.push(metrics.acRms); ambientZeroRatioTotal += metrics.zeroSampleRatio; ambientFrameCount += 1; await wait(config.pollIntervalMs, signal); - } while (now() - ambientStartedAt < config.ambientDurationMs); + } while (now() < ambientDeadline); const noiseFloorAcRms = percentile(ambientLevels, 0.8); const vad = new EnergyVoiceActivityDetector({ @@ -299,14 +326,24 @@ export const runSpeechPreflight = async ({ onPhaseChange?.("waitingForVoice"); const voiceStartedAt = now(); + let voiceDeadline = voiceStartedAt + config.voiceTimeoutMs; + let voicePausedMs = 0; let maximumVoiceAcRms = 0; let maximumClippedSampleRatio = 0; do { throwIfAborted(signal); - assertUsableTrack(session); + const pausedMs = await waitForUsableTrack( + session, + config, + now, + wait, + signal, + ); + voiceDeadline += pausedMs; + voicePausedMs += pausedMs; session.readFrame(frame); - const timestampMs = now() - voiceStartedAt; + const timestampMs = now() - voiceStartedAt - voicePausedMs; const decision = vad.process(frame, timestampMs); maximumVoiceAcRms = Math.max(maximumVoiceAcRms, decision.signal.acRms); maximumClippedSampleRatio = Math.max( @@ -327,7 +364,7 @@ export const runSpeechPreflight = async ({ }; } await wait(config.pollIntervalMs, signal); - } while (now() - voiceStartedAt < config.voiceTimeoutMs); + } while (now() < voiceDeadline); const averageAmbientZeroRatio = ambientFrameCount === 0 ? 1 : ambientZeroRatioTotal / ambientFrameCount; diff --git a/components/speech/speechSession.ts b/components/speech/speechSession.ts new file mode 100644 index 00000000..6d58beee --- /dev/null +++ b/components/speech/speechSession.ts @@ -0,0 +1,540 @@ +import { + TranscriberError, + type PcmAudioChunk, + type StreamingTranscriber, + type TranscriberConnectionInfo, + type TranscriberEvent, +} from "./transcriber"; + +export type SpeechSessionState = + | "idle" + | "connecting" + | "ready" + | "listening" + | "finalizing" + | "failed" + | "closed"; + +export type SpeechSessionErrorCode = + | "invalidState" + | "invalidUtterance" + | "utteranceCancelled" + | "finalizationTimeout" + | "transcriberFailure" + | "closed"; + +export class SpeechSessionError extends Error { + readonly code: SpeechSessionErrorCode; + readonly retryable: boolean; + readonly originalError?: unknown; + + constructor( + code: SpeechSessionErrorCode, + message: string, + options: { retryable?: boolean; originalError?: unknown } = {}, + ) { + super(message); + this.name = "SpeechSessionError"; + this.code = code; + this.retryable = options.retryable ?? false; + this.originalError = options.originalError; + } +} + +export type UtteranceFinalizationTrigger = + | "providerVad" + | "manual" + | "maxDuration"; + +export interface SpeechCommittedSegment { + readonly text: string; + readonly receivedAtMs: number; +} + +export interface SpeechUtteranceResult { + readonly utteranceId: string; + readonly text: string; + readonly committedSegments: readonly SpeechCommittedSegment[]; + readonly startedAtMs: number; + readonly completedAtMs: number; + readonly durationMs: number; + readonly finalizationTrigger: UtteranceFinalizationTrigger; +} + +export interface SpeechSessionStateEvent { + readonly type: "state"; + readonly state: SpeechSessionState; + readonly utteranceId?: string; +} + +export interface SpeechSessionPartialEvent { + readonly type: "partial"; + readonly utteranceId: string; + readonly text: string; + readonly settled: boolean; + readonly receivedAtMs: number; +} + +export interface SpeechSessionFinalEvent { + readonly type: "final"; + readonly result: SpeechUtteranceResult; +} + +export interface SpeechSessionFailureEvent { + readonly type: "error"; + readonly utteranceId?: string; + readonly error: SpeechSessionError; +} + +export type SpeechSessionEvent = + | SpeechSessionStateEvent + | SpeechSessionPartialEvent + | SpeechSessionFinalEvent + | SpeechSessionFailureEvent; + +export type SpeechSessionListener = (event: SpeechSessionEvent) => void; + +export interface SpeechSessionOptions { + readonly maximumUtteranceDurationMs: number; + readonly finalizationTimeoutMs?: number; + readonly now?: () => number; +} + +interface PendingUtterance { + readonly id: string; + readonly startedAtMs: number; + readonly promise: Promise; + readonly resolve: (result: SpeechUtteranceResult) => void; + readonly reject: (error: SpeechSessionError) => void; + readonly committedSegments: SpeechCommittedSegment[]; + finalizationTrigger: UtteranceFinalizationTrigger; + providerFinalizationAllowed: boolean; +} + +const DEFAULT_FINALIZATION_TIMEOUT_MS = 3000; + +const positiveDuration = (value: number, name: string): number => { + if (!Number.isFinite(value) || value <= 0) { + throw new RangeError(`${name} must be a finite positive number.`); + } + return value; +}; + +const deferredUtterance = ( + id: string, + startedAtMs: number, +): PendingUtterance => { + let resolve!: (result: SpeechUtteranceResult) => void; + let reject!: (error: SpeechSessionError) => void; + const promise = new Promise( + (resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }, + ); + return { + id, + startedAtMs, + promise, + resolve, + reject, + committedSegments: [], + finalizationTrigger: "providerVad", + providerFinalizationAllowed: false, + }; +}; + +const asSessionError = (error: unknown): SpeechSessionError => { + if (error instanceof SpeechSessionError) return error; + if (error instanceof TranscriberError) { + return new SpeechSessionError( + error.code === "closed" ? "closed" : "transcriberFailure", + error.message, + { retryable: error.retryable, originalError: error }, + ); + } + return new SpeechSessionError( + "transcriberFailure", + "The realtime transcription session failed.", + { retryable: true, originalError: error }, + ); +}; + +/** + * Owns the audio gate and one-at-a-time utterance lifecycle. Audio is + * forwarded only while an utterance is listening; callbacks for stale + * utterance identifiers are ignored. + */ +export class SpeechSession { + private readonly transcriber: StreamingTranscriber; + private readonly maximumUtteranceDurationMs: number; + private readonly finalizationTimeoutMs: number; + private readonly now: () => number; + private readonly listeners = new Set(); + private readonly unsubscribeTranscriber: () => void; + private stateValue: SpeechSessionState = "idle"; + private info?: TranscriberConnectionInfo; + private pending?: PendingUtterance; + private maximumDurationTimer?: ReturnType; + private finalizationTimer?: ReturnType; + private closePromise?: Promise; + private expectedClose = false; + + constructor( + transcriber: StreamingTranscriber, + options: SpeechSessionOptions, + ) { + this.transcriber = transcriber; + this.maximumUtteranceDurationMs = positiveDuration( + options.maximumUtteranceDurationMs, + "maximumUtteranceDurationMs", + ); + this.finalizationTimeoutMs = positiveDuration( + options.finalizationTimeoutMs ?? DEFAULT_FINALIZATION_TIMEOUT_MS, + "finalizationTimeoutMs", + ); + this.now = options.now ?? (() => performance.now()); + this.unsubscribeTranscriber = transcriber.subscribe( + this.handleTranscriberEvent, + ); + } + + get state(): SpeechSessionState { + return this.stateValue; + } + + get connectionInfo(): TranscriberConnectionInfo | undefined { + return this.info; + } + + get activeUtteranceId(): string | undefined { + return this.pending?.id; + } + + async connect(): Promise { + if (this.stateValue !== "idle") { + throw new SpeechSessionError( + "invalidState", + "The speech session can only connect from its idle state.", + ); + } + + this.setState("connecting"); + try { + this.info = await this.transcriber.connect(); + if (this.state === "closed") { + throw new SpeechSessionError( + "closed", + "The speech session was closed while connecting.", + ); + } + this.setState("ready"); + return this.info; + } catch (error) { + const mapped = asSessionError(error); + this.fail(mapped); + throw mapped; + } + } + + beginUtterance(utteranceId: string): Promise { + if (this.stateValue !== "ready" || this.pending) { + throw new SpeechSessionError( + "invalidState", + "A speech utterance can only begin when the session is ready.", + ); + } + const id = utteranceId.trim(); + if (!id) { + throw new SpeechSessionError( + "invalidUtterance", + "A non-empty speech utterance identifier is required.", + ); + } + + const pending = deferredUtterance(id, this.now()); + this.pending = pending; + try { + this.transcriber.beginUtterance(id); + } catch (error) { + this.pending = undefined; + const mapped = asSessionError(error); + this.fail(mapped); + throw mapped; + } + + this.setState("listening", id); + this.maximumDurationTimer = globalThis.setTimeout(() => { + if (this.pending?.id !== id || this.stateValue !== "listening") return; + this.requestCommit("maxDuration"); + }, this.maximumUtteranceDurationMs); + return pending.promise; + } + + pushAudio(chunk: PcmAudioChunk): boolean { + if (this.stateValue !== "listening" || !this.pending) return false; + try { + this.transcriber.sendAudio(this.pending.id, chunk); + return true; + } catch (error) { + this.fail(asSessionError(error)); + return false; + } + } + + /** + * Allows a provider endpoint to finish the utterance. RSVP calls this only + * after the final target has stopped drawing; earlier provider commits are + * retained as transcript text while capture continues. + */ + allowProviderFinalization(): void { + if (this.stateValue !== "listening" || !this.pending) { + throw new SpeechSessionError( + "invalidState", + "Provider finalization can only be enabled while listening.", + ); + } + this.pending.providerFinalizationAllowed = true; + } + + requestCommit( + trigger: Exclude = "manual", + ): void { + if (this.stateValue !== "listening" || !this.pending) { + throw new SpeechSessionError( + "invalidState", + "Only a listening speech utterance can be committed.", + ); + } + + const utteranceId = this.pending.id; + this.pending.finalizationTrigger = trigger; + this.clearMaximumDurationTimer(); + try { + this.transcriber.requestCommit(utteranceId); + } catch (error) { + this.fail(asSessionError(error)); + return; + } + + this.setState("finalizing", utteranceId); + this.finalizationTimer = globalThis.setTimeout(() => { + if (this.pending?.id !== utteranceId) return; + const error = new SpeechSessionError( + "finalizationTimeout", + "The speech utterance did not produce a committed transcript in time.", + { retryable: true }, + ); + this.fail(error); + try { + this.transcriber.cancelUtterance(utteranceId); + } catch { + // The session is already failed; cancellation is best-effort cleanup. + } + }, this.finalizationTimeoutMs); + } + + cancelUtterance(): void { + if (!this.pending) return; + const utteranceId = this.pending.id; + const error = new SpeechSessionError( + "utteranceCancelled", + "The active speech utterance was cancelled.", + { retryable: true }, + ); + this.clearUtteranceTimers(); + this.pending.reject(error); + this.pending = undefined; + this.expectedClose = true; + try { + this.transcriber.cancelUtterance(utteranceId); + } finally { + // Cancellation invalidates the connection so provider context cannot + // leak into a later utterance. + this.setState("closed"); + } + } + + subscribe(listener: SpeechSessionListener): () => void { + this.listeners.add(listener); + return () => this.listeners.delete(listener); + } + + close(): Promise { + this.closePromise ??= this.performClose(); + return this.closePromise; + } + + private readonly handleTranscriberEvent = (event: TranscriberEvent): void => { + if (this.stateValue === "closed") return; + + switch (event.type) { + case "partial": + if (event.utteranceId !== this.pending?.id) return; + this.emit({ + type: "partial", + utteranceId: event.utteranceId, + text: this.withCommittedText(event.text), + settled: event.settled, + receivedAtMs: event.receivedAtMs, + }); + break; + case "commit": + this.handleProviderCommit(event); + break; + case "error": + if (event.utteranceId && event.utteranceId !== this.pending?.id) return; + this.fail(asSessionError(event.error)); + break; + case "closed": + if (!event.expected && !this.expectedClose) { + this.fail( + new SpeechSessionError( + "transcriberFailure", + "The realtime transcription connection closed unexpectedly.", + { retryable: true }, + ), + ); + } + break; + } + }; + + private handleProviderCommit( + event: Extract, + ): void { + if (!this.pending || event.utteranceId !== this.pending.id) return; + const text = event.text.trim(); + if (text) { + this.pending.committedSegments.push({ + text, + receivedAtMs: event.receivedAtMs, + }); + } + const accumulatedText = this.accumulatedCommittedText(); + this.emit({ + type: "partial", + utteranceId: event.utteranceId, + text: accumulatedText, + settled: true, + receivedAtMs: event.receivedAtMs, + }); + + if ( + this.stateValue === "finalizing" || + this.pending.providerFinalizationAllowed + ) { + this.completeUtterance( + event.utteranceId, + accumulatedText, + event.receivedAtMs, + ); + } + } + + private withCommittedText(partialText: string): string { + const committedText = this.accumulatedCommittedText(); + return [committedText, partialText.trim()].filter(Boolean).join(" "); + } + + private accumulatedCommittedText(): string { + return ( + this.pending?.committedSegments + .map((segment) => segment.text) + .join(" ") ?? "" + ).trim(); + } + + private completeUtterance( + utteranceId: string, + text: string, + completedAtMs: number, + ): void { + if (!this.pending || utteranceId !== this.pending.id) return; + const pending = this.pending; + try { + this.transcriber.endUtterance(utteranceId); + } catch (error) { + this.fail(asSessionError(error)); + return; + } + const result: SpeechUtteranceResult = { + utteranceId, + text, + committedSegments: pending.committedSegments.map((segment) => ({ + ...segment, + })), + startedAtMs: pending.startedAtMs, + completedAtMs, + durationMs: Math.max(0, completedAtMs - pending.startedAtMs), + finalizationTrigger: pending.finalizationTrigger, + }; + + this.clearUtteranceTimers(); + this.pending = undefined; + this.setState("ready"); + pending.resolve(result); + this.emit({ type: "final", result }); + } + + private fail(error: SpeechSessionError): void { + if (this.stateValue === "closed" || this.stateValue === "failed") return; + const utteranceId = this.pending?.id; + this.clearUtteranceTimers(); + this.pending?.reject(error); + this.pending = undefined; + this.setState("failed", utteranceId); + this.emit({ type: "error", utteranceId, error }); + } + + private async performClose(): Promise { + if (this.stateValue === "closed") { + this.unsubscribeTranscriber(); + this.listeners.clear(); + return; + } + this.expectedClose = true; + this.clearUtteranceTimers(); + if (this.pending) { + this.pending.reject( + new SpeechSessionError("closed", "The speech session was closed."), + ); + this.pending = undefined; + } + await this.transcriber.close(); + this.unsubscribeTranscriber(); + this.setState("closed"); + this.listeners.clear(); + } + + private setState(state: SpeechSessionState, utteranceId?: string): void { + if (this.stateValue === state) return; + this.stateValue = state; + this.emit({ type: "state", state, utteranceId }); + } + + private clearMaximumDurationTimer(): void { + if (this.maximumDurationTimer !== undefined) { + globalThis.clearTimeout(this.maximumDurationTimer); + this.maximumDurationTimer = undefined; + } + } + + private clearUtteranceTimers(): void { + this.clearMaximumDurationTimer(); + if (this.finalizationTimer !== undefined) { + globalThis.clearTimeout(this.finalizationTimer); + this.finalizationTimer = undefined; + } + } + + private emit(event: SpeechSessionEvent): void { + for (const listener of this.listeners) { + try { + listener(event); + } catch { + // A UI observer cannot interrupt utterance or cleanup state transitions. + } + } + } +} diff --git a/components/speech/speechToken.ts b/components/speech/speechToken.ts new file mode 100644 index 00000000..74f97e01 --- /dev/null +++ b/components/speech/speechToken.ts @@ -0,0 +1,137 @@ +import { getEasyEyesBaseUrl } from "../easyeyesBaseUrl"; +import { TranscriberError } from "./transcriber"; + +export type SpeechProvider = "elevenlabs" | "deepgram"; + +export interface SpeechTokenRequestContext { + readonly experimentFullPath: string; + readonly pavloviaSessionToken: string; +} + +export interface SpeechTokenProviderOptions { + readonly requestContext: () => SpeechTokenRequestContext; + readonly endpoint?: string; + readonly timeoutMs?: number; + readonly fetchImpl?: typeof fetch; +} + +const SPEECH_TOKEN_PATH = "/.netlify/functions/speech-token"; +const SPEECH_TOKEN_PROTOCOL_VERSION = 1; +const DEFAULT_TIMEOUT_MS = 5000; +const PRODUCTION_ORIGIN = "https://easyeyes.app"; +const NETLIFY_PREVIEW_ORIGIN = /^https:\/\/[a-z0-9-]+--easyeyes\.netlify\.app$/; +const LOCAL_DEVELOPMENT_ORIGIN = /^http:\/\/localhost:\d+$/; + +export const buildSpeechTokenEndpoint = (baseUrl: string): string => { + let base: URL; + try { + base = new URL(baseUrl); + } catch { + throw new TranscriberError( + "invalidConfiguration", + "The speech credential service URL is invalid.", + ); + } + + const trusted = + base.origin === PRODUCTION_ORIGIN || + NETLIFY_PREVIEW_ORIGIN.test(base.origin) || + LOCAL_DEVELOPMENT_ORIGIN.test(base.origin); + if (!trusted || base.username || base.password) { + throw new TranscriberError( + "invalidConfiguration", + "The speech credential service URL is not trusted.", + ); + } + + return `${base.origin}${SPEECH_TOKEN_PATH}`; +}; + +export const createSpeechTokenProvider = ( + provider: SpeechProvider, + options: SpeechTokenProviderOptions, +): (() => Promise) => { + if (!options || typeof options.requestContext !== "function") { + throw new TranscriberError( + "invalidConfiguration", + "A Pavlovia session context provider is required.", + ); + } + const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { + throw new TranscriberError( + "invalidConfiguration", + "Token request timeout must be a finite positive number.", + ); + } + + return async () => { + const fetchImpl = options.fetchImpl ?? globalThis.fetch; + if (typeof fetchImpl !== "function") { + throw new TranscriberError( + "unsupported", + "Credential requests are unavailable in this browser.", + ); + } + const endpoint = + options.endpoint ?? buildSpeechTokenEndpoint(await getEasyEyesBaseUrl()); + const context = options.requestContext(); + if ( + !context.experimentFullPath.trim() || + !context.pavloviaSessionToken.trim() + ) { + throw new TranscriberError( + "credentialFailure", + "The active Pavlovia session is unavailable.", + ); + } + + const controller = new AbortController(); + const timeoutId = globalThis.setTimeout( + () => controller.abort(), + timeoutMs, + ); + try { + const response = await fetchImpl(endpoint, { + method: "POST", + headers: { + Accept: "application/json", + "Content-Type": "application/json", + }, + body: JSON.stringify({ + protocolVersion: SPEECH_TOKEN_PROTOCOL_VERSION, + provider, + experimentFullPath: context.experimentFullPath, + pavloviaSessionToken: context.pavloviaSessionToken, + }), + cache: "no-store", + signal: controller.signal, + }); + if (!response.ok) { + throw new TranscriberError( + "credentialFailure", + "A realtime transcription credential could not be issued.", + { retryable: response.status === 429 || response.status >= 500 }, + ); + } + const body = (await response.json()) as { token?: unknown }; + if (typeof body.token !== "string" || !body.token.trim()) { + throw new TranscriberError( + "credentialFailure", + "The credential service returned an invalid response.", + { retryable: true }, + ); + } + return body.token; + } catch (error) { + if (error instanceof TranscriberError) throw error; + throw new TranscriberError( + "credentialFailure", + "The realtime transcription credential request failed.", + { retryable: true, originalError: error }, + ); + } finally { + globalThis.clearTimeout(timeoutId); + } + }; +}; diff --git a/components/speech/transcriber.ts b/components/speech/transcriber.ts new file mode 100644 index 00000000..291654db --- /dev/null +++ b/components/speech/transcriber.ts @@ -0,0 +1,114 @@ +import { + TRANSCRIPTION_PCM_SAMPLE_RATE, + type PcmAudioChunk, +} from "./audioCapture"; + +export { TRANSCRIPTION_PCM_SAMPLE_RATE, type PcmAudioChunk }; + +export type TranscriberState = + | "idle" + | "connecting" + | "ready" + | "closed" + | "failed"; + +export type TranscriberErrorCode = + | "unsupported" + | "invalidConfiguration" + | "credentialFailure" + | "connectionFailure" + | "connectionTimeout" + | "notConnected" + | "utteranceConflict" + | "noActiveUtterance" + | "invalidAudio" + | "authenticationFailure" + | "quotaExceeded" + | "rateLimited" + | "termsNotAccepted" + | "providerUnavailable" + | "sessionLimit" + | "inputRejected" + | "closed" + | "unexpected"; + +export class TranscriberError extends Error { + readonly code: TranscriberErrorCode; + readonly retryable: boolean; + readonly originalError?: unknown; + + constructor( + code: TranscriberErrorCode, + message: string, + options: { retryable?: boolean; originalError?: unknown } = {}, + ) { + super(message); + this.name = "TranscriberError"; + this.code = code; + this.retryable = options.retryable ?? false; + this.originalError = options.originalError; + } +} + +export interface TranscriberConnectionInfo { + readonly provider: string; + readonly model: string; + readonly sessionId: string; + readonly languageCode: string; + readonly sampleRate: typeof TRANSCRIPTION_PCM_SAMPLE_RATE; +} + +export interface TranscriberPartialEvent { + readonly type: "partial"; + readonly utteranceId: string; + readonly text: string; + readonly settled: boolean; + readonly receivedAtMs: number; +} + +export interface TranscriberCommitEvent { + readonly type: "commit"; + readonly utteranceId: string; + readonly text: string; + readonly receivedAtMs: number; +} + +export interface TranscriberErrorEvent { + readonly type: "error"; + readonly utteranceId?: string; + readonly error: TranscriberError; + readonly receivedAtMs: number; +} + +export interface TranscriberClosedEvent { + readonly type: "closed"; + readonly expected: boolean; + readonly receivedAtMs: number; +} + +export type TranscriberEvent = + | TranscriberPartialEvent + | TranscriberCommitEvent + | TranscriberErrorEvent + | TranscriberClosedEvent; + +export type TranscriberListener = (event: TranscriberEvent) => void; + +/** + * Provider-neutral boundary for one realtime transcription connection. + * A connection may contain sequential utterances, but never more than one + * active utterance. Provider commits are exposed separately so a task can + * retain early endpointed text without ending its application-level response. + */ +export interface StreamingTranscriber { + readonly state: TranscriberState; + readonly connectionInfo?: TranscriberConnectionInfo; + connect(): Promise; + beginUtterance(utteranceId: string): void; + sendAudio(utteranceId: string, chunk: PcmAudioChunk): void; + requestCommit(utteranceId: string): void; + endUtterance(utteranceId: string): void; + cancelUtterance(utteranceId: string): void; + subscribe(listener: TranscriberListener): () => void; + close(): Promise; +} diff --git a/tests/audioCapture.test.ts b/tests/audioCapture.test.ts new file mode 100644 index 00000000..8bfa8d93 --- /dev/null +++ b/tests/audioCapture.test.ts @@ -0,0 +1,106 @@ +import { + PcmMicrophoneCapture, + StreamingLinearResampler, + float32ToPcm16, +} from "../components/speech/audioCapture"; +import type { + MicrophoneAudioFrameListener, + MicrophoneHealth, + MicrophoneSession, +} from "../components/speech/microphone"; + +class FakeMicrophoneSession implements MicrophoneSession { + readonly stream = {} as MediaStream; + readonly track = {} as MediaStreamTrack; + readonly frameSize = 128; + readonly sampleRate = 48000; + readonly frameDurationMs = (this.frameSize / this.sampleRate) * 1000; + readonly startFrameSource = jest.fn(); + readonly stopFrameSource = jest.fn(); + readonly closeFrameSource = jest.fn(() => { + this.listener = undefined; + }); + private listener?: MicrophoneAudioFrameListener; + + getHealth(): MicrophoneHealth { + return { + state: "ready", + readyState: "live", + enabled: true, + muted: false, + }; + } + + getTrackSettings(): MediaTrackSettings { + return { sampleRate: this.sampleRate }; + } + + readFrame(): void {} + + async subscribeToAudioFrames(listener: MicrophoneAudioFrameListener) { + this.listener = listener; + return { + start: this.startFrameSource, + stop: this.stopFrameSource, + close: this.closeFrameSource, + }; + } + + subscribeToHealth(): () => void { + return () => undefined; + } + + async close(): Promise {} + + emit(samples: Float32Array, capturedAtMs = 100): void { + this.listener?.({ samples, sampleRate: this.sampleRate, capturedAtMs }); + } +} + +describe("generic microphone PCM capture", () => { + it("clips and converts normalized floating-point samples to PCM16", () => { + expect([...float32ToPcm16(Float32Array.from([-2, -1, 0, 1, 2]))]).toEqual([ + -32768, -32768, 0, 32767, 32767, + ]); + }); + + it("preserves streaming resampler continuity across frame boundaries", () => { + const resampler = new StreamingLinearResampler(48000, 16000); + const first = resampler.process( + Float32Array.from({ length: 48 }, (_, index) => index / 100), + ); + const second = resampler.process( + Float32Array.from({ length: 48 }, (_, index) => (index + 48) / 100), + ); + + expect(first.length + second.length).toBe(32); + expect(first[0]).toBeCloseTo(0); + expect(second[0]).toBeCloseTo(0.48); + }); + + it("drops pre-onset frames and emits only while the task gate is open", async () => { + const microphone = new FakeMicrophoneSession(); + const capture = new PcmMicrophoneCapture(microphone, { + outputChunkDurationMs: 1, + }); + const chunks: Int16Array[] = []; + capture.subscribe((chunk) => chunks.push(chunk.samples)); + await capture.initialize(); + + microphone.emit(new Float32Array(48).fill(0.5)); + expect(chunks).toHaveLength(0); + + capture.start(); + expect(microphone.startFrameSource).toHaveBeenCalledTimes(1); + microphone.emit(new Float32Array(48).fill(0.25)); + expect(chunks).toHaveLength(1); + expect(chunks[0]).toHaveLength(16); + + capture.stop(); + expect(microphone.stopFrameSource).toHaveBeenCalledTimes(1); + microphone.emit(new Float32Array(48).fill(0.75)); + expect(chunks).toHaveLength(1); + await capture.close(); + expect(microphone.closeFrameSource).toHaveBeenCalledTimes(1); + }); +}); diff --git a/tests/deepgramRealtimeTranscriber.test.ts b/tests/deepgramRealtimeTranscriber.test.ts new file mode 100644 index 00000000..7efb1be0 --- /dev/null +++ b/tests/deepgramRealtimeTranscriber.test.ts @@ -0,0 +1,161 @@ +import { + DeepgramRealtimeTranscriber, + buildDeepgramRealtimeUrl, + type DeepgramWebSocketLike, +} from "../components/speech/deepgramRealtimeTranscriber"; +import { + TRANSCRIPTION_PCM_SAMPLE_RATE, + type TranscriberEvent, +} from "../components/speech/transcriber"; + +class FakeWebSocket implements DeepgramWebSocketLike { + readyState = 0; + readonly sent: Array = []; + readonly close = jest.fn(() => { + this.readyState = 3; + }); + private readonly listeners = new Map< + string, + Set<(event?: unknown) => void> + >(); + + addEventListener(type: "open", listener: () => void): void; + addEventListener( + type: "message", + listener: (event: MessageEvent) => void, + ): void; + addEventListener(type: "error", listener: (event: unknown) => void): void; + addEventListener(type: "close", listener: (event: CloseEvent) => void): void; + addEventListener(type: string, listener: (event?: unknown) => void): void { + const listeners = this.listeners.get(type) ?? new Set(); + listeners.add(listener); + this.listeners.set(type, listeners); + } + + send(data: string | ArrayBuffer): void { + this.sent.push(data); + } + + open(): void { + this.readyState = 1; + for (const listener of this.listeners.get("open") ?? []) listener(); + } + + message(message: unknown): void { + const event = { data: JSON.stringify(message) }; + for (const listener of this.listeners.get("message") ?? []) listener(event); + } +} + +const options = { + tokenProvider: async () => "temporary-jwt", + languageCode: "en-US", + keyterms: ["cat", "dog", "cat"], + targetKeytermBiasEnabled: true, + endpointingMs: 500, +} as const; + +const deferred = () => { + let resolve!: (value: T) => void; + const promise = new Promise((resolvePromise) => { + resolve = resolvePromise; + }); + return { promise, resolve }; +}; + +describe("Deepgram realtime transcription", () => { + it("builds an explicit Nova-3 PCM configuration", () => { + const url = new URL(buildDeepgramRealtimeUrl(options)); + + expect(url.searchParams.get("model")).toBe("nova-3"); + expect(url.searchParams.get("encoding")).toBe("linear16"); + expect(url.searchParams.get("sample_rate")).toBe("16000"); + expect(url.searchParams.get("endpointing")).toBe("500"); + expect(url.searchParams.get("mip_opt_out")).toBe("true"); + expect(url.searchParams.getAll("keyterm")).toEqual(["cat", "dog"]); + }); + + it("authenticates with a short-lived bearer token and accumulates final pieces", async () => { + const socket = new FakeWebSocket(); + let protocols: readonly string[] = []; + const transcriber = new DeepgramRealtimeTranscriber({ + ...options, + webSocketFactory: (_url, receivedProtocols) => { + protocols = receivedProtocols; + return socket; + }, + now: () => 100, + }); + const events: TranscriberEvent[] = []; + transcriber.subscribe((event) => events.push(event)); + + const connecting = transcriber.connect(); + await Promise.resolve(); + socket.open(); + await connecting; + expect(protocols).toEqual(["bearer", "temporary-jwt"]); + + transcriber.beginUtterance("trial-1"); + transcriber.sendAudio("trial-1", { + samples: new Int16Array([1, -2]), + sampleRate: TRANSCRIPTION_PCM_SAMPLE_RATE, + capturedAtMs: 50, + }); + expect(socket.sent[0]).toBeInstanceOf(ArrayBuffer); + + socket.message({ + type: "Results", + is_final: true, + speech_final: false, + channel: { alternatives: [{ transcript: "cat dog" }] }, + }); + socket.message({ + type: "Results", + is_final: true, + speech_final: true, + channel: { alternatives: [{ transcript: "fish" }] }, + }); + + expect(events.at(-1)).toEqual( + expect.objectContaining({ + type: "commit", + utteranceId: "trial-1", + text: "cat dog fish", + }), + ); + }); + + it("sends an explicit finalize message at the safety bound", async () => { + const socket = new FakeWebSocket(); + const transcriber = new DeepgramRealtimeTranscriber({ + ...options, + webSocketFactory: () => socket, + }); + const connecting = transcriber.connect(); + await Promise.resolve(); + socket.open(); + await connecting; + + transcriber.beginUtterance("trial-1"); + transcriber.requestCommit("trial-1"); + + expect(socket.sent.at(-1)).toBe(JSON.stringify({ type: "Finalize" })); + }); + + it("does not open a socket when closed during credential retrieval", async () => { + const token = deferred(); + const webSocketFactory = jest.fn(() => new FakeWebSocket()); + const transcriber = new DeepgramRealtimeTranscriber({ + ...options, + tokenProvider: () => token.promise, + webSocketFactory, + }); + + const connecting = transcriber.connect(); + await transcriber.close(); + token.resolve("late-token"); + + await expect(connecting).rejects.toMatchObject({ code: "closed" }); + expect(webSocketFactory).not.toHaveBeenCalled(); + }); +}); diff --git a/tests/elevenLabsRealtimeTranscriber.test.ts b/tests/elevenLabsRealtimeTranscriber.test.ts new file mode 100644 index 00000000..f19c6d30 --- /dev/null +++ b/tests/elevenLabsRealtimeTranscriber.test.ts @@ -0,0 +1,445 @@ +import { + DEFAULT_ELEVENLABS_VAD_CONFIG, + ElevenLabsRealtimeTranscriber, + buildElevenLabsRealtimeUrl, + createElevenLabsTokenProvider, + normalizeElevenLabsKeyterms, + type WebSocketCloseEventLike, + type WebSocketLike, + type WebSocketMessageEventLike, +} from "../components/speech/elevenLabsRealtimeTranscriber"; +import { + TRANSCRIPTION_PCM_SAMPLE_RATE, + TranscriberError, + type TranscriberEvent, +} from "../components/speech/transcriber"; + +type MessageListener = (event: WebSocketMessageEventLike) => void; +type ErrorListener = (event: unknown) => void; +type CloseListener = (event: WebSocketCloseEventLike) => void; + +class FakeWebSocket implements WebSocketLike { + readyState = 1; + readonly sent: string[] = []; + readonly close = jest.fn((code?: number, reason?: string) => { + this.readyState = 3; + this.emitClose({ code, reason }); + }); + + private readonly messageListeners = new Set(); + private readonly errorListeners = new Set(); + private readonly closeListeners = new Set(); + + addEventListener(type: "message", listener: MessageListener): void; + addEventListener(type: "error", listener: ErrorListener): void; + addEventListener(type: "close", listener: CloseListener): void; + addEventListener( + type: "message" | "error" | "close", + listener: MessageListener | ErrorListener | CloseListener, + ): void { + if (type === "message") + this.messageListeners.add(listener as MessageListener); + else if (type === "error") + this.errorListeners.add(listener as ErrorListener); + else this.closeListeners.add(listener as CloseListener); + } + + send(data: string): void { + this.sent.push(data); + } + + emitMessage(message: unknown): void { + const event = { data: JSON.stringify(message) }; + for (const listener of this.messageListeners) listener(event); + } + + emitError(error: unknown): void { + for (const listener of this.errorListeners) listener(error); + } + + emitClose(event: WebSocketCloseEventLike): void { + for (const listener of this.closeListeners) listener(event); + } +} + +const startConnection = async ( + options: Partial< + ConstructorParameters[0] + > = {}, +) => { + const socket = new FakeWebSocket(); + let connectionUrl = ""; + const transcriber = new ElevenLabsRealtimeTranscriber({ + tokenProvider: async () => "single-use-token", + languageCode: "en", + keyterms: ["cat", "dog", "fish"], + targetKeytermBiasEnabled: true, + noVerbatim: true, + webSocketFactory: (url) => { + connectionUrl = url; + return socket; + }, + now: () => 100, + ...options, + }); + + const connecting = transcriber.connect(); + await Promise.resolve(); + socket.emitMessage({ + message_type: "session_started", + session_id: "scribe-session", + }); + const info = await connecting; + return { transcriber, socket, connectionUrl, info }; +}; + +describe("ElevenLabs realtime configuration", () => { + it("uses provider VAD, cleaned output, PCM16 and repeated trial keyterms", () => { + const url = new URL( + buildElevenLabsRealtimeUrl("secret-token", { + languageCode: "EN", + keyterms: ["cat", "dog", "cat"], + targetKeytermBiasEnabled: true, + noVerbatim: true, + }), + ); + + expect(url.origin).toBe("wss://api.elevenlabs.io"); + expect(url.searchParams.get("model_id")).toBe("scribe_v2_realtime"); + expect(url.searchParams.get("audio_format")).toBe("pcm_16000"); + expect(url.searchParams.get("language_code")).toBe("en"); + expect(url.searchParams.get("commit_strategy")).toBe("vad"); + expect(url.searchParams.get("no_verbatim")).toBe("true"); + expect(url.searchParams.getAll("keyterms")).toEqual(["cat", "dog"]); + expect(url.searchParams.get("vad_silence_threshold_secs")).toBe( + String(DEFAULT_ELEVENLABS_VAD_CONFIG.silenceThresholdSecs), + ); + }); + + it("can disable target-keyterm bias without changing the audio path", () => { + const url = new URL( + buildElevenLabsRealtimeUrl("secret-token", { + languageCode: "en", + keyterms: ["cat", "dog", "fish"], + targetKeytermBiasEnabled: false, + noVerbatim: true, + }), + ); + + expect(url.searchParams.getAll("keyterms")).toEqual([]); + expect(url.searchParams.get("no_verbatim")).toBe("true"); + expect(url.searchParams.get("commit_strategy")).toBe("vad"); + }); + + it("uses cleaned output by default and permits a task-specific override", () => { + const defaultUrl = new URL( + buildElevenLabsRealtimeUrl("secret-token", { + languageCode: "en", + targetKeytermBiasEnabled: false, + }), + ); + const url = new URL( + buildElevenLabsRealtimeUrl("secret-token", { + languageCode: "en", + targetKeytermBiasEnabled: false, + noVerbatim: false, + }), + ); + + expect(defaultUrl.searchParams.get("no_verbatim")).toBe("true"); + expect(url.searchParams.get("no_verbatim")).toBe("false"); + }); + + it("always requests zero retention", () => { + const url = new URL( + buildElevenLabsRealtimeUrl("secret-token", { + languageCode: "en", + targetKeytermBiasEnabled: false, + }), + ); + + expect(url.searchParams.get("enable_logging")).toBe("false"); + }); + + it("rejects unsupported realtime keyterms without truncating them", () => { + expect(() => normalizeElevenLabsKeyterms(["x".repeat(21)])).toThrow( + TranscriberError, + ); + expect(() => + normalizeElevenLabsKeyterms( + Array.from({ length: 51 }, (_, index) => `word-${index}`), + ), + ).toThrow("at most 50"); + }); + + it("requires a resolved ISO language code", () => { + expect(() => + buildElevenLabsRealtimeUrl("token", { + languageCode: "en-US", + targetKeytermBiasEnabled: false, + noVerbatim: true, + }), + ).toThrow("ISO-639"); + }); +}); + +describe("createElevenLabsTokenProvider", () => { + it("requests a no-store token without accepting an empty response", async () => { + const fetchImpl = jest.fn( + async () => + ({ + ok: true, + status: 200, + json: async () => ({ token: "sutkn-test" }), + }) as Response, + ); + const provider = createElevenLabsTokenProvider({ + endpoint: "/token", + fetchImpl: fetchImpl as typeof fetch, + requestContext: () => ({ + experimentFullPath: "owner/experiment", + pavloviaSessionToken: "session-token", + }), + }); + + await expect(provider()).resolves.toBe("sutkn-test"); + expect(fetchImpl).toHaveBeenCalledWith( + "/token", + expect.objectContaining({ method: "POST", cache: "no-store" }), + ); + expect( + JSON.parse(fetchImpl.mock.calls[0][1]?.body as string), + ).toMatchObject({ + protocolVersion: 1, + provider: "elevenlabs", + experimentFullPath: "owner/experiment", + }); + }); + + it("redacts upstream credential response details", async () => { + const fetchImpl = jest.fn( + async () => + ({ + ok: false, + status: 401, + json: async () => ({ error: "secret upstream detail" }), + }) as Response, + ); + const provider = createElevenLabsTokenProvider({ + endpoint: "/token", + fetchImpl: fetchImpl as typeof fetch, + requestContext: () => ({ + experimentFullPath: "owner/experiment", + pavloviaSessionToken: "session-token", + }), + }); + + await expect(provider()).rejects.toMatchObject({ + code: "credentialFailure", + message: expect.not.stringContaining("secret upstream detail"), + }); + }); + + it("uses the EasyEyes function host when the experiment runs elsewhere", async () => { + const originalWindow = globalThis.window; + Object.defineProperty(globalThis, "window", { + configurable: true, + value: { + location: { + hostname: "run.pavlovia.org", + origin: "https://run.pavlovia.org", + search: "", + }, + }, + }); + const fetchImpl = jest.fn( + async () => + ({ + ok: true, + status: 200, + json: async () => ({ token: "sutkn-test" }), + }) as Response, + ); + + try { + const provider = createElevenLabsTokenProvider({ + fetchImpl: fetchImpl as typeof fetch, + requestContext: () => ({ + experimentFullPath: "owner/experiment", + pavloviaSessionToken: "session-token", + }), + }); + + await expect(provider()).resolves.toBe("sutkn-test"); + expect(fetchImpl).toHaveBeenCalledWith( + "https://easyeyes.app/.netlify/functions/speech-token", + expect.any(Object), + ); + } finally { + if (originalWindow === undefined) { + Reflect.deleteProperty(globalThis, "window"); + } else { + Object.defineProperty(globalThis, "window", { + configurable: true, + value: originalWindow, + }); + } + } + }); +}); + +describe("ElevenLabsRealtimeTranscriber", () => { + it("waits for session_started before becoming ready", async () => { + const { transcriber, connectionUrl, info } = await startConnection(); + + expect(transcriber.state).toBe("ready"); + expect(info).toEqual({ + provider: "elevenlabs", + model: "scribe_v2_realtime", + sessionId: "scribe-session", + languageCode: "en", + sampleRate: TRANSCRIPTION_PCM_SAMPLE_RATE, + }); + expect(new URL(connectionUrl).searchParams.getAll("keyterms")).toEqual([ + "cat", + "dog", + "fish", + ]); + }); + + it("rejects a pending connect when cleanup closes the socket", async () => { + const socket = new FakeWebSocket(); + const transcriber = new ElevenLabsRealtimeTranscriber({ + tokenProvider: async () => "single-use-token", + languageCode: "en", + targetKeytermBiasEnabled: false, + noVerbatim: true, + webSocketFactory: () => socket, + }); + + const connecting = transcriber.connect(); + await Promise.resolve(); + await transcriber.close(); + + await expect(connecting).rejects.toMatchObject({ code: "closed" }); + expect(transcriber.state).toBe("closed"); + }); + + it("maps provider commits without ending the application utterance", async () => { + const { transcriber, socket } = await startConnection(); + const events: TranscriberEvent[] = []; + transcriber.subscribe((event) => events.push(event)); + transcriber.beginUtterance("trial-1"); + + socket.emitMessage({ message_type: "partial_transcript", text: "ca" }); + socket.emitMessage({ message_type: "final_transcript", text: "cat" }); + expect(events).toEqual([ + expect.objectContaining({ + type: "partial", + utteranceId: "trial-1", + text: "ca", + settled: false, + }), + expect.objectContaining({ + type: "partial", + utteranceId: "trial-1", + text: "cat", + settled: true, + }), + ]); + + socket.emitMessage({ message_type: "committed_transcript", text: "cat" }); + expect(events.at(-1)).toEqual( + expect.objectContaining({ + type: "commit", + utteranceId: "trial-1", + text: "cat", + }), + ); + + expect(() => transcriber.beginUtterance("trial-2")).toThrow( + "already active", + ); + transcriber.endUtterance("trial-1"); + expect(() => transcriber.beginUtterance("trial-2")).not.toThrow(); + }); + + it("sends little-endian PCM only for the active utterance", async () => { + const { transcriber, socket } = await startConnection(); + const chunk = { + samples: new Int16Array([1, -2]), + sampleRate: TRANSCRIPTION_PCM_SAMPLE_RATE, + capturedAtMs: 10, + } as const; + + expect(() => transcriber.sendAudio("trial-1", chunk)).toThrow( + "no active transcription utterance", + ); + transcriber.beginUtterance("trial-1"); + transcriber.sendAudio("trial-1", chunk); + + const payload = JSON.parse(socket.sent[0]) as { + audio_base_64: string; + sample_rate: number; + }; + const bytes = Uint8Array.from(atob(payload.audio_base_64), (value) => + value.charCodeAt(0), + ); + expect([...bytes]).toEqual([1, 0, 254, 255]); + expect(payload.sample_rate).toBe(16000); + expect(() => transcriber.sendAudio("trial-old", chunk)).toThrow( + "stale transcription utterance", + ); + }); + + it("supports a bounded manual commit without treating it as final", async () => { + const { transcriber, socket } = await startConnection(); + transcriber.beginUtterance("trial-1"); + transcriber.requestCommit("trial-1"); + + expect(JSON.parse(socket.sent[0])).toEqual({ + message_type: "input_audio_chunk", + audio_base_64: "", + commit: true, + sample_rate: 16000, + }); + expect(() => transcriber.beginUtterance("trial-2")).toThrow( + "already active", + ); + }); + + it("maps provider failures to typed redacted events", async () => { + const { transcriber, socket } = await startConnection(); + const events: TranscriberEvent[] = []; + transcriber.subscribe((event) => events.push(event)); + transcriber.beginUtterance("trial-1"); + + socket.emitMessage({ + message_type: "rate_limited", + error: "account-sensitive provider response", + }); + + expect(transcriber.state).toBe("failed"); + expect(events[0]).toEqual( + expect.objectContaining({ + type: "error", + utteranceId: "trial-1", + error: expect.objectContaining({ + code: "rateLimited", + retryable: true, + message: expect.not.stringContaining("account-sensitive"), + }), + }), + ); + }); + + it("closes the connection when an utterance is cancelled", async () => { + const { transcriber, socket } = await startConnection(); + transcriber.beginUtterance("trial-1"); + + transcriber.cancelUtterance("trial-1"); + + expect(socket.close).toHaveBeenCalledWith(1000, "utterance-cancelled"); + expect(transcriber.state).toBe("closed"); + }); +}); diff --git a/tests/speechPreflight.test.ts b/tests/speechPreflight.test.ts index 0fd24cbb..9a4a6e4c 100644 --- a/tests/speechPreflight.test.ts +++ b/tests/speechPreflight.test.ts @@ -66,9 +66,21 @@ class FakeMicrophoneSession implements MicrophoneSession { this.readCount += 1; } + async subscribeToAudioFrames() { + return { + start: () => undefined, + stop: () => undefined, + close: () => undefined, + }; + } + subscribeToHealth(): () => void { return () => undefined; } + + setHealth(health: MicrophoneHealth): void { + this.health = health; + } } const runtime = { @@ -184,6 +196,60 @@ describe("runSpeechPreflight", () => { expect(result).toEqual({ ok: false, code: "clippedInput" }); expect(session.close).toHaveBeenCalledTimes(1); }); + + it("allows a transient muted track to recover within the grace period", async () => { + let nowMs = 0; + const session = new FakeMicrophoneSession( + (readIndex) => + readIndex < 2 ? alternatingFrame(0.001) : alternatingFrame(0.05), + { + state: "muted", + readyState: "live", + enabled: true, + muted: true, + }, + ); + + const result = await runSpeechPreflight({ + runtime: { ...runtime, muteRecoveryGraceMs: 150 }, + openMicrophone: jest.fn(async () => session), + now: () => nowMs, + wait: async (durationMs) => { + nowMs += durationMs; + if (nowMs === 50) { + session.setHealth({ + state: "ready", + readyState: "live", + enabled: true, + muted: false, + }); + } + }, + }); + + expect(result.ok).toBe(true); + }); + + it("rejects a track that remains muted beyond the recovery grace", async () => { + let nowMs = 0; + const session = new FakeMicrophoneSession(() => alternatingFrame(0.01), { + state: "muted", + readyState: "live", + enabled: true, + muted: true, + }); + + const result = await runSpeechPreflight({ + runtime: { ...runtime, muteRecoveryGraceMs: 100 }, + openMicrophone: jest.fn(async () => session), + now: () => nowMs, + wait: async (durationMs) => { + nowMs += durationMs; + }, + }); + + expect(result).toEqual({ ok: false, code: "microphoneUnavailable" }); + }); }); describe("mountRsvpSpeechPreflight", () => { diff --git a/tests/speechSession.test.ts b/tests/speechSession.test.ts new file mode 100644 index 00000000..c0685184 --- /dev/null +++ b/tests/speechSession.test.ts @@ -0,0 +1,324 @@ +import { + SpeechSession, + SpeechSessionError, + type SpeechSessionEvent, +} from "../components/speech/speechSession"; +import { + TRANSCRIPTION_PCM_SAMPLE_RATE, + TranscriberError, + type PcmAudioChunk, + type StreamingTranscriber, + type TranscriberConnectionInfo, + type TranscriberEvent, + type TranscriberListener, + type TranscriberState, +} from "../components/speech/transcriber"; + +const connectionInfo: TranscriberConnectionInfo = { + provider: "fake", + model: "fake-realtime", + sessionId: "session-1", + languageCode: "en", + sampleRate: TRANSCRIPTION_PCM_SAMPLE_RATE, +}; + +const audioChunk: PcmAudioChunk = { + samples: new Int16Array([1, -1, 2, -2]), + sampleRate: TRANSCRIPTION_PCM_SAMPLE_RATE, + capturedAtMs: 100, +}; + +class FakeTranscriber implements StreamingTranscriber { + state: TranscriberState = "idle"; + connectionInfo?: TranscriberConnectionInfo; + readonly beginUtterance = jest.fn((utteranceId: string) => { + this.activeUtteranceId = utteranceId; + }); + readonly sendAudio = jest.fn(); + readonly requestCommit = jest.fn(); + readonly endUtterance = jest.fn((utteranceId: string) => { + if (this.activeUtteranceId === utteranceId) { + this.activeUtteranceId = undefined; + } + }); + readonly cancelUtterance = jest.fn(() => { + this.state = "closed"; + this.activeUtteranceId = undefined; + }); + readonly close = jest.fn(async () => { + this.state = "closed"; + }); + + private readonly listeners = new Set(); + private activeUtteranceId?: string; + + async connect(): Promise { + this.state = "ready"; + this.connectionInfo = connectionInfo; + return connectionInfo; + } + + subscribe(listener: TranscriberListener): () => void { + this.listeners.add(listener); + return () => this.listeners.delete(listener); + } + + emit(event: TranscriberEvent): void { + for (const listener of this.listeners) listener(event); + } +} + +describe("SpeechSession", () => { + afterEach(() => { + jest.useRealTimers(); + }); + + it("forwards audio only while the current utterance is listening", async () => { + const transcriber = new FakeTranscriber(); + const session = new SpeechSession(transcriber, { + maximumUtteranceDurationMs: 12000, + now: () => 100, + }); + const events: SpeechSessionEvent[] = []; + session.subscribe((event) => events.push(event)); + + await session.connect(); + expect(session.pushAudio(audioChunk)).toBe(false); + + const resultPromise = session.beginUtterance("trial-1"); + expect(session.state).toBe("listening"); + expect(session.pushAudio(audioChunk)).toBe(true); + expect(transcriber.sendAudio).toHaveBeenCalledWith("trial-1", audioChunk); + + transcriber.emit({ + type: "partial", + utteranceId: "trial-1", + text: "ca", + settled: false, + receivedAtMs: 120, + }); + session.allowProviderFinalization(); + transcriber.emit({ + type: "commit", + utteranceId: "trial-1", + text: "cat dog fish", + receivedAtMs: 150, + }); + + await expect(resultPromise).resolves.toMatchObject({ + utteranceId: "trial-1", + text: "cat dog fish", + finalizationTrigger: "providerVad", + }); + expect(session.state).toBe("ready"); + expect(session.pushAudio(audioChunk)).toBe(false); + expect(events).toContainEqual( + expect.objectContaining({ + type: "partial", + utteranceId: "trial-1", + text: "ca", + }), + ); + }); + + it("ignores a commit carrying a stale utterance identifier", async () => { + const transcriber = new FakeTranscriber(); + const session = new SpeechSession(transcriber, { + maximumUtteranceDurationMs: 12000, + }); + await session.connect(); + + const resultPromise = session.beginUtterance("trial-2"); + transcriber.emit({ + type: "commit", + utteranceId: "trial-old", + text: "wrong callback", + receivedAtMs: 10, + }); + expect(session.state).toBe("listening"); + + session.allowProviderFinalization(); + transcriber.emit({ + type: "commit", + utteranceId: "trial-2", + text: "dog", + receivedAtMs: 20, + }); + await expect(resultPromise).resolves.toMatchObject({ text: "dog" }); + }); + + it("accumulates provider commits before presentation completion", async () => { + const transcriber = new FakeTranscriber(); + const session = new SpeechSession(transcriber, { + maximumUtteranceDurationMs: 12000, + }); + await session.connect(); + + const resultPromise = session.beginUtterance("trial-1"); + transcriber.emit({ + type: "commit", + utteranceId: "trial-1", + text: "cat", + receivedAtMs: 100, + }); + expect(session.state).toBe("listening"); + + session.allowProviderFinalization(); + transcriber.emit({ + type: "commit", + utteranceId: "trial-1", + text: "dog fish", + receivedAtMs: 200, + }); + + await expect(resultPromise).resolves.toMatchObject({ + text: "cat dog fish", + finalizationTrigger: "providerVad", + committedSegments: [ + { text: "cat", receivedAtMs: 100 }, + { text: "dog fish", receivedAtMs: 200 }, + ], + }); + expect(transcriber.endUtterance).toHaveBeenCalledWith("trial-1"); + }); + + it("starts each sequential utterance with an empty transcript buffer", async () => { + const transcriber = new FakeTranscriber(); + const session = new SpeechSession(transcriber, { + maximumUtteranceDurationMs: 12000, + }); + await session.connect(); + + const firstPromise = session.beginUtterance("trial-1"); + session.allowProviderFinalization(); + transcriber.emit({ + type: "commit", + utteranceId: "trial-1", + text: "cat", + receivedAtMs: 100, + }); + await expect(firstPromise).resolves.toMatchObject({ text: "cat" }); + + const secondPromise = session.beginUtterance("trial-2"); + transcriber.emit({ + type: "commit", + utteranceId: "trial-1", + text: "stale callback", + receivedAtMs: 150, + }); + session.allowProviderFinalization(); + transcriber.emit({ + type: "commit", + utteranceId: "trial-2", + text: "dog", + receivedAtMs: 200, + }); + + await expect(secondPromise).resolves.toMatchObject({ + text: "dog", + committedSegments: [{ text: "dog", receivedAtMs: 200 }], + }); + }); + + it("uses a manual commit without accepting later audio", async () => { + const transcriber = new FakeTranscriber(); + const session = new SpeechSession(transcriber, { + maximumUtteranceDurationMs: 12000, + }); + await session.connect(); + + const resultPromise = session.beginUtterance("trial-1"); + session.requestCommit("manual"); + + expect(session.state).toBe("finalizing"); + expect(session.pushAudio(audioChunk)).toBe(false); + expect(transcriber.requestCommit).toHaveBeenCalledWith("trial-1"); + + transcriber.emit({ + type: "commit", + utteranceId: "trial-1", + text: "fish", + receivedAtMs: performance.now(), + }); + await expect(resultPromise).resolves.toMatchObject({ + text: "fish", + finalizationTrigger: "manual", + }); + }); + + it("commits at the hard duration bound and fails if no final arrives", async () => { + jest.useFakeTimers(); + const transcriber = new FakeTranscriber(); + const session = new SpeechSession(transcriber, { + maximumUtteranceDurationMs: 12000, + finalizationTimeoutMs: 1000, + }); + await session.connect(); + + const resultPromise = session.beginUtterance("trial-timeout"); + jest.advanceTimersByTime(12000); + expect(transcriber.requestCommit).toHaveBeenCalledWith("trial-timeout"); + expect(session.state).toBe("finalizing"); + + jest.advanceTimersByTime(1000); + await expect(resultPromise).rejects.toMatchObject({ + code: "finalizationTimeout", + retryable: true, + }); + expect(transcriber.cancelUtterance).toHaveBeenCalledWith("trial-timeout"); + expect(session.state).toBe("failed"); + }); + + it("turns provider errors into a rejected technical utterance", async () => { + const transcriber = new FakeTranscriber(); + const session = new SpeechSession(transcriber, { + maximumUtteranceDurationMs: 12000, + }); + await session.connect(); + const resultPromise = session.beginUtterance("trial-error"); + + transcriber.emit({ + type: "error", + utteranceId: "trial-error", + error: new TranscriberError( + "providerUnavailable", + "Provider unavailable", + { retryable: true }, + ), + receivedAtMs: 10, + }); + + await expect(resultPromise).rejects.toMatchObject({ + code: "transcriberFailure", + retryable: true, + }); + expect(session.state).toBe("failed"); + }); + + it("closes a cancelled utterance so provider context cannot leak", async () => { + const transcriber = new FakeTranscriber(); + const session = new SpeechSession(transcriber, { + maximumUtteranceDurationMs: 12000, + }); + await session.connect(); + const resultPromise = session.beginUtterance("trial-cancelled"); + + session.cancelUtterance(); + + await expect(resultPromise).rejects.toBeInstanceOf(SpeechSessionError); + expect(transcriber.cancelUtterance).toHaveBeenCalledWith("trial-cancelled"); + expect(session.state).toBe("closed"); + }); + + it("closes idempotently", async () => { + const transcriber = new FakeTranscriber(); + const session = new SpeechSession(transcriber, { + maximumUtteranceDurationMs: 12000, + }); + await session.connect(); + + await Promise.all([session.close(), session.close()]); + expect(transcriber.close).toHaveBeenCalledTimes(1); + expect(session.state).toBe("closed"); + }); +}); diff --git a/tests/speechToken.test.ts b/tests/speechToken.test.ts new file mode 100644 index 00000000..79b70121 --- /dev/null +++ b/tests/speechToken.test.ts @@ -0,0 +1,61 @@ +/** + * @jest-environment jsdom + */ + +import { + buildSpeechTokenEndpoint, + createSpeechTokenProvider, +} from "../components/speech/speechToken"; +import { TranscriberError } from "../components/speech/transcriber"; + +describe("buildSpeechTokenEndpoint", () => { + it.each([ + [ + "https://easyeyes.app", + "https://easyeyes.app/.netlify/functions/speech-token", + ], + [ + "https://deploy-preview-123--easyeyes.netlify.app", + "https://deploy-preview-123--easyeyes.netlify.app/.netlify/functions/speech-token", + ], + [ + "http://localhost:8888", + "http://localhost:8888/.netlify/functions/speech-token", + ], + ])("accepts a trusted EasyEyes service origin", (base, expected) => { + expect(buildSpeechTokenEndpoint(base)).toBe(expected); + }); + + it.each([ + "https://attacker.example", + "https://easyeyes.app.attacker.example", + "http://easyeyes.app", + "https://user:password@easyeyes.app", + "not a URL", + ])("rejects an untrusted service origin", (base) => { + expect(() => buildSpeechTokenEndpoint(base)).toThrow(TranscriberError); + }); +}); + +describe("createSpeechTokenProvider", () => { + it("does not send a Pavlovia session to an untrusted default endpoint", async () => { + window.history.replaceState( + {}, + "", + "/?preview-deploy=https%3A%2F%2Fattacker.example", + ); + const fetchImpl = jest.fn(); + const provider = createSpeechTokenProvider("elevenlabs", { + requestContext: () => ({ + experimentFullPath: "scientist/study", + pavloviaSessionToken: "private-session-token", + }), + fetchImpl: fetchImpl as typeof fetch, + }); + + await expect(provider()).rejects.toMatchObject({ + code: "invalidConfiguration", + }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); +});