diff --git a/Client/tauri-client/src/lib/livekitSession.ts b/Client/tauri-client/src/lib/livekitSession.ts index 73757b11..524c8863 100644 --- a/Client/tauri-client/src/lib/livekitSession.ts +++ b/Client/tauri-client/src/lib/livekitSession.ts @@ -6,6 +6,7 @@ import { type RemoteTrack, type RemoteTrackPublication, type RemoteParticipant, + type Participant, DisconnectReason, } from "livekit-client"; import type { WsClient } from "@lib/ws"; @@ -18,8 +19,7 @@ import { } from "@stores/voice.store"; import { loadPref, savePref } from "@components/settings/helpers"; import { createLogger } from "@lib/logger"; -import { createNoiseSuppressor } from "@lib/noise-suppression"; -import type { NoiseSuppressor } from "@lib/noise-suppression"; +import { createRNNoiseProcessor } from "@lib/noise-suppression"; const log = createLogger("livekitSession"); @@ -32,81 +32,52 @@ export function parseUserId(identity: string): number { return 0; } -/** Compute RMS audio level (0-1) from frequency data. */ -export function computeRms(data: Uint8Array): number { - let sum = 0; - for (let i = 0; i < data.length; i++) { - const v = (data[i] ?? 0) / 255; - sum += v * v; - } - return Math.sqrt(sum / data.length); -} - -/** Get saved per-user volume (0-200 range, default 100). */ +/** Get saved per-user volume (0-200 range, default 100). Applied via LiveKit's GainNode-backed setVolume(). */ function getSavedUserVolume(userId: number): number { return loadPref(`userVolume_${userId}`, 100); } -/** Compare two pre-sorted speaker ID arrays. Both must be sorted ascending. */ -function speakerSetsEqual(a: readonly number[], b: readonly number[]): boolean { - if (a.length !== b.length) return false; - for (let i = 0; i < a.length; i++) { - if (a[i] !== b[i]) return false; - } - return true; -} - // --- Types --- type RemoteVideoCallback = (userId: number, stream: MediaStream) => void; type RemoteVideoRemovedCallback = (userId: number) => void; -interface RemoteAnalyser { - analyser: AnalyserNode; - source: MediaStreamAudioSourceNode; - data: Uint8Array; -} - // --- LiveKitSession class --- export class LiveKitSession { private room: Room | null = null; private ws: WsClient | null = null; - private noiseSuppressor: NoiseSuppressor | null = null; private onErrorCallback: ((message: string) => void) | null = null; private currentChannelId: number | null = null; private serverHost: string | null = null; - private speakingPollInterval: ReturnType | null = null; - private speakingThreshold = ((100 - 50) / 100) * 0.15; - private rawMicStream: MediaStream | null = null; - private readonly audioElements = new Map(); - private audioContainer: HTMLDivElement | null = null; private onRemoteVideoCallback: RemoteVideoCallback | null = null; private onRemoteVideoRemovedCallback: RemoteVideoRemovedCallback | null = null; - private sharedAudioCtx: AudioContext | null = null; - private readonly remoteAnalysers = new Map(); - private localMicGated = false; - private localAnalyser: AnalyserNode | null = null; - private localAnalyserSource: MediaStreamAudioSourceNode | null = null; - private localAnalyserClonedTrack: MediaStreamTrack | null = null; - private readonly localAnalyserData = new Uint8Array(128); - private previousSpeakerIds: readonly number[] = []; private tokenRefreshTimer: ReturnType | null = null; + /** Master output volume multiplier (0-2.0). Per-user volumes are scaled by this. */ + private outputVolumeMultiplier = loadPref("outputVolume", 100) / 100; - // --- Shared AudioContext (lazy) --- + // --- RNNoise processor (LiveKit TrackProcessor API) --- - private getSharedAudioCtx(): AudioContext { - if (this.sharedAudioCtx === null || this.sharedAudioCtx.state === "closed") { - this.sharedAudioCtx = new AudioContext({ sampleRate: 48000 }); - } - return this.sharedAudioCtx; + /** Attach RNNoise processor to the local mic track. Safe to call if already attached. */ + private async applyNoiseSuppressor(): Promise { + if (this.room === null) return; + const micPub = this.room.localParticipant.getTrackPublication(Track.Source.Microphone); + if (micPub?.track === undefined) return; + if (micPub.track.getProcessor() !== undefined) return; + const processor = createRNNoiseProcessor(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- LocalTrack.setProcessor uses wide generic, but AudioProcessorOptions is guaranteed at runtime with webAudioMix + await micPub.track.setProcessor(processor as any); + log.info("RNNoise processor attached to mic track"); } - private closeSharedAudioCtx(): void { - if (this.sharedAudioCtx !== null) { - void this.sharedAudioCtx.close(); - this.sharedAudioCtx = null; - } + /** Remove RNNoise processor from the local mic track. Safe to call if none attached. */ + private async removeNoiseSuppressor(): Promise { + if (this.room === null) return; + const micPub = this.room.localParticipant.getTrackPublication(Track.Source.Microphone); + if (micPub?.track === undefined) return; + if (micPub.track.getProcessor() === undefined) return; + await micPub.track.stopProcessor(); + log.info("RNNoise processor removed from mic track"); } // --- Room factory --- @@ -115,6 +86,7 @@ export class LiveKitSession { const newRoom = new Room({ adaptiveStream: true, dynacast: true, + webAudioMix: true, audioCaptureDefaults: { echoCancellation: loadPref("echoCancellation", true), noiseSuppression: loadPref("noiseSuppression", true), @@ -124,70 +96,10 @@ export class LiveKitSession { newRoom.on(RoomEvent.TrackSubscribed, this.handleTrackSubscribed); newRoom.on(RoomEvent.TrackUnsubscribed, this.handleTrackUnsubscribed); newRoom.on(RoomEvent.Disconnected, this.handleDisconnected); + newRoom.on(RoomEvent.ActiveSpeakersChanged, this.handleActiveSpeakersChanged); return newRoom; } - // --- Audio container --- - - private getOrCreateAudioContainer(): HTMLDivElement { - if (this.audioContainer !== null) return this.audioContainer; - const existing = document.getElementById("voice-audio-container"); - if (existing instanceof HTMLDivElement) { - this.audioContainer = existing; - return this.audioContainer; - } - const div = document.createElement("div"); - div.id = "voice-audio-container"; - div.style.display = "none"; - document.body.appendChild(div); - this.audioContainer = div; - return this.audioContainer; - } - - private cleanupAudioElements(): void { - for (const el of this.audioElements.values()) { - el.srcObject = null; - el.remove(); - } - this.audioElements.clear(); - } - - // --- RNNoise --- - - private async publishWithNoiseSuppression(): Promise { - if (this.room === null) return; - const savedDevice = loadPref("audioInputDevice", ""); - const constraints: MediaStreamConstraints = { - audio: { - deviceId: savedDevice ? { exact: savedDevice } : undefined, - echoCancellation: loadPref("echoCancellation", true), - noiseSuppression: loadPref("noiseSuppression", true), - autoGainControl: loadPref("autoGainControl", true), - }, - video: false, - }; - const rawStream = await navigator.mediaDevices.getUserMedia(constraints); - this.rawMicStream = rawStream; - try { - this.noiseSuppressor = createNoiseSuppressor(); - const processedStream = await this.noiseSuppressor.process(rawStream); - const processedTrack = processedStream.getAudioTracks()[0]; - if (processedTrack) { - await this.room.localParticipant.publishTrack(processedTrack, { - source: Track.Source.Microphone, - }); - } - } catch (err) { - for (const track of rawStream.getTracks()) track.stop(); - this.rawMicStream = null; - if (this.noiseSuppressor !== null) { - this.noiseSuppressor.destroy(); - this.noiseSuppressor = null; - } - throw err; - } - } - // --- Room event handlers (arrow fns to preserve `this`) --- private handleTrackSubscribed = ( @@ -197,22 +109,20 @@ export class LiveKitSession { ): void => { const userId = parseUserId(participant.identity); if (track.kind === Track.Kind.Audio) { - const container = this.getOrCreateAudioContainer(); + // Attach creates an