mirror of
https://github.com/J3vb/OwnCord.git
synced 2026-09-03 03:50:00 +03:00
fix: voice audio pipeline — autoplay unlock and GainNode-based VAD
- Add --autoplay-policy=no-user-gesture-required to WebView2 config
so remote participants' audio plays immediately on join (desktop app
doesn't need browser autoplay restrictions)
- Add AudioPlaybackStatusChanged handler with click-to-unlock fallback
for browsers that still block autoplay
- Replace broken VAD implementation that used setMicrophoneEnabled/
mediaStreamTrack.enabled (both fought LiveKit's track lifecycle) with
a unified GainNode audio pipeline:
rawMic → AnalyserNode (VAD) → GainNode (volume × gate) → sender
- VAD now gates by setting gain=0 instead of touching the track —
analyser always sees real audio, no stale track references
- Merge input volume and VAD into single pipeline (always active)
- Add voice settings UI: draggable sensitivity threshold on mic meter,
input/output volume sliders (0-200%), audio processing toggles
Co-Authored-By: claude-flow <ruv@ruv.net>
This commit is contained in:
jevb
@@ -18,7 +18,8 @@
|
||||
"minHeight": 500,
|
||||
"decorations": true,
|
||||
"resizable": true,
|
||||
"center": true
|
||||
"center": true,
|
||||
"additionalBrowserArgs": "--autoplay-policy=no-user-gesture-required"
|
||||
}
|
||||
],
|
||||
"withGlobalTauri": true,
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
import { createElement, appendChildren, setText } from "@lib/dom";
|
||||
import { loadPref, savePref, createToggle } from "./helpers";
|
||||
import { switchInputDevice, switchOutputDevice, setVoiceSensitivity, setInputVolume, setOutputVolume } from "@lib/livekitSession";
|
||||
import { switchInputDevice, switchOutputDevice, setVoiceSensitivity, setInputVolume, setOutputVolume, reapplyAudioProcessing } from "@lib/livekitSession";
|
||||
|
||||
export interface VoiceAudioTabHandle {
|
||||
build(): HTMLDivElement;
|
||||
@@ -96,6 +96,59 @@ function buildVoiceAudioTabInner(signal: AbortSignal, registerMic: MicRegistrar,
|
||||
appendChildren(inputVolumeRow, inputVolumeSlider, inputVolumeLabel);
|
||||
section.appendChild(inputVolumeRow);
|
||||
|
||||
// ── Mic level meter with draggable sensitivity threshold ────────
|
||||
const sensitivityHeader = createElement("h3", {}, "Input Sensitivity");
|
||||
section.appendChild(sensitivityHeader);
|
||||
|
||||
// Real-time mic level bar with embedded draggable threshold handle
|
||||
const meterWrap = createElement("div", { class: "mic-meter-wrap" });
|
||||
const meterBar = createElement("div", { class: "mic-meter-bar" });
|
||||
const meterLevel = createElement("div", { class: "mic-meter-level" });
|
||||
const meterThreshold = createElement("div", { class: "mic-meter-threshold" });
|
||||
meterBar.appendChild(meterLevel);
|
||||
meterBar.appendChild(meterThreshold);
|
||||
meterWrap.appendChild(meterBar);
|
||||
section.appendChild(meterWrap);
|
||||
|
||||
let currentSensitivity = loadPref<number>("voiceSensitivity", 50);
|
||||
|
||||
function updateThresholdIndicator(sensitivity: number): void {
|
||||
meterThreshold.style.left = `${sensitivity}%`;
|
||||
}
|
||||
updateThresholdIndicator(currentSensitivity);
|
||||
|
||||
/** Compute sensitivity % from a mouse/touch X position relative to the meter bar. */
|
||||
function sensitivityFromPointer(clientX: number): number {
|
||||
const rect = meterBar.getBoundingClientRect();
|
||||
const ratio = Math.max(0, Math.min(1, (clientX - rect.left) / rect.width));
|
||||
return Math.round(ratio * 100);
|
||||
}
|
||||
|
||||
function applySensitivity(val: number): void {
|
||||
currentSensitivity = val;
|
||||
savePref("voiceSensitivity", val);
|
||||
setVoiceSensitivity(val);
|
||||
updateThresholdIndicator(val);
|
||||
}
|
||||
|
||||
// Drag the threshold handle
|
||||
meterThreshold.addEventListener("pointerdown", (e: PointerEvent) => {
|
||||
e.preventDefault();
|
||||
meterThreshold.setPointerCapture(e.pointerId);
|
||||
const onMove = (ev: PointerEvent): void => { applySensitivity(sensitivityFromPointer(ev.clientX)); };
|
||||
const onUp = (): void => {
|
||||
meterThreshold.removeEventListener("pointermove", onMove);
|
||||
meterThreshold.removeEventListener("pointerup", onUp);
|
||||
};
|
||||
meterThreshold.addEventListener("pointermove", onMove, { signal });
|
||||
meterThreshold.addEventListener("pointerup", onUp, { signal });
|
||||
}, { signal });
|
||||
|
||||
// Click on the meter bar to jump the threshold
|
||||
meterBar.addEventListener("click", (e: MouseEvent) => {
|
||||
applySensitivity(sensitivityFromPointer(e.clientX));
|
||||
}, { signal });
|
||||
|
||||
// Output device selector
|
||||
const outputHeader = createElement("h3", {}, "Output Device");
|
||||
const outputSelect = createElement("select", {
|
||||
@@ -253,49 +306,6 @@ function buildVoiceAudioTabInner(signal: AbortSignal, registerMic: MicRegistrar,
|
||||
stopCameraPreview();
|
||||
});
|
||||
|
||||
// ── Mic level meter + sensitivity slider ──────────────────────────
|
||||
const sensitivityHeader = createElement("h3", {}, "Input Sensitivity");
|
||||
section.appendChild(sensitivityHeader);
|
||||
|
||||
// Real-time mic level bar
|
||||
const meterWrap = createElement("div", { class: "mic-meter-wrap" });
|
||||
const meterBar = createElement("div", { class: "mic-meter-bar" });
|
||||
const meterLevel = createElement("div", { class: "mic-meter-level" });
|
||||
const meterThreshold = createElement("div", { class: "mic-meter-threshold" });
|
||||
meterBar.appendChild(meterLevel);
|
||||
meterBar.appendChild(meterThreshold);
|
||||
meterWrap.appendChild(meterBar);
|
||||
section.appendChild(meterWrap);
|
||||
|
||||
// Sensitivity slider
|
||||
const sensitivityRow = createElement("div", { class: "slider-row" });
|
||||
const savedSensitivity = loadPref<number>("voiceSensitivity", 50);
|
||||
const sensitivitySlider = createElement("input", {
|
||||
class: "settings-slider",
|
||||
type: "range",
|
||||
min: "0",
|
||||
max: "100",
|
||||
value: String(savedSensitivity),
|
||||
});
|
||||
const sensitivityLabel = createElement("span", { class: "slider-val" }, `${savedSensitivity}%`);
|
||||
|
||||
// Position threshold indicator — matches slider direction:
|
||||
// slider left (low sensitivity) = indicator left, slider right = indicator right
|
||||
function updateThresholdIndicator(sensitivity: number): void {
|
||||
meterThreshold.style.left = `${sensitivity}%`;
|
||||
}
|
||||
updateThresholdIndicator(savedSensitivity);
|
||||
|
||||
sensitivitySlider.addEventListener("input", () => {
|
||||
const val = Number(sensitivitySlider.value);
|
||||
setText(sensitivityLabel, `${val}%`);
|
||||
savePref("voiceSensitivity", val);
|
||||
setVoiceSensitivity(val);
|
||||
updateThresholdIndicator(val);
|
||||
}, { signal });
|
||||
appendChildren(sensitivityRow, sensitivitySlider, sensitivityLabel);
|
||||
section.appendChild(sensitivityRow);
|
||||
|
||||
// Start mic level monitoring for visual feedback
|
||||
void (async () => {
|
||||
try {
|
||||
@@ -330,7 +340,7 @@ function buildVoiceAudioTabInner(signal: AbortSignal, registerMic: MicRegistrar,
|
||||
meterLevel.style.width = `${visual * 100}%`;
|
||||
|
||||
// Color: green if above threshold, yellow/red if below
|
||||
const threshold = ((100 - Number(sensitivitySlider.value)) / 100) * 0.15;
|
||||
const threshold = ((100 - currentSensitivity) / 100) * 0.15;
|
||||
if (rms >= threshold) {
|
||||
meterLevel.style.background = "#43b581"; // green — voice detected
|
||||
} else {
|
||||
@@ -367,8 +377,8 @@ function buildVoiceAudioTabInner(signal: AbortSignal, registerMic: MicRegistrar,
|
||||
signal,
|
||||
onChange: (nowOn) => {
|
||||
savePref(item.key, nowOn);
|
||||
const currentDevice = loadPref<string>("audioInputDevice", "");
|
||||
void switchInputDevice(currentDevice);
|
||||
// Reapply audio processing constraints to the live mic track
|
||||
void reapplyAudioProcessing();
|
||||
},
|
||||
});
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
type RemoteTrackPublication,
|
||||
type RemoteParticipant,
|
||||
type Participant,
|
||||
type LocalAudioTrack,
|
||||
DisconnectReason,
|
||||
} from "livekit-client";
|
||||
import type { WsClient } from "@lib/ws";
|
||||
@@ -61,6 +62,19 @@ export class LiveKitSession {
|
||||
/** Master output volume multiplier (0-2.0). Per-user volumes are scaled by this. */
|
||||
private outputVolumeMultiplier = loadPref<number>("outputVolume", 100) / 100;
|
||||
|
||||
// --- Unified audio pipeline: input volume + VAD gating ---
|
||||
// Pipeline: rawMicTrack → source → analyser (VAD reads here)
|
||||
// → gainNode (volume × vadGate) → dest → WebRTC sender
|
||||
private audioPipelineCtx: AudioContext | null = null;
|
||||
private audioPipelineGain: GainNode | null = null;
|
||||
private audioPipelineAnalyser: AnalyserNode | null = null;
|
||||
private audioPipelineDest: MediaStreamAudioDestinationNode | null = null;
|
||||
private vadAnimFrame: number = 0;
|
||||
/** When true, mic is currently gated (muted by VAD — gain set to 0). */
|
||||
private vadGated = false;
|
||||
/** The user's input volume gain (0-2.0). VAD multiplies this by 0 or 1. */
|
||||
private currentInputGain = 1.0;
|
||||
|
||||
// --- RNNoise processor (LiveKit TrackProcessor API) ---
|
||||
|
||||
/** Attach RNNoise processor to the local mic track. Safe to call if already attached. */
|
||||
@@ -101,6 +115,7 @@ export class LiveKitSession {
|
||||
newRoom.on(RoomEvent.TrackUnsubscribed, this.handleTrackUnsubscribed);
|
||||
newRoom.on(RoomEvent.Disconnected, this.handleDisconnected);
|
||||
newRoom.on(RoomEvent.ActiveSpeakersChanged, this.handleActiveSpeakersChanged);
|
||||
newRoom.on(RoomEvent.AudioPlaybackStatusChanged, this.handleAudioPlaybackChanged);
|
||||
return newRoom;
|
||||
}
|
||||
|
||||
@@ -165,6 +180,41 @@ export class LiveKitSession {
|
||||
setSpeakers({ channel_id: this.currentChannelId, speakers: speakerIds });
|
||||
};
|
||||
|
||||
/**
|
||||
* Autoplay unlock: browsers block audio playback without user interaction.
|
||||
* When LiveKit reports audio can't play, we register a one-time click handler
|
||||
* on document that calls room.startAudio() — the next click anywhere unlocks audio.
|
||||
*/
|
||||
private autoplayUnlockHandler: (() => void) | null = null;
|
||||
|
||||
private handleAudioPlaybackChanged = (): void => {
|
||||
if (this.room === null) return;
|
||||
if (this.room.canPlaybackAudio) {
|
||||
log.info("Audio playback is now allowed");
|
||||
this.removeAutoplayUnlock();
|
||||
return;
|
||||
}
|
||||
log.warn("Audio playback blocked by browser — registering click-to-unlock");
|
||||
// Remove previous handler if any, then register a new one
|
||||
this.removeAutoplayUnlock();
|
||||
this.autoplayUnlockHandler = () => {
|
||||
if (this.room !== null) {
|
||||
void this.room.startAudio().then(() => {
|
||||
log.info("Audio playback unlocked via user gesture");
|
||||
});
|
||||
}
|
||||
this.removeAutoplayUnlock();
|
||||
};
|
||||
document.addEventListener("click", this.autoplayUnlockHandler, { once: true });
|
||||
};
|
||||
|
||||
private removeAutoplayUnlock(): void {
|
||||
if (this.autoplayUnlockHandler !== null) {
|
||||
document.removeEventListener("click", this.autoplayUnlockHandler);
|
||||
this.autoplayUnlockHandler = null;
|
||||
}
|
||||
}
|
||||
|
||||
private handleDisconnected = (reason?: DisconnectReason): void => {
|
||||
log.info("LiveKit room disconnected", { reason });
|
||||
const isUnexpected = reason !== DisconnectReason.CLIENT_INITIATED;
|
||||
@@ -297,6 +347,12 @@ export class LiveKitSession {
|
||||
}
|
||||
}
|
||||
log.info("Connected to LiveKit room", { channelId, url: resolvedUrl });
|
||||
// Optimistic startAudio — may succeed if the join was triggered by a
|
||||
// recent user gesture. If not, the AudioPlaybackStatusChanged handler
|
||||
// will register a click-to-unlock fallback.
|
||||
this.room.startAudio().catch(() => {
|
||||
log.debug("Optimistic startAudio failed — waiting for user gesture");
|
||||
});
|
||||
try {
|
||||
await this.room.localParticipant.setMicrophoneEnabled(true);
|
||||
log.info("Published mic via LiveKit native capture");
|
||||
@@ -319,8 +375,9 @@ export class LiveKitSession {
|
||||
if (savedInput) await this.room.switchActiveDevice("audioinput", savedInput);
|
||||
const savedOutput = loadPref<string>("audioOutputDevice", "");
|
||||
if (savedOutput) await this.room.switchActiveDevice("audiooutput", savedOutput);
|
||||
// Apply saved input volume
|
||||
this.applyInputVolume(loadPref<number>("inputVolume", 100));
|
||||
// Set up unified audio pipeline (input volume + VAD gating via GainNode).
|
||||
// VAD polling only starts if saved sensitivity < 100.
|
||||
this.setupAudioPipeline();
|
||||
this.currentChannelId = channelId;
|
||||
this.startTokenRefreshTimer();
|
||||
log.info("Voice session active", { channelId });
|
||||
@@ -337,10 +394,11 @@ export class LiveKitSession {
|
||||
|
||||
leaveVoice(sendWs = true): void {
|
||||
this.clearTokenRefreshTimer();
|
||||
this.teardownAudioPipeline();
|
||||
this.removeAutoplayUnlock();
|
||||
if (sendWs && this.ws !== null) {
|
||||
this.ws.send({ type: "voice_leave", payload: {} });
|
||||
}
|
||||
this.cleanupInputGain();
|
||||
if (this.room !== null) {
|
||||
const r = this.room;
|
||||
this.room = null;
|
||||
@@ -426,9 +484,8 @@ export class LiveKitSession {
|
||||
await this.room.localParticipant.setMicrophoneEnabled(false);
|
||||
await this.room.localParticipant.setMicrophoneEnabled(true);
|
||||
}
|
||||
// Reset and re-apply input volume after device switch (source track changed)
|
||||
this.cleanupInputGain();
|
||||
this.applyInputVolume(loadPref<number>("inputVolume", 100));
|
||||
// Rebuild audio pipeline (source track changed after device switch)
|
||||
this.setupAudioPipeline();
|
||||
// Re-apply or remove RNNoise processor based on current setting
|
||||
const enhancedNS = loadPref<boolean>("enhancedNoiseSuppression", false);
|
||||
if (enhancedNS) {
|
||||
@@ -462,110 +519,245 @@ export class LiveKitSession {
|
||||
|
||||
getUserVolume(userId: number): number { return getSavedUserVolume(userId); }
|
||||
|
||||
/** Input volume GainNode — adjusts mic gain via the WebRTC sender. */
|
||||
private inputGainNode: GainNode | null = null;
|
||||
private inputGainCtx: AudioContext | null = null;
|
||||
private inputGainDest: MediaStreamAudioDestinationNode | null = null;
|
||||
// ── Unified audio pipeline: input volume + VAD gating ─────────────
|
||||
//
|
||||
// Architecture:
|
||||
// rawMicTrack → AudioContext source
|
||||
// ├──→ AnalyserNode (VAD reads raw audio here — always sees real signal)
|
||||
// └──→ GainNode (inputVolume × vadGate) → MediaStreamDestination → WebRTC sender
|
||||
//
|
||||
// The pipeline is always active while in a voice session. This avoids
|
||||
// creating/destroying it when volume changes, and gives the VAD a stable
|
||||
// analyser that's independent of LiveKit's track lifecycle.
|
||||
|
||||
/** Apply input volume gain to the local mic track via a Web Audio GainNode. */
|
||||
private applyInputVolume(volume: number): void {
|
||||
/** Build or rebuild the audio pipeline on the current mic track. */
|
||||
private setupAudioPipeline(): void {
|
||||
this.teardownAudioPipeline();
|
||||
if (this.room === null) return;
|
||||
const micPub = this.room.localParticipant.getTrackPublication(Track.Source.Microphone);
|
||||
if (micPub?.track === undefined) return;
|
||||
const gain = Math.max(0, Math.min(200, volume)) / 100;
|
||||
|
||||
// At 100% (gain=1.0), tear down the pipeline — no processing needed
|
||||
if (gain === 1 && this.inputGainNode !== null) {
|
||||
this.restoreOriginalSenderTrack();
|
||||
this.cleanupInputGain();
|
||||
log.info("Input volume reset to 100% — gain pipeline removed");
|
||||
return;
|
||||
}
|
||||
|
||||
// No gain node and volume is default — nothing to do
|
||||
if (gain === 1) return;
|
||||
|
||||
if (this.inputGainNode !== null) {
|
||||
this.inputGainNode.gain.setTargetAtTime(gain, 0, 0.05);
|
||||
log.debug("Input volume adjusted", { gain });
|
||||
return;
|
||||
}
|
||||
|
||||
// Build GainNode pipeline and replace the sender's track
|
||||
try {
|
||||
const mediaTrack = micPub.track.mediaStreamTrack;
|
||||
const ctx = new AudioContext({ sampleRate: 48000 });
|
||||
void ctx.resume(); // Ensure not suspended (WebView2 autoplay policy)
|
||||
|
||||
const source = ctx.createMediaStreamSource(new MediaStream([mediaTrack]));
|
||||
|
||||
// Analyser: VAD reads time-domain data from here (always real audio)
|
||||
const analyser = ctx.createAnalyser();
|
||||
analyser.fftSize = 2048;
|
||||
analyser.smoothingTimeConstant = 0.3;
|
||||
|
||||
// GainNode: controls both input volume and VAD gating
|
||||
const gainNode = ctx.createGain();
|
||||
gainNode.gain.setTargetAtTime(gain, 0, 0.05);
|
||||
this.currentInputGain = loadPref<number>("inputVolume", 100) / 100;
|
||||
gainNode.gain.setValueAtTime(this.currentInputGain, ctx.currentTime);
|
||||
|
||||
const dest = ctx.createMediaStreamDestination();
|
||||
|
||||
// Wire: source → analyser (tap) and source → gain → dest
|
||||
source.connect(analyser);
|
||||
source.connect(gainNode);
|
||||
gainNode.connect(dest);
|
||||
|
||||
this.inputGainNode = gainNode;
|
||||
this.inputGainCtx = ctx;
|
||||
this.inputGainDest = dest;
|
||||
this.audioPipelineCtx = ctx;
|
||||
this.audioPipelineGain = gainNode;
|
||||
this.audioPipelineAnalyser = analyser;
|
||||
this.audioPipelineDest = dest;
|
||||
|
||||
// Replace the WebRTC sender's track with the gain-adjusted one
|
||||
// Replace the WebRTC sender's track with the pipeline output
|
||||
const adjustedTrack = dest.stream.getAudioTracks()[0];
|
||||
if (adjustedTrack !== undefined && micPub.track.sender) {
|
||||
void micPub.track.sender.replaceTrack(adjustedTrack).catch((err) => {
|
||||
log.warn("Failed to replace sender track with gain-adjusted track", err);
|
||||
log.warn("Failed to replace sender track with pipeline output", err);
|
||||
});
|
||||
}
|
||||
log.info("Input volume GainNode created", { gain });
|
||||
|
||||
log.info("Audio pipeline created", { inputGain: this.currentInputGain });
|
||||
|
||||
// Start VAD polling if sensitivity < 100
|
||||
this.startVadPolling();
|
||||
} catch (err) {
|
||||
log.warn("Failed to set up input volume gain", err);
|
||||
log.warn("Failed to set up audio pipeline", err);
|
||||
}
|
||||
}
|
||||
|
||||
/** Restore the original mic track on the WebRTC sender (undo gain pipeline). */
|
||||
private restoreOriginalSenderTrack(): void {
|
||||
if (this.room === null) return;
|
||||
const micPub = this.room.localParticipant.getTrackPublication(Track.Source.Microphone);
|
||||
if (micPub?.track === undefined) return;
|
||||
const originalTrack = micPub.track.mediaStreamTrack;
|
||||
if (micPub.track.sender) {
|
||||
void micPub.track.sender.replaceTrack(originalTrack).catch((err) => {
|
||||
log.warn("Failed to restore original sender track", err);
|
||||
});
|
||||
/** Tear down the audio pipeline and restore the original sender track. */
|
||||
private teardownAudioPipeline(): void {
|
||||
this.stopVadPolling();
|
||||
|
||||
// Restore original mic track on the WebRTC sender
|
||||
if (this.room !== null) {
|
||||
const micPub = this.room.localParticipant.getTrackPublication(Track.Source.Microphone);
|
||||
if (micPub?.track?.sender !== undefined) {
|
||||
const originalTrack = micPub.track.mediaStreamTrack;
|
||||
void micPub.track.sender.replaceTrack(originalTrack).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
if (this.audioPipelineGain !== null) { this.audioPipelineGain.disconnect(); this.audioPipelineGain = null; }
|
||||
if (this.audioPipelineAnalyser !== null) { this.audioPipelineAnalyser.disconnect(); this.audioPipelineAnalyser = null; }
|
||||
if (this.audioPipelineDest !== null) { this.audioPipelineDest.disconnect(); this.audioPipelineDest = null; }
|
||||
if (this.audioPipelineCtx !== null) { void this.audioPipelineCtx.close(); this.audioPipelineCtx = null; }
|
||||
this.vadGated = false;
|
||||
}
|
||||
|
||||
private cleanupInputGain(): void {
|
||||
if (this.inputGainNode !== null) {
|
||||
this.inputGainNode.disconnect();
|
||||
this.inputGainNode = null;
|
||||
}
|
||||
if (this.inputGainDest !== null) {
|
||||
this.inputGainDest.disconnect();
|
||||
this.inputGainDest = null;
|
||||
}
|
||||
if (this.inputGainCtx !== null) {
|
||||
void this.inputGainCtx.close();
|
||||
this.inputGainCtx = null;
|
||||
}
|
||||
/** Update the effective gain on the pipeline (inputVolume × vadGate). */
|
||||
private updatePipelineGain(): void {
|
||||
if (this.audioPipelineGain === null || this.audioPipelineCtx === null) return;
|
||||
const effectiveGain = this.vadGated ? 0 : this.currentInputGain;
|
||||
this.audioPipelineGain.gain.setTargetAtTime(effectiveGain, this.audioPipelineCtx.currentTime, 0.015);
|
||||
}
|
||||
|
||||
setInputVolume(volume: number): void {
|
||||
const clamped = Math.max(0, Math.min(200, volume));
|
||||
savePref("inputVolume", clamped);
|
||||
this.applyInputVolume(clamped);
|
||||
this.currentInputGain = clamped / 100;
|
||||
this.updatePipelineGain();
|
||||
}
|
||||
|
||||
setOutputVolume(volume: number): void {
|
||||
const clamped = Math.max(0, Math.min(200, volume));
|
||||
savePref("outputVolume", clamped);
|
||||
this.outputVolumeMultiplier = clamped / 100;
|
||||
// Re-apply all per-user volumes scaled by the new master output
|
||||
this.applyAllVolumes();
|
||||
}
|
||||
|
||||
setVoiceSensitivity(_sensitivity: number): void {
|
||||
// Voice sensitivity is now handled by LiveKit's built-in speaking detection.
|
||||
// The sensitivity parameter is saved in preferences by the UI but
|
||||
// LiveKit's server-side VAD determines speaking state.
|
||||
log.debug("Voice sensitivity setting saved (handled by LiveKit VAD)");
|
||||
/**
|
||||
* Apply voice sensitivity as a client-side VAD gate.
|
||||
* Sensitivity 0 = gate everything (threshold impossibly high).
|
||||
* Sensitivity 100 = gate nothing (no VAD polling).
|
||||
* VAD sets gain to 0 when gated, restores inputVolume when ungated.
|
||||
*/
|
||||
setVoiceSensitivity(sensitivity: number): void {
|
||||
const clamped = Math.max(0, Math.min(100, sensitivity));
|
||||
savePref("voiceSensitivity", clamped);
|
||||
// Restart VAD polling with the new threshold (pipeline stays intact)
|
||||
this.stopVadPolling();
|
||||
if (clamped >= 100) {
|
||||
// Ensure ungated
|
||||
if (this.vadGated) { this.vadGated = false; this.updatePipelineGain(); }
|
||||
} else {
|
||||
this.startVadPolling();
|
||||
}
|
||||
log.debug("Voice sensitivity updated", { sensitivity: clamped });
|
||||
}
|
||||
|
||||
/** Start VAD polling loop — reads from the pipeline's analyser. */
|
||||
private startVadPolling(): void {
|
||||
this.stopVadPolling();
|
||||
if (this.audioPipelineAnalyser === null) return;
|
||||
|
||||
const sensitivity = loadPref<number>("voiceSensitivity", 50);
|
||||
if (sensitivity >= 100) return;
|
||||
|
||||
// Convert sensitivity to an RMS threshold (time-domain):
|
||||
// sensitivity 0 → threshold ~0.10, sensitivity 50 → ~0.05, sensitivity 99 → ~0.001
|
||||
const threshold = ((100 - sensitivity) / 100) * 0.10;
|
||||
const analyser = this.audioPipelineAnalyser;
|
||||
const dataArray = new Float32Array(analyser.fftSize);
|
||||
|
||||
let silentFrames = 0;
|
||||
let speechFrames = 0;
|
||||
const GATE_ON_FRAMES = 12; // ~200ms of silence before gating
|
||||
const GATE_OFF_FRAMES = 2; // ~33ms of speech before ungating
|
||||
// Grace period: don't gate for the first ~500ms to let audio settle
|
||||
let startupFrames = 0;
|
||||
const STARTUP_GRACE = 30;
|
||||
|
||||
const poll = (): void => {
|
||||
if (this.audioPipelineAnalyser === null) return;
|
||||
|
||||
analyser.getFloatTimeDomainData(dataArray);
|
||||
let sum = 0;
|
||||
for (let i = 0; i < dataArray.length; i++) {
|
||||
const v = dataArray[i] ?? 0;
|
||||
sum += v * v;
|
||||
}
|
||||
const rms = Math.sqrt(sum / dataArray.length);
|
||||
|
||||
if (startupFrames < STARTUP_GRACE) {
|
||||
startupFrames++;
|
||||
this.vadAnimFrame = requestAnimationFrame(poll);
|
||||
return;
|
||||
}
|
||||
|
||||
if (rms < threshold) {
|
||||
speechFrames = 0;
|
||||
silentFrames++;
|
||||
if (!this.vadGated && silentFrames >= GATE_ON_FRAMES) {
|
||||
this.vadGated = true;
|
||||
this.updatePipelineGain(); // gain → 0
|
||||
}
|
||||
} else {
|
||||
silentFrames = 0;
|
||||
speechFrames++;
|
||||
if (this.vadGated && speechFrames >= GATE_OFF_FRAMES) {
|
||||
this.vadGated = false;
|
||||
this.updatePipelineGain(); // gain → inputVolume
|
||||
}
|
||||
}
|
||||
|
||||
this.vadAnimFrame = requestAnimationFrame(poll);
|
||||
};
|
||||
this.vadAnimFrame = requestAnimationFrame(poll);
|
||||
log.info("VAD polling started", { sensitivity, threshold });
|
||||
}
|
||||
|
||||
/** Stop VAD polling loop (pipeline stays intact). */
|
||||
private stopVadPolling(): void {
|
||||
if (this.vadAnimFrame !== 0) {
|
||||
cancelAnimationFrame(this.vadAnimFrame);
|
||||
this.vadAnimFrame = 0;
|
||||
}
|
||||
// Ungate if was gated
|
||||
if (this.vadGated) {
|
||||
this.vadGated = false;
|
||||
this.updatePipelineGain();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-apply audio processing settings (echo cancellation, noise suppression, AGC)
|
||||
* to the live mic track by restarting it with updated constraints.
|
||||
*/
|
||||
async reapplyAudioProcessing(): Promise<void> {
|
||||
if (this.room === null) {
|
||||
log.debug("Skipping audio processing reapply — no active voice session");
|
||||
return;
|
||||
}
|
||||
const micPub = this.room.localParticipant.getTrackPublication(Track.Source.Microphone);
|
||||
if (micPub?.track === undefined) {
|
||||
log.debug("Skipping audio processing reapply — no mic track");
|
||||
return;
|
||||
}
|
||||
|
||||
const captureOptions = {
|
||||
echoCancellation: loadPref("echoCancellation", true),
|
||||
noiseSuppression: loadPref("noiseSuppression", true),
|
||||
autoGainControl: loadPref("autoGainControl", true),
|
||||
};
|
||||
|
||||
try {
|
||||
// restartTrack re-acquires the mic with new constraints without unpublishing
|
||||
await (micPub.track as LocalAudioTrack).restartTrack(captureOptions);
|
||||
log.info("Audio processing reapplied via restartTrack", captureOptions);
|
||||
|
||||
// Rebuild audio pipeline (underlying track changed)
|
||||
this.setupAudioPipeline();
|
||||
|
||||
// Re-apply or remove RNNoise processor
|
||||
const enhancedNS = loadPref<boolean>("enhancedNoiseSuppression", false);
|
||||
if (enhancedNS) {
|
||||
await this.applyNoiseSuppressor();
|
||||
} else {
|
||||
await this.removeNoiseSuppressor();
|
||||
}
|
||||
} catch (err) {
|
||||
log.error("Failed to reapply audio processing", err);
|
||||
this.onErrorCallback?.("Failed to update audio settings");
|
||||
}
|
||||
}
|
||||
|
||||
getLocalCameraStream(): MediaStream | null {
|
||||
@@ -600,9 +792,11 @@ export class LiveKitSession {
|
||||
hasRNNoiseProcessor: this.room.localParticipant.getTrackPublication(Track.Source.Microphone)?.track?.getProcessor() !== undefined,
|
||||
currentChannelId: this.currentChannelId,
|
||||
outputVolumeMultiplier: this.outputVolumeMultiplier,
|
||||
inputGainActive: this.inputGainNode !== null,
|
||||
inputGainValue: this.inputGainNode?.gain.value ?? null,
|
||||
inputGainCtxState: this.inputGainCtx?.state ?? null,
|
||||
audioPipelineActive: this.audioPipelineGain !== null,
|
||||
audioPipelineGain: this.audioPipelineGain?.gain.value ?? null,
|
||||
audioPipelineCtxState: this.audioPipelineCtx?.state ?? null,
|
||||
vadGated: this.vadGated,
|
||||
currentInputGain: this.currentInputGain,
|
||||
localParticipant: this.room.localParticipant.identity, localTracks,
|
||||
remoteParticipants,
|
||||
};
|
||||
@@ -638,5 +832,6 @@ export const getUserVolume = session.getUserVolume.bind(session);
|
||||
export const setInputVolume = session.setInputVolume.bind(session);
|
||||
export const setOutputVolume = session.setOutputVolume.bind(session);
|
||||
export const setVoiceSensitivity = session.setVoiceSensitivity.bind(session);
|
||||
export const reapplyAudioProcessing = session.reapplyAudioProcessing.bind(session);
|
||||
export const getLocalCameraStream = session.getLocalCameraStream.bind(session);
|
||||
export const getSessionDebugInfo = session.getSessionDebugInfo.bind(session);
|
||||
|
||||
@@ -1013,20 +1013,27 @@
|
||||
padding: 8px 12px; font-size: 14px; width: 200px; cursor: pointer;
|
||||
}
|
||||
/* ── Mic Level Meter ── */
|
||||
.mic-meter-wrap { margin-bottom: 8px; }
|
||||
.mic-meter-wrap { margin-bottom: 12px; }
|
||||
.mic-meter-bar {
|
||||
position: relative; height: 8px; border-radius: 4px;
|
||||
background: var(--bg-tertiary); overflow: visible;
|
||||
cursor: pointer;
|
||||
}
|
||||
.mic-meter-level {
|
||||
height: 100%; border-radius: 4px; width: 0%;
|
||||
background: #43b581; transition: width 50ms linear;
|
||||
pointer-events: none;
|
||||
}
|
||||
.mic-meter-threshold {
|
||||
position: absolute; top: -3px; width: 2px; height: 14px;
|
||||
background: #fff; border-radius: 1px; left: 50%;
|
||||
pointer-events: none; opacity: 0.8;
|
||||
position: absolute; top: -5px; width: 12px; height: 18px;
|
||||
background: #fff; border-radius: 6px; left: 50%;
|
||||
transform: translateX(-50%); opacity: 0.9;
|
||||
cursor: grab; pointer-events: auto; z-index: 1;
|
||||
box-shadow: 0 1px 3px rgba(0,0,0,0.4);
|
||||
transition: opacity 0.1s;
|
||||
}
|
||||
.mic-meter-threshold:hover { opacity: 1; }
|
||||
.mic-meter-threshold:active { cursor: grabbing; opacity: 1; }
|
||||
.slider-row { display: flex; align-items: center; gap: 12px; }
|
||||
.settings-slider {
|
||||
flex: 1; -webkit-appearance: none; appearance: none;
|
||||
|
||||
Reference in New Issue
Block a user