mirror of
https://github.com/J3vb/OwnCord.git
synced 2026-09-03 03:50:00 +03:00
fix: voice audio pipeline — autoplay unlock and GainNode-based VAD
- Add --autoplay-policy=no-user-gesture-required to WebView2 config
so remote participants' audio plays immediately on join (desktop app
doesn't need browser autoplay restrictions)
- Add AudioPlaybackStatusChanged handler with click-to-unlock fallback
for browsers that still block autoplay
- Replace broken VAD implementation that used setMicrophoneEnabled/
mediaStreamTrack.enabled (both fought LiveKit's track lifecycle) with
a unified GainNode audio pipeline:
rawMic → AnalyserNode (VAD) → GainNode (volume × gate) → sender
- VAD now gates by setting gain=0 instead of touching the track —
analyser always sees real audio, no stale track references
- Merge input volume and VAD into single pipeline (always active)
- Add voice settings UI: draggable sensitivity threshold on mic meter,
input/output volume sliders (0-200%), audio processing toggles
Co-Authored-By: claude-flow <ruv@ruv.net>
This commit is contained in:
@@ -18,7 +18,8 @@
|
|||||||
"minHeight": 500,
|
"minHeight": 500,
|
||||||
"decorations": true,
|
"decorations": true,
|
||||||
"resizable": true,
|
"resizable": true,
|
||||||
"center": true
|
"center": true,
|
||||||
|
"additionalBrowserArgs": "--autoplay-policy=no-user-gesture-required"
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"withGlobalTauri": true,
|
"withGlobalTauri": true,
|
||||||
|
|||||||
@@ -4,7 +4,7 @@
|
|||||||
|
|
||||||
import { createElement, appendChildren, setText } from "@lib/dom";
|
import { createElement, appendChildren, setText } from "@lib/dom";
|
||||||
import { loadPref, savePref, createToggle } from "./helpers";
|
import { loadPref, savePref, createToggle } from "./helpers";
|
||||||
import { switchInputDevice, switchOutputDevice, setVoiceSensitivity, setInputVolume, setOutputVolume } from "@lib/livekitSession";
|
import { switchInputDevice, switchOutputDevice, setVoiceSensitivity, setInputVolume, setOutputVolume, reapplyAudioProcessing } from "@lib/livekitSession";
|
||||||
|
|
||||||
export interface VoiceAudioTabHandle {
|
export interface VoiceAudioTabHandle {
|
||||||
build(): HTMLDivElement;
|
build(): HTMLDivElement;
|
||||||
@@ -96,6 +96,59 @@ function buildVoiceAudioTabInner(signal: AbortSignal, registerMic: MicRegistrar,
|
|||||||
appendChildren(inputVolumeRow, inputVolumeSlider, inputVolumeLabel);
|
appendChildren(inputVolumeRow, inputVolumeSlider, inputVolumeLabel);
|
||||||
section.appendChild(inputVolumeRow);
|
section.appendChild(inputVolumeRow);
|
||||||
|
|
||||||
|
// ── Mic level meter with draggable sensitivity threshold ────────
|
||||||
|
const sensitivityHeader = createElement("h3", {}, "Input Sensitivity");
|
||||||
|
section.appendChild(sensitivityHeader);
|
||||||
|
|
||||||
|
// Real-time mic level bar with embedded draggable threshold handle
|
||||||
|
const meterWrap = createElement("div", { class: "mic-meter-wrap" });
|
||||||
|
const meterBar = createElement("div", { class: "mic-meter-bar" });
|
||||||
|
const meterLevel = createElement("div", { class: "mic-meter-level" });
|
||||||
|
const meterThreshold = createElement("div", { class: "mic-meter-threshold" });
|
||||||
|
meterBar.appendChild(meterLevel);
|
||||||
|
meterBar.appendChild(meterThreshold);
|
||||||
|
meterWrap.appendChild(meterBar);
|
||||||
|
section.appendChild(meterWrap);
|
||||||
|
|
||||||
|
let currentSensitivity = loadPref<number>("voiceSensitivity", 50);
|
||||||
|
|
||||||
|
function updateThresholdIndicator(sensitivity: number): void {
|
||||||
|
meterThreshold.style.left = `${sensitivity}%`;
|
||||||
|
}
|
||||||
|
updateThresholdIndicator(currentSensitivity);
|
||||||
|
|
||||||
|
/** Compute sensitivity % from a mouse/touch X position relative to the meter bar. */
|
||||||
|
function sensitivityFromPointer(clientX: number): number {
|
||||||
|
const rect = meterBar.getBoundingClientRect();
|
||||||
|
const ratio = Math.max(0, Math.min(1, (clientX - rect.left) / rect.width));
|
||||||
|
return Math.round(ratio * 100);
|
||||||
|
}
|
||||||
|
|
||||||
|
function applySensitivity(val: number): void {
|
||||||
|
currentSensitivity = val;
|
||||||
|
savePref("voiceSensitivity", val);
|
||||||
|
setVoiceSensitivity(val);
|
||||||
|
updateThresholdIndicator(val);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Drag the threshold handle
|
||||||
|
meterThreshold.addEventListener("pointerdown", (e: PointerEvent) => {
|
||||||
|
e.preventDefault();
|
||||||
|
meterThreshold.setPointerCapture(e.pointerId);
|
||||||
|
const onMove = (ev: PointerEvent): void => { applySensitivity(sensitivityFromPointer(ev.clientX)); };
|
||||||
|
const onUp = (): void => {
|
||||||
|
meterThreshold.removeEventListener("pointermove", onMove);
|
||||||
|
meterThreshold.removeEventListener("pointerup", onUp);
|
||||||
|
};
|
||||||
|
meterThreshold.addEventListener("pointermove", onMove, { signal });
|
||||||
|
meterThreshold.addEventListener("pointerup", onUp, { signal });
|
||||||
|
}, { signal });
|
||||||
|
|
||||||
|
// Click on the meter bar to jump the threshold
|
||||||
|
meterBar.addEventListener("click", (e: MouseEvent) => {
|
||||||
|
applySensitivity(sensitivityFromPointer(e.clientX));
|
||||||
|
}, { signal });
|
||||||
|
|
||||||
// Output device selector
|
// Output device selector
|
||||||
const outputHeader = createElement("h3", {}, "Output Device");
|
const outputHeader = createElement("h3", {}, "Output Device");
|
||||||
const outputSelect = createElement("select", {
|
const outputSelect = createElement("select", {
|
||||||
@@ -253,49 +306,6 @@ function buildVoiceAudioTabInner(signal: AbortSignal, registerMic: MicRegistrar,
|
|||||||
stopCameraPreview();
|
stopCameraPreview();
|
||||||
});
|
});
|
||||||
|
|
||||||
// ── Mic level meter + sensitivity slider ──────────────────────────
|
|
||||||
const sensitivityHeader = createElement("h3", {}, "Input Sensitivity");
|
|
||||||
section.appendChild(sensitivityHeader);
|
|
||||||
|
|
||||||
// Real-time mic level bar
|
|
||||||
const meterWrap = createElement("div", { class: "mic-meter-wrap" });
|
|
||||||
const meterBar = createElement("div", { class: "mic-meter-bar" });
|
|
||||||
const meterLevel = createElement("div", { class: "mic-meter-level" });
|
|
||||||
const meterThreshold = createElement("div", { class: "mic-meter-threshold" });
|
|
||||||
meterBar.appendChild(meterLevel);
|
|
||||||
meterBar.appendChild(meterThreshold);
|
|
||||||
meterWrap.appendChild(meterBar);
|
|
||||||
section.appendChild(meterWrap);
|
|
||||||
|
|
||||||
// Sensitivity slider
|
|
||||||
const sensitivityRow = createElement("div", { class: "slider-row" });
|
|
||||||
const savedSensitivity = loadPref<number>("voiceSensitivity", 50);
|
|
||||||
const sensitivitySlider = createElement("input", {
|
|
||||||
class: "settings-slider",
|
|
||||||
type: "range",
|
|
||||||
min: "0",
|
|
||||||
max: "100",
|
|
||||||
value: String(savedSensitivity),
|
|
||||||
});
|
|
||||||
const sensitivityLabel = createElement("span", { class: "slider-val" }, `${savedSensitivity}%`);
|
|
||||||
|
|
||||||
// Position threshold indicator — matches slider direction:
|
|
||||||
// slider left (low sensitivity) = indicator left, slider right = indicator right
|
|
||||||
function updateThresholdIndicator(sensitivity: number): void {
|
|
||||||
meterThreshold.style.left = `${sensitivity}%`;
|
|
||||||
}
|
|
||||||
updateThresholdIndicator(savedSensitivity);
|
|
||||||
|
|
||||||
sensitivitySlider.addEventListener("input", () => {
|
|
||||||
const val = Number(sensitivitySlider.value);
|
|
||||||
setText(sensitivityLabel, `${val}%`);
|
|
||||||
savePref("voiceSensitivity", val);
|
|
||||||
setVoiceSensitivity(val);
|
|
||||||
updateThresholdIndicator(val);
|
|
||||||
}, { signal });
|
|
||||||
appendChildren(sensitivityRow, sensitivitySlider, sensitivityLabel);
|
|
||||||
section.appendChild(sensitivityRow);
|
|
||||||
|
|
||||||
// Start mic level monitoring for visual feedback
|
// Start mic level monitoring for visual feedback
|
||||||
void (async () => {
|
void (async () => {
|
||||||
try {
|
try {
|
||||||
@@ -330,7 +340,7 @@ function buildVoiceAudioTabInner(signal: AbortSignal, registerMic: MicRegistrar,
|
|||||||
meterLevel.style.width = `${visual * 100}%`;
|
meterLevel.style.width = `${visual * 100}%`;
|
||||||
|
|
||||||
// Color: green if above threshold, yellow/red if below
|
// Color: green if above threshold, yellow/red if below
|
||||||
const threshold = ((100 - Number(sensitivitySlider.value)) / 100) * 0.15;
|
const threshold = ((100 - currentSensitivity) / 100) * 0.15;
|
||||||
if (rms >= threshold) {
|
if (rms >= threshold) {
|
||||||
meterLevel.style.background = "#43b581"; // green — voice detected
|
meterLevel.style.background = "#43b581"; // green — voice detected
|
||||||
} else {
|
} else {
|
||||||
@@ -367,8 +377,8 @@ function buildVoiceAudioTabInner(signal: AbortSignal, registerMic: MicRegistrar,
|
|||||||
signal,
|
signal,
|
||||||
onChange: (nowOn) => {
|
onChange: (nowOn) => {
|
||||||
savePref(item.key, nowOn);
|
savePref(item.key, nowOn);
|
||||||
const currentDevice = loadPref<string>("audioInputDevice", "");
|
// Reapply audio processing constraints to the live mic track
|
||||||
void switchInputDevice(currentDevice);
|
void reapplyAudioProcessing();
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import {
|
|||||||
type RemoteTrackPublication,
|
type RemoteTrackPublication,
|
||||||
type RemoteParticipant,
|
type RemoteParticipant,
|
||||||
type Participant,
|
type Participant,
|
||||||
|
type LocalAudioTrack,
|
||||||
DisconnectReason,
|
DisconnectReason,
|
||||||
} from "livekit-client";
|
} from "livekit-client";
|
||||||
import type { WsClient } from "@lib/ws";
|
import type { WsClient } from "@lib/ws";
|
||||||
@@ -61,6 +62,19 @@ export class LiveKitSession {
|
|||||||
/** Master output volume multiplier (0-2.0). Per-user volumes are scaled by this. */
|
/** Master output volume multiplier (0-2.0). Per-user volumes are scaled by this. */
|
||||||
private outputVolumeMultiplier = loadPref<number>("outputVolume", 100) / 100;
|
private outputVolumeMultiplier = loadPref<number>("outputVolume", 100) / 100;
|
||||||
|
|
||||||
|
// --- Unified audio pipeline: input volume + VAD gating ---
|
||||||
|
// Pipeline: rawMicTrack → source → analyser (VAD reads here)
|
||||||
|
// → gainNode (volume × vadGate) → dest → WebRTC sender
|
||||||
|
private audioPipelineCtx: AudioContext | null = null;
|
||||||
|
private audioPipelineGain: GainNode | null = null;
|
||||||
|
private audioPipelineAnalyser: AnalyserNode | null = null;
|
||||||
|
private audioPipelineDest: MediaStreamAudioDestinationNode | null = null;
|
||||||
|
private vadAnimFrame: number = 0;
|
||||||
|
/** When true, mic is currently gated (muted by VAD — gain set to 0). */
|
||||||
|
private vadGated = false;
|
||||||
|
/** The user's input volume gain (0-2.0). VAD multiplies this by 0 or 1. */
|
||||||
|
private currentInputGain = 1.0;
|
||||||
|
|
||||||
// --- RNNoise processor (LiveKit TrackProcessor API) ---
|
// --- RNNoise processor (LiveKit TrackProcessor API) ---
|
||||||
|
|
||||||
/** Attach RNNoise processor to the local mic track. Safe to call if already attached. */
|
/** Attach RNNoise processor to the local mic track. Safe to call if already attached. */
|
||||||
@@ -101,6 +115,7 @@ export class LiveKitSession {
|
|||||||
newRoom.on(RoomEvent.TrackUnsubscribed, this.handleTrackUnsubscribed);
|
newRoom.on(RoomEvent.TrackUnsubscribed, this.handleTrackUnsubscribed);
|
||||||
newRoom.on(RoomEvent.Disconnected, this.handleDisconnected);
|
newRoom.on(RoomEvent.Disconnected, this.handleDisconnected);
|
||||||
newRoom.on(RoomEvent.ActiveSpeakersChanged, this.handleActiveSpeakersChanged);
|
newRoom.on(RoomEvent.ActiveSpeakersChanged, this.handleActiveSpeakersChanged);
|
||||||
|
newRoom.on(RoomEvent.AudioPlaybackStatusChanged, this.handleAudioPlaybackChanged);
|
||||||
return newRoom;
|
return newRoom;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -165,6 +180,41 @@ export class LiveKitSession {
|
|||||||
setSpeakers({ channel_id: this.currentChannelId, speakers: speakerIds });
|
setSpeakers({ channel_id: this.currentChannelId, speakers: speakerIds });
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Autoplay unlock: browsers block audio playback without user interaction.
|
||||||
|
* When LiveKit reports audio can't play, we register a one-time click handler
|
||||||
|
* on document that calls room.startAudio() — the next click anywhere unlocks audio.
|
||||||
|
*/
|
||||||
|
private autoplayUnlockHandler: (() => void) | null = null;
|
||||||
|
|
||||||
|
private handleAudioPlaybackChanged = (): void => {
|
||||||
|
if (this.room === null) return;
|
||||||
|
if (this.room.canPlaybackAudio) {
|
||||||
|
log.info("Audio playback is now allowed");
|
||||||
|
this.removeAutoplayUnlock();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
log.warn("Audio playback blocked by browser — registering click-to-unlock");
|
||||||
|
// Remove previous handler if any, then register a new one
|
||||||
|
this.removeAutoplayUnlock();
|
||||||
|
this.autoplayUnlockHandler = () => {
|
||||||
|
if (this.room !== null) {
|
||||||
|
void this.room.startAudio().then(() => {
|
||||||
|
log.info("Audio playback unlocked via user gesture");
|
||||||
|
});
|
||||||
|
}
|
||||||
|
this.removeAutoplayUnlock();
|
||||||
|
};
|
||||||
|
document.addEventListener("click", this.autoplayUnlockHandler, { once: true });
|
||||||
|
};
|
||||||
|
|
||||||
|
private removeAutoplayUnlock(): void {
|
||||||
|
if (this.autoplayUnlockHandler !== null) {
|
||||||
|
document.removeEventListener("click", this.autoplayUnlockHandler);
|
||||||
|
this.autoplayUnlockHandler = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private handleDisconnected = (reason?: DisconnectReason): void => {
|
private handleDisconnected = (reason?: DisconnectReason): void => {
|
||||||
log.info("LiveKit room disconnected", { reason });
|
log.info("LiveKit room disconnected", { reason });
|
||||||
const isUnexpected = reason !== DisconnectReason.CLIENT_INITIATED;
|
const isUnexpected = reason !== DisconnectReason.CLIENT_INITIATED;
|
||||||
@@ -297,6 +347,12 @@ export class LiveKitSession {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
log.info("Connected to LiveKit room", { channelId, url: resolvedUrl });
|
log.info("Connected to LiveKit room", { channelId, url: resolvedUrl });
|
||||||
|
// Optimistic startAudio — may succeed if the join was triggered by a
|
||||||
|
// recent user gesture. If not, the AudioPlaybackStatusChanged handler
|
||||||
|
// will register a click-to-unlock fallback.
|
||||||
|
this.room.startAudio().catch(() => {
|
||||||
|
log.debug("Optimistic startAudio failed — waiting for user gesture");
|
||||||
|
});
|
||||||
try {
|
try {
|
||||||
await this.room.localParticipant.setMicrophoneEnabled(true);
|
await this.room.localParticipant.setMicrophoneEnabled(true);
|
||||||
log.info("Published mic via LiveKit native capture");
|
log.info("Published mic via LiveKit native capture");
|
||||||
@@ -319,8 +375,9 @@ export class LiveKitSession {
|
|||||||
if (savedInput) await this.room.switchActiveDevice("audioinput", savedInput);
|
if (savedInput) await this.room.switchActiveDevice("audioinput", savedInput);
|
||||||
const savedOutput = loadPref<string>("audioOutputDevice", "");
|
const savedOutput = loadPref<string>("audioOutputDevice", "");
|
||||||
if (savedOutput) await this.room.switchActiveDevice("audiooutput", savedOutput);
|
if (savedOutput) await this.room.switchActiveDevice("audiooutput", savedOutput);
|
||||||
// Apply saved input volume
|
// Set up unified audio pipeline (input volume + VAD gating via GainNode).
|
||||||
this.applyInputVolume(loadPref<number>("inputVolume", 100));
|
// VAD polling only starts if saved sensitivity < 100.
|
||||||
|
this.setupAudioPipeline();
|
||||||
this.currentChannelId = channelId;
|
this.currentChannelId = channelId;
|
||||||
this.startTokenRefreshTimer();
|
this.startTokenRefreshTimer();
|
||||||
log.info("Voice session active", { channelId });
|
log.info("Voice session active", { channelId });
|
||||||
@@ -337,10 +394,11 @@ export class LiveKitSession {
|
|||||||
|
|
||||||
leaveVoice(sendWs = true): void {
|
leaveVoice(sendWs = true): void {
|
||||||
this.clearTokenRefreshTimer();
|
this.clearTokenRefreshTimer();
|
||||||
|
this.teardownAudioPipeline();
|
||||||
|
this.removeAutoplayUnlock();
|
||||||
if (sendWs && this.ws !== null) {
|
if (sendWs && this.ws !== null) {
|
||||||
this.ws.send({ type: "voice_leave", payload: {} });
|
this.ws.send({ type: "voice_leave", payload: {} });
|
||||||
}
|
}
|
||||||
this.cleanupInputGain();
|
|
||||||
if (this.room !== null) {
|
if (this.room !== null) {
|
||||||
const r = this.room;
|
const r = this.room;
|
||||||
this.room = null;
|
this.room = null;
|
||||||
@@ -426,9 +484,8 @@ export class LiveKitSession {
|
|||||||
await this.room.localParticipant.setMicrophoneEnabled(false);
|
await this.room.localParticipant.setMicrophoneEnabled(false);
|
||||||
await this.room.localParticipant.setMicrophoneEnabled(true);
|
await this.room.localParticipant.setMicrophoneEnabled(true);
|
||||||
}
|
}
|
||||||
// Reset and re-apply input volume after device switch (source track changed)
|
// Rebuild audio pipeline (source track changed after device switch)
|
||||||
this.cleanupInputGain();
|
this.setupAudioPipeline();
|
||||||
this.applyInputVolume(loadPref<number>("inputVolume", 100));
|
|
||||||
// Re-apply or remove RNNoise processor based on current setting
|
// Re-apply or remove RNNoise processor based on current setting
|
||||||
const enhancedNS = loadPref<boolean>("enhancedNoiseSuppression", false);
|
const enhancedNS = loadPref<boolean>("enhancedNoiseSuppression", false);
|
||||||
if (enhancedNS) {
|
if (enhancedNS) {
|
||||||
@@ -462,110 +519,245 @@ export class LiveKitSession {
|
|||||||
|
|
||||||
getUserVolume(userId: number): number { return getSavedUserVolume(userId); }
|
getUserVolume(userId: number): number { return getSavedUserVolume(userId); }
|
||||||
|
|
||||||
/** Input volume GainNode — adjusts mic gain via the WebRTC sender. */
|
// ── Unified audio pipeline: input volume + VAD gating ─────────────
|
||||||
private inputGainNode: GainNode | null = null;
|
//
|
||||||
private inputGainCtx: AudioContext | null = null;
|
// Architecture:
|
||||||
private inputGainDest: MediaStreamAudioDestinationNode | null = null;
|
// rawMicTrack → AudioContext source
|
||||||
|
// ├──→ AnalyserNode (VAD reads raw audio here — always sees real signal)
|
||||||
|
// └──→ GainNode (inputVolume × vadGate) → MediaStreamDestination → WebRTC sender
|
||||||
|
//
|
||||||
|
// The pipeline is always active while in a voice session. This avoids
|
||||||
|
// creating/destroying it when volume changes, and gives the VAD a stable
|
||||||
|
// analyser that's independent of LiveKit's track lifecycle.
|
||||||
|
|
||||||
/** Apply input volume gain to the local mic track via a Web Audio GainNode. */
|
/** Build or rebuild the audio pipeline on the current mic track. */
|
||||||
private applyInputVolume(volume: number): void {
|
private setupAudioPipeline(): void {
|
||||||
|
this.teardownAudioPipeline();
|
||||||
if (this.room === null) return;
|
if (this.room === null) return;
|
||||||
const micPub = this.room.localParticipant.getTrackPublication(Track.Source.Microphone);
|
const micPub = this.room.localParticipant.getTrackPublication(Track.Source.Microphone);
|
||||||
if (micPub?.track === undefined) return;
|
if (micPub?.track === undefined) return;
|
||||||
const gain = Math.max(0, Math.min(200, volume)) / 100;
|
|
||||||
|
|
||||||
// At 100% (gain=1.0), tear down the pipeline — no processing needed
|
|
||||||
if (gain === 1 && this.inputGainNode !== null) {
|
|
||||||
this.restoreOriginalSenderTrack();
|
|
||||||
this.cleanupInputGain();
|
|
||||||
log.info("Input volume reset to 100% — gain pipeline removed");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// No gain node and volume is default — nothing to do
|
|
||||||
if (gain === 1) return;
|
|
||||||
|
|
||||||
if (this.inputGainNode !== null) {
|
|
||||||
this.inputGainNode.gain.setTargetAtTime(gain, 0, 0.05);
|
|
||||||
log.debug("Input volume adjusted", { gain });
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Build GainNode pipeline and replace the sender's track
|
|
||||||
try {
|
try {
|
||||||
const mediaTrack = micPub.track.mediaStreamTrack;
|
const mediaTrack = micPub.track.mediaStreamTrack;
|
||||||
const ctx = new AudioContext({ sampleRate: 48000 });
|
const ctx = new AudioContext({ sampleRate: 48000 });
|
||||||
|
void ctx.resume(); // Ensure not suspended (WebView2 autoplay policy)
|
||||||
|
|
||||||
const source = ctx.createMediaStreamSource(new MediaStream([mediaTrack]));
|
const source = ctx.createMediaStreamSource(new MediaStream([mediaTrack]));
|
||||||
|
|
||||||
|
// Analyser: VAD reads time-domain data from here (always real audio)
|
||||||
|
const analyser = ctx.createAnalyser();
|
||||||
|
analyser.fftSize = 2048;
|
||||||
|
analyser.smoothingTimeConstant = 0.3;
|
||||||
|
|
||||||
|
// GainNode: controls both input volume and VAD gating
|
||||||
const gainNode = ctx.createGain();
|
const gainNode = ctx.createGain();
|
||||||
gainNode.gain.setTargetAtTime(gain, 0, 0.05);
|
this.currentInputGain = loadPref<number>("inputVolume", 100) / 100;
|
||||||
|
gainNode.gain.setValueAtTime(this.currentInputGain, ctx.currentTime);
|
||||||
|
|
||||||
const dest = ctx.createMediaStreamDestination();
|
const dest = ctx.createMediaStreamDestination();
|
||||||
|
|
||||||
|
// Wire: source → analyser (tap) and source → gain → dest
|
||||||
|
source.connect(analyser);
|
||||||
source.connect(gainNode);
|
source.connect(gainNode);
|
||||||
gainNode.connect(dest);
|
gainNode.connect(dest);
|
||||||
|
|
||||||
this.inputGainNode = gainNode;
|
this.audioPipelineCtx = ctx;
|
||||||
this.inputGainCtx = ctx;
|
this.audioPipelineGain = gainNode;
|
||||||
this.inputGainDest = dest;
|
this.audioPipelineAnalyser = analyser;
|
||||||
|
this.audioPipelineDest = dest;
|
||||||
|
|
||||||
// Replace the WebRTC sender's track with the gain-adjusted one
|
// Replace the WebRTC sender's track with the pipeline output
|
||||||
const adjustedTrack = dest.stream.getAudioTracks()[0];
|
const adjustedTrack = dest.stream.getAudioTracks()[0];
|
||||||
if (adjustedTrack !== undefined && micPub.track.sender) {
|
if (adjustedTrack !== undefined && micPub.track.sender) {
|
||||||
void micPub.track.sender.replaceTrack(adjustedTrack).catch((err) => {
|
void micPub.track.sender.replaceTrack(adjustedTrack).catch((err) => {
|
||||||
log.warn("Failed to replace sender track with gain-adjusted track", err);
|
log.warn("Failed to replace sender track with pipeline output", err);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
log.info("Input volume GainNode created", { gain });
|
|
||||||
|
log.info("Audio pipeline created", { inputGain: this.currentInputGain });
|
||||||
|
|
||||||
|
// Start VAD polling if sensitivity < 100
|
||||||
|
this.startVadPolling();
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
log.warn("Failed to set up input volume gain", err);
|
log.warn("Failed to set up audio pipeline", err);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Restore the original mic track on the WebRTC sender (undo gain pipeline). */
|
/** Tear down the audio pipeline and restore the original sender track. */
|
||||||
private restoreOriginalSenderTrack(): void {
|
private teardownAudioPipeline(): void {
|
||||||
if (this.room === null) return;
|
this.stopVadPolling();
|
||||||
const micPub = this.room.localParticipant.getTrackPublication(Track.Source.Microphone);
|
|
||||||
if (micPub?.track === undefined) return;
|
// Restore original mic track on the WebRTC sender
|
||||||
const originalTrack = micPub.track.mediaStreamTrack;
|
if (this.room !== null) {
|
||||||
if (micPub.track.sender) {
|
const micPub = this.room.localParticipant.getTrackPublication(Track.Source.Microphone);
|
||||||
void micPub.track.sender.replaceTrack(originalTrack).catch((err) => {
|
if (micPub?.track?.sender !== undefined) {
|
||||||
log.warn("Failed to restore original sender track", err);
|
const originalTrack = micPub.track.mediaStreamTrack;
|
||||||
});
|
void micPub.track.sender.replaceTrack(originalTrack).catch(() => {});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (this.audioPipelineGain !== null) { this.audioPipelineGain.disconnect(); this.audioPipelineGain = null; }
|
||||||
|
if (this.audioPipelineAnalyser !== null) { this.audioPipelineAnalyser.disconnect(); this.audioPipelineAnalyser = null; }
|
||||||
|
if (this.audioPipelineDest !== null) { this.audioPipelineDest.disconnect(); this.audioPipelineDest = null; }
|
||||||
|
if (this.audioPipelineCtx !== null) { void this.audioPipelineCtx.close(); this.audioPipelineCtx = null; }
|
||||||
|
this.vadGated = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
private cleanupInputGain(): void {
|
/** Update the effective gain on the pipeline (inputVolume × vadGate). */
|
||||||
if (this.inputGainNode !== null) {
|
private updatePipelineGain(): void {
|
||||||
this.inputGainNode.disconnect();
|
if (this.audioPipelineGain === null || this.audioPipelineCtx === null) return;
|
||||||
this.inputGainNode = null;
|
const effectiveGain = this.vadGated ? 0 : this.currentInputGain;
|
||||||
}
|
this.audioPipelineGain.gain.setTargetAtTime(effectiveGain, this.audioPipelineCtx.currentTime, 0.015);
|
||||||
if (this.inputGainDest !== null) {
|
|
||||||
this.inputGainDest.disconnect();
|
|
||||||
this.inputGainDest = null;
|
|
||||||
}
|
|
||||||
if (this.inputGainCtx !== null) {
|
|
||||||
void this.inputGainCtx.close();
|
|
||||||
this.inputGainCtx = null;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
setInputVolume(volume: number): void {
|
setInputVolume(volume: number): void {
|
||||||
const clamped = Math.max(0, Math.min(200, volume));
|
const clamped = Math.max(0, Math.min(200, volume));
|
||||||
savePref("inputVolume", clamped);
|
savePref("inputVolume", clamped);
|
||||||
this.applyInputVolume(clamped);
|
this.currentInputGain = clamped / 100;
|
||||||
|
this.updatePipelineGain();
|
||||||
}
|
}
|
||||||
|
|
||||||
setOutputVolume(volume: number): void {
|
setOutputVolume(volume: number): void {
|
||||||
const clamped = Math.max(0, Math.min(200, volume));
|
const clamped = Math.max(0, Math.min(200, volume));
|
||||||
savePref("outputVolume", clamped);
|
savePref("outputVolume", clamped);
|
||||||
this.outputVolumeMultiplier = clamped / 100;
|
this.outputVolumeMultiplier = clamped / 100;
|
||||||
// Re-apply all per-user volumes scaled by the new master output
|
|
||||||
this.applyAllVolumes();
|
this.applyAllVolumes();
|
||||||
}
|
}
|
||||||
|
|
||||||
setVoiceSensitivity(_sensitivity: number): void {
|
/**
|
||||||
// Voice sensitivity is now handled by LiveKit's built-in speaking detection.
|
* Apply voice sensitivity as a client-side VAD gate.
|
||||||
// The sensitivity parameter is saved in preferences by the UI but
|
* Sensitivity 0 = gate everything (threshold impossibly high).
|
||||||
// LiveKit's server-side VAD determines speaking state.
|
* Sensitivity 100 = gate nothing (no VAD polling).
|
||||||
log.debug("Voice sensitivity setting saved (handled by LiveKit VAD)");
|
* VAD sets gain to 0 when gated, restores inputVolume when ungated.
|
||||||
|
*/
|
||||||
|
setVoiceSensitivity(sensitivity: number): void {
|
||||||
|
const clamped = Math.max(0, Math.min(100, sensitivity));
|
||||||
|
savePref("voiceSensitivity", clamped);
|
||||||
|
// Restart VAD polling with the new threshold (pipeline stays intact)
|
||||||
|
this.stopVadPolling();
|
||||||
|
if (clamped >= 100) {
|
||||||
|
// Ensure ungated
|
||||||
|
if (this.vadGated) { this.vadGated = false; this.updatePipelineGain(); }
|
||||||
|
} else {
|
||||||
|
this.startVadPolling();
|
||||||
|
}
|
||||||
|
log.debug("Voice sensitivity updated", { sensitivity: clamped });
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Start VAD polling loop — reads from the pipeline's analyser. */
|
||||||
|
private startVadPolling(): void {
|
||||||
|
this.stopVadPolling();
|
||||||
|
if (this.audioPipelineAnalyser === null) return;
|
||||||
|
|
||||||
|
const sensitivity = loadPref<number>("voiceSensitivity", 50);
|
||||||
|
if (sensitivity >= 100) return;
|
||||||
|
|
||||||
|
// Convert sensitivity to an RMS threshold (time-domain):
|
||||||
|
// sensitivity 0 → threshold ~0.10, sensitivity 50 → ~0.05, sensitivity 99 → ~0.001
|
||||||
|
const threshold = ((100 - sensitivity) / 100) * 0.10;
|
||||||
|
const analyser = this.audioPipelineAnalyser;
|
||||||
|
const dataArray = new Float32Array(analyser.fftSize);
|
||||||
|
|
||||||
|
let silentFrames = 0;
|
||||||
|
let speechFrames = 0;
|
||||||
|
const GATE_ON_FRAMES = 12; // ~200ms of silence before gating
|
||||||
|
const GATE_OFF_FRAMES = 2; // ~33ms of speech before ungating
|
||||||
|
// Grace period: don't gate for the first ~500ms to let audio settle
|
||||||
|
let startupFrames = 0;
|
||||||
|
const STARTUP_GRACE = 30;
|
||||||
|
|
||||||
|
const poll = (): void => {
|
||||||
|
if (this.audioPipelineAnalyser === null) return;
|
||||||
|
|
||||||
|
analyser.getFloatTimeDomainData(dataArray);
|
||||||
|
let sum = 0;
|
||||||
|
for (let i = 0; i < dataArray.length; i++) {
|
||||||
|
const v = dataArray[i] ?? 0;
|
||||||
|
sum += v * v;
|
||||||
|
}
|
||||||
|
const rms = Math.sqrt(sum / dataArray.length);
|
||||||
|
|
||||||
|
if (startupFrames < STARTUP_GRACE) {
|
||||||
|
startupFrames++;
|
||||||
|
this.vadAnimFrame = requestAnimationFrame(poll);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (rms < threshold) {
|
||||||
|
speechFrames = 0;
|
||||||
|
silentFrames++;
|
||||||
|
if (!this.vadGated && silentFrames >= GATE_ON_FRAMES) {
|
||||||
|
this.vadGated = true;
|
||||||
|
this.updatePipelineGain(); // gain → 0
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
silentFrames = 0;
|
||||||
|
speechFrames++;
|
||||||
|
if (this.vadGated && speechFrames >= GATE_OFF_FRAMES) {
|
||||||
|
this.vadGated = false;
|
||||||
|
this.updatePipelineGain(); // gain → inputVolume
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
this.vadAnimFrame = requestAnimationFrame(poll);
|
||||||
|
};
|
||||||
|
this.vadAnimFrame = requestAnimationFrame(poll);
|
||||||
|
log.info("VAD polling started", { sensitivity, threshold });
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Stop VAD polling loop (pipeline stays intact). */
|
||||||
|
private stopVadPolling(): void {
|
||||||
|
if (this.vadAnimFrame !== 0) {
|
||||||
|
cancelAnimationFrame(this.vadAnimFrame);
|
||||||
|
this.vadAnimFrame = 0;
|
||||||
|
}
|
||||||
|
// Ungate if was gated
|
||||||
|
if (this.vadGated) {
|
||||||
|
this.vadGated = false;
|
||||||
|
this.updatePipelineGain();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Re-apply audio processing settings (echo cancellation, noise suppression, AGC)
|
||||||
|
* to the live mic track by restarting it with updated constraints.
|
||||||
|
*/
|
||||||
|
async reapplyAudioProcessing(): Promise<void> {
|
||||||
|
if (this.room === null) {
|
||||||
|
log.debug("Skipping audio processing reapply — no active voice session");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const micPub = this.room.localParticipant.getTrackPublication(Track.Source.Microphone);
|
||||||
|
if (micPub?.track === undefined) {
|
||||||
|
log.debug("Skipping audio processing reapply — no mic track");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const captureOptions = {
|
||||||
|
echoCancellation: loadPref("echoCancellation", true),
|
||||||
|
noiseSuppression: loadPref("noiseSuppression", true),
|
||||||
|
autoGainControl: loadPref("autoGainControl", true),
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
// restartTrack re-acquires the mic with new constraints without unpublishing
|
||||||
|
await (micPub.track as LocalAudioTrack).restartTrack(captureOptions);
|
||||||
|
log.info("Audio processing reapplied via restartTrack", captureOptions);
|
||||||
|
|
||||||
|
// Rebuild audio pipeline (underlying track changed)
|
||||||
|
this.setupAudioPipeline();
|
||||||
|
|
||||||
|
// Re-apply or remove RNNoise processor
|
||||||
|
const enhancedNS = loadPref<boolean>("enhancedNoiseSuppression", false);
|
||||||
|
if (enhancedNS) {
|
||||||
|
await this.applyNoiseSuppressor();
|
||||||
|
} else {
|
||||||
|
await this.removeNoiseSuppressor();
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
log.error("Failed to reapply audio processing", err);
|
||||||
|
this.onErrorCallback?.("Failed to update audio settings");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
getLocalCameraStream(): MediaStream | null {
|
getLocalCameraStream(): MediaStream | null {
|
||||||
@@ -600,9 +792,11 @@ export class LiveKitSession {
|
|||||||
hasRNNoiseProcessor: this.room.localParticipant.getTrackPublication(Track.Source.Microphone)?.track?.getProcessor() !== undefined,
|
hasRNNoiseProcessor: this.room.localParticipant.getTrackPublication(Track.Source.Microphone)?.track?.getProcessor() !== undefined,
|
||||||
currentChannelId: this.currentChannelId,
|
currentChannelId: this.currentChannelId,
|
||||||
outputVolumeMultiplier: this.outputVolumeMultiplier,
|
outputVolumeMultiplier: this.outputVolumeMultiplier,
|
||||||
inputGainActive: this.inputGainNode !== null,
|
audioPipelineActive: this.audioPipelineGain !== null,
|
||||||
inputGainValue: this.inputGainNode?.gain.value ?? null,
|
audioPipelineGain: this.audioPipelineGain?.gain.value ?? null,
|
||||||
inputGainCtxState: this.inputGainCtx?.state ?? null,
|
audioPipelineCtxState: this.audioPipelineCtx?.state ?? null,
|
||||||
|
vadGated: this.vadGated,
|
||||||
|
currentInputGain: this.currentInputGain,
|
||||||
localParticipant: this.room.localParticipant.identity, localTracks,
|
localParticipant: this.room.localParticipant.identity, localTracks,
|
||||||
remoteParticipants,
|
remoteParticipants,
|
||||||
};
|
};
|
||||||
@@ -638,5 +832,6 @@ export const getUserVolume = session.getUserVolume.bind(session);
|
|||||||
export const setInputVolume = session.setInputVolume.bind(session);
|
export const setInputVolume = session.setInputVolume.bind(session);
|
||||||
export const setOutputVolume = session.setOutputVolume.bind(session);
|
export const setOutputVolume = session.setOutputVolume.bind(session);
|
||||||
export const setVoiceSensitivity = session.setVoiceSensitivity.bind(session);
|
export const setVoiceSensitivity = session.setVoiceSensitivity.bind(session);
|
||||||
|
export const reapplyAudioProcessing = session.reapplyAudioProcessing.bind(session);
|
||||||
export const getLocalCameraStream = session.getLocalCameraStream.bind(session);
|
export const getLocalCameraStream = session.getLocalCameraStream.bind(session);
|
||||||
export const getSessionDebugInfo = session.getSessionDebugInfo.bind(session);
|
export const getSessionDebugInfo = session.getSessionDebugInfo.bind(session);
|
||||||
|
|||||||
@@ -1013,20 +1013,27 @@
|
|||||||
padding: 8px 12px; font-size: 14px; width: 200px; cursor: pointer;
|
padding: 8px 12px; font-size: 14px; width: 200px; cursor: pointer;
|
||||||
}
|
}
|
||||||
/* ── Mic Level Meter ── */
|
/* ── Mic Level Meter ── */
|
||||||
.mic-meter-wrap { margin-bottom: 8px; }
|
.mic-meter-wrap { margin-bottom: 12px; }
|
||||||
.mic-meter-bar {
|
.mic-meter-bar {
|
||||||
position: relative; height: 8px; border-radius: 4px;
|
position: relative; height: 8px; border-radius: 4px;
|
||||||
background: var(--bg-tertiary); overflow: visible;
|
background: var(--bg-tertiary); overflow: visible;
|
||||||
|
cursor: pointer;
|
||||||
}
|
}
|
||||||
.mic-meter-level {
|
.mic-meter-level {
|
||||||
height: 100%; border-radius: 4px; width: 0%;
|
height: 100%; border-radius: 4px; width: 0%;
|
||||||
background: #43b581; transition: width 50ms linear;
|
background: #43b581; transition: width 50ms linear;
|
||||||
|
pointer-events: none;
|
||||||
}
|
}
|
||||||
.mic-meter-threshold {
|
.mic-meter-threshold {
|
||||||
position: absolute; top: -3px; width: 2px; height: 14px;
|
position: absolute; top: -5px; width: 12px; height: 18px;
|
||||||
background: #fff; border-radius: 1px; left: 50%;
|
background: #fff; border-radius: 6px; left: 50%;
|
||||||
pointer-events: none; opacity: 0.8;
|
transform: translateX(-50%); opacity: 0.9;
|
||||||
|
cursor: grab; pointer-events: auto; z-index: 1;
|
||||||
|
box-shadow: 0 1px 3px rgba(0,0,0,0.4);
|
||||||
|
transition: opacity 0.1s;
|
||||||
}
|
}
|
||||||
|
.mic-meter-threshold:hover { opacity: 1; }
|
||||||
|
.mic-meter-threshold:active { cursor: grabbing; opacity: 1; }
|
||||||
.slider-row { display: flex; align-items: center; gap: 12px; }
|
.slider-row { display: flex; align-items: center; gap: 12px; }
|
||||||
.settings-slider {
|
.settings-slider {
|
||||||
flex: 1; -webkit-appearance: none; appearance: none;
|
flex: 1; -webkit-appearance: none; appearance: none;
|
||||||
|
|||||||
Reference in New Issue
Block a user