J3vb/OwnCord
1
0
Fork 0
mirror of https://github.com/J3vb/OwnCord.git synced 2026-09-03 03:50:00 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix: voice mute pipeline, security hardening, and video tile controls

Browse Source 
- Deafen now also mutes the local microphone (privacy fix)
- Mute/deafen zero the audio pipeline GainNode to guarantee silence
  when the replaced sender track bypasses LiveKit's track disable
- Send voice_leave on failed auto-reconnect to prevent ghost states
- Gate voice_join on LiveKit availability (reject if h.livekit == nil)
- IP-restrict /api/v1/livekit/webhook to admin CIDRs
- Warn at startup when LiveKit API keys are auto-generated (ephemeral)
- Fix connecting guard: move pending-join dispatch outside finally block
- Add volume slider + mute button overlay on remote video tiles
- Document token refresh limitation for 4h+ sessions
- Fix TS2306 in rnnoise-worklet test (ts-expect-error for worklet import)
This commit is contained in:
jevb
2026-03-26 20:53:09 +01:00
parent c13d5a67a5
commit 15b779f86b
7 changed files with 640 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Client/tauri-client/src/components/VideoGrid.ts
+184 -22
View File
@@ -4,14 +4,30 @@
*/
import { createElement, appendChildren } from "@lib/dom";
import { muteScreenshareAudio, setUserVolume } from "@lib/livekitSession";
import type { MountableComponent } from "@lib/safe-render";
export interface TileConfig {
/** True if this is the local user's own tile (no audio controls) */
readonly isSelf: boolean;
/** The real userId for audio control (differs from tile ID for screenshare tiles) */
readonly audioUserId: number;
/** True if this tile represents a screenshare (vs camera) */
readonly isScreenshare: boolean;
}
export interface VideoGridComponent extends MountableComponent {
addStream(userId: number, username: string, stream: MediaStream): void;
addStream(userId: number, username: string, stream: MediaStream, config?: TileConfig): void;
removeStream(userId: number): void;
hasStreams(): boolean;
setFocusedTile(tileId: number): void;
getFocusedTileId(): number | null;
}
const ICON_VOLUME = `<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polygon points="11 5 6 9 2 9 2 15 6 15 11 19 11 5"/><path d="M19.07 4.93a10 10 0 0 1 0 14.14"/><path d="M15.54 8.46a5 5 0 0 1 0 7.07"/></svg>`;
const ICON_VOLUME_X = `<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polygon points="11 5 6 9 2 9 2 15 6 15 11 19 11 5"/><line x1="23" y1="9" x2="17" y2="15"/><line x1="17" y1="9" x2="23" y2="15"/></svg>`;
function computeGridColumns(count: number): string {
if (count <= 1) return "1fr";
if (count <= 4) return "1fr 1fr";
@@ -21,20 +37,80 @@ function computeGridColumns(count: number): string {
export function createVideoGrid(): VideoGridComponent {
let root: HTMLDivElement | null = null;
const cells = new Map<number, HTMLDivElement>();
const cells = new Map<number, { el: HTMLDivElement; config?: TileConfig }>();
let focusedTileId: number | null = null;
function rebuildFocusLayout(): void {
if (root === null) return;
// Clear root children (we'll re-append in focus layout order)
while (root.firstChild) root.removeChild(root.firstChild);
if (focusedTileId === null || cells.size === 0) {
// No focus — use regular grid layout
root.classList.remove("focus-mode");
root.style.gridTemplateColumns = computeGridColumns(cells.size);
for (const entry of cells.values()) {
entry.el.classList.remove("focused", "thumb");
root.appendChild(entry.el);
}
return;
}
root.classList.add("focus-mode");
root.style.gridTemplateColumns = ""; // Clear grid columns, focus uses flex
// Main area
const mainArea = createElement("div", { class: "video-focus-main" });
// Strip area
const stripArea = createElement("div", { class: "video-focus-strip" });
const focusedEntry = cells.get(focusedTileId);
if (focusedEntry !== undefined) {
focusedEntry.el.classList.add("focused");
focusedEntry.el.classList.remove("thumb");
mainArea.appendChild(focusedEntry.el);
}
for (const [id, entry] of cells) {
if (id === focusedTileId) continue;
entry.el.classList.remove("focused");
entry.el.classList.add("thumb");
stripArea.appendChild(entry.el);
}
root.appendChild(mainArea);
// Only show strip if there are thumbnails
if (stripArea.childElementCount > 0) {
root.appendChild(stripArea);
}
}
function setFocusedTile(tileId: number): void {
focusedTileId = tileId;
rebuildFocusLayout();
}
function getFocusedTileIdFn(): number | null {
return focusedTileId;
}
function updateLayout(): void {
if (root === null) return;
if (focusedTileId !== null) {
rebuildFocusLayout();
return;
}
root.style.gridTemplateColumns = computeGridColumns(cells.size);
}
function addStream(userId: number, username: string, stream: MediaStream): void {
function addStream(userId: number, username: string, stream: MediaStream, config?: TileConfig): void {
if (root === null) return;
// If a cell already exists for this user, update it in place
const existing = cells.get(userId);
if (existing) {
const video = existing.querySelector("video");
if (existing !== undefined) {
const video = existing.el.querySelector("video");
if (video !== null) {
// Only replace srcObject if the underlying tracks changed
const oldTracks = (video.srcObject as MediaStream | null)?.getTracks() ?? [];
@@ -47,7 +123,7 @@ export function createVideoGrid(): VideoGridComponent {
}
}
// Update username label in case it changed
const label = existing.querySelector(".video-username");
const label = existing.el.querySelector(".video-username");
if (label !== null) {
label.textContent = username;
}
@@ -69,23 +145,110 @@ export function createVideoGrid(): VideoGridComponent {
});
appendChildren(cell, video, label);
cells.set(userId, cell);
cell.addEventListener("click", (e) => {
// Don't switch focus if clicking the mute button
if ((e.target as Element).closest(".tile-mute-btn")) return;
if (focusedTileId !== null && focusedTileId !== userId) {
focusedTileId = userId;
rebuildFocusLayout();
}
});
// Add audio control overlay for remote tiles
if (config !== undefined && !config.isSelf) {
let muted = false;
let currentVolume = 100;
const overlay = createElement("div", { class: "video-tile-overlay" });
// Volume slider
const volumeSlider = createElement("input", {
type: "range",
min: "0",
max: "200",
value: "100",
class: "tile-volume-slider",
"aria-label": "Volume",
}) as HTMLInputElement;
volumeSlider.addEventListener("input", () => {
currentVolume = Number(volumeSlider.value);
const wasMuted = muted;
muted = currentVolume === 0;
if (config.isScreenshare) {
muteScreenshareAudio(config.audioUserId, muted);
} else {
setUserVolume(config.audioUserId, currentVolume);
}
muteBtn.innerHTML = muted ? ICON_VOLUME_X : ICON_VOLUME;
muteBtn.setAttribute("aria-label", muted ? "Unmute" : "Mute");
if (muted !== wasMuted) {
overlay.classList.toggle("muted", muted);
}
});
// Mute button
const muteBtn = createElement("button", {
class: "tile-mute-btn",
"aria-label": "Mute",
});
muteBtn.innerHTML = ICON_VOLUME;
muteBtn.addEventListener("click", () => {
muted = !muted;
if (config.isScreenshare) {
muteScreenshareAudio(config.audioUserId, muted);
} else {
setUserVolume(config.audioUserId, muted ? 0 : 100);
}
muteBtn.innerHTML = muted ? ICON_VOLUME_X : ICON_VOLUME;
muteBtn.setAttribute("aria-label", muted ? "Unmute" : "Mute");
overlay.classList.toggle("muted", muted);
if (muted) {
volumeSlider.value = "0";
} else {
volumeSlider.value = String(currentVolume > 0 ? currentVolume : 100);
if (currentVolume === 0) currentVolume = 100;
if (!config.isScreenshare) setUserVolume(config.audioUserId, currentVolume);
}
});
overlay.appendChild(volumeSlider);
overlay.appendChild(muteBtn);
cell.appendChild(overlay);
}
cells.set(userId, { el: cell, config });
root.appendChild(cell);
updateLayout();
if (focusedTileId !== null) {
rebuildFocusLayout();
} else {
updateLayout();
}
}
function removeStream(userId: number): void {
const cell = cells.get(userId);
if (cell === undefined) return;
const entry = cells.get(userId);
if (entry === undefined) return;
const video = cell.querySelector("video");
if (video !== null) {
video.srcObject = null;
const video = entry.el.querySelector("video");
if (video !== null) video.srcObject = null;
entry.el.remove();
cells.delete(userId);
// If focused tile was removed, focus the first remaining tile or clear
const wasFocusMode = focusedTileId !== null;
if (focusedTileId === userId) {
const firstKey = cells.keys().next().value;
focusedTileId = firstKey ?? null;
}
cell.remove();
cells.delete(userId);
updateLayout();
if (focusedTileId !== null || wasFocusMode) {
rebuildFocusLayout();
} else {
updateLayout();
}
}
function hasStreams(): boolean {
@@ -101,13 +264,12 @@ export function createVideoGrid(): VideoGridComponent {
}
function destroy(): void {
for (const [, cell] of cells) {
const video = cell.querySelector("video");
if (video !== null) {
video.srcObject = null;
}
for (const [, entry] of cells) {
const video = entry.el.querySelector("video");
if (video !== null) video.srcObject = null;
}
cells.clear();
focusedTileId = null;
if (root !== null) {
root.remove();
@@ -115,5 +277,5 @@ export function createVideoGrid(): VideoGridComponent {
}
}
return { mount, destroy, addStream, removeStream, hasStreams };
return { mount, destroy, addStream, removeStream, hasStreams, setFocusedTile, getFocusedTileId: getFocusedTileIdFn };
}
Client/tauri-client/src/lib/livekitSession.ts
+255 -80
View File
@@ -42,8 +42,14 @@ function getSavedUserVolume(userId: number): number {
// --- Types ---
type RemoteVideoCallback = (userId: number, stream: MediaStream) => void;
type RemoteVideoRemovedCallback = (userId: number) => void;
type RemoteVideoCallback = (userId: number, stream: MediaStream, isScreenshare: boolean) => void;
type RemoteVideoRemovedCallback = (userId: number, isScreenshare: boolean) => void;
type PendingVoiceJoin = {
readonly token: string;
readonly url: string;
readonly channelId: number;
readonly directUrl?: string;
};
// --- LiveKitSession class ---
@@ -60,6 +66,8 @@ export class LiveKitSession {
private latestToken: string | null = null;
/** Guard: true while handleVoiceToken is connecting — prevents concurrent joins. */
private connecting = false;
/** Latest join request received while a connection attempt is already running. */
private pendingJoin: PendingVoiceJoin | null = null;
/** Last known LiveKit URL and directUrl for auto-reconnect on unexpected disconnect. */
private lastUrl: string | null = null;
private lastDirectUrl: string | undefined = undefined;
@@ -68,6 +76,10 @@ export class LiveKitSession {
private static readonly RECONNECT_DELAY_MS = 3000;
/** Master output volume multiplier (0-2.0). Per-user volumes are scaled by this. */
private outputVolumeMultiplier = loadPref<number>("outputVolume", 100) / 100;
/** Screenshare audio elements keyed by userId — separate from mic audio pipeline. */
private screenshareAudioElements = new Map<number, Set<HTMLAudioElement>>();
/** Persisted mute state for screenshare audio so replacement tracks inherit UI state. */
private screenshareAudioMutedByUser = new Map<number, boolean>();
// --- Unified audio pipeline: input volume + VAD gating ---
// Pipeline: rawMicTrack → source → analyser (VAD reads here)
@@ -130,30 +142,55 @@ export class LiveKitSession {
private handleTrackSubscribed = (
track: RemoteTrack,
_publication: RemoteTrackPublication,
publication: RemoteTrackPublication,
participant: RemoteParticipant,
): void => {
const userId = parseUserId(participant.identity);
if (track.kind === Track.Kind.Audio) {
// Detach any previous <audio> elements to prevent duplicate playback
// on fast reconnects (new subscription fires before old unsubscription)
for (const el of track.detach()) el.remove();
const audioEl = track.attach();
audioEl.style.display = "none";
document.body.appendChild(audioEl);
// Apply saved per-user volume via LiveKit's setVolume (supports 0-2.0 range)
participant.setVolume(this.getEffectiveVolume(userId));
const savedOutput = loadPref<string>("audioOutputDevice", "");
if (savedOutput !== "" && typeof audioEl.setSinkId === "function") {
audioEl.setSinkId(savedOutput).catch((err) => {
log.warn("Failed to set output device on remote audio", err);
});
if (publication.source === Track.Source.ScreenShareAudio) {
// Screenshare audio: manage via HTMLAudioElement volume (not participant.setVolume)
for (const el of track.detach()) el.remove();
const audioEl = track.attach();
audioEl.style.display = "none";
document.body.appendChild(audioEl);
audioEl.volume = this.getScreenshareOutputVolume();
audioEl.muted = this.screenshareAudioMutedByUser.get(userId) ?? false;
let audioEls = this.screenshareAudioElements.get(userId);
if (audioEls === undefined) {
audioEls = new Set();
this.screenshareAudioElements.set(userId, audioEls);
}
audioEls.add(audioEl);
const savedOutput = loadPref<string>("audioOutputDevice", "");
if (savedOutput !== "" && typeof audioEl.setSinkId === "function") {
audioEl.setSinkId(savedOutput).catch((err) => {
log.warn("Failed to set output device on screenshare audio", err);
});
}
log.debug("Screenshare audio track subscribed and attached", { userId, trackSid: track.sid });
} else {
// Microphone audio: use LiveKit's GainNode-backed setVolume
// Detach any previous <audio> elements to prevent duplicate playback
// on fast reconnects (new subscription fires before old unsubscription)
for (const el of track.detach()) el.remove();
const audioEl = track.attach();
audioEl.style.display = "none";
document.body.appendChild(audioEl);
// Apply saved per-user volume via LiveKit's setVolume (supports 0-2.0 range)
participant.setVolume(this.getEffectiveVolume(userId));
const savedOutput = loadPref<string>("audioOutputDevice", "");
if (savedOutput !== "" && typeof audioEl.setSinkId === "function") {
audioEl.setSinkId(savedOutput).catch((err) => {
log.warn("Failed to set output device on remote audio", err);
});
}
log.debug("Remote audio track subscribed and attached", { userId, trackSid: track.sid });
}
log.debug("Remote audio track subscribed and attached", { userId, trackSid: track.sid });
} else if (track.kind === Track.Kind.Video) {
if (userId > 0 && this.onRemoteVideoCallback !== null) {
const stream = new MediaStream([track.mediaStreamTrack]);
this.onRemoteVideoCallback(userId, stream);
const isScreenshare = publication.source === Track.Source.ScreenShare;
this.onRemoteVideoCallback(userId, stream, isScreenshare);
}
log.debug("Remote video track subscribed", { userId, trackSid: track.sid });
}
@@ -161,16 +198,28 @@ export class LiveKitSession {
private handleTrackUnsubscribed = (
track: RemoteTrack,
_publication: RemoteTrackPublication,
publication: RemoteTrackPublication,
participant: RemoteParticipant,
): void => {
const userId = parseUserId(participant.identity);
if (track.kind === Track.Kind.Audio) {
for (const el of track.detach()) el.remove();
log.debug("Remote audio track unsubscribed and detached", { userId, trackSid: track.sid });
if (publication.source === Track.Source.ScreenShareAudio) {
const detachedEls = track.detach() as HTMLAudioElement[];
for (const el of detachedEls) el.remove();
const audioEls = this.screenshareAudioElements.get(userId);
if (audioEls !== undefined) {
for (const el of detachedEls) audioEls.delete(el);
if (audioEls.size === 0) this.screenshareAudioElements.delete(userId);
}
log.debug("Screenshare audio track unsubscribed and detached", { userId, trackSid: track.sid });
} else {
for (const el of track.detach()) el.remove();
log.debug("Remote audio track unsubscribed and detached", { userId, trackSid: track.sid });
}
} else if (track.kind === Track.Kind.Video) {
track.detach();
if (userId > 0) this.onRemoteVideoRemovedCallback?.(userId);
const isScreenshare = publication.source === Track.Source.ScreenShare;
if (userId > 0) this.onRemoteVideoRemovedCallback?.(userId, isScreenshare);
log.debug("Remote video track unsubscribed", { userId, trackSid: track.sid });
}
};
@@ -267,14 +316,7 @@ export class LiveKitSession {
await this.room.connect(resolvedUrl, token);
log.info("Auto-reconnect succeeded", { attempt, channelId });
this.room.startAudio().catch(() => {});
try {
await this.room.localParticipant.setMicrophoneEnabled(true);
if (loadPref<boolean>("enhancedNoiseSuppression", false)) {
await this.applyNoiseSuppressor();
}
} catch (micErr) {
log.warn("Auto-reconnect: mic unavailable — listen-only mode", micErr);
}
await this.restoreLocalVoiceState("reconnect");
this.setupAudioPipeline();
this.startTokenRefreshTimer();
// Request a fresh token since the stored one may be close to expiry.
@@ -290,8 +332,10 @@ export class LiveKitSession {
}
}
// All attempts exhausted — give up and clean up.
// Send voice_leave over WS so the server removes our voice state;
// without this the server and other clients see us as a ghost participant.
log.error("Auto-reconnect exhausted all attempts, giving up");
this.leaveVoice(false);
this.leaveVoice(true);
leaveVoiceChannel();
this.onErrorCallback?.("Voice connection lost — failed to reconnect");
}
@@ -339,11 +383,16 @@ export class LiveKitSession {
}
handleVoiceTokenRefresh(token?: string): void {
// The installed livekit-client SDK version does not expose a refreshToken
// method on Room. Store the fresh token so that if LiveKit disconnects
// (e.g. token expiry), the reconnection path in handleVoiceToken can use
// it automatically. For now, the 4h TTL with 3.5h refresh request ensures
// a fresh token is always available before expiry.
// KNOWN LIMITATION: The livekit-client SDK does not expose a method to
// rotate the token on an active connection. We store the fresh token so
// that reconnection (auto-reconnect or manual rejoin) uses it, but the
// live session continues with the original token. This means:
// - Sessions longer than the 4h TTL remain connected (LiveKit keeps
// active connections alive) but lose the ability to reconnect after a
// network blip once the original token expires.
// - The 3.5h refresh timer ensures a fresh token is always ready
// *before* the original expires, so reconnects within the window work.
// See also: Server/ws/livekit.go tokenTTL constant.
if (token) {
this.latestToken = token;
}
@@ -359,6 +408,61 @@ export class LiveKitSession {
return (userVol / 100) * this.outputVolumeMultiplier;
}
private getScreenshareOutputVolume(): number {
return Math.max(0, Math.min(1, this.outputVolumeMultiplier));
}
private getLocalVoiceFlags(): { muted: boolean; deafened: boolean } {
const state = voiceStore.getState();
return {
muted: state.localMuted || state.localDeafened,
deafened: state.localDeafened,
};
}
private applyRemoteAudioSubscriptionState(deafened: boolean): void {
if (this.room === null) return;
for (const participant of this.room.remoteParticipants.values()) {
for (const publication of participant.audioTrackPublications.values()) {
publication.setSubscribed(!deafened);
}
}
}
private async restoreLocalVoiceState(mode: "join" | "reconnect"): Promise<void> {
if (this.room === null) return;
const { muted, deafened } = this.getLocalVoiceFlags();
const shouldEnableMicrophone = !muted;
try {
await this.room.localParticipant.setMicrophoneEnabled(shouldEnableMicrophone);
if (shouldEnableMicrophone) {
log.info(mode === "join"
? "Published mic via LiveKit native capture"
: "Auto-reconnect restored live microphone");
if (loadPref<boolean>("enhancedNoiseSuppression", false)) {
await this.applyNoiseSuppressor();
}
}
} catch (micErr) {
if (mode === "reconnect") {
log.warn("Auto-reconnect: mic unavailable — listen-only mode", micErr);
} else if (micErr instanceof DOMException && micErr.name === "NotAllowedError") {
log.warn("Microphone permission denied — joined in listen-only mode");
this.onErrorCallback?.("Microphone permission denied — joined in listen-only mode");
} else if (micErr instanceof DOMException && micErr.name === "NotFoundError") {
log.warn("No microphone found — joined in listen-only mode");
this.onErrorCallback?.("No microphone found — joined in listen-only mode");
} else {
log.warn("Microphone unavailable — joined in listen-only mode", micErr);
this.onErrorCallback?.("Microphone unavailable — joined in listen-only mode");
}
}
this.applyRemoteAudioSubscriptionState(deafened);
}
/** Apply effective volume to all remote participants. */
private applyAllVolumes(): void {
if (this.room === null) return;
@@ -390,13 +494,10 @@ export class LiveKitSession {
this.handleVoiceTokenRefresh(token);
return;
}
// Store URL/token for auto-reconnect on unexpected disconnect.
this.latestToken = token;
this.lastUrl = url;
this.lastDirectUrl = directUrl;
// Prevent concurrent connect attempts (rapid channel switching).
if (this.connecting) {
log.warn("handleVoiceToken: already connecting, ignoring duplicate call");
this.pendingJoin = { token, url, channelId, directUrl };
log.warn("handleVoiceToken: already connecting, queued latest join request", { channelId });
return;
}
if (this.room !== null) this.leaveVoice(false);
@@ -409,6 +510,25 @@ export class LiveKitSession {
for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
try {
await this.room.connect(resolvedUrl, token);
const queuedJoin = this.pendingJoin;
if (queuedJoin !== null
&& (queuedJoin.token !== token
|| queuedJoin.url !== url
|| queuedJoin.channelId !== channelId
|| queuedJoin.directUrl !== directUrl)) {
log.info("Discarding stale voice join in favor of queued request", {
channelId,
queuedChannelId: queuedJoin.channelId,
});
if (this.room !== null) {
const room = this.room;
this.room = null;
room.removeAllListeners();
room.disconnect().catch(() => {});
}
// Don't return — fall through to finally + pending-join dispatch.
break;
}
break;
} catch (connectErr) {
if (attempt < MAX_RETRIES) {
@@ -422,41 +542,30 @@ export class LiveKitSession {
}
}
}
log.info("Connected to LiveKit room", { channelId, url: resolvedUrl });
// Optimistic startAudio — may succeed if the join was triggered by a
// recent user gesture. If not, the AudioPlaybackStatusChanged handler
// will register a click-to-unlock fallback.
this.room.startAudio().catch(() => {
log.debug("Optimistic startAudio failed — waiting for user gesture");
});
try {
await this.room.localParticipant.setMicrophoneEnabled(true);
log.info("Published mic via LiveKit native capture");
if (loadPref<boolean>("enhancedNoiseSuppression", false)) {
await this.applyNoiseSuppressor();
}
} catch (micErr) {
if (micErr instanceof DOMException && micErr.name === "NotAllowedError") {
log.warn("Microphone permission denied — joined in listen-only mode");
this.onErrorCallback?.("Microphone permission denied — joined in listen-only mode");
} else if (micErr instanceof DOMException && micErr.name === "NotFoundError") {
log.warn("No microphone found — joined in listen-only mode");
this.onErrorCallback?.("No microphone found — joined in listen-only mode");
} else {
log.warn("Microphone unavailable — joined in listen-only mode", micErr);
this.onErrorCallback?.("Microphone unavailable — joined in listen-only mode");
}
// If the room was discarded (stale join superseded by pending), skip setup.
if (this.room !== null) {
log.info("Connected to LiveKit room", { channelId, url: resolvedUrl });
this.currentChannelId = channelId;
this.latestToken = token;
this.lastUrl = url;
this.lastDirectUrl = directUrl;
// Optimistic startAudio — may succeed if the join was triggered by a
// recent user gesture. If not, the AudioPlaybackStatusChanged handler
// will register a click-to-unlock fallback.
this.room.startAudio().catch(() => {
log.debug("Optimistic startAudio failed — waiting for user gesture");
});
await this.restoreLocalVoiceState("join");
const savedInput = loadPref<string>("audioInputDevice", "");
if (savedInput) await this.room.switchActiveDevice("audioinput", savedInput);
const savedOutput = loadPref<string>("audioOutputDevice", "");
if (savedOutput) await this.room.switchActiveDevice("audiooutput", savedOutput);
// Set up unified audio pipeline (input volume + VAD gating via GainNode).
// VAD polling only starts if saved sensitivity < 100.
this.setupAudioPipeline();
this.startTokenRefreshTimer();
log.info("Voice session active", { channelId });
}
const savedInput = loadPref<string>("audioInputDevice", "");
if (savedInput) await this.room.switchActiveDevice("audioinput", savedInput);
const savedOutput = loadPref<string>("audioOutputDevice", "");
if (savedOutput) await this.room.switchActiveDevice("audiooutput", savedOutput);
// Set up unified audio pipeline (input volume + VAD gating via GainNode).
// VAD polling only starts if saved sensitivity < 100.
this.setupAudioPipeline();
this.currentChannelId = channelId;
this.startTokenRefreshTimer();
log.info("Voice session active", { channelId });
} catch (err) {
log.error("Failed to connect to LiveKit", err);
if (this.room !== null) {
@@ -466,15 +575,33 @@ export class LiveKitSession {
} finally {
this.connecting = false;
}
// Dispatch pending join *after* the try/finally so that a throw inside
// the recursive call doesn't interfere with the outer finally's flag reset.
const pendingJoin = this.pendingJoin;
this.pendingJoin = null;
if (pendingJoin !== null) {
await this.handleVoiceToken(
pendingJoin.token,
pendingJoin.url,
pendingJoin.channelId,
pendingJoin.directUrl,
);
}
}
leaveVoice(sendWs = true): void {
this.clearTokenRefreshTimer();
this.teardownAudioPipeline();
this.removeAutoplayUnlock();
this.pendingJoin = null;
if (sendWs && this.ws !== null) {
this.ws.send({ type: "voice_leave", payload: {} });
}
for (const audioEls of this.screenshareAudioElements.values()) {
for (const el of audioEls) el.remove();
}
this.screenshareAudioElements.clear();
this.screenshareAudioMutedByUser.clear();
if (this.room !== null) {
const r = this.room;
this.room = null;
@@ -501,14 +628,30 @@ export class LiveKitSession {
setMuted(muted: boolean): void {
setLocalMuted(muted);
if (this.room !== null) void this.room.localParticipant.setMicrophoneEnabled(!muted);
if (this.room !== null) {
void this.room.localParticipant.setMicrophoneEnabled(!muted);
// When the audio pipeline is active, LiveKit's track disable may not
// silence the replaced sender track. Zero the gain to guarantee silence.
if (this.audioPipelineGain !== null && this.audioPipelineCtx !== null) {
const gain = muted ? 0 : this.currentInputGain;
this.audioPipelineGain.gain.setTargetAtTime(gain, this.audioPipelineCtx.currentTime, 0.015);
}
}
}
setDeafened(deafened: boolean): void {
setLocalDeafened(deafened);
if (this.room === null) return;
for (const participant of this.room.remoteParticipants.values()) {
for (const pub of participant.audioTrackPublications.values()) pub.setSubscribed(!deafened);
this.applyRemoteAudioSubscriptionState(deafened);
// Deafen implies mute — stop publishing audio so other participants
// don't hear us while we can't hear them (matches Discord behaviour).
if (this.room !== null) {
const shouldPublish = !deafened && !voiceStore.getState().localMuted;
void this.room.localParticipant.setMicrophoneEnabled(shouldPublish);
// Also zero the pipeline gain to guarantee silence on the replaced track.
if (this.audioPipelineGain !== null && this.audioPipelineCtx !== null) {
const gain = shouldPublish ? this.currentInputGain : 0;
this.audioPipelineGain.gain.setTargetAtTime(gain, this.audioPipelineCtx.currentTime, 0.015);
}
}
log.debug("Deafen state changed", { deafened });
}
@@ -559,7 +702,7 @@ export class LiveKitSession {
}
setLocalScreenshare(true);
try {
await this.room.localParticipant.setScreenShareEnabled(true);
await this.room.localParticipant.setScreenShareEnabled(true, { audio: true });
this.ws.send({ type: "voice_screenshare", payload: { enabled: true } });
log.info("Screenshare enabled");
} catch (err) {
@@ -632,6 +775,29 @@ export class LiveKitSession {
getUserVolume(userId: number): number { return getSavedUserVolume(userId); }
setScreenshareAudioVolume(userId: number, volume: number): void {
const audioEls = this.screenshareAudioElements.get(userId);
if (audioEls === undefined) return;
const clamped = Math.max(0, Math.min(1, volume));
for (const el of audioEls) el.volume = clamped;
}
muteScreenshareAudio(userId: number, muted: boolean): void {
this.screenshareAudioMutedByUser.set(userId, muted);
const audioEls = this.screenshareAudioElements.get(userId);
if (audioEls === undefined) return;
for (const el of audioEls) el.muted = muted;
}
getScreenshareAudioMuted(userId: number): boolean {
const storedMuted = this.screenshareAudioMutedByUser.get(userId);
if (storedMuted !== undefined) return storedMuted;
const audioEls = this.screenshareAudioElements.get(userId);
if (audioEls === undefined) return false;
for (const el of audioEls) return el.muted;
return false;
}
// ── Unified audio pipeline: input volume + VAD gating ─────────────
//
// Architecture:
@@ -735,6 +901,12 @@ export class LiveKitSession {
savePref("outputVolume", clamped);
this.outputVolumeMultiplier = clamped / 100;
this.applyAllVolumes();
const screenshareVolume = this.getScreenshareOutputVolume();
for (const audioEls of this.screenshareAudioElements.values()) {
for (const audioEl of audioEls) {
audioEl.volume = screenshareVolume;
}
}
}
/**
@@ -958,3 +1130,6 @@ export const reapplyAudioProcessing = session.reapplyAudioProcessing.bind(sessio
export const getLocalCameraStream = session.getLocalCameraStream.bind(session);
export const getLocalScreenshareStream = session.getLocalScreenshareStream.bind(session);
export const getSessionDebugInfo = session.getSessionDebugInfo.bind(session);
export const setScreenshareAudioVolume = session.setScreenshareAudioVolume.bind(session);
export const muteScreenshareAudio = session.muteScreenshareAudio.bind(session);
export const getScreenshareAudioMuted = session.getScreenshareAudioMuted.bind(session);
Client/tauri-client/src/styles/app.css
+105
View File
@@ -147,6 +147,17 @@
.voice-user-item .vu-name + .vu-status,
.voice-user-item .vu-name + .vu-muted { margin-left: auto; }
/* ── LIVE badge (voice channel sidebar) ── */
.vu-live-badge {
background: #ed4245;
color: white;
font-size: 10px;
font-weight: 700;
padding: 1px 4px;
border-radius: 3px;
margin-left: 2px;
}
/* Voice widget (above user bar, when connected) */
.voice-widget {
background: var(--bg-secondary); border-top: 1px solid var(--border);
@@ -1970,6 +1981,100 @@
color: #fff;
}
/* ── Video tile overlay (mute button) ── */
.video-tile-overlay {
position: absolute;
bottom: 0;
right: 0;
padding: 6px;
display: flex;
gap: 4px;
align-items: center;
z-index: 2;
opacity: 0;
transition: opacity 0.15s;
}
.video-cell:hover .video-tile-overlay,
.video-tile-overlay.muted {
opacity: 1;
}
.tile-mute-btn {
background: rgba(0, 0, 0, 0.6);
border: none;
border-radius: 4px;
color: var(--text-normal);
padding: 4px;
cursor: pointer;
display: flex;
align-items: center;
justify-content: center;
}
.tile-mute-btn:hover {
background: rgba(0, 0, 0, 0.8);
}
.tile-volume-slider {
width: 80px;
height: 4px;
appearance: none;
background: rgba(255, 255, 255, 0.3);
border-radius: 2px;
cursor: pointer;
}
.tile-volume-slider::-webkit-slider-thumb {
appearance: none;
width: 12px;
height: 12px;
border-radius: 50%;
background: #fff;
cursor: pointer;
}
.tile-volume-slider::-moz-range-thumb {
width: 12px;
height: 12px;
border-radius: 50%;
background: #fff;
border: none;
cursor: pointer;
}
/* ── Video focus mode layout ── */
.video-grid.focus-mode {
display: flex;
flex-direction: column;
height: 100%;
}
.video-focus-main {
flex: 1;
display: flex;
align-items: center;
justify-content: center;
min-height: 0;
}
.video-focus-main .video-cell {
width: 100%;
height: 100%;
}
.video-focus-strip {
display: flex;
gap: 4px;
padding: 4px;
overflow-x: auto;
flex-shrink: 0;
height: 90px;
background: var(--bg-tertiary);
}
.video-focus-strip .video-cell {
width: 120px;
min-width: 120px;
height: 100%;
cursor: pointer;
border: 2px solid transparent;
border-radius: 4px;
}
.video-focus-strip .video-cell:hover {
border-color: var(--accent);
}
/* ── Invite Manager ── */
.invite-manager__list {
display: flex;
Client/tauri-client/tests/unit/rnnoise-worklet.test.ts
+81
View File
@@ -0,0 +1,81 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const REGISTERED_NAME = "rnnoise-processor";
describe("rnnoise-worklet", () => {
let registerProcessorMock: ReturnType<typeof vi.fn>;
let processorCtor: (new () => {
_processFrame(): void;
_outAvailable: number;
_outReadPos: number;
_outWritePos: number;
_outSampleOffset: number;
_inputPtr: number;
_outputPtr: number;
_state: number;
_inputRing: Float32Array;
_outBuffer: Float32Array;
_heapF32: Float32Array | null;
_instance: { exports: { rnnoise_process_frame: ReturnType<typeof vi.fn> } } | null;
}) | null;
beforeEach(() => {
vi.resetModules();
processorCtor = null;
registerProcessorMock = vi.fn((name: string, ctor: unknown) => {
if (name === REGISTERED_NAME) {
processorCtor = ctor as typeof processorCtor;
}
});
class FakeAudioWorkletProcessor {
readonly port = {
onmessage: null,
postMessage: vi.fn(),
};
}
Object.assign(globalThis, {
registerProcessor: registerProcessorMock,
AudioWorkletProcessor: FakeAudioWorkletProcessor,
});
});
afterEach(() => {
delete (globalThis as Record<string, unknown>).registerProcessor;
delete (globalThis as Record<string, unknown>).AudioWorkletProcessor;
});
it("registers the RNNoise processor once", async () => {
// @ts-expect-error — worklet script has no module exports
await import("../../public/rnnoise-worklet.js");
expect(registerProcessorMock).toHaveBeenCalledTimes(1);
expect(registerProcessorMock).toHaveBeenCalledWith(REGISTERED_NAME, expect.any(Function));
});
it("resets the output sample offset when overwriting the oldest buffered frame", async () => {
// @ts-expect-error — worklet script has no module exports
await import("../../public/rnnoise-worklet.js");
expect(processorCtor).not.toBeNull();
const processor = new processorCtor!();
processor._instance = {
exports: {
rnnoise_process_frame: vi.fn(),
},
};
processor._heapF32 = new Float32Array(960);
processor._state = 1;
processor._inputPtr = 0;
processor._outputPtr = 480 * 4;
processor._inputRing.fill(0.5);
processor._outAvailable = 50;
processor._outReadPos = 3;
processor._outWritePos = 4;
processor._outSampleOffset = 123;
processor._processFrame();
expect(processor._outReadPos).toBe(4);
expect(processor._outSampleOffset).toBe(0);
});
});
Server/api/router.go
+3 -2
View File
@@ -86,8 +86,9 @@ func NewRouter(cfg *config.Config, database *db.DB, ver string, logBuf *admin.Ri
// LiveKit webhook endpoint (no auth middleware — uses LiveKit JWT verification).
if lkErr == nil {
r.Post("/api/v1/livekit/webhook",
ws.MountWebhookRoute(hub, cfg.Voice.LiveKitAPIKey, cfg.Voice.LiveKitAPISecret))
r.With(AdminIPRestrict(cfg.Server.AdminAllowedCIDRs)).
Post("/api/v1/livekit/webhook",
ws.MountWebhookRoute(hub, cfg.Voice.LiveKitAPIKey, cfg.Voice.LiveKitAPISecret))
// LiveKit health check — admin-IP-restricted.
r.With(AdminIPRestrict(cfg.Server.AdminAllowedCIDRs)).
Server/config/config.go
+2 -2
View File
@@ -252,11 +252,11 @@ func generateRandomKey(byteLen int) string {
func applyVoiceDefaults(v *VoiceConfig) {
if v.LiveKitAPIKey == "" {
v.LiveKitAPIKey = "key-" + generateRandomKey(8)
slog.Info("generated random LiveKit API key (no key configured)")
slog.Warn("generated random LiveKit API key — voice tokens will break on restart; set voice.livekit_api_key in config.yaml for stable operation")
}
if v.LiveKitAPISecret == "" {
v.LiveKitAPISecret = generateRandomKey(32) // 64 hex chars, well above 32-char minimum
slog.Info("generated random LiveKit API secret (no secret configured)")
slog.Warn("generated random LiveKit API secret — set voice.livekit_api_secret in config.yaml for stable operation")
}
if v.LiveKitURL == "" {
v.LiveKitURL = "ws://localhost:7880"
Server/ws/voice_join.go
+10 -4
View File
@@ -37,10 +37,16 @@ func (h *Hub) handleVoiceJoin(c *Client, payload json.RawMessage) {
return
}
// Guard: reject voice join if LiveKit is configured but the companion
// process is not running (e.g. crashed 10 times and gave up).
// When livekit is nil, voice still works — just without SFU tokens.
if h.livekit != nil && h.lkProcess != nil && !h.lkProcess.IsRunning() {
// Hard-fail when LiveKit is not configured — without an SFU the client
// cannot connect to voice, so persisting state would create a ghost.
if h.livekit == nil {
c.sendMsg(buildErrorMsg(ErrCodeVoiceError, "voice is not configured on this server"))
return
}
// Guard: reject voice join if the companion LiveKit process is not running
// (e.g. crashed 10 times and gave up).
if h.lkProcess != nil && !h.lkProcess.IsRunning() {
slog.Warn("handleVoiceJoin: LiveKit process not running", "user_id", c.userID)
c.sendMsg(buildErrorMsg(ErrCodeVoiceError, "voice is temporarily unavailable — LiveKit is not running"))
return