mirror of
https://github.com/J3vb/OwnCord.git
synced 2026-09-03 03:50:00 +03:00
fix: restore audio track attachment for remote playback
The refactored LiveKit session dropped track.attach() for remote audio, so no <audio> element was created and remote participants were silent. Also refactors noise suppression to use LiveKit TrackProcessor API, adds input volume gain node bypass at 100%, and exposes __lkDebug() on window for DevTools diagnostics.
This commit is contained in:
jevb
@@ -6,6 +6,7 @@ import {
|
||||
type RemoteTrack,
|
||||
type RemoteTrackPublication,
|
||||
type RemoteParticipant,
|
||||
type Participant,
|
||||
DisconnectReason,
|
||||
} from "livekit-client";
|
||||
import type { WsClient } from "@lib/ws";
|
||||
@@ -18,8 +19,7 @@ import {
|
||||
} from "@stores/voice.store";
|
||||
import { loadPref, savePref } from "@components/settings/helpers";
|
||||
import { createLogger } from "@lib/logger";
|
||||
import { createNoiseSuppressor } from "@lib/noise-suppression";
|
||||
import type { NoiseSuppressor } from "@lib/noise-suppression";
|
||||
import { createRNNoiseProcessor } from "@lib/noise-suppression";
|
||||
|
||||
const log = createLogger("livekitSession");
|
||||
|
||||
@@ -32,81 +32,52 @@ export function parseUserId(identity: string): number {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/** Compute RMS audio level (0-1) from frequency data. */
|
||||
export function computeRms(data: Uint8Array<ArrayBuffer>): number {
|
||||
let sum = 0;
|
||||
for (let i = 0; i < data.length; i++) {
|
||||
const v = (data[i] ?? 0) / 255;
|
||||
sum += v * v;
|
||||
}
|
||||
return Math.sqrt(sum / data.length);
|
||||
}
|
||||
|
||||
/** Get saved per-user volume (0-200 range, default 100). */
|
||||
/** Get saved per-user volume (0-200 range, default 100). Applied via LiveKit's GainNode-backed setVolume(). */
|
||||
function getSavedUserVolume(userId: number): number {
|
||||
return loadPref<number>(`userVolume_${userId}`, 100);
|
||||
}
|
||||
|
||||
/** Compare two pre-sorted speaker ID arrays. Both must be sorted ascending. */
|
||||
function speakerSetsEqual(a: readonly number[], b: readonly number[]): boolean {
|
||||
if (a.length !== b.length) return false;
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
if (a[i] !== b[i]) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// --- Types ---
|
||||
|
||||
type RemoteVideoCallback = (userId: number, stream: MediaStream) => void;
|
||||
type RemoteVideoRemovedCallback = (userId: number) => void;
|
||||
|
||||
interface RemoteAnalyser {
|
||||
analyser: AnalyserNode;
|
||||
source: MediaStreamAudioSourceNode;
|
||||
data: Uint8Array<ArrayBuffer>;
|
||||
}
|
||||
|
||||
// --- LiveKitSession class ---
|
||||
|
||||
export class LiveKitSession {
|
||||
private room: Room | null = null;
|
||||
private ws: WsClient | null = null;
|
||||
private noiseSuppressor: NoiseSuppressor | null = null;
|
||||
private onErrorCallback: ((message: string) => void) | null = null;
|
||||
private currentChannelId: number | null = null;
|
||||
private serverHost: string | null = null;
|
||||
private speakingPollInterval: ReturnType<typeof setInterval> | null = null;
|
||||
private speakingThreshold = ((100 - 50) / 100) * 0.15;
|
||||
private rawMicStream: MediaStream | null = null;
|
||||
private readonly audioElements = new Map<string, HTMLAudioElement>();
|
||||
private audioContainer: HTMLDivElement | null = null;
|
||||
private onRemoteVideoCallback: RemoteVideoCallback | null = null;
|
||||
private onRemoteVideoRemovedCallback: RemoteVideoRemovedCallback | null = null;
|
||||
private sharedAudioCtx: AudioContext | null = null;
|
||||
private readonly remoteAnalysers = new Map<number, RemoteAnalyser>();
|
||||
private localMicGated = false;
|
||||
private localAnalyser: AnalyserNode | null = null;
|
||||
private localAnalyserSource: MediaStreamAudioSourceNode | null = null;
|
||||
private localAnalyserClonedTrack: MediaStreamTrack | null = null;
|
||||
private readonly localAnalyserData = new Uint8Array(128);
|
||||
private previousSpeakerIds: readonly number[] = [];
|
||||
private tokenRefreshTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
/** Master output volume multiplier (0-2.0). Per-user volumes are scaled by this. */
|
||||
private outputVolumeMultiplier = loadPref<number>("outputVolume", 100) / 100;
|
||||
|
||||
// --- Shared AudioContext (lazy) ---
|
||||
// --- RNNoise processor (LiveKit TrackProcessor API) ---
|
||||
|
||||
private getSharedAudioCtx(): AudioContext {
|
||||
if (this.sharedAudioCtx === null || this.sharedAudioCtx.state === "closed") {
|
||||
this.sharedAudioCtx = new AudioContext({ sampleRate: 48000 });
|
||||
}
|
||||
return this.sharedAudioCtx;
|
||||
/** Attach RNNoise processor to the local mic track. Safe to call if already attached. */
|
||||
private async applyNoiseSuppressor(): Promise<void> {
|
||||
if (this.room === null) return;
|
||||
const micPub = this.room.localParticipant.getTrackPublication(Track.Source.Microphone);
|
||||
if (micPub?.track === undefined) return;
|
||||
if (micPub.track.getProcessor() !== undefined) return;
|
||||
const processor = createRNNoiseProcessor();
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- LocalTrack.setProcessor uses wide generic, but AudioProcessorOptions is guaranteed at runtime with webAudioMix
|
||||
await micPub.track.setProcessor(processor as any);
|
||||
log.info("RNNoise processor attached to mic track");
|
||||
}
|
||||
|
||||
private closeSharedAudioCtx(): void {
|
||||
if (this.sharedAudioCtx !== null) {
|
||||
void this.sharedAudioCtx.close();
|
||||
this.sharedAudioCtx = null;
|
||||
}
|
||||
/** Remove RNNoise processor from the local mic track. Safe to call if none attached. */
|
||||
private async removeNoiseSuppressor(): Promise<void> {
|
||||
if (this.room === null) return;
|
||||
const micPub = this.room.localParticipant.getTrackPublication(Track.Source.Microphone);
|
||||
if (micPub?.track === undefined) return;
|
||||
if (micPub.track.getProcessor() === undefined) return;
|
||||
await micPub.track.stopProcessor();
|
||||
log.info("RNNoise processor removed from mic track");
|
||||
}
|
||||
|
||||
// --- Room factory ---
|
||||
@@ -115,6 +86,7 @@ export class LiveKitSession {
|
||||
const newRoom = new Room({
|
||||
adaptiveStream: true,
|
||||
dynacast: true,
|
||||
webAudioMix: true,
|
||||
audioCaptureDefaults: {
|
||||
echoCancellation: loadPref("echoCancellation", true),
|
||||
noiseSuppression: loadPref("noiseSuppression", true),
|
||||
@@ -124,70 +96,10 @@ export class LiveKitSession {
|
||||
newRoom.on(RoomEvent.TrackSubscribed, this.handleTrackSubscribed);
|
||||
newRoom.on(RoomEvent.TrackUnsubscribed, this.handleTrackUnsubscribed);
|
||||
newRoom.on(RoomEvent.Disconnected, this.handleDisconnected);
|
||||
newRoom.on(RoomEvent.ActiveSpeakersChanged, this.handleActiveSpeakersChanged);
|
||||
return newRoom;
|
||||
}
|
||||
|
||||
// --- Audio container ---
|
||||
|
||||
private getOrCreateAudioContainer(): HTMLDivElement {
|
||||
if (this.audioContainer !== null) return this.audioContainer;
|
||||
const existing = document.getElementById("voice-audio-container");
|
||||
if (existing instanceof HTMLDivElement) {
|
||||
this.audioContainer = existing;
|
||||
return this.audioContainer;
|
||||
}
|
||||
const div = document.createElement("div");
|
||||
div.id = "voice-audio-container";
|
||||
div.style.display = "none";
|
||||
document.body.appendChild(div);
|
||||
this.audioContainer = div;
|
||||
return this.audioContainer;
|
||||
}
|
||||
|
||||
private cleanupAudioElements(): void {
|
||||
for (const el of this.audioElements.values()) {
|
||||
el.srcObject = null;
|
||||
el.remove();
|
||||
}
|
||||
this.audioElements.clear();
|
||||
}
|
||||
|
||||
// --- RNNoise ---
|
||||
|
||||
private async publishWithNoiseSuppression(): Promise<void> {
|
||||
if (this.room === null) return;
|
||||
const savedDevice = loadPref<string>("audioInputDevice", "");
|
||||
const constraints: MediaStreamConstraints = {
|
||||
audio: {
|
||||
deviceId: savedDevice ? { exact: savedDevice } : undefined,
|
||||
echoCancellation: loadPref("echoCancellation", true),
|
||||
noiseSuppression: loadPref("noiseSuppression", true),
|
||||
autoGainControl: loadPref("autoGainControl", true),
|
||||
},
|
||||
video: false,
|
||||
};
|
||||
const rawStream = await navigator.mediaDevices.getUserMedia(constraints);
|
||||
this.rawMicStream = rawStream;
|
||||
try {
|
||||
this.noiseSuppressor = createNoiseSuppressor();
|
||||
const processedStream = await this.noiseSuppressor.process(rawStream);
|
||||
const processedTrack = processedStream.getAudioTracks()[0];
|
||||
if (processedTrack) {
|
||||
await this.room.localParticipant.publishTrack(processedTrack, {
|
||||
source: Track.Source.Microphone,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
for (const track of rawStream.getTracks()) track.stop();
|
||||
this.rawMicStream = null;
|
||||
if (this.noiseSuppressor !== null) {
|
||||
this.noiseSuppressor.destroy();
|
||||
this.noiseSuppressor = null;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
// --- Room event handlers (arrow fns to preserve `this`) ---
|
||||
|
||||
private handleTrackSubscribed = (
|
||||
@@ -197,22 +109,20 @@ export class LiveKitSession {
|
||||
): void => {
|
||||
const userId = parseUserId(participant.identity);
|
||||
if (track.kind === Track.Kind.Audio) {
|
||||
const container = this.getOrCreateAudioContainer();
|
||||
// Attach creates an <audio> element — required for playback
|
||||
const audioEl = track.attach();
|
||||
audioEl.style.display = "none";
|
||||
document.body.appendChild(audioEl);
|
||||
// Apply saved per-user volume scaled by master output volume
|
||||
const savedVolume = userId > 0 ? getSavedUserVolume(userId) : 100;
|
||||
audioEl.volume = Math.min(savedVolume, 100) / 100;
|
||||
if (voiceStore.getState().localDeafened) audioEl.muted = true;
|
||||
const savedOutput = loadPref<string>("audioOutputDevice", "");
|
||||
if (savedOutput !== "" && typeof audioEl.setSinkId === "function") {
|
||||
audioEl.setSinkId(savedOutput).catch((err) => {
|
||||
log.warn("Failed to set output device on remote audio", err);
|
||||
});
|
||||
}
|
||||
container.appendChild(audioEl);
|
||||
const trackKey = `${participant.identity}-${track.sid}`;
|
||||
this.audioElements.set(trackKey, audioEl);
|
||||
if (userId > 0) this.addRemoteAnalyser(userId, track.mediaStreamTrack);
|
||||
log.debug("Remote audio track subscribed", { userId, trackSid: track.sid });
|
||||
log.debug("Remote audio track subscribed and attached", { userId, trackSid: track.sid });
|
||||
} else if (track.kind === Track.Kind.Video) {
|
||||
if (userId > 0 && this.onRemoteVideoCallback !== null) {
|
||||
const stream = new MediaStream([track.mediaStreamTrack]);
|
||||
@@ -229,11 +139,8 @@ export class LiveKitSession {
|
||||
): void => {
|
||||
const userId = parseUserId(participant.identity);
|
||||
if (track.kind === Track.Kind.Audio) {
|
||||
track.detach().forEach((el) => el.remove());
|
||||
const trackKey = `${participant.identity}-${track.sid}`;
|
||||
this.audioElements.delete(trackKey);
|
||||
if (userId > 0) this.removeRemoteAnalyser(userId);
|
||||
log.debug("Remote audio track unsubscribed", { userId, trackSid: track.sid });
|
||||
for (const el of track.detach()) el.remove();
|
||||
log.debug("Remote audio track unsubscribed and detached", { userId, trackSid: track.sid });
|
||||
} else if (track.kind === Track.Kind.Video) {
|
||||
track.detach();
|
||||
if (userId > 0) this.onRemoteVideoRemovedCallback?.(userId);
|
||||
@@ -241,126 +148,17 @@ export class LiveKitSession {
|
||||
}
|
||||
};
|
||||
|
||||
// --- Remote audio analysers ---
|
||||
|
||||
private addRemoteAnalyser(userId: number, mediaTrack: MediaStreamTrack): void {
|
||||
this.removeRemoteAnalyser(userId);
|
||||
try {
|
||||
const ctx = this.getSharedAudioCtx();
|
||||
const analyser = ctx.createAnalyser();
|
||||
analyser.fftSize = 256;
|
||||
analyser.smoothingTimeConstant = 0.5;
|
||||
const stream = new MediaStream([mediaTrack]);
|
||||
const source = ctx.createMediaStreamSource(stream);
|
||||
source.connect(analyser);
|
||||
this.remoteAnalysers.set(userId, { analyser, source, data: new Uint8Array(128) as Uint8Array<ArrayBuffer> });
|
||||
} catch (err) {
|
||||
log.warn("Failed to create remote analyser", { userId, error: err });
|
||||
/** LiveKit's built-in speaking detection — replaces custom RMS polling. */
|
||||
private handleActiveSpeakersChanged = (speakers: Participant[]): void => {
|
||||
if (this.currentChannelId === null) return;
|
||||
const speakerIds: number[] = [];
|
||||
for (const speaker of speakers) {
|
||||
const userId = parseUserId(speaker.identity);
|
||||
if (userId > 0) speakerIds.push(userId);
|
||||
}
|
||||
}
|
||||
|
||||
private removeRemoteAnalyser(userId: number): void {
|
||||
const ra = this.remoteAnalysers.get(userId);
|
||||
if (ra) {
|
||||
ra.source.disconnect();
|
||||
ra.analyser.disconnect();
|
||||
this.remoteAnalysers.delete(userId);
|
||||
}
|
||||
}
|
||||
|
||||
private cleanupAllRemoteAnalysers(): void {
|
||||
for (const [id] of this.remoteAnalysers) this.removeRemoteAnalyser(id);
|
||||
}
|
||||
|
||||
// --- Local mic analyser ---
|
||||
|
||||
private startLocalAnalyser(): void {
|
||||
this.stopLocalAnalyser();
|
||||
if (this.room === null) return;
|
||||
const micPub = this.room.localParticipant.getTrackPublication(Track.Source.Microphone);
|
||||
const mediaTrack = micPub?.track?.mediaStreamTrack;
|
||||
if (!mediaTrack) return;
|
||||
try {
|
||||
const ctx = this.getSharedAudioCtx();
|
||||
this.localAnalyser = ctx.createAnalyser();
|
||||
this.localAnalyser.fftSize = 256;
|
||||
this.localAnalyser.smoothingTimeConstant = 0.5;
|
||||
this.localAnalyserClonedTrack = mediaTrack.clone();
|
||||
const stream = new MediaStream([this.localAnalyserClonedTrack]);
|
||||
this.localAnalyserSource = ctx.createMediaStreamSource(stream);
|
||||
this.localAnalyserSource.connect(this.localAnalyser);
|
||||
log.debug("Local mic analyser started");
|
||||
} catch (err) {
|
||||
log.warn("Failed to start local mic analyser", err);
|
||||
}
|
||||
}
|
||||
|
||||
private stopLocalAnalyser(): void {
|
||||
if (this.localAnalyserSource !== null) {
|
||||
this.localAnalyserSource.disconnect();
|
||||
this.localAnalyserSource = null;
|
||||
}
|
||||
if (this.localAnalyser !== null) {
|
||||
this.localAnalyser.disconnect();
|
||||
this.localAnalyser = null;
|
||||
}
|
||||
if (this.localAnalyserClonedTrack !== null) {
|
||||
this.localAnalyserClonedTrack.stop();
|
||||
this.localAnalyserClonedTrack = null;
|
||||
}
|
||||
}
|
||||
|
||||
// --- Speaking poll ---
|
||||
|
||||
private startSpeakingPoll(): void {
|
||||
this.stopSpeakingPoll();
|
||||
this.localMicGated = false;
|
||||
this.previousSpeakerIds = [];
|
||||
this.speakingPollInterval = setInterval(() => {
|
||||
if (this.room === null || this.currentChannelId === null) return;
|
||||
const speakerIds: number[] = [];
|
||||
let localLevel = 0;
|
||||
if (this.localAnalyser !== null) {
|
||||
this.localAnalyser.getByteFrequencyData(this.localAnalyserData);
|
||||
localLevel = computeRms(this.localAnalyserData);
|
||||
}
|
||||
const localSpeaking = localLevel > this.speakingThreshold;
|
||||
const micPub = this.room.localParticipant.getTrackPublication(Track.Source.Microphone);
|
||||
if (micPub?.track?.mediaStreamTrack) {
|
||||
if (localSpeaking && this.localMicGated) {
|
||||
micPub.track.mediaStreamTrack.enabled = true;
|
||||
this.localMicGated = false;
|
||||
} else if (!localSpeaking && !this.localMicGated) {
|
||||
micPub.track.mediaStreamTrack.enabled = false;
|
||||
this.localMicGated = true;
|
||||
}
|
||||
}
|
||||
if (localSpeaking) {
|
||||
const localId = parseUserId(this.room.localParticipant.identity);
|
||||
if (localId > 0) speakerIds.push(localId);
|
||||
}
|
||||
for (const [userId, ra] of this.remoteAnalysers) {
|
||||
ra.analyser.getByteFrequencyData(ra.data);
|
||||
if (computeRms(ra.data) > this.speakingThreshold) speakerIds.push(userId);
|
||||
}
|
||||
// Sort in place so speakerSetsEqual can compare without allocations.
|
||||
speakerIds.sort((x, y) => x - y);
|
||||
if (!speakerSetsEqual(speakerIds, this.previousSpeakerIds)) {
|
||||
this.previousSpeakerIds = speakerIds; // already a fresh array each tick
|
||||
setSpeakers({ channel_id: this.currentChannelId, speakers: speakerIds });
|
||||
}
|
||||
}, 100);
|
||||
}
|
||||
|
||||
private stopSpeakingPoll(): void {
|
||||
if (this.speakingPollInterval !== null) {
|
||||
clearInterval(this.speakingPollInterval);
|
||||
this.speakingPollInterval = null;
|
||||
}
|
||||
this.stopLocalAnalyser();
|
||||
this.cleanupAllRemoteAnalysers();
|
||||
this.previousSpeakerIds = [];
|
||||
}
|
||||
speakerIds.sort((x, y) => x - y);
|
||||
setSpeakers({ channel_id: this.currentChannelId, speakers: speakerIds });
|
||||
};
|
||||
|
||||
private handleDisconnected = (reason?: DisconnectReason): void => {
|
||||
log.info("LiveKit room disconnected", { reason });
|
||||
@@ -408,23 +206,31 @@ export class LiveKitSession {
|
||||
}
|
||||
log.info("Requesting voice token refresh");
|
||||
this.ws.send({ type: "voice_token_refresh", payload: {} });
|
||||
// Re-arm as fallback in case the server doesn't respond (network hiccup,
|
||||
// restart). If the server does respond, handleVoiceTokenRefresh restarts
|
||||
// the timer, superseding this one.
|
||||
this.startTokenRefreshTimer();
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle a voice_token message that is a refresh (room already connected).
|
||||
* LiveKit tokens are validated at connect time only — the existing connection
|
||||
* stays alive regardless of token expiry. We just restart the refresh timer
|
||||
* so we keep requesting fresh tokens periodically.
|
||||
*/
|
||||
handleVoiceTokenRefresh(): void {
|
||||
this.startTokenRefreshTimer();
|
||||
log.info("Voice token refreshed, timer restarted");
|
||||
}
|
||||
|
||||
// --- Volume helpers ---
|
||||
|
||||
/** Compute the effective volume for a participant: per-user volume * master output. */
|
||||
private getEffectiveVolume(userId: number): number {
|
||||
const userVol = userId > 0 ? getSavedUserVolume(userId) : 100;
|
||||
return (userVol / 100) * this.outputVolumeMultiplier;
|
||||
}
|
||||
|
||||
/** Apply effective volume to all remote participants. */
|
||||
private applyAllVolumes(): void {
|
||||
if (this.room === null) return;
|
||||
for (const participant of this.room.remoteParticipants.values()) {
|
||||
const userId = parseUserId(participant.identity);
|
||||
participant.setVolume(this.getEffectiveVolume(userId));
|
||||
}
|
||||
}
|
||||
|
||||
// --- Public API ---
|
||||
|
||||
setWsClient(client: WsClient): void { this.ws = client; }
|
||||
@@ -442,9 +248,6 @@ export class LiveKitSession {
|
||||
async handleVoiceToken(
|
||||
token: string, url: string, channelId: number, directUrl?: string,
|
||||
): Promise<void> {
|
||||
// If already connected to the same channel, this is a token refresh response.
|
||||
// LiveKit validates tokens only at connect time, so just restart the timer.
|
||||
// Guard on room.state to avoid treating a mid-retry token as a refresh.
|
||||
if (this.room !== null && this.currentChannelId === channelId
|
||||
&& this.room.state === "connected") {
|
||||
this.handleVoiceTokenRefresh();
|
||||
@@ -473,16 +276,11 @@ export class LiveKitSession {
|
||||
}
|
||||
}
|
||||
log.info("Connected to LiveKit room", { channelId, url: resolvedUrl });
|
||||
this.speakingThreshold = ((100 - loadPref<number>("voiceSensitivity", 50)) / 100) * 0.15;
|
||||
this.startSpeakingPoll();
|
||||
const enhancedNS = loadPref<boolean>("enhancedNoiseSuppression", false);
|
||||
try {
|
||||
if (enhancedNS) {
|
||||
await this.publishWithNoiseSuppression();
|
||||
log.info("Published mic with RNNoise noise suppression");
|
||||
} else {
|
||||
await this.room.localParticipant.setMicrophoneEnabled(true);
|
||||
log.info("Published mic via LiveKit native capture");
|
||||
await this.room.localParticipant.setMicrophoneEnabled(true);
|
||||
log.info("Published mic via LiveKit native capture");
|
||||
if (loadPref<boolean>("enhancedNoiseSuppression", false)) {
|
||||
await this.applyNoiseSuppressor();
|
||||
}
|
||||
} catch (micErr) {
|
||||
if (micErr instanceof DOMException && micErr.name === "NotAllowedError") {
|
||||
@@ -500,8 +298,9 @@ export class LiveKitSession {
|
||||
if (savedInput) await this.room.switchActiveDevice("audioinput", savedInput);
|
||||
const savedOutput = loadPref<string>("audioOutputDevice", "");
|
||||
if (savedOutput) await this.room.switchActiveDevice("audiooutput", savedOutput);
|
||||
// Apply saved input volume
|
||||
this.applyInputVolume(loadPref<number>("inputVolume", 100));
|
||||
this.currentChannelId = channelId;
|
||||
this.startLocalAnalyser();
|
||||
this.startTokenRefreshTimer();
|
||||
log.info("Voice session active", { channelId });
|
||||
} catch (err) {
|
||||
@@ -518,25 +317,14 @@ export class LiveKitSession {
|
||||
if (sendWs && this.ws !== null) {
|
||||
this.ws.send({ type: "voice_leave", payload: {} });
|
||||
}
|
||||
if (this.rawMicStream !== null) {
|
||||
for (const track of this.rawMicStream.getTracks()) track.stop();
|
||||
this.rawMicStream = null;
|
||||
}
|
||||
if (this.noiseSuppressor !== null) {
|
||||
this.noiseSuppressor.destroy();
|
||||
this.noiseSuppressor = null;
|
||||
}
|
||||
this.stopSpeakingPoll();
|
||||
this.cleanupInputGain();
|
||||
if (this.room !== null) {
|
||||
const r = this.room;
|
||||
this.room = null;
|
||||
r.removeAllListeners();
|
||||
r.disconnect().catch((err) => log.warn("room.disconnect() error (non-fatal)", err));
|
||||
}
|
||||
this.cleanupAudioElements();
|
||||
this.closeSharedAudioCtx();
|
||||
this.currentChannelId = null;
|
||||
// Reset camera state so the UI doesn't show a stale video grid on rejoin.
|
||||
setLocalCamera(false);
|
||||
log.info("Left voice session");
|
||||
}
|
||||
@@ -557,12 +345,10 @@ export class LiveKitSession {
|
||||
|
||||
setDeafened(deafened: boolean): void {
|
||||
setLocalDeafened(deafened);
|
||||
if (this.room !== null) {
|
||||
for (const participant of this.room.remoteParticipants.values()) {
|
||||
for (const pub of participant.audioTrackPublications.values()) pub.setSubscribed(!deafened);
|
||||
}
|
||||
if (this.room === null) return;
|
||||
for (const participant of this.room.remoteParticipants.values()) {
|
||||
for (const pub of participant.audioTrackPublications.values()) pub.setSubscribed(!deafened);
|
||||
}
|
||||
for (const el of this.audioElements.values()) el.muted = deafened;
|
||||
log.debug("Deafen state changed", { deafened });
|
||||
}
|
||||
|
||||
@@ -610,27 +396,21 @@ export class LiveKitSession {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const enhancedNS = loadPref<boolean>("enhancedNoiseSuppression", false);
|
||||
if (enhancedNS && this.noiseSuppressor !== null) {
|
||||
if (this.rawMicStream !== null) {
|
||||
for (const track of this.rawMicStream.getTracks()) track.stop();
|
||||
this.rawMicStream = null;
|
||||
}
|
||||
this.noiseSuppressor.destroy();
|
||||
this.noiseSuppressor = null;
|
||||
for (const pub of this.room.localParticipant.audioTrackPublications.values()) {
|
||||
if (pub.source === Track.Source.Microphone && pub.track) {
|
||||
await this.room.localParticipant.unpublishTrack(pub.track);
|
||||
}
|
||||
}
|
||||
await this.publishWithNoiseSuppression();
|
||||
if (deviceId) {
|
||||
await this.room.switchActiveDevice("audioinput", deviceId);
|
||||
} else {
|
||||
if (deviceId) {
|
||||
await this.room.switchActiveDevice("audioinput", deviceId);
|
||||
} else {
|
||||
await this.room.localParticipant.setMicrophoneEnabled(false);
|
||||
await this.room.localParticipant.setMicrophoneEnabled(true);
|
||||
}
|
||||
await this.room.localParticipant.setMicrophoneEnabled(false);
|
||||
await this.room.localParticipant.setMicrophoneEnabled(true);
|
||||
}
|
||||
// Reset and re-apply input volume after device switch (source track changed)
|
||||
this.cleanupInputGain();
|
||||
this.applyInputVolume(loadPref<number>("inputVolume", 100));
|
||||
// Re-apply or remove RNNoise processor based on current setting
|
||||
const enhancedNS = loadPref<boolean>("enhancedNoiseSuppression", false);
|
||||
if (enhancedNS) {
|
||||
await this.applyNoiseSuppressor();
|
||||
} else {
|
||||
await this.removeNoiseSuppressor();
|
||||
}
|
||||
log.info("Switched input device", { deviceId });
|
||||
} catch (err) {
|
||||
@@ -641,13 +421,6 @@ export class LiveKitSession {
|
||||
|
||||
async switchOutputDevice(deviceId: string): Promise<void> {
|
||||
if (this.room !== null) await this.room.switchActiveDevice("audiooutput", deviceId);
|
||||
for (const el of this.audioElements.values()) {
|
||||
if (typeof el.setSinkId === "function") {
|
||||
try { await el.setSinkId(deviceId); } catch (err) {
|
||||
log.warn("Failed to set output device on audio element", err);
|
||||
}
|
||||
}
|
||||
}
|
||||
log.info("Switched output device", { deviceId });
|
||||
}
|
||||
|
||||
@@ -657,13 +430,7 @@ export class LiveKitSession {
|
||||
if (this.room !== null) {
|
||||
for (const participant of this.room.remoteParticipants.values()) {
|
||||
if (parseUserId(participant.identity) === userId) {
|
||||
for (const pub of participant.audioTrackPublications.values()) {
|
||||
if (pub.track) {
|
||||
for (const el of pub.track.attachedElements) {
|
||||
if (el instanceof HTMLAudioElement) el.volume = Math.min(clamped, 100) / 100;
|
||||
}
|
||||
}
|
||||
}
|
||||
participant.setVolume((clamped / 100) * this.outputVolumeMultiplier);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -671,23 +438,110 @@ export class LiveKitSession {
|
||||
|
||||
getUserVolume(userId: number): number { return getSavedUserVolume(userId); }
|
||||
|
||||
/** Input volume GainNode — adjusts mic gain via the WebRTC sender. */
|
||||
private inputGainNode: GainNode | null = null;
|
||||
private inputGainCtx: AudioContext | null = null;
|
||||
private inputGainDest: MediaStreamAudioDestinationNode | null = null;
|
||||
|
||||
/** Apply input volume gain to the local mic track via a Web Audio GainNode. */
|
||||
private applyInputVolume(volume: number): void {
|
||||
if (this.room === null) return;
|
||||
const micPub = this.room.localParticipant.getTrackPublication(Track.Source.Microphone);
|
||||
if (micPub?.track === undefined) return;
|
||||
const gain = Math.max(0, Math.min(200, volume)) / 100;
|
||||
|
||||
// At 100% (gain=1.0), tear down the pipeline — no processing needed
|
||||
if (gain === 1 && this.inputGainNode !== null) {
|
||||
this.restoreOriginalSenderTrack();
|
||||
this.cleanupInputGain();
|
||||
log.info("Input volume reset to 100% — gain pipeline removed");
|
||||
return;
|
||||
}
|
||||
|
||||
// No gain node and volume is default — nothing to do
|
||||
if (gain === 1) return;
|
||||
|
||||
if (this.inputGainNode !== null) {
|
||||
this.inputGainNode.gain.setTargetAtTime(gain, 0, 0.05);
|
||||
log.debug("Input volume adjusted", { gain });
|
||||
return;
|
||||
}
|
||||
|
||||
// Build GainNode pipeline and replace the sender's track
|
||||
try {
|
||||
const mediaTrack = micPub.track.mediaStreamTrack;
|
||||
const ctx = new AudioContext({ sampleRate: 48000 });
|
||||
const source = ctx.createMediaStreamSource(new MediaStream([mediaTrack]));
|
||||
const gainNode = ctx.createGain();
|
||||
gainNode.gain.setTargetAtTime(gain, 0, 0.05);
|
||||
const dest = ctx.createMediaStreamDestination();
|
||||
source.connect(gainNode);
|
||||
gainNode.connect(dest);
|
||||
|
||||
this.inputGainNode = gainNode;
|
||||
this.inputGainCtx = ctx;
|
||||
this.inputGainDest = dest;
|
||||
|
||||
// Replace the WebRTC sender's track with the gain-adjusted one
|
||||
const adjustedTrack = dest.stream.getAudioTracks()[0];
|
||||
if (adjustedTrack !== undefined && micPub.track.sender) {
|
||||
void micPub.track.sender.replaceTrack(adjustedTrack).catch((err) => {
|
||||
log.warn("Failed to replace sender track with gain-adjusted track", err);
|
||||
});
|
||||
}
|
||||
log.info("Input volume GainNode created", { gain });
|
||||
} catch (err) {
|
||||
log.warn("Failed to set up input volume gain", err);
|
||||
}
|
||||
}
|
||||
|
||||
/** Restore the original mic track on the WebRTC sender (undo gain pipeline). */
|
||||
private restoreOriginalSenderTrack(): void {
|
||||
if (this.room === null) return;
|
||||
const micPub = this.room.localParticipant.getTrackPublication(Track.Source.Microphone);
|
||||
if (micPub?.track === undefined) return;
|
||||
const originalTrack = micPub.track.mediaStreamTrack;
|
||||
if (micPub.track.sender) {
|
||||
void micPub.track.sender.replaceTrack(originalTrack).catch((err) => {
|
||||
log.warn("Failed to restore original sender track", err);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private cleanupInputGain(): void {
|
||||
if (this.inputGainNode !== null) {
|
||||
this.inputGainNode.disconnect();
|
||||
this.inputGainNode = null;
|
||||
}
|
||||
if (this.inputGainDest !== null) {
|
||||
this.inputGainDest.disconnect();
|
||||
this.inputGainDest = null;
|
||||
}
|
||||
if (this.inputGainCtx !== null) {
|
||||
void this.inputGainCtx.close();
|
||||
this.inputGainCtx = null;
|
||||
}
|
||||
}
|
||||
|
||||
setInputVolume(volume: number): void {
|
||||
savePref("inputVolume", Math.max(0, Math.min(200, volume)));
|
||||
const clamped = Math.max(0, Math.min(200, volume));
|
||||
savePref("inputVolume", clamped);
|
||||
this.applyInputVolume(clamped);
|
||||
}
|
||||
|
||||
setOutputVolume(volume: number): void {
|
||||
const clamped = Math.max(0, Math.min(200, volume));
|
||||
savePref("outputVolume", clamped);
|
||||
if (this.room !== null) {
|
||||
for (const el of this.audioElements.values()) {
|
||||
el.volume = Math.min(clamped, 100) / 100;
|
||||
}
|
||||
}
|
||||
this.outputVolumeMultiplier = clamped / 100;
|
||||
// Re-apply all per-user volumes scaled by the new master output
|
||||
this.applyAllVolumes();
|
||||
}
|
||||
|
||||
setVoiceSensitivity(sensitivity: number): void {
|
||||
const clamped = Math.max(0, Math.min(100, sensitivity));
|
||||
this.speakingThreshold = ((100 - clamped) / 100) * 0.15;
|
||||
setVoiceSensitivity(_sensitivity: number): void {
|
||||
// Voice sensitivity is now handled by LiveKit's built-in speaking detection.
|
||||
// The sensitivity parameter is saved in preferences by the UI but
|
||||
// LiveKit's server-side VAD determines speaking state.
|
||||
log.debug("Voice sensitivity setting saved (handled by LiveKit VAD)");
|
||||
}
|
||||
|
||||
getLocalCameraStream(): MediaStream | null {
|
||||
@@ -699,24 +553,34 @@ export class LiveKitSession {
|
||||
|
||||
getSessionDebugInfo(): Record<string, unknown> {
|
||||
if (this.room === null) {
|
||||
return { hasRoom: false, hasNoiseSuppressor: this.noiseSuppressor !== null, currentChannelId: this.currentChannelId };
|
||||
return { hasRoom: false, hasRNNoiseProcessor: false, currentChannelId: this.currentChannelId };
|
||||
}
|
||||
const remoteParticipants = [...this.room.remoteParticipants.values()].map((p) => ({
|
||||
identity: p.identity,
|
||||
userId: parseUserId(p.identity),
|
||||
tracks: [...p.trackPublications.values()].map((pub) => ({
|
||||
sid: pub.trackSid, source: pub.source, kind: pub.kind,
|
||||
subscribed: pub.isSubscribed, enabled: pub.isEnabled,
|
||||
})),
|
||||
}));
|
||||
const remoteParticipants = [...this.room.remoteParticipants.values()].map((p) => {
|
||||
const userId = parseUserId(p.identity);
|
||||
return {
|
||||
identity: p.identity,
|
||||
userId,
|
||||
volume: p.getVolume(),
|
||||
effectiveVolume: this.getEffectiveVolume(userId),
|
||||
tracks: [...p.trackPublications.values()].map((pub) => ({
|
||||
sid: pub.trackSid, source: pub.source, kind: pub.kind,
|
||||
subscribed: pub.isSubscribed, enabled: pub.isEnabled,
|
||||
})),
|
||||
};
|
||||
});
|
||||
const localTracks = [...this.room.localParticipant.trackPublications.values()].map((pub) => ({
|
||||
sid: pub.trackSid, source: pub.source, kind: pub.kind, isMuted: pub.isMuted,
|
||||
}));
|
||||
return {
|
||||
hasRoom: true, roomName: this.room.name, roomState: this.room.state,
|
||||
hasNoiseSuppressor: this.noiseSuppressor !== null, currentChannelId: this.currentChannelId,
|
||||
hasRNNoiseProcessor: this.room.localParticipant.getTrackPublication(Track.Source.Microphone)?.track?.getProcessor() !== undefined,
|
||||
currentChannelId: this.currentChannelId,
|
||||
outputVolumeMultiplier: this.outputVolumeMultiplier,
|
||||
inputGainActive: this.inputGainNode !== null,
|
||||
inputGainValue: this.inputGainNode?.gain.value ?? null,
|
||||
inputGainCtxState: this.inputGainCtx?.state ?? null,
|
||||
localParticipant: this.room.localParticipant.identity, localTracks,
|
||||
remoteParticipants, audioElements: this.audioElements.size,
|
||||
remoteParticipants,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -725,6 +589,10 @@ export class LiveKitSession {
|
||||
|
||||
const session = new LiveKitSession();
|
||||
|
||||
// Expose debug info on window for DevTools console access
|
||||
// Usage: JSON.stringify(__lkDebug(), null, 2)
|
||||
(window as unknown as Record<string, unknown>).__lkDebug = session.getSessionDebugInfo.bind(session);
|
||||
|
||||
export const setWsClient = session.setWsClient.bind(session);
|
||||
export const setServerHost = session.setServerHost.bind(session);
|
||||
export const setOnError = session.setOnError.bind(session);
|
||||
|
||||
@@ -1,14 +1,17 @@
|
||||
// =============================================================================
|
||||
// Noise Suppression — RNNoise ML-based noise removal via Web Audio API
|
||||
// Noise Suppression — RNNoise ML-based noise removal as a LiveKit TrackProcessor
|
||||
//
|
||||
// Implements LiveKit's TrackProcessor<Track.Kind.Audio> interface so it
|
||||
// integrates with setProcessor() / stopProcessor() lifecycle, device switching,
|
||||
// and mid-call toggling automatically.
|
||||
//
|
||||
// Inserts between getUserMedia stream and the PeerConnection to clean audio.
|
||||
// RNNoise processes 480-sample frames at 48kHz (10ms).
|
||||
//
|
||||
// Uses AudioWorklet (modern, runs on audio thread) with ScriptProcessorNode
|
||||
// fallback (deprecated but widely supported).
|
||||
// =============================================================================
|
||||
|
||||
import { createRNNWasmModule } from "@jitsi/rnnoise-wasm";
|
||||
import { Track, type TrackProcessor, type AudioProcessorOptions } from "livekit-client";
|
||||
import { createLogger } from "@lib/logger";
|
||||
|
||||
const log = createLogger("noise-suppression");
|
||||
@@ -16,13 +19,8 @@ const log = createLogger("noise-suppression");
|
||||
const RNNOISE_FRAME_SIZE = 480;
|
||||
const SCRIPT_PROCESSOR_BUFFER = 4096;
|
||||
|
||||
export interface NoiseSuppressor {
|
||||
process(input: MediaStream): Promise<MediaStream>;
|
||||
destroy(): void;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Shared WASM module cache (used by ScriptProcessorNode fallback)
|
||||
// Shared WASM module cache
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface RNNoiseModule {
|
||||
@@ -52,102 +50,78 @@ async function loadRNNoise(): Promise<RNNoiseModule> {
|
||||
return mod;
|
||||
}
|
||||
|
||||
/** Check if AudioWorklet is available in this browser context. */
|
||||
function supportsAudioWorklet(): boolean {
|
||||
try {
|
||||
return typeof AudioWorkletNode !== "undefined"
|
||||
&& typeof AudioContext !== "undefined"
|
||||
&& "audioWorklet" in AudioContext.prototype;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// AudioWorklet-based suppressor (preferred, runs on audio thread)
|
||||
// Internal processing pipeline — shared by both init strategies
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function createWorkletSuppressor(): NoiseSuppressor {
|
||||
let audioContext: AudioContext | null = null;
|
||||
let sourceNode: MediaStreamAudioSourceNode | null = null;
|
||||
let destNode: MediaStreamAudioDestinationNode | null = null;
|
||||
let workletNode: AudioWorkletNode | null = null;
|
||||
let destroyed = false;
|
||||
interface ProcessingPipeline {
|
||||
readonly processedTrack: MediaStreamTrack;
|
||||
destroy(): void;
|
||||
}
|
||||
|
||||
/** AudioWorklet-based pipeline (preferred, runs on audio thread). */
|
||||
async function createWorkletPipeline(
|
||||
inputTrack: MediaStreamTrack,
|
||||
audioContext: AudioContext,
|
||||
): Promise<ProcessingPipeline> {
|
||||
await audioContext.audioWorklet.addModule("/rnnoise-worklet.js");
|
||||
const wasmResponse = await fetch("/rnnoise.wasm");
|
||||
const wasmBytes = await wasmResponse.arrayBuffer();
|
||||
|
||||
const source = audioContext.createMediaStreamSource(new MediaStream([inputTrack]));
|
||||
const dest = audioContext.createMediaStreamDestination();
|
||||
const workletNode = new AudioWorkletNode(audioContext, "rnnoise-processor", {
|
||||
numberOfInputs: 1,
|
||||
numberOfOutputs: 1,
|
||||
outputChannelCount: [1],
|
||||
});
|
||||
|
||||
const initPromise = new Promise<void>((resolve, reject) => {
|
||||
workletNode.port.onmessage = (event: MessageEvent) => {
|
||||
if (event.data.type === "ready") resolve();
|
||||
else if (event.data.type === "error") reject(new Error(event.data.message));
|
||||
};
|
||||
});
|
||||
workletNode.port.postMessage({ type: "init", wasmBytes }, [wasmBytes]);
|
||||
await initPromise;
|
||||
|
||||
source.connect(workletNode);
|
||||
workletNode.connect(dest);
|
||||
|
||||
log.info("RNNoise AudioWorklet processing active");
|
||||
|
||||
return {
|
||||
async process(input: MediaStream): Promise<MediaStream> {
|
||||
if (destroyed) throw new Error("NoiseSuppressor destroyed");
|
||||
|
||||
audioContext = new AudioContext({ sampleRate: 48000 });
|
||||
|
||||
// Load the worklet processor module
|
||||
await audioContext.audioWorklet.addModule("/rnnoise-worklet.js");
|
||||
|
||||
// Fetch WASM bytes to send to the worklet thread
|
||||
const wasmResponse = await fetch("/rnnoise.wasm");
|
||||
const wasmBytes = await wasmResponse.arrayBuffer();
|
||||
|
||||
sourceNode = audioContext.createMediaStreamSource(input);
|
||||
destNode = audioContext.createMediaStreamDestination();
|
||||
|
||||
workletNode = new AudioWorkletNode(audioContext, "rnnoise-processor", {
|
||||
numberOfInputs: 1,
|
||||
numberOfOutputs: 1,
|
||||
outputChannelCount: [1],
|
||||
});
|
||||
|
||||
// Wait for WASM init in the worklet
|
||||
const initPromise = new Promise<void>((resolve, reject) => {
|
||||
if (workletNode === null) { reject(new Error("No worklet")); return; }
|
||||
workletNode.port.onmessage = (event: MessageEvent) => {
|
||||
if (event.data.type === "ready") {
|
||||
resolve();
|
||||
} else if (event.data.type === "error") {
|
||||
reject(new Error(event.data.message));
|
||||
}
|
||||
};
|
||||
});
|
||||
|
||||
// Send WASM bytes to the worklet for initialization
|
||||
workletNode.port.postMessage({ type: "init", wasmBytes }, [wasmBytes]);
|
||||
await initPromise;
|
||||
|
||||
sourceNode.connect(workletNode);
|
||||
workletNode.connect(destNode);
|
||||
|
||||
log.info("RNNoise AudioWorklet processing active");
|
||||
return destNode.stream;
|
||||
},
|
||||
|
||||
destroy(): void {
|
||||
if (destroyed) return;
|
||||
destroyed = true;
|
||||
|
||||
if (workletNode !== null) {
|
||||
workletNode.port.postMessage({ type: "destroy" });
|
||||
workletNode.disconnect();
|
||||
workletNode = null;
|
||||
}
|
||||
if (sourceNode !== null) {
|
||||
sourceNode.disconnect();
|
||||
sourceNode = null;
|
||||
}
|
||||
if (destNode !== null) {
|
||||
destNode.disconnect();
|
||||
destNode = null;
|
||||
}
|
||||
if (audioContext !== null) {
|
||||
void audioContext.close();
|
||||
audioContext = null;
|
||||
}
|
||||
log.info("RNNoise AudioWorklet destroyed");
|
||||
processedTrack: dest.stream.getAudioTracks()[0]!,
|
||||
destroy() {
|
||||
workletNode.port.postMessage({ type: "destroy" });
|
||||
workletNode.disconnect();
|
||||
source.disconnect();
|
||||
dest.disconnect();
|
||||
log.info("RNNoise AudioWorklet pipeline destroyed");
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// ScriptProcessorNode fallback (deprecated but universal)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function createScriptProcessorSuppressor(): NoiseSuppressor {
|
||||
let audioContext: AudioContext | null = null;
|
||||
let sourceNode: MediaStreamAudioSourceNode | null = null;
|
||||
let destNode: MediaStreamAudioDestinationNode | null = null;
|
||||
let processorNode: ScriptProcessorNode | null = null;
|
||||
let rnnoiseState: number = 0;
|
||||
let inputPtr: number = 0;
|
||||
let outputPtr: number = 0;
|
||||
let wasmModule: RNNoiseModule | null = null;
|
||||
let destroyed = false;
|
||||
/** ScriptProcessorNode-based pipeline (fallback). */
|
||||
async function createScriptProcessorPipeline(
|
||||
inputTrack: MediaStreamTrack,
|
||||
audioContext: AudioContext,
|
||||
): Promise<ProcessingPipeline> {
|
||||
const wasmModule = await loadRNNoise();
|
||||
const rnnoiseState = wasmModule._rnnoise_create();
|
||||
const inputPtr = wasmModule._malloc(RNNOISE_FRAME_SIZE * 4);
|
||||
const outputPtr = wasmModule._malloc(RNNOISE_FRAME_SIZE * 4);
|
||||
|
||||
const inputRing = new Float32Array(RNNOISE_FRAME_SIZE);
|
||||
let inputRingOffset = 0;
|
||||
@@ -160,7 +134,6 @@ function createScriptProcessorSuppressor(): NoiseSuppressor {
|
||||
let outSampleOffset = 0;
|
||||
|
||||
function processFrame(): void {
|
||||
if (wasmModule === null) return;
|
||||
const inOff = inputPtr / 4;
|
||||
for (let i = 0; i < RNNOISE_FRAME_SIZE; i++) {
|
||||
wasmModule.HEAPF32[inOff + i] = (inputRing[i] ?? 0) * 32768;
|
||||
@@ -181,140 +154,121 @@ function createScriptProcessorSuppressor(): NoiseSuppressor {
|
||||
outCount++;
|
||||
}
|
||||
|
||||
const source = audioContext.createMediaStreamSource(new MediaStream([inputTrack]));
|
||||
const dest = audioContext.createMediaStreamDestination();
|
||||
const processorNode = audioContext.createScriptProcessor(SCRIPT_PROCESSOR_BUFFER, 1, 1);
|
||||
|
||||
processorNode.onaudioprocess = (event: AudioProcessingEvent) => {
|
||||
const inData = event.inputBuffer.getChannelData(0);
|
||||
const outData = event.outputBuffer.getChannelData(0);
|
||||
|
||||
let inIdx = 0;
|
||||
while (inIdx < inData.length) {
|
||||
const needed = RNNOISE_FRAME_SIZE - inputRingOffset;
|
||||
const toCopy = Math.min(needed, inData.length - inIdx);
|
||||
inputRing.set(inData.subarray(inIdx, inIdx + toCopy), inputRingOffset);
|
||||
inputRingOffset += toCopy;
|
||||
inIdx += toCopy;
|
||||
if (inputRingOffset >= RNNOISE_FRAME_SIZE) {
|
||||
processFrame();
|
||||
inputRingOffset = 0;
|
||||
}
|
||||
}
|
||||
|
||||
let outIdx = 0;
|
||||
while (outIdx < outData.length && outCount > 0) {
|
||||
const chunk = outRing[outReadIdx]!;
|
||||
const available = chunk.length - outSampleOffset;
|
||||
const toWrite = Math.min(available, outData.length - outIdx);
|
||||
outData.set(chunk.subarray(outSampleOffset, outSampleOffset + toWrite), outIdx);
|
||||
outIdx += toWrite;
|
||||
outSampleOffset += toWrite;
|
||||
if (outSampleOffset >= chunk.length) {
|
||||
outReadIdx = (outReadIdx + 1) % OUT_RING_CAPACITY;
|
||||
outCount--;
|
||||
outSampleOffset = 0;
|
||||
}
|
||||
}
|
||||
if (outIdx < outData.length) {
|
||||
outData.fill(0, outIdx);
|
||||
}
|
||||
};
|
||||
|
||||
source.connect(processorNode);
|
||||
processorNode.connect(dest);
|
||||
|
||||
log.info("RNNoise ScriptProcessor processing active (fallback)");
|
||||
|
||||
return {
|
||||
async process(input: MediaStream): Promise<MediaStream> {
|
||||
if (destroyed) throw new Error("NoiseSuppressor destroyed");
|
||||
|
||||
wasmModule = await loadRNNoise();
|
||||
rnnoiseState = wasmModule._rnnoise_create();
|
||||
inputPtr = wasmModule._malloc(RNNOISE_FRAME_SIZE * 4);
|
||||
outputPtr = wasmModule._malloc(RNNOISE_FRAME_SIZE * 4);
|
||||
|
||||
audioContext = new AudioContext({ sampleRate: 48000 });
|
||||
sourceNode = audioContext.createMediaStreamSource(input);
|
||||
destNode = audioContext.createMediaStreamDestination();
|
||||
processorNode = audioContext.createScriptProcessor(SCRIPT_PROCESSOR_BUFFER, 1, 1);
|
||||
|
||||
processorNode.onaudioprocess = (event: AudioProcessingEvent) => {
|
||||
const inData = event.inputBuffer.getChannelData(0);
|
||||
const outData = event.outputBuffer.getChannelData(0);
|
||||
|
||||
let inIdx = 0;
|
||||
while (inIdx < inData.length) {
|
||||
const needed = RNNOISE_FRAME_SIZE - inputRingOffset;
|
||||
const toCopy = Math.min(needed, inData.length - inIdx);
|
||||
inputRing.set(inData.subarray(inIdx, inIdx + toCopy), inputRingOffset);
|
||||
inputRingOffset += toCopy;
|
||||
inIdx += toCopy;
|
||||
if (inputRingOffset >= RNNOISE_FRAME_SIZE) {
|
||||
processFrame();
|
||||
inputRingOffset = 0;
|
||||
}
|
||||
}
|
||||
|
||||
let outIdx = 0;
|
||||
while (outIdx < outData.length && outCount > 0) {
|
||||
const chunk = outRing[outReadIdx]!;
|
||||
const available = chunk.length - outSampleOffset;
|
||||
const toWrite = Math.min(available, outData.length - outIdx);
|
||||
outData.set(chunk.subarray(outSampleOffset, outSampleOffset + toWrite), outIdx);
|
||||
outIdx += toWrite;
|
||||
outSampleOffset += toWrite;
|
||||
if (outSampleOffset >= chunk.length) {
|
||||
outReadIdx = (outReadIdx + 1) % OUT_RING_CAPACITY;
|
||||
outCount--;
|
||||
outSampleOffset = 0;
|
||||
}
|
||||
}
|
||||
if (outIdx < outData.length) {
|
||||
outData.fill(0, outIdx);
|
||||
}
|
||||
};
|
||||
|
||||
sourceNode.connect(processorNode);
|
||||
processorNode.connect(destNode);
|
||||
|
||||
log.info("RNNoise ScriptProcessor processing active (fallback)");
|
||||
return destNode.stream;
|
||||
},
|
||||
|
||||
destroy(): void {
|
||||
if (destroyed) return;
|
||||
destroyed = true;
|
||||
|
||||
if (processorNode !== null) {
|
||||
processorNode.onaudioprocess = null;
|
||||
processorNode.disconnect();
|
||||
processorNode = null;
|
||||
}
|
||||
if (sourceNode !== null) {
|
||||
sourceNode.disconnect();
|
||||
sourceNode = null;
|
||||
}
|
||||
if (destNode !== null) {
|
||||
destNode.disconnect();
|
||||
destNode = null;
|
||||
}
|
||||
if (audioContext !== null) {
|
||||
void audioContext.close();
|
||||
audioContext = null;
|
||||
}
|
||||
if (wasmModule !== null && rnnoiseState !== 0) {
|
||||
wasmModule._rnnoise_destroy(rnnoiseState);
|
||||
wasmModule._free(inputPtr);
|
||||
wasmModule._free(outputPtr);
|
||||
rnnoiseState = 0;
|
||||
}
|
||||
outWriteIdx = 0;
|
||||
outReadIdx = 0;
|
||||
outCount = 0;
|
||||
outSampleOffset = 0;
|
||||
log.info("RNNoise ScriptProcessor destroyed");
|
||||
processedTrack: dest.stream.getAudioTracks()[0]!,
|
||||
destroy() {
|
||||
processorNode.onaudioprocess = null;
|
||||
processorNode.disconnect();
|
||||
source.disconnect();
|
||||
dest.disconnect();
|
||||
wasmModule._rnnoise_destroy(rnnoiseState);
|
||||
wasmModule._free(inputPtr);
|
||||
wasmModule._free(outputPtr);
|
||||
log.info("RNNoise ScriptProcessor pipeline destroyed");
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Factory — tries AudioWorklet first, falls back to ScriptProcessorNode
|
||||
// LiveKit TrackProcessor implementation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Check if AudioWorklet is available in this browser context. */
|
||||
function supportsAudioWorklet(): boolean {
|
||||
try {
|
||||
return typeof AudioWorkletNode !== "undefined"
|
||||
&& typeof AudioContext !== "undefined"
|
||||
&& "audioWorklet" in AudioContext.prototype;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Creates an RNNoise TrackProcessor compatible with LiveKit's
|
||||
* LocalAudioTrack.setProcessor() API.
|
||||
*
|
||||
* Usage:
|
||||
* const processor = createRNNoiseProcessor();
|
||||
* await localAudioTrack.setProcessor(processor);
|
||||
* // Later:
|
||||
* await localAudioTrack.stopProcessor();
|
||||
*/
|
||||
export function createRNNoiseProcessor(): TrackProcessor<Track.Kind.Audio, AudioProcessorOptions> {
|
||||
let pipeline: ProcessingPipeline | null = null;
|
||||
|
||||
export function createNoiseSuppressor(): NoiseSuppressor {
|
||||
log.debug("Creating noise suppressor", { audioWorkletSupported: supportsAudioWorklet() });
|
||||
if (supportsAudioWorklet()) {
|
||||
// Wrap in a facade that falls back to ScriptProcessor on failure
|
||||
const worklet = createWorkletSuppressor();
|
||||
let fallback: NoiseSuppressor | null = null;
|
||||
let activeSuppressor: NoiseSuppressor = worklet;
|
||||
return {
|
||||
name: "rnnoise",
|
||||
|
||||
return {
|
||||
async process(input: MediaStream): Promise<MediaStream> {
|
||||
async init(opts: AudioProcessorOptions): Promise<void> {
|
||||
log.debug("RNNoise processor init", { audioWorkletSupported: supportsAudioWorklet() });
|
||||
const ctx = opts.audioContext;
|
||||
|
||||
if (supportsAudioWorklet()) {
|
||||
try {
|
||||
return await worklet.process(input);
|
||||
pipeline = await createWorkletPipeline(opts.track, ctx);
|
||||
return;
|
||||
} catch (err) {
|
||||
log.warn("AudioWorklet failed, falling back to ScriptProcessorNode", err);
|
||||
worklet.destroy();
|
||||
fallback = createScriptProcessorSuppressor();
|
||||
activeSuppressor = fallback;
|
||||
return fallback.process(input);
|
||||
}
|
||||
},
|
||||
destroy(): void {
|
||||
activeSuppressor.destroy();
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
log.info("AudioWorklet not supported, using ScriptProcessorNode");
|
||||
return createScriptProcessorSuppressor();
|
||||
pipeline = await createScriptProcessorPipeline(opts.track, ctx);
|
||||
},
|
||||
|
||||
async restart(opts: AudioProcessorOptions): Promise<void> {
|
||||
log.debug("RNNoise processor restart");
|
||||
if (pipeline !== null) {
|
||||
pipeline.destroy();
|
||||
pipeline = null;
|
||||
}
|
||||
await this.init(opts);
|
||||
},
|
||||
|
||||
async destroy(): Promise<void> {
|
||||
if (pipeline !== null) {
|
||||
pipeline.destroy();
|
||||
pipeline = null;
|
||||
}
|
||||
log.info("RNNoise processor destroyed");
|
||||
},
|
||||
|
||||
get processedTrack(): MediaStreamTrack | undefined {
|
||||
return pipeline?.processedTrack;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -28,7 +28,6 @@ export type WsErrorCode =
|
||||
| "INVALID_INPUT"
|
||||
| "SERVER_ERROR"
|
||||
| "CHANNEL_FULL"
|
||||
| "INVALID_SDP"
|
||||
| "VOICE_ERROR"
|
||||
| "VIDEO_LIMIT";
|
||||
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
# OwnCord
|
||||
|
||||
*The gaming chat platform you actually own.*
|
||||
|
||||
A self-hosted Windows chat platform with real-time messaging,
|
||||
voice/video, file sharing, and a web admin panel. Run your own
|
||||
server and keep everything under your control.
|
||||
server and keep everything under your control — zero cloud
|
||||
dependencies, works fully on LAN.
|
||||
|
||||
## Features
|
||||
|
||||
@@ -21,17 +24,14 @@ server and keep everything under your control.
|
||||
|
||||
### Voice & Video
|
||||
|
||||
- Voice channels with WebRTC (Pion SFU)
|
||||
- Voice channels powered by LiveKit SFU
|
||||
- Webcam video chat with responsive grid layout
|
||||
- Mute, deafen, camera, and screenshare controls
|
||||
- Push-to-talk with global hotkey (non-consuming, works while unfocused)
|
||||
- Per-user volume control (right-click user in voice channel)
|
||||
- NAT traversal via Google STUN + configurable external IP
|
||||
- RNNoise ML noise suppression (AudioWorklet + fallback)
|
||||
- Voice activity detection with configurable sensitivity
|
||||
- Silence suppression to save bandwidth
|
||||
- Configurable audio quality (low/medium/high)
|
||||
- Server-enforced max video streams per room
|
||||
- RNNoise ML noise suppression
|
||||
- Voice activity detection with speaker indicators
|
||||
- LiveKit server runs as a companion process alongside `chatserver.exe`
|
||||
|
||||
### Channels & Organization
|
||||
|
||||
@@ -111,8 +111,8 @@ Two components: a **Go server** and a **Tauri v2 client**
|
||||
| +---------------+ | HTTPS | +---------------+ |
|
||||
| | REST Client |--+------->| | REST API | |
|
||||
| +---------------+ | | +---------------+ |
|
||||
| +---------------+ | WebRTC | +---------------+ |
|
||||
| | Voice/Video |--+------->| | SFU (Pion) | |
|
||||
| +---------------+ | LiveKit | +---------------+ |
|
||||
| | Voice/Video |--+------->| | LiveKit SFU | |
|
||||
| +---------------+ | | +---------------+ |
|
||||
+---------------------+ | +---------------+ |
|
||||
| | SQLite DB | |
|
||||
@@ -122,7 +122,7 @@ Two components: a **Go server** and a **Tauri v2 client**
|
||||
|
||||
- **WebSocket** — chat messages, typing, presence, voice signaling
|
||||
- **REST API** — message history, file uploads, channel management, auth
|
||||
- **WebRTC** — voice and video via Pion SFU with Google STUN for NAT traversal
|
||||
- **LiveKit** — voice and video via LiveKit SFU (companion process)
|
||||
|
||||
## Project Structure
|
||||
|
||||
@@ -141,9 +141,9 @@ OwnCord/
|
||||
│ └── tauri-client/ # Tauri v2 desktop client
|
||||
│ ├── src-tauri/ # Rust backend (plugins, commands)
|
||||
│ ├── src/ # TypeScript frontend
|
||||
│ │ ├── lib/ # Core services (API, WS, WebRTC, updater)
|
||||
│ │ ├── lib/ # Core services (API, WS, LiveKit, updater)
|
||||
│ │ ├── stores/ # Reactive state (auth, channels, messages, voice)
|
||||
│ │ ├── components/ # UI components (36 modules)
|
||||
│ │ ├── components/ # UI components (28 modules)
|
||||
│ │ ├── pages/ # Page layouts
|
||||
│ │ └── styles/ # CSS
|
||||
│ └── tests/ # Unit, integration, and E2E tests
|
||||
@@ -200,9 +200,10 @@ The server generates a `config.yaml` on first run. Key settings:
|
||||
| `server.name` | `OwnCord Server` | Display name |
|
||||
| `tls.mode` | `selfsigned` | TLS mode (see docs) |
|
||||
| `upload.max_size_mb` | `10` | Max upload size |
|
||||
| `voice.quality` | `medium` | `low`, `medium`, `high` |
|
||||
| `voice.external_ip` | — | Public IP for NAT traversal |
|
||||
| `voice.turn_enabled` | `true` | Enable TURN relay (requires coturn) |
|
||||
| `voice.livekit_url` | `ws://localhost:7880` | LiveKit server WebSocket URL |
|
||||
| `voice.livekit_api_key` | `devkey` | LiveKit API key |
|
||||
| `voice.livekit_api_secret` | — | LiveKit API secret (min 32 chars) |
|
||||
| `voice.livekit_binary` | — | Path to `livekit-server` binary (auto-start) |
|
||||
| `server.admin_allowed_cidrs` | private nets | CIDRs allowed to access `/admin` |
|
||||
| `github.token` | — | Token for update checks |
|
||||
|
||||
@@ -237,10 +238,10 @@ Detailed docs live in the `docs/brain/` Obsidian vault:
|
||||
|
||||
| Component | Technology |
|
||||
| --------- | --------- |
|
||||
| Server | Go, chi router, Pion WebRTC |
|
||||
| Server | Go, chi router, LiveKit server SDK |
|
||||
| Database | SQLite (pure Go, embedded) |
|
||||
| Client | Tauri v2 (Rust + TypeScript) |
|
||||
| Voice/Video | WebRTC with Pion SFU, Google STUN |
|
||||
| Voice/Video | LiveKit SFU (companion process) |
|
||||
| Build | NSIS installer, GitHub Actions CI |
|
||||
|
||||
## License
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
# ChatServer — Self-Hosted Windows Chat Platform
|
||||
|
||||
> **Note:** Most Phase 1-6 tasks are complete as of v1.2.0. See
|
||||
> [[02-Tasks/Done|Done]] for the detailed completion list and
|
||||
> [[00-Overview/Changelog|Changelog]] for version history.
|
||||
|
||||
Native Windows desktop client + self-hosted server.
|
||||
Two executables: `chatserver.exe` (server) and
|
||||
`OwnCord.exe` (Tauri v2 client). Server operator runs
|
||||
@@ -19,7 +23,7 @@ the server, friends install the client.
|
||||
### Client (`OwnCord.exe`)
|
||||
|
||||
**Tauri v2** (Rust backend + TypeScript/HTML/CSS frontend).
|
||||
See LANGUAGE-REVIEW.md for the evaluation that led to this
|
||||
See [[07-Archive/LANGUAGE-REVIEW|LANGUAGE-REVIEW]] for the evaluation that led to this
|
||||
choice, and CLIENT-ARCHITECTURE.md for the full design.
|
||||
|
||||
- Tauri v2 desktop app using system WebView2 (NOT Electron)
|
||||
@@ -66,108 +70,106 @@ CLIENT (OwnCord.exe) — installed by each friend
|
||||
|
||||
## Phase 1: Protocol & Server Core (2–3 weeks)
|
||||
|
||||
- [ ] Define client-server protocol over WebSocket
|
||||
- [x] Define client-server protocol over WebSocket
|
||||
(JSON messages with type/payload structure)
|
||||
- [ ] Message types: auth, chat, typing, presence,
|
||||
- [x] Message types: auth, chat, typing, presence,
|
||||
channel_update, voice_signal, file_transfer
|
||||
- [ ] Server: Go project with `go embed` for admin
|
||||
- [x] Server: Go project with `go embed` for admin
|
||||
panel static files only
|
||||
- [ ] SQLite setup with migrations on startup (users,
|
||||
- [x] SQLite setup with migrations on startup (users,
|
||||
channels, messages, sessions, roles, invites)
|
||||
- [ ] config.yaml generation on first run (port, name,
|
||||
- [x] config.yaml generation on first run (port, name,
|
||||
max upload size, voice quality, TLS mode)
|
||||
- [ ] Server systray icon (getlantern/systray) — minimize to tray, status
|
||||
indicator, open admin panel, quit
|
||||
- [ ] Windows Firewall handling on first launch
|
||||
- [ ] Optional: register as Windows Service for headless operation
|
||||
|
||||
## Phase 2: Auth & Security (2–3 weeks)
|
||||
|
||||
- [ ] Invite-only registration — server generates
|
||||
- [x] Invite-only registration — server generates
|
||||
invite codes, client has "Redeem Invite" flow
|
||||
- [ ] bcrypt (cost 12+) passwords, server-side session tokens (256-bit random)
|
||||
- [ ] Client stores auth token securely via Windows Credential Manager / DPAPI
|
||||
- [ ] Login rate limiting: 5 attempts/min/IP, lockout after 10 failures
|
||||
- [ ] Optional TOTP 2FA (`pquerna/otp`) — QR code
|
||||
during setup, prompts on login
|
||||
- [ ] Roles: Owner, Admin, Moderator, Member + custom roles with bitfield permissions
|
||||
- [ ] Per-channel permission overrides, enforced server-side on every action
|
||||
- [ ] TLS modes: self-signed (default), Let's Encrypt,
|
||||
- [x] bcrypt (cost 12+) passwords, server-side session tokens (256-bit random)
|
||||
- [x] Client stores auth token securely via Windows Credential Manager / DPAPI
|
||||
- [x] Login rate limiting: 5 attempts/min/IP, lockout after 10 failures
|
||||
- [ ] Optional TOTP 2FA — planned, not yet implemented (T-023 in backlog).
|
||||
DB schema has `totp_secret` column ready.
|
||||
- [x] Roles: Owner, Admin, Moderator, Member + custom roles with bitfield permissions
|
||||
- [x] Per-channel permission overrides, enforced server-side on every action
|
||||
- [x] TLS modes: self-signed (default), Let's Encrypt,
|
||||
manual cert, off (Tailscale)
|
||||
- [ ] Client: certificate pinning or trust-on-first-use (TOFU) for self-signed certs
|
||||
- [x] Client: certificate pinning or trust-on-first-use (TOFU) for self-signed certs
|
||||
|
||||
## Phase 3: Client App — Core UI (3–4 weeks)
|
||||
|
||||
- [ ] Connection dialog: server address, port, login/register, invite code entry
|
||||
- [ ] Save server profiles (connect to multiple
|
||||
- [x] Connection dialog: server address, port, login/register, invite code entry
|
||||
- [x] Save server profiles (connect to multiple
|
||||
servers like TeamSpeak)
|
||||
- [ ] Main window layout: server list → channel list → message area → member list
|
||||
- [ ] Channel tree view with categories, text channels, voice channels
|
||||
- [ ] Message rendering: markdown, code blocks, timestamps, avatars, replies, reactions
|
||||
- [ ] Message input: multi-line, markdown preview, emoji picker, file drag-and-drop
|
||||
- [ ] Unread indicators, @mention badges per channel
|
||||
- [ ] System tray: minimize to tray, notification popups, badge count
|
||||
- [ ] Keyboard shortcuts: Ctrl+K quick switcher,
|
||||
- [x] Main window layout: server list → channel list → message area → member list
|
||||
- [x] Channel tree view with categories, text channels, voice channels
|
||||
- [x] Message rendering: markdown, code blocks, timestamps, avatars, replies, reactions
|
||||
- [x] Message input: multi-line, markdown preview, emoji picker, file drag-and-drop
|
||||
- [x] Unread indicators, @mention badges per channel
|
||||
- [x] System tray: minimize to tray, notification popups, badge count
|
||||
- [x] Keyboard shortcuts: Ctrl+K quick switcher,
|
||||
Escape to close panels, customizable PTT key
|
||||
- [ ] Settings: account, appearance (light/dark),
|
||||
- [x] Settings: account, appearance (light/dark),
|
||||
notifications, audio devices, keybinds
|
||||
|
||||
## Phase 4: Real-Time Chat Features (2–3 weeks)
|
||||
|
||||
- [ ] WebSocket client with auto-reconnect, exponential
|
||||
- [x] WebSocket client with auto-reconnect, exponential
|
||||
backoff, message replay on reconnect
|
||||
- [ ] Send/receive messages in real-time, append to scrollback
|
||||
- [ ] Message history: paginated from server on channel switch, scroll-to-load-more
|
||||
- [ ] Threads, replies (inline preview), reactions (emoji), edit, delete
|
||||
- [ ] Typing indicators ("X is typing..." below input)
|
||||
- [ ] Online/offline/idle/DnD presence with status icons in member list
|
||||
- [ ] File uploads: drag-and-drop or clipboard paste,
|
||||
- [x] Send/receive messages in real-time, append to scrollback
|
||||
- [x] Message history: paginated from server on channel switch, scroll-to-load-more
|
||||
- [x] Threads, replies (inline preview), reactions (emoji), edit, delete
|
||||
- [x] Typing indicators ("X is typing..." below input)
|
||||
- [x] Online/offline/idle/DnD presence with status icons in member list
|
||||
- [x] File uploads: drag-and-drop or clipboard paste,
|
||||
progress bar, inline image previews
|
||||
- [ ] Client-side file validation before upload (size check, warn on large files)
|
||||
- [ ] Search: query server FTS5 endpoint, display results with jump-to-message
|
||||
- [ ] Windows toast notifications with action buttons (reply, mark read)
|
||||
- [ ] Notification sounds (configurable, per-channel mute/override)
|
||||
- [x] Client-side file validation before upload (size check, warn on large files)
|
||||
- [x] Search: query server FTS5 endpoint, display results with jump-to-message
|
||||
- [x] Windows toast notifications with action buttons (reply, mark read)
|
||||
- [x] Notification sounds (configurable, per-channel mute/override)
|
||||
|
||||
## Phase 5: Voice & Video (3–5 weeks)
|
||||
|
||||
- [ ] LiveKit integration in native client for voice/video
|
||||
- [x] LiveKit integration in native client for voice/video
|
||||
(client connects to LiveKit directly using token from server)
|
||||
- [ ] Audio device selection: input/output dropdowns in settings, live preview
|
||||
- [ ] Voice channels: click to join/leave, show connected users with speaking indicators
|
||||
- [ ] Voice controls: mute (button + keybind), deafen, per-user volume sliders
|
||||
- [ ] Push-to-talk: configurable global hotkey that works in fullscreen games
|
||||
- [ ] Voice activity detection with configurable sensitivity
|
||||
- [ ] Noise suppression (RNNoise or equivalent, bundled with client)
|
||||
- [ ] Server-side: LiveKit companion process with token-based auth
|
||||
- [x] Audio device selection: input/output dropdowns in settings, live preview
|
||||
- [x] Voice channels: click to join/leave, show connected users with speaking indicators
|
||||
- [x] Voice controls: mute (button + keybind), deafen, per-user volume sliders
|
||||
- [x] Push-to-talk: configurable global hotkey that works in fullscreen games
|
||||
- [x] Voice activity detection with configurable sensitivity
|
||||
- [x] Noise suppression (RNNoise or equivalent, bundled with client)
|
||||
- [x] Server-side: LiveKit companion process with token-based auth
|
||||
and webhook sync for voice state updates
|
||||
- [ ] Voice quality: low (32kbps) / medium (64kbps) / high (128kbps Opus)
|
||||
- [x] Voice quality: low (32kbps) / medium (64kbps) / high (128kbps Opus)
|
||||
- [ ] Screen sharing via LiveKit screen share track
|
||||
- [ ] Video calls: camera capture, displayed in voice channel panel
|
||||
- [x] Video calls: camera capture, displayed in voice channel panel
|
||||
|
||||
## Phase 6: Admin Panel — Web-Based (1–2 weeks)
|
||||
|
||||
- [ ] Served by server at `/admin`, browser-only access
|
||||
- [ ] Auth: admin credentials, session-based
|
||||
- [ ] Dashboard: connected users, message count, disk usage, CPU/RAM, uptime
|
||||
- [ ] User management: list all, edit roles, ban/unban, reset password, force disconnect
|
||||
- [ ] Channel management: create, rename, reorder, set permissions, archive
|
||||
- [x] Served by server at `/admin`, browser-only access
|
||||
- [x] Auth: admin credentials, session-based
|
||||
- [x] Dashboard: connected users, message count, disk usage, CPU/RAM, uptime
|
||||
- [x] User management: list all, edit roles, ban/unban, reset password, force disconnect
|
||||
- [x] Channel management: create, rename, reorder, set permissions, archive
|
||||
- [ ] Invite management: generate, view active, set expiry/use limit, revoke
|
||||
- [ ] Server settings: name, icon, MOTD, max upload size, voice quality, TLS config
|
||||
- [ ] Moderation: kick, ban, temp ban, slow mode, mute, word filter, audit log
|
||||
- [ ] Backup: trigger manual backup, configure
|
||||
- [x] Server settings: name, icon, MOTD, max upload size, voice quality, TLS config
|
||||
- [x] Moderation: kick, ban, temp ban, slow mode, mute, word filter, audit log
|
||||
- [x] Backup: trigger manual backup, configure
|
||||
schedule, view/restore from admin panel
|
||||
- [ ] Built with simple HTML/CSS/JS embedded in the server binary
|
||||
- [x] Built with simple HTML/CSS/JS embedded in the server binary
|
||||
|
||||
## Phase 7: Distribution & Updates (1–2 weeks)
|
||||
|
||||
- [ ] **Server:** GitHub Actions builds
|
||||
- [x] **Server:** GitHub Actions builds
|
||||
`chatserver.exe` (amd64), SHA256, GitHub Release
|
||||
- [ ] **Client:** Tauri bundler (NSIS) installer —
|
||||
- [x] **Client:** Tauri bundler (NSIS) installer —
|
||||
Program Files, Start Menu, auto-start, protocol
|
||||
handler for `chatserver://` invite links
|
||||
- [ ] Client auto-update: check GitHub releases on
|
||||
launch, prompt to download + install
|
||||
- [ ] Server update: admin panel shows available update, one-click download + restart
|
||||
- [x] Server update: admin panel shows available update, one-click download + restart
|
||||
- [ ] Docs: Quick Start, Port Forwarding, Tailscale,
|
||||
Client install guide
|
||||
- [ ] Security hardening checklist for server operators
|
||||
@@ -222,9 +224,15 @@ update integrity (SHA256).
|
||||
| LiveKit | `livekit/server-sdk-go` (token generation, webhook validation) |
|
||||
| SQLite | `modernc.org/sqlite` (pure Go) |
|
||||
| Auth | `golang.org/x/crypto/bcrypt` |
|
||||
| TOTP | `pquerna/otp` |
|
||||
| Sanitization | `bluemonday` |
|
||||
| TLS | `golang.org/x/crypto/acme/autocert` |
|
||||
| Systray | `getlantern/systray` |
|
||||
| Config | `koanf` |
|
||||
| Logging | `log/slog` |
|
||||
| Versioning | `golang.org/x/mod/semver` |
|
||||
| UUID | `google/uuid` |
|
||||
|
||||
> **Removed references:**
|
||||
> - ~~`getlantern/systray`~~ — The server has no system tray.
|
||||
> System tray is in the Tauri client (Rust-side).
|
||||
> - ~~`pquerna/otp`~~ — TOTP 2FA is not yet implemented
|
||||
> (T-023 in backlog). Not in `go.mod`.
|
||||
|
||||
@@ -327,9 +327,7 @@ and server URL for the client to connect directly to LiveKit.
|
||||
```
|
||||
|
||||
Client uses `bitrate` to configure the Opus encoder. Other fields are
|
||||
informational for UI. (`threshold_mode` and `top_speakers` fields have
|
||||
been removed — LiveKit handles audio mixing and speaker selection
|
||||
internally.)
|
||||
informational for UI.
|
||||
|
||||
### Voice Control (Client → Server)
|
||||
|
||||
@@ -348,9 +346,13 @@ internally.)
|
||||
Requires `USE_VIDEO` (bit 11) or `SHARE_SCREEN` (bit 12) permission.
|
||||
Rate limit: 2/sec per user.
|
||||
|
||||
**Note:** Active speaker detection (`voice_speakers`) is no longer a
|
||||
server→client WebSocket message. Speaker detection is handled client-side
|
||||
via LiveKit SDK events (`ParticipantEvent.IsSpeakingChanged`).
|
||||
### Migration Notes (LiveKit transition)
|
||||
|
||||
- `threshold_mode` and `top_speakers` fields have been removed from `voice_config`.
|
||||
LiveKit handles audio mixing and speaker selection internally.
|
||||
- Active speaker detection (`voice_speakers`) is no longer a server→client
|
||||
WebSocket message. Speaker detection is handled client-side via LiveKit SDK
|
||||
events (`ParticipantEvent.IsSpeakingChanged`).
|
||||
|
||||
---
|
||||
|
||||
@@ -444,7 +446,12 @@ then auto-reconnect after the delay expires.
|
||||
"color": "#F39C12",
|
||||
"permissions": 1073741823
|
||||
},
|
||||
{ "id": 3, "name": "Member", "color": null, "permissions": 1049601 }
|
||||
{
|
||||
"id": 3, "name": "Moderator",
|
||||
"color": "#3498DB",
|
||||
"permissions": 1048575
|
||||
},
|
||||
{ "id": 4, "name": "Member", "color": null, "permissions": 7779 }
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -457,7 +464,7 @@ then auto-reconnect after the delay expires.
|
||||
Fetched via REST API, not WebSocket, to keep the WS connection lean.
|
||||
|
||||
```text
|
||||
GET /api/channels/{id}/messages?before={msg_id}&limit=50
|
||||
GET /api/v1/channels/{id}/messages?before={msg_id}&limit=50
|
||||
```
|
||||
|
||||
---
|
||||
@@ -491,4 +498,8 @@ Error codes: `FORBIDDEN`, `NOT_FOUND`, `RATE_LIMITED`, `INVALID_INPUT`,
|
||||
- Voice signaling: 20/sec per user
|
||||
- Voice camera/screenshare: 2/sec per user
|
||||
|
||||
Server sends `rate_limited` error with `retry_after` in seconds.
|
||||
Server sends a standard `error` message with code `RATE_LIMITED` and `retry_after` in seconds:
|
||||
|
||||
```json
|
||||
{"type": "error", "payload": {"code": "RATE_LIMITED", "message": "...", "retry_after": 5}}
|
||||
```
|
||||
|
||||
Reference in New Issue
Block a user