fluxerapp/fluxer
1
0
Fork 0
mirror of https://github.com/fluxerapp/fluxer.git synced 2026-09-03 05:10:25 +03:00
Code Issues Packages Projects Releases Wiki Activity

refactor(voice): bound the screen-share codec wire formats (#2388)

Browse Source
This commit is contained in:
Hampus
2026-09-02 17:33:57 +02:00
committed by GitHub
parent 90f4a222b7
commit 59f6217267
6 changed files with 285 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.test.ts
+182
View File
@@ -0,0 +1,182 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {FluxerCodecAdvertisement} from '@app/features/voice/engine/ScreenShareCodecNegotiation';
import type {HardwareEncodeReport} from '@app/features/voice/utils/GpuEncoderCapabilities';
import {beforeEach, describe, expect, it, vi} from 'vitest';
let av1OptIn = false;
let hevcOptIn = false;
let preferredScreenShareCodec = 'auto';
let gpuReport: HardwareEncodeReport | null = null;
vi.mock('@app/features/voice/state/VoiceSettings', () => ({
default: {
getScreenShareAv1OptIn: () => av1OptIn,
getScreenShareHevcOptIn: () => hevcOptIn,
getPreferredScreenShareCodec: () => preferredScreenShareCodec,
getScreenShareEncoderMode: () => 'auto',
},
}));
vi.mock('@app/features/devtools/utils/DesktopTroubleshootingUtils', () => ({
getCachedDesktopTroubleshootingSettings: () => null,
}));
vi.mock('@app/features/ui/utils/NativeUtils', () => ({
guessPlatform: () => 'windows',
isChromiumBrowser: () => true,
isDesktop: () => true,
isFirefoxBrowser: () => false,
}));
vi.mock('@app/features/voice/utils/GpuEncoderCapabilities', () => ({
getGpuEncoderReportSync: () => gpuReport,
loadGpuEncoderReport: async () => gpuReport,
}));
vi.mock('@app/features/voice/utils/NativeHardwareEncoderCapabilities', () => ({
getNativeHardwareEncoderCapabilitiesSync: () => null,
hasNativeHardwareEncoder: () => false,
resetNativeHardwareEncoderCapabilities: () => undefined,
loadNativeHardwareEncoderCapabilities: async () => null,
}));
vi.mock('@app/features/voice/utils/OpenH264Status', () => ({
getOpenH264StatusSync: () => null,
resetOpenH264Status: () => undefined,
loadOpenH264Status: async () => null,
}));
vi.mock('@app/features/voice/utils/VideoDecoderCapabilities', () => ({
getVideoDecoderExclusionsSync: () => [],
loadVideoDecoderExclusions: async () => [],
}));
const VIDEO_CAPABILITIES = {
codecs: [
{mimeType: 'video/VP8'},
{mimeType: 'video/VP9'},
{mimeType: 'video/H264'},
{mimeType: 'video/H265'},
{mimeType: 'video/AV1'},
],
};
Object.defineProperty(globalThis, 'RTCRtpSender', {
configurable: true,
writable: true,
value: {getCapabilities: () => VIDEO_CAPABILITIES},
});
Object.defineProperty(globalThis, 'RTCRtpReceiver', {
configurable: true,
writable: true,
value: {getCapabilities: () => VIDEO_CAPABILITIES},
});
const {buildLocalCodecAdvertisements, computeNegotiatedVideoCodec, getScreenShareCodecPreferenceOrder} = await import(
'./ScreenShareCodecNegotiation'
);
const {resetCachedCodecCapabilities} = await import('@app/features/voice/utils/CodecCapabilityDetector');
function videoAdvertisement(name: 'AV1' | 'VP9' | 'H265', encode: boolean, decode: boolean): FluxerCodecAdvertisement {
const payloadType = name === 'AV1' ? 101 : name === 'H265' ? 105 : 109;
return {name, type: 'video', payload_type: payloadType, priority: 1, encode, decode};
}
describe('screen-share codec negotiation with the AV1 opt-in off', () => {
beforeEach(() => {
av1OptIn = false;
hevcOptIn = false;
preferredScreenShareCodec = 'auto';
gpuReport = {av1: 'hardware', h265: 'software', h264: 'hardware', vp9: 'software', vp8: 'software'};
resetCachedCodecCapabilities();
});
it('keeps AV1 out of the negotiated preference order the hardware tail would refill', () => {
expect(getScreenShareCodecPreferenceOrder()).not.toContain('av1');
av1OptIn = true;
resetCachedCodecCapabilities();
expect(getScreenShareCodecPreferenceOrder()).toContain('av1');
});
it('drops an explicitly requested AV1 preference from the order', () => {
expect(getScreenShareCodecPreferenceOrder('av1')).not.toContain('av1');
av1OptIn = true;
resetCachedCodecCapabilities();
expect(getScreenShareCodecPreferenceOrder('av1')[0]).toBe('av1');
});
it('stops advertising AV1 encode while still advertising AV1 decode', () => {
const av1 = buildLocalCodecAdvertisements().find((codec) => codec.name === 'AV1');
expect(av1).toMatchObject({encode: false, decode: true});
av1OptIn = true;
resetCachedCodecCapabilities();
expect(buildLocalCodecAdvertisements().find((codec) => codec.name === 'AV1')).toMatchObject({
encode: true,
decode: true,
});
});
it('negotiates away from AV1 even when both ends can encode and decode it', () => {
const local = [videoAdvertisement('AV1', true, true), videoAdvertisement('VP9', true, true)];
const remote = [[videoAdvertisement('AV1', true, true), videoAdvertisement('VP9', true, true)]];
expect(computeNegotiatedVideoCodec(local, remote, 0, getScreenShareCodecPreferenceOrder()).codec).toBe('vp9');
av1OptIn = true;
resetCachedCodecCapabilities();
expect(computeNegotiatedVideoCodec(local, remote, 0, getScreenShareCodecPreferenceOrder()).codec).toBe('av1');
});
it('avoids exotic codecs while any participant codec set is still unknown', () => {
av1OptIn = true;
resetCachedCodecCapabilities();
const local = [videoAdvertisement('AV1', true, true), videoAdvertisement('VP9', true, true)];
const remote = [[videoAdvertisement('AV1', true, true), videoAdvertisement('VP9', true, true)]];
expect(computeNegotiatedVideoCodec(local, remote, 0, getScreenShareCodecPreferenceOrder()).codec).toBe('av1');
expect(computeNegotiatedVideoCodec(local, remote, 1, getScreenShareCodecPreferenceOrder()).codec).toBe('vp9');
});
});
describe('screen-share codec negotiation with the HEVC opt-in off', () => {
beforeEach(() => {
av1OptIn = false;
hevcOptIn = false;
preferredScreenShareCodec = 'auto';
gpuReport = {av1: 'hardware', h265: 'hardware', h264: 'hardware', vp9: 'software', vp8: 'software'};
resetCachedCodecCapabilities();
});
it('keeps HEVC out of the negotiated preference order the hardware tail would refill', () => {
expect(getScreenShareCodecPreferenceOrder()).not.toContain('h265');
hevcOptIn = true;
resetCachedCodecCapabilities();
expect(getScreenShareCodecPreferenceOrder()).toContain('h265');
});
it('drops an explicitly requested HEVC preference from the order', () => {
expect(getScreenShareCodecPreferenceOrder('h265')).not.toContain('h265');
hevcOptIn = true;
resetCachedCodecCapabilities();
expect(getScreenShareCodecPreferenceOrder('h265')[0]).toBe('h265');
});
it('stops advertising HEVC encode while still advertising HEVC decode', () => {
const h265 = buildLocalCodecAdvertisements().find((codec) => codec.name === 'H265');
expect(h265).toMatchObject({encode: false, decode: true});
hevcOptIn = true;
resetCachedCodecCapabilities();
expect(buildLocalCodecAdvertisements().find((codec) => codec.name === 'H265')).toMatchObject({
encode: true,
decode: true,
});
});
it('negotiates away from HEVC even when both ends can encode and decode it', () => {
const local = [videoAdvertisement('H265', true, true), videoAdvertisement('VP9', true, true)];
const remote = [[videoAdvertisement('H265', true, true), videoAdvertisement('VP9', true, true)]];
expect(computeNegotiatedVideoCodec(local, remote, 0, getScreenShareCodecPreferenceOrder()).codec).toBe('vp9');
hevcOptIn = true;
resetCachedCodecCapabilities();
expect(computeNegotiatedVideoCodec(local, remote, 0, getScreenShareCodecPreferenceOrder()).codec).toBe('h265');
});
});
fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiation.ts
+35 -11
View File
@@ -28,10 +28,17 @@ const PROTOCOL_TOPIC = 'fluxer.rtc.codec-negotiation.v1';
const SELECT_PROTOCOL_OP = 1;
const SESSION_UPDATE_OP = 14;
const TEXT_ENCODER = new TextEncoder();
const TEXT_DECODER = new TextDecoder();
const TEXT_DECODER = new TextDecoder('utf-8', {fatal: true});
const NEGOTIATION_MESSAGE_BYTES_MAX = 16 * 1024;
const CODEC_ADVERTISEMENTS_MAX = 16;
const NEGOTIATION_IDENTIFIER_CHARS_MAX = 256;
const EXPERIMENTS_MAX = 16;
const EXPERIMENT_NAME_CHARS_MAX = 128;
const RTP_PAYLOAD_TYPE_MAX = 255;
const CODEC_PRIORITY_MAX = 65_535;
const CODEC_PREFERENCE: ReadonlyArray<VideoCodec> = ['av1', 'h265', 'h264', 'vp9', 'vp8'];
const SOFTWARE_CODEC_PREFERENCE: ReadonlyArray<VideoCodec> = ['av1', 'vp9', 'h264', 'vp8', 'h265'];
const COMPATIBILITY_FALLBACK_CODEC_PREFERENCE: ReadonlyArray<VideoCodec> = ['vp9', 'vp8'];
const COMPATIBILITY_FALLBACK_CODEC_PREFERENCE: ReadonlyArray<VideoCodec> = ['h264', 'vp9', 'vp8'];
const BASELINE_VIDEO_CODEC: VideoCodec = 'vp8';
const VIDEO_CODEC_NAMES: Record<VideoCodec, FluxerVideoCodecName> = {
av1: 'AV1',
@@ -393,8 +400,16 @@ function isBooleanOrUndefined(value: unknown): value is boolean | undefined {
return value === undefined || typeof value === 'boolean';
}
function isNumberOrUndefined(value: unknown): value is number | undefined {
return value === undefined || typeof value === 'number';
function isBoundedInteger(value: unknown, maximum: number): value is number {
return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 && value <= maximum;
}
function isBoundedIntegerOrUndefined(value: unknown, maximum: number): value is number | undefined {
return value === undefined || isBoundedInteger(value, maximum);
}
function isBoundedString(value: unknown, maximumLength: number): value is string {
return typeof value === 'string' && value.length > 0 && value.length <= maximumLength;
}
function isFluxerVideoCodecName(value: unknown): value is FluxerVideoCodecName {
@@ -414,16 +429,22 @@ function isCodecAdvertisement(value: unknown): value is FluxerCodecAdvertisement
return (
isFluxerCodecName(value.name) &&
isFluxerCodecType(value.type) &&
typeof value.payload_type === 'number' &&
isNumberOrUndefined(value.rtx_payload_type) &&
typeof value.priority === 'number' &&
((value.name === 'opus' && value.type === 'audio') || (value.name !== 'opus' && value.type === 'video')) &&
isBoundedInteger(value.payload_type, RTP_PAYLOAD_TYPE_MAX) &&
isBoundedIntegerOrUndefined(value.rtx_payload_type, RTP_PAYLOAD_TYPE_MAX) &&
isBoundedInteger(value.priority, CODEC_PRIORITY_MAX) &&
isBooleanOrUndefined(value.encode) &&
isBooleanOrUndefined(value.decode)
);
}
function isCodecAdvertisementList(value: unknown): value is Array<FluxerCodecAdvertisement> {
return Array.isArray(value) && value.every(isCodecAdvertisement);
return (
Array.isArray(value) &&
value.length > 0 &&
value.length <= CODEC_ADVERTISEMENTS_MAX &&
value.every(isCodecAdvertisement)
);
}
function isSelectProtocolMessage(value: unknown): value is FluxerSelectProtocolMessage {
@@ -434,9 +455,11 @@ function isSelectProtocolMessage(value: unknown): value is FluxerSelectProtocolM
isObject(data) &&
data.mode === 'livekit-sfu' &&
isCodecAdvertisementList(value.d.codecs) &&
(typeof value.d.rtc_connection_id === 'string' || value.d.rtc_connection_id === null) &&
(isBoundedString(value.d.rtc_connection_id, NEGOTIATION_IDENTIFIER_CHARS_MAX) ||
value.d.rtc_connection_id === null) &&
Array.isArray(value.d.experiments) &&
value.d.experiments.every((experiment) => typeof experiment === 'string')
value.d.experiments.length <= EXPERIMENTS_MAX &&
value.d.experiments.every((experiment) => isBoundedString(experiment, EXPERIMENT_NAME_CHARS_MAX))
);
}
@@ -455,13 +478,14 @@ function isSessionUpdateMessage(value: unknown): value is FluxerSessionUpdateMes
if (!isObject(value) || value.op !== SESSION_UPDATE_OP || !isObject(value.d)) return false;
return (
isFluxerVideoCodecName(value.d.video_codec) &&
typeof value.d.media_session_id === 'string' &&
isBoundedString(value.d.media_session_id, NEGOTIATION_IDENTIFIER_CHARS_MAX) &&
isNegotiationReason(value.d.reason) &&
isCodecAdvertisementList(value.d.codecs)
);
}
function parseMessage(payload: Uint8Array): FluxerCodecNegotiationMessage | null {
if (payload.byteLength === 0 || payload.byteLength > NEGOTIATION_MESSAGE_BYTES_MAX) return null;
try {
const parsed = JSON.parse(TEXT_DECODER.decode(payload)) as unknown;
if (isSelectProtocolMessage(parsed)) return parsed;
fluxer_app/src/features/voice/engine/ScreenShareCodecNegotiationOptIn.test.ts
-43
View File
@@ -1,43 +0,0 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {beforeEach, describe, expect, it, vi} from 'vitest';
let av1OptIn = false;
let hevcOptIn = false;
vi.mock('@app/features/voice/state/VoiceSettings', () => ({
default: {
getScreenShareEncoderMode: () => 'software',
getPreferredScreenShareCodec: () => 'auto',
getScreenShareAv1OptIn: () => av1OptIn,
getScreenShareHevcOptIn: () => hevcOptIn,
},
}));
const {getScreenShareCodecPreferenceOrder} = await import('./ScreenShareCodecNegotiation');
describe('screen-share codec preference order opt-in filter', () => {
beforeEach(() => {
av1OptIn = false;
hevcOptIn = false;
});
it('excludes AV1 and HEVC from the automatic preference order by default', () => {
const order = getScreenShareCodecPreferenceOrder('auto');
expect(order).not.toContain('av1');
expect(order).not.toContain('h265');
expect(order).toContain('vp9');
});
it('includes AV1 only once its opt-in is on', () => {
expect(getScreenShareCodecPreferenceOrder('auto')).not.toContain('av1');
av1OptIn = true;
expect(getScreenShareCodecPreferenceOrder('auto')).toContain('av1');
});
it('includes HEVC only once its opt-in is on', () => {
expect(getScreenShareCodecPreferenceOrder('auto')).not.toContain('h265');
hevcOptIn = true;
expect(getScreenShareCodecPreferenceOrder('auto')).toContain('h265');
});
});
fluxer_app/src/features/voice/engine/ScreenSharePublicationMigration.ts
+39 -30
View File
@@ -16,7 +16,9 @@ import {
import {Store} from '@app/features/voice/engine/Store';
import {selectVoiceMediaGraphViewerStreamKeys} from '@app/features/voice/engine/VoiceMediaGraph';
import {voiceMediaGraphStore} from '@app/features/voice/engine/VoiceMediaGraphStore';
import {createVoiceMediaIdentity} from '@app/features/voice/engine/VoiceMediaIdentity';
import {getStreamKeyForParticipantIdentity} from '@app/features/voice/engine/VoiceStreamWatchState';
import {isScreenShareVideoCodecValue} from '@app/features/voice/engine/v2/VoiceEngineV2AppScreenShareNativePublishOptions';
import {
type LocalParticipant,
type Participant,
@@ -40,7 +42,11 @@ const COMMIT_OP = 3;
const ABORT_OP = 4;
const BREAK_OP = 5;
const TEXT_ENCODER = new TextEncoder();
const TEXT_DECODER = new TextDecoder();
const TEXT_DECODER = new TextDecoder('utf-8', {fatal: true});
const MIGRATION_MESSAGE_BYTES_MAX = 4096;
const MIGRATION_IDENTIFIER_CHARS_MAX = 256;
const TRACK_SID_CHARS_MAX = 256;
const MIGRATION_REASON_CHARS_MAX = 512;
const DEFAULT_READY_TIMEOUT_MS = 5000;
const REMOTE_READY_PROBE_TIMEOUT_MS = 6500;
const REMOTE_MIGRATION_STATE_TIMEOUT_MS = 10000;
@@ -134,33 +140,35 @@ interface ReadyProbe {
}
function createId(prefix: string): string {
const cryptoObject = globalThis.crypto as Crypto | undefined;
if (typeof cryptoObject?.randomUUID === 'function') return `${prefix}_${cryptoObject.randomUUID()}`;
return `${prefix}_${Date.now().toString(36)}_${Math.random().toString(36).slice(2)}`;
return `${prefix}_${createVoiceMediaIdentity()}`;
}
function isObject(value: unknown): value is Record<string, unknown> {
return value !== null && typeof value === 'object';
}
function isVideoCodec(value: unknown): value is VideoCodec {
return value === 'av1' || value === 'h265' || value === 'h264' || value === 'vp9' || value === 'vp8';
function isBoundedString(value: unknown, maximumLength: number): value is string {
return typeof value === 'string' && value.length > 0 && value.length <= maximumLength;
}
function isStringOrNull(value: unknown): value is string | null {
return typeof value === 'string' || value === null;
function isBoundedStringOrNull(value: unknown, maximumLength: number): value is string | null {
return value === null || isBoundedString(value, maximumLength);
}
function isMigrationGeneration(value: unknown): value is number {
return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0;
}
function isCandidateMessage(message: unknown): message is ScreenShareMigrationCandidateMessage {
if (!isObject(message)) return false;
if (message.op !== CANDIDATE_OP || !isObject(message.d)) return false;
return (
typeof message.d.migration_id === 'string' &&
typeof message.d.generation === 'number' &&
isStringOrNull(message.d.previous_track_sid) &&
typeof message.d.candidate_track_sid === 'string' &&
isVideoCodec(message.d.codec) &&
typeof message.d.reason === 'string'
isBoundedString(message.d.migration_id, MIGRATION_IDENTIFIER_CHARS_MAX) &&
isMigrationGeneration(message.d.generation) &&
isBoundedStringOrNull(message.d.previous_track_sid, TRACK_SID_CHARS_MAX) &&
isBoundedString(message.d.candidate_track_sid, TRACK_SID_CHARS_MAX) &&
isScreenShareVideoCodecValue(message.d.codec) &&
isBoundedString(message.d.reason, MIGRATION_REASON_CHARS_MAX)
);
}
@@ -168,11 +176,11 @@ function isBreakMessage(message: unknown): message is ScreenShareMigrationBreakM
if (!isObject(message)) return false;
if (message.op !== BREAK_OP || !isObject(message.d)) return false;
return (
typeof message.d.migration_id === 'string' &&
typeof message.d.generation === 'number' &&
isStringOrNull(message.d.previous_track_sid) &&
isVideoCodec(message.d.codec) &&
typeof message.d.reason === 'string'
isBoundedString(message.d.migration_id, MIGRATION_IDENTIFIER_CHARS_MAX) &&
isMigrationGeneration(message.d.generation) &&
isBoundedStringOrNull(message.d.previous_track_sid, TRACK_SID_CHARS_MAX) &&
isScreenShareVideoCodecValue(message.d.codec) &&
isBoundedString(message.d.reason, MIGRATION_REASON_CHARS_MAX)
);
}
@@ -180,9 +188,9 @@ function isReadyMessage(message: unknown): message is ScreenShareMigrationReadyM
if (!isObject(message)) return false;
if (message.op !== READY_OP || !isObject(message.d)) return false;
return (
typeof message.d.migration_id === 'string' &&
typeof message.d.generation === 'number' &&
typeof message.d.candidate_track_sid === 'string'
isBoundedString(message.d.migration_id, MIGRATION_IDENTIFIER_CHARS_MAX) &&
isMigrationGeneration(message.d.generation) &&
isBoundedString(message.d.candidate_track_sid, TRACK_SID_CHARS_MAX)
);
}
@@ -190,10 +198,10 @@ function isCommitMessage(message: unknown): message is ScreenShareMigrationCommi
if (!isObject(message)) return false;
if (message.op !== COMMIT_OP || !isObject(message.d)) return false;
return (
typeof message.d.migration_id === 'string' &&
typeof message.d.generation === 'number' &&
isStringOrNull(message.d.previous_track_sid) &&
typeof message.d.candidate_track_sid === 'string'
isBoundedString(message.d.migration_id, MIGRATION_IDENTIFIER_CHARS_MAX) &&
isMigrationGeneration(message.d.generation) &&
isBoundedStringOrNull(message.d.previous_track_sid, TRACK_SID_CHARS_MAX) &&
isBoundedString(message.d.candidate_track_sid, TRACK_SID_CHARS_MAX)
);
}
@@ -201,14 +209,15 @@ function isAbortMessage(message: unknown): message is ScreenShareMigrationAbortM
if (!isObject(message)) return false;
if (message.op !== ABORT_OP || !isObject(message.d)) return false;
return (
typeof message.d.migration_id === 'string' &&
typeof message.d.generation === 'number' &&
isStringOrNull(message.d.candidate_track_sid) &&
typeof message.d.reason === 'string'
isBoundedString(message.d.migration_id, MIGRATION_IDENTIFIER_CHARS_MAX) &&
isMigrationGeneration(message.d.generation) &&
isBoundedStringOrNull(message.d.candidate_track_sid, TRACK_SID_CHARS_MAX) &&
isBoundedString(message.d.reason, MIGRATION_REASON_CHARS_MAX)
);
}
export function parseScreenShareMigrationMessage(payload: Uint8Array): ScreenShareMigrationMessage | null {
if (payload.byteLength === 0 || payload.byteLength > MIGRATION_MESSAGE_BYTES_MAX) return null;
try {
const parsed = JSON.parse(TEXT_DECODER.decode(payload)) as unknown;
if (!isObject(parsed)) return null;
fluxer_app/src/features/voice/engine/VoiceMediaIdentity.ts
+29
View File
@@ -0,0 +1,29 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
export class VoiceMediaIdentityCapabilityError extends Error {
constructor() {
super('Voice media identity generation requires crypto.randomUUID');
this.name = 'VoiceMediaIdentityCapabilityError';
}
}
const RANDOM_UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u;
export class VoiceMediaIdentityInvariantError extends Error {
constructor() {
super('crypto.randomUUID must return a canonical version 4 UUID');
this.name = 'VoiceMediaIdentityInvariantError';
}
}
export function createVoiceMediaIdentity(): string {
const cryptoPort = globalThis.crypto;
if (cryptoPort == null || typeof cryptoPort.randomUUID !== 'function') {
throw new VoiceMediaIdentityCapabilityError();
}
const identity = cryptoPort.randomUUID();
if (!RANDOM_UUID_PATTERN.test(identity)) {
throw new VoiceMediaIdentityInvariantError();
}
return identity;
}
fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppCodecGossipAdapter.ts
BIN
View File
Binary file not shown.