refactor(voice): request mic and camera permission separately (#2387)

This commit is contained in:
Hampus
2026-09-02 17:33:45 +02:00
committed by GitHub
parent 749d2091eb
commit 90f4a222b7
13 changed files with 592 additions and 157 deletions
@@ -102,7 +102,7 @@ class NewDeviceMonitoring {
private handleDeviceStateChange(state: VoiceDeviceState): void {
if (!this.isStarted) return;
if (state.permissionStatus !== 'granted') {
if (state.permissionStatus.audio !== 'granted') {
return;
}
if (this.suppressAlerts) {
@@ -0,0 +1,178 @@
// @vitest-environment happy-dom
// SPDX-License-Identifier: AGPL-3.0-or-later
import type {
EnsureVoiceDevicesOptions,
VoiceDeviceState,
VoiceMediaPermissionType,
} from '@app/features/voice/utils/VoiceDeviceManager';
import {act, createElement, useEffect} from 'react';
import {createRoot, type Root} from 'react-dom/client';
import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';
const deviceManagerMock = vi.hoisted(() => {
const idleState = (): VoiceDeviceState => ({
inputDevices: [],
outputDevices: [],
videoDevices: [],
permissionStatus: {audio: 'idle', video: 'idle'},
});
const listeners = new Set<(state: VoiceDeviceState) => void>();
const mock = {
state: idleState(),
permissionRequests: [] as Array<ReadonlyArray<VoiceMediaPermissionType>>,
pending: [] as Array<(state: VoiceDeviceState) => void>,
reset(): void {
mock.state = idleState();
mock.permissionRequests = [];
mock.pending = [];
listeners.clear();
},
settlePendingWith(next: VoiceDeviceState): void {
const pending = mock.pending;
mock.pending = [];
mock.state = next;
for (const listener of listeners) listener(next);
for (const resolve of pending) resolve(next);
},
manager: {
getState: (): VoiceDeviceState => mock.state,
subscribe: (listener: (state: VoiceDeviceState) => void): (() => void) => {
listeners.add(listener);
listener(mock.state);
return () => listeners.delete(listener);
},
ensureDevices: (options: EnsureVoiceDevicesOptions = {}): Promise<VoiceDeviceState> => {
const requestPermissionTypes =
options.requestPermissionTypes ?? (options.requestPermissions === true ? ['audio', 'video'] : []);
if (requestPermissionTypes.length === 0) return Promise.resolve(mock.state);
mock.permissionRequests.push([...requestPermissionTypes]);
return new Promise<VoiceDeviceState>((resolve) => {
mock.pending.push(resolve);
});
},
},
};
return mock;
});
const mediaPermissionMock = vi.hoisted(() => ({
microphoneExplicitlyDenied: false,
cameraExplicitlyDenied: false,
microphonePermissionState: null as PermissionState | null,
cameraPermissionState: null as PermissionState | null,
isMicrophoneGranted: vi.fn(() => false),
isCameraGranted: vi.fn(() => false),
updateMicrophonePermissionGranted: vi.fn(),
updateCameraPermissionGranted: vi.fn(),
markMicrophoneExplicitlyDenied: vi.fn(),
markCameraExplicitlyDenied: vi.fn(),
}));
const handleMediaPermissionBlocked = vi.hoisted(() => vi.fn());
vi.mock('@app/features/voice/utils/VoiceDeviceManager', () => ({voiceDeviceManager: deviceManagerMock.manager}));
vi.mock('@app/features/permissions/system/state/MediaPermission', () => ({default: mediaPermissionMock}));
vi.mock('@app/features/permissions/system/commands/MacPermissionsModalCommands', () => ({
handleMediaPermissionBlocked,
}));
const {useMediaPermission} = await import('@app/features/user/components/modals/tabs/hooks/useMediaPermission');
(globalThis as {IS_REACT_ACT_ENVIRONMENT?: boolean}).IS_REACT_ACT_ENVIRONMENT = true;
const MICROPHONE: MediaDeviceInfo = {
deviceId: 'mic-1',
groupId: 'group-1',
kind: 'audioinput',
label: 'Microphone',
toJSON: () => ({}),
};
const grantedState = (): VoiceDeviceState => ({
inputDevices: [MICROPHONE],
outputDevices: [],
videoDevices: [],
permissionStatus: {audio: 'granted', video: 'idle'},
});
const deniedState = (): VoiceDeviceState => ({
inputDevices: [],
outputDevices: [],
videoDevices: [],
permissionStatus: {audio: 'denied', video: 'idle'},
});
let host: HTMLDivElement;
let root: Root;
const requesters: Array<() => Promise<boolean>> = [];
function Consumer(): null {
const {requestPermission} = useMediaPermission('audio');
useEffect(() => {
requesters.push(requestPermission);
return () => {
const index = requesters.indexOf(requestPermission);
if (index !== -1) requesters.splice(index, 1);
};
}, [requestPermission]);
return null;
}
function renderConsumers(count: number): void {
act(() => {
root.render(Array.from({length: count}, (_, index) => createElement(Consumer, {key: index})));
});
}
beforeEach(() => {
deviceManagerMock.reset();
requesters.length = 0;
mediaPermissionMock.microphoneExplicitlyDenied = false;
mediaPermissionMock.microphonePermissionState = null;
mediaPermissionMock.isMicrophoneGranted.mockReturnValue(false);
host = document.createElement('div');
document.body.append(host);
root = createRoot(host);
});
afterEach(() => {
act(() => {
root.unmount();
});
document.body.replaceChildren();
vi.clearAllMocks();
});
describe('useMediaPermission', () => {
it('raises a single device permission request when two consumers ask at once', async () => {
renderConsumers(2);
expect(requesters).toHaveLength(2);
const results = await act(async () => {
const pending = Promise.all([requesters[0](), requesters[1]()]);
await Promise.resolve();
deviceManagerMock.settlePendingWith(grantedState());
return pending;
});
expect(deviceManagerMock.permissionRequests).toEqual([['audio']]);
expect(results).toEqual([true, true]);
expect(mediaPermissionMock.updateMicrophonePermissionGranted).toHaveBeenCalledTimes(1);
});
it('reports a denied request through the shared permission state', async () => {
renderConsumers(1);
const granted = await act(async () => {
const pending = requesters[0]();
await Promise.resolve();
deviceManagerMock.settlePendingWith(deniedState());
return pending;
});
expect(granted).toBe(false);
expect(mediaPermissionMock.markMicrophoneExplicitlyDenied).toHaveBeenCalledTimes(1);
expect(handleMediaPermissionBlocked).toHaveBeenCalledWith('microphone');
});
});
@@ -2,10 +2,13 @@
import {handleMediaPermissionBlocked} from '@app/features/permissions/system/commands/MacPermissionsModalCommands';
import MediaPermission from '@app/features/permissions/system/state/MediaPermission';
import {Logger} from '@app/features/platform/utils/AppLogger';
import VoiceDevicePermissionState from '@app/features/voice/engine/VoiceDevicePermissionState';
import type {VoiceDeviceState} from '@app/features/voice/utils/VoiceDeviceManager';
import {useCallback, useEffect, useLayoutEffect, useRef, useState} from 'react';
const logger = new Logger('useMediaPermission');
type PermissionType = 'audio' | 'video';
type BrowserPermissionState = 'denied' | 'granted' | 'prompt';
@@ -34,21 +37,27 @@ const resolveStatusFromVoiceState = ({
deviceState,
isExplicitlyDenied,
osPermissionLooksGranted,
type,
}: {
cachedPermissionState: BrowserPermissionState | null;
deviceState: VoiceDeviceState;
isExplicitlyDenied: boolean;
osPermissionLooksGranted: boolean;
type: PermissionType;
}): PermissionState['status'] => {
if (isExplicitlyDenied) return 'denied';
if (deviceState.permissionStatus === 'loading') return 'loading';
if (deviceState.permissionStatus === 'granted') return 'granted';
if (deviceState.permissionStatus === 'denied') return 'denied';
if (cachedPermissionState === 'granted' || osPermissionLooksGranted) return 'loading';
if (cachedPermissionState === 'denied') return 'denied';
const permissionStatus = deviceState.permissionStatus[type];
if (permissionStatus === 'denied') return 'denied';
if (cachedPermissionState === 'granted' || osPermissionLooksGranted) {
return permissionStatus === 'loading' ? 'loading' : 'granted';
}
if (permissionStatus === 'loading') return 'loading';
if (permissionStatus === 'granted') return 'granted';
return 'idle';
};
export const useMediaPermission = (type: PermissionType, options: UseMediaPermissionOptions = {}) => {
const {autoRequest = true} = options;
const {autoRequest = false} = options;
const micExplicitlyDenied = MediaPermission.microphoneExplicitlyDenied;
const cameraExplicitlyDenied = MediaPermission.cameraExplicitlyDenied;
const isExplicitlyDenied = type === 'audio' ? micExplicitlyDenied : cameraExplicitlyDenied;
@@ -62,6 +71,7 @@ export const useMediaPermission = (type: PermissionType, options: UseMediaPermis
deviceState: initialDeviceState,
isExplicitlyDenied,
osPermissionLooksGranted,
type,
}),
devices: devicesFromVoiceState(initialDeviceState, type),
deviceState: initialDeviceState,
@@ -74,6 +84,7 @@ export const useMediaPermission = (type: PermissionType, options: UseMediaPermis
deviceState,
isExplicitlyDenied,
osPermissionLooksGranted,
type,
}),
devices: devicesFromVoiceState(deviceState, type),
deviceState,
@@ -85,25 +96,16 @@ export const useMediaPermission = (type: PermissionType, options: UseMediaPermis
);
const unlockDevices = useCallback(async (): Promise<PermissionState> => {
setState((prev) => ({...prev, status: 'loading'}));
const deviceState = await VoiceDevicePermissionState.ensureDevices({
requestPermissions: true,
forceRefresh: true,
});
if (deviceState.permissionStatus === 'granted') {
if (type === 'audio') {
MediaPermission.updateMicrophonePermissionGranted({refreshDevices: false});
} else {
MediaPermission.updateCameraPermissionGranted({refreshDevices: false});
}
} else if (deviceState.permissionStatus === 'denied') {
if (type === 'audio') {
MediaPermission.markMicrophoneExplicitlyDenied();
} else {
MediaPermission.markCameraExplicitlyDenied();
}
}
return applyVoiceDeviceState(deviceState);
}, [applyVoiceDeviceState, type]);
const granted = await VoiceDevicePermissionState.requestPermissionFor(type);
const deviceState = VoiceDevicePermissionState.getState();
const nextState = {
status: granted ? 'granted' : deviceState.permissionStatus[type] === 'denied' ? 'denied' : 'idle',
devices: devicesFromVoiceState(deviceState, type),
deviceState,
} satisfies PermissionState;
setState(nextState);
return nextState;
}, [type]);
const requestPermission = useCallback(async () => {
if (isExplicitlyDenied) {
handleMediaPermissionBlocked(type === 'audio' ? 'microphone' : 'camera');
@@ -111,7 +113,9 @@ export const useMediaPermission = (type: PermissionType, options: UseMediaPermis
}
const currentDeviceState = VoiceDevicePermissionState.getState();
const currentDevices = devicesFromVoiceState(currentDeviceState, type);
if (currentDeviceState.permissionStatus === 'granted' && hasPrimaryDeviceForType(currentDevices, type)) {
const permissionAlreadyGranted =
type === 'audio' ? MediaPermission.isMicrophoneGranted() : MediaPermission.isCameraGranted();
if (permissionAlreadyGranted && hasPrimaryDeviceForType(currentDevices, type)) {
applyVoiceDeviceState(currentDeviceState);
return true;
}
@@ -122,7 +126,8 @@ export const useMediaPermission = (type: PermissionType, options: UseMediaPermis
return false;
}
return nextState.status === 'granted' && hasPrimaryDeviceForType(nextState.devices, type);
} catch {
} catch (error) {
logger.error('Media permission request failed', {type, error});
return false;
}
}, [type, isExplicitlyDenied, applyVoiceDeviceState, unlockDevices]);
@@ -136,11 +141,16 @@ export const useMediaPermission = (type: PermissionType, options: UseMediaPermis
setState((prev) => ({...prev, status: 'denied'}));
return;
}
if (!autoRequest && !osPermissionLooksGranted) {
if (!autoRequest) {
void VoiceDevicePermissionState.ensureDevices({requestPermissions: false})
.then(applyVoiceDeviceState)
.catch((error) => logger.warn('Passive media device enumeration failed', {type, error}));
return;
}
void unlockDevicesRef.current().catch(() => {});
}, [isExplicitlyDenied, type, autoRequest, osPermissionLooksGranted]);
void unlockDevicesRef
.current()
.catch((error) => logger.error('Automatic media permission request failed', {type, error}));
}, [applyVoiceDeviceState, isExplicitlyDenied, type, autoRequest]);
return {
...state,
isExplicitlyDenied,
@@ -2029,7 +2029,7 @@ class MediaEngineFacade extends Store {
syncLocalVoiceStateWithServer(partial?: VoiceStateSyncPartial): void {
const devicePermission = VoiceDevicePermissionState.getState().permissionStatus;
const micGranted = MediaPermission.isMicrophoneGranted() || devicePermission === 'granted';
const micGranted = MediaPermission.isMicrophoneGranted() || devicePermission.audio === 'granted';
if (!micGranted || LocalVoiceState.getMutedByPermission()) {
LocalVoiceState.ensurePermissionMute();
}
@@ -15,7 +15,7 @@ type DeviceListener = (state: VoiceDeviceState) => void;
class VoiceDevicePermissionState {
deviceState: VoiceDeviceState = voiceDeviceManager.getState();
private deviceListeners = new Set<DeviceListener>();
private permissionRequestInFlight: Promise<boolean> | null = null;
private permissionRequestsInFlight = new Map<'audio' | 'video', Promise<boolean>>();
constructor() {
voiceDeviceManager.subscribe((state) => this.handleDeviceStateChange(state));
@@ -57,12 +57,20 @@ class VoiceDevicePermissionState {
}
async requestPermissionFor(type: 'audio' | 'video'): Promise<boolean> {
if (this.permissionRequestInFlight) {
return this.permissionRequestInFlight;
const permissionGranted =
type === 'audio' ? MediaPermission.isMicrophoneGranted() : MediaPermission.isCameraGranted();
const hasPrimaryDevice =
type === 'audio'
? this.deviceState.inputDevices.some((device) => device.kind === 'audioinput')
: this.deviceState.videoDevices.some((device) => device.kind === 'videoinput');
if (permissionGranted && hasPrimaryDevice) return true;
const inFlightRequest = this.permissionRequestsInFlight.get(type);
if (inFlightRequest) {
return inFlightRequest;
}
const requestPromise = (async (): Promise<boolean> => {
const state = await this.ensureDevices({requestPermissions: true});
if (state.permissionStatus === 'granted') {
const state = await this.ensureDevices({requestPermissionTypes: [type], forceRefresh: true});
if (state.permissionStatus[type] === 'granted') {
if (type === 'audio') {
MediaPermission.updateMicrophonePermissionGranted({refreshDevices: false});
} else {
@@ -70,7 +78,7 @@ class VoiceDevicePermissionState {
}
return true;
}
if (state.permissionStatus === 'denied') {
if (state.permissionStatus[type] === 'denied') {
if (type === 'audio') {
MediaPermission.markMicrophoneExplicitlyDenied();
} else {
@@ -78,16 +86,16 @@ class VoiceDevicePermissionState {
}
return false;
}
return type === 'audio' ? MediaPermission.isMicrophoneGranted() : MediaPermission.isCameraGranted();
return false;
})()
.catch((error) => {
logger.error('Failed to request media permission', {type, error});
return false;
})
.finally(() => {
this.permissionRequestInFlight = null;
this.permissionRequestsInFlight.delete(type);
});
this.permissionRequestInFlight = requestPromise;
this.permissionRequestsInFlight.set(type, requestPromise);
return requestPromise;
}
}
@@ -2,7 +2,10 @@
import assert from 'node:assert/strict';
import {SoundType} from '@app/features/notification/utils/SoundUtils';
import {handleMediaPermissionBlocked} from '@app/features/permissions/system/commands/MacPermissionsModalCommands';
import MediaPermission from '@app/features/permissions/system/state/MediaPermission';
import * as SoundCommands from '@app/features/ui/commands/SoundCommands';
import {isPermissionDeniedError} from '@app/features/voice/engine/v2/VoiceEngineV2AppAdapterAssertions';
import VoiceEngineV2AppMediaStateAdapter from '@app/features/voice/engine/v2/VoiceEngineV2AppMediaStateAdapter';
import {ensureNativeMediaPermission} from '@app/features/voice/engine/voice_screen_share_manager/NativePermissionGate';
@@ -62,6 +65,7 @@ export async function runCameraTransition(
}
await args.publish();
if (args.enabled) {
MediaPermission.updateCameraPermissionGranted();
VoiceEngineV2AppMediaStateAdapter.applyCameraState(true, {
forceSync: syncSettledState,
sendUpdate: args.sendUpdate,
@@ -71,6 +75,12 @@ export async function runCameraTransition(
SoundCommands.playSound(args.enabled ? SoundType.CameraOn : SoundType.CameraOff);
return 'applied';
} catch (error) {
if (args.enabled && isPermissionDeniedError(error)) {
MediaPermission.markCameraExplicitlyDenied();
if (args.sendUpdate) handleMediaPermissionBlocked('camera');
args.onPermissionDenied?.();
return 'denied';
}
const actual = args.readActualEnabled();
assert.equal(typeof actual, 'boolean', 'actual-state reader must return a boolean');
args.onFailure?.(actual, error);
@@ -367,11 +367,11 @@ export class VoiceEngineV2AppMediaExecutionAdapter extends Store {
);
this.transitionMediaState({type: 'permission.warmup.start'});
const devicePermission = VoiceDevicePermissionState.getState().permissionStatus;
if (MediaPermission.isMicrophoneGranted() || devicePermission === 'granted') {
if (MediaPermission.isMicrophoneGranted() || devicePermission.audio === 'granted') {
this.transitionMediaState({type: 'permission.warmup.granted'});
return true;
}
if (MediaPermission.isMicrophoneExplicitlyDenied() || devicePermission === 'denied') {
if (MediaPermission.isMicrophoneExplicitlyDenied() || devicePermission.audio === 'denied') {
this.transitionMediaState({type: 'permission.warmup.denied'});
this.handleMicrophonePermissionDenied();
return false;
@@ -13,7 +13,7 @@ import {
type LocalVoiceStateSnapshot,
transitionLocalVoiceStateSnapshot,
} from '@app/features/voice/state/LocalVoiceStateMachine';
import type {VoiceDeviceState} from '@app/features/voice/utils/VoiceDeviceManager';
import type {VoiceDeviceState, VoiceMediaPermissionStatus} from '@app/features/voice/utils/VoiceDeviceManager';
import {makeAutoObservable, observable, reaction, runInAction} from 'mobx';
const logger = new Logger('LocalVoiceState');
@@ -61,8 +61,8 @@ class LocalVoiceState implements LocalVoiceConnectionState {
private persistenceHydrationPromise: Promise<void>;
private _disposers: Array<() => void> = [];
private listeners = new Set<() => void>();
private lastDevicePermissionStatus: VoiceDeviceState['permissionStatus'] | null =
VoiceDevicePermissionState.getState().permissionStatus;
private lastDeviceAudioPermissionStatus: VoiceMediaPermissionStatus | null =
VoiceDevicePermissionState.getState().permissionStatus.audio;
private isNotifyingServerOfPermissionMute = false;
private connectionStates = observable.object<Record<string, LocalVoiceConnectionState>>({});
private machineSnapshot: LocalVoiceStateSnapshot;
@@ -367,20 +367,21 @@ class LocalVoiceState implements LocalVoiceConnectionState {
}
private handleDevicePermissionStatus(status: VoiceDeviceState['permissionStatus']): void {
if (status === this.lastDevicePermissionStatus) {
const audioStatus = status.audio;
if (audioStatus === this.lastDeviceAudioPermissionStatus) {
return;
}
this.lastDevicePermissionStatus = status;
if (status === 'granted') {
this.lastDeviceAudioPermissionStatus = audioStatus;
if (audioStatus === 'granted') {
void this.applyPermissionGrant();
} else if (status === 'denied') {
} else if (audioStatus === 'denied') {
this.applyTransientPermissionMute();
}
}
private enforcePermissionMuteIfNeeded(): void {
const devicePermission = VoiceDevicePermissionState.getState().permissionStatus;
const granted = MediaPermission.isMicrophoneGranted() || devicePermission === 'granted';
const granted = MediaPermission.isMicrophoneGranted() || devicePermission.audio === 'granted';
if (granted) {
runInAction(() => {
this.transitionLocalState({type: 'permission.grant', activeConnectionId: this.getActiveConnectionId()});
@@ -0,0 +1,36 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {beforeEach, describe, expect, test, vi} from 'vitest';
const ensureDevices = vi.fn(async () => ({}));
const invalidate = vi.fn();
vi.mock('@app/features/voice/engine/VoiceDevicePermissionState', () => ({
default: {ensureDevices},
}));
vi.mock('@app/features/voice/devices/MediaDeviceCache', () => ({
mediaDeviceCache: {invalidate},
}));
const {refreshMediaDeviceLists, MediaDeviceRefreshType} = await import('@app/features/voice/utils/MediaDeviceRefresh');
beforeEach(() => {
vi.clearAllMocks();
});
describe('refreshMediaDeviceLists', () => {
test('an audio refresh only requests audio permission, never video', async () => {
await refreshMediaDeviceLists({type: MediaDeviceRefreshType.audio});
expect(invalidate).toHaveBeenCalledWith(MediaDeviceRefreshType.audio);
expect(ensureDevices).toHaveBeenCalledTimes(1);
expect(ensureDevices).toHaveBeenCalledWith({requestPermissionTypes: ['audio'], forceRefresh: true});
});
test('a video refresh only requests video permission, never audio', async () => {
await refreshMediaDeviceLists({type: MediaDeviceRefreshType.video});
expect(invalidate).toHaveBeenCalledWith(MediaDeviceRefreshType.video);
expect(ensureDevices).toHaveBeenCalledTimes(1);
expect(ensureDevices).toHaveBeenCalledWith({requestPermissionTypes: ['video'], forceRefresh: true});
});
});
@@ -19,7 +19,10 @@ export async function refreshMediaDeviceLists(options: RefreshMediaDeviceListsOp
const {type} = options;
mediaDeviceCache.invalidate(type);
try {
await VoiceDevicePermissionState.ensureDevices({requestPermissions: true, forceRefresh: true});
await VoiceDevicePermissionState.ensureDevices({
requestPermissionTypes: [type === MediaDeviceRefreshType.audio ? 'audio' : 'video'],
forceRefresh: true,
});
} catch (error) {
logger.error('Failed to refresh media device lists', error);
}
@@ -3,6 +3,7 @@
import MediaPermission from '@app/features/permissions/system/state/MediaPermission';
import {Logger} from '@app/features/platform/utils/AppLogger';
import VoiceDevicePermissionState from '@app/features/voice/engine/VoiceDevicePermissionState';
import type {VoiceMediaPermissionType} from '@app/features/voice/utils/VoiceDeviceManager';
const logger = new Logger('MediaDeviceStartupPreload');
@@ -11,7 +12,9 @@ export function startMediaDeviceStartupPreload(): () => void {
let lastPermissionStateKey: string | null = null;
const preloadDevices = () => {
if (stopped) return;
const requestPermissions = MediaPermission.isMicrophoneGranted() || MediaPermission.isCameraGranted();
const requestPermissionTypes: Array<VoiceMediaPermissionType> = [];
if (MediaPermission.isMicrophoneGranted()) requestPermissionTypes.push('audio');
if (MediaPermission.isCameraGranted()) requestPermissionTypes.push('video');
const permissionStateKey = [
MediaPermission.isInitialized() ? 'initialized' : 'pending',
MediaPermission.getMicrophonePermissionState() ?? 'unknown',
@@ -19,11 +22,14 @@ export function startMediaDeviceStartupPreload(): () => void {
].join(':');
const deviceState = VoiceDevicePermissionState.getState();
const forceRefresh = lastPermissionStateKey !== null && lastPermissionStateKey !== permissionStateKey;
if (!forceRefresh && lastPermissionStateKey === permissionStateKey && deviceState.permissionStatus !== 'idle') {
const requestedPermissionStatesSettled = requestPermissionTypes.every(
(type) => deviceState.permissionStatus[type] !== 'idle',
);
if (!forceRefresh && lastPermissionStateKey === permissionStateKey && requestedPermissionStatesSettled) {
return;
}
lastPermissionStateKey = permissionStateKey;
void VoiceDevicePermissionState.ensureDevices({forceRefresh, requestPermissions}).catch((error) => {
void VoiceDevicePermissionState.ensureDevices({forceRefresh, requestPermissionTypes}).catch((error) => {
logger.debug('Failed to preload media devices', {error});
});
};
@@ -26,10 +26,14 @@ export function hasDeviceLabels(devices: ReadonlyArray<MediaDeviceInfo>): boolea
return devices.some((d) => d.label && d.label.trim().length > 0);
}
type PermissionStatus = 'idle' | 'loading' | 'granted' | 'denied';
export type VoiceMediaPermissionType = 'audio' | 'video';
export type VoiceMediaPermissionStatus = 'idle' | 'loading' | 'granted' | 'denied';
const MAX_ENUMERATION_CHAIN_PASSES = 3;
export interface EnsureVoiceDevicesOptions {
requestPermissions?: boolean;
requestPermissionTypes?: ReadonlyArray<VoiceMediaPermissionType>;
forceRefresh?: boolean;
}
@@ -37,7 +41,7 @@ export interface VoiceDeviceState {
inputDevices: Array<MediaDeviceInfo>;
outputDevices: Array<MediaDeviceInfo>;
videoDevices: Array<MediaDeviceInfo>;
permissionStatus: PermissionStatus;
permissionStatus: Record<VoiceMediaPermissionType, VoiceMediaPermissionStatus>;
}
type Listener = (state: VoiceDeviceState) => void;
@@ -382,13 +386,15 @@ class VoiceDeviceManager {
inputDevices: [],
outputDevices: [],
videoDevices: [],
permissionStatus: 'idle',
permissionStatus: {
audio: 'idle',
video: 'idle',
},
};
private listeners = new Set<Listener>();
private enumeratingPromise: Promise<VoiceDeviceState> | null = null;
private queuedPermissionEnumerationPromise: Promise<VoiceDeviceState> | null = null;
private currentEnumerationRequestsPermissions = false;
private shouldRequestPermissions = false;
private enumerationChainPromise: Promise<VoiceDeviceState> | null = null;
private scheduledEnumerationPermissionTypes = new Set<VoiceMediaPermissionType>();
private enumerationChainPermissionTypes = new Set<VoiceMediaPermissionType>();
private hasEnumeratedDevices = false;
constructor() {
@@ -410,86 +416,116 @@ class VoiceDeviceManager {
}
public async ensureDevices(options: EnsureVoiceDevicesOptions = {}): Promise<VoiceDeviceState> {
const requestPermissions = options.requestPermissions ?? false;
const requestPermissionTypes = this.resolveRequestedPermissionTypes(options);
const requestPermissions = requestPermissionTypes.length > 0;
const forceRefresh = options.forceRefresh ?? false;
logger.debug('ensureDevices called', {
requestPermissions,
requestPermissionTypes,
forceRefresh,
shouldRequestPermissions: this.shouldRequestPermissions,
hasEnumeratingPromise: !!this.enumeratingPromise,
hasEnumeratingPromise: !!this.enumerationChainPromise,
currentState: {
inputDeviceCount: this.state.inputDevices.length,
permissionStatus: this.state.permissionStatus,
},
});
if (requestPermissions) {
this.shouldRequestPermissions = true;
}
const shouldRequest = this.shouldRequestPermissions || requestPermissions;
if (!forceRefresh && !this.enumeratingPromise && this.canUseCachedState(shouldRequest)) {
if (!forceRefresh && !this.enumerationChainPromise && this.canUseCachedState(requestPermissions)) {
logger.debug('Using cached device state');
return this.state;
}
if (this.enumeratingPromise) {
if (shouldRequest && !this.currentEnumerationRequestsPermissions) {
logger.debug('Queueing permissioned enumeration after current enumeration');
if (!this.queuedPermissionEnumerationPromise) {
this.queuedPermissionEnumerationPromise = this.enumeratingPromise
.catch(() => this.state)
.then(() => this.startEnumeration(true))
.finally(() => {
this.queuedPermissionEnumerationPromise = null;
});
}
return this.queuedPermissionEnumerationPromise;
}
if (this.enumerationChainPromise) {
this.scheduleMissingPermissionTypes(requestPermissionTypes);
logger.debug('Joining existing enumeration promise');
return this.enumeratingPromise;
return this.enumerationChainPromise;
}
logger.debug('Creating new enumeration promise');
return this.startEnumeration(shouldRequest);
return this.startEnumerationChain(requestPermissionTypes);
}
private scheduleMissingPermissionTypes(requestPermissionTypes: ReadonlyArray<VoiceMediaPermissionType>): void {
for (const type of requestPermissionTypes) {
if (this.enumerationChainPermissionTypes.has(type)) continue;
this.enumerationChainPermissionTypes.add(type);
this.scheduledEnumerationPermissionTypes.add(type);
}
}
private resolveRequestedPermissionTypes(options: EnsureVoiceDevicesOptions): Array<VoiceMediaPermissionType> {
if (options.requestPermissionTypes) {
return [...new Set(options.requestPermissionTypes)];
}
return options.requestPermissions === true ? ['audio', 'video'] : [];
}
private canUseCachedState(requestPermissions: boolean): boolean {
if (!this.hasEnumeratedDevices) {
return false;
}
if (!requestPermissions) {
return true;
return !requestPermissions;
}
private updatePermissionStatusForTypes(
requestPermissionTypes: ReadonlyArray<VoiceMediaPermissionType>,
status: VoiceMediaPermissionStatus,
): void {
const permissionStatus = {...this.state.permissionStatus};
for (const type of requestPermissionTypes) {
permissionStatus[type] = status;
}
return this.state.permissionStatus === 'granted';
this.updateState({permissionStatus});
}
private startEnumeration(requestPermissions: boolean): Promise<VoiceDeviceState> {
this.currentEnumerationRequestsPermissions = requestPermissions;
const pendingPromise = this.enumerateDevices(requestPermissions).catch((error) => {
logger.debug('Failed to enumerate media devices:', error);
throw error;
});
this.enumeratingPromise = pendingPromise;
return pendingPromise.finally(() => {
if (this.enumeratingPromise === pendingPromise) {
logger.debug('Enumeration promise completed');
this.enumeratingPromise = null;
this.currentEnumerationRequestsPermissions = false;
}
private startEnumerationChain(
requestPermissionTypes: ReadonlyArray<VoiceMediaPermissionType>,
): Promise<VoiceDeviceState> {
this.enumerationChainPermissionTypes = new Set(requestPermissionTypes);
const pendingPromise = this.runEnumerationChain(requestPermissionTypes).finally(() => {
if (this.enumerationChainPromise !== pendingPromise) return;
logger.debug('Enumeration promise completed');
this.enumerationChainPromise = null;
this.scheduledEnumerationPermissionTypes.clear();
this.enumerationChainPermissionTypes.clear();
});
this.enumerationChainPromise = pendingPromise;
return pendingPromise;
}
private async enumerateDevices(requestPermissions: boolean): Promise<VoiceDeviceState> {
logger.debug('enumerateDevices started', {requestPermissions});
private async runEnumerationChain(
initialPermissionTypes: ReadonlyArray<VoiceMediaPermissionType>,
): Promise<VoiceDeviceState> {
let requestPermissionTypes = [...initialPermissionTypes];
let state = this.state;
for (let pass = 0; pass < MAX_ENUMERATION_CHAIN_PASSES; pass += 1) {
state = await this.enumerateDevices(requestPermissionTypes);
if (this.scheduledEnumerationPermissionTypes.size === 0) return state;
requestPermissionTypes = [...this.scheduledEnumerationPermissionTypes];
this.scheduledEnumerationPermissionTypes.clear();
}
if (this.scheduledEnumerationPermissionTypes.size > 0) {
throw new Error(`Voice device enumeration exceeded ${MAX_ENUMERATION_CHAIN_PASSES} bounded passes`);
}
return state;
}
private async enumerateDevices(
requestPermissionTypes: ReadonlyArray<VoiceMediaPermissionType>,
): Promise<VoiceDeviceState> {
const requestPermissions = requestPermissionTypes.length > 0;
logger.debug('enumerateDevices started', {requestPermissionTypes});
if (!navigator.mediaDevices?.enumerateDevices) {
logger.debug('Navigator or mediaDevices API not available');
return this.state;
}
if (requestPermissions && this.state.permissionStatus !== 'granted') {
const permissionTypesToLoad = requestPermissionTypes.filter(
(type) => this.state.permissionStatus[type] !== 'granted',
);
if (permissionTypesToLoad.length > 0) {
logger.debug('Setting permission status to loading');
this.updateState({permissionStatus: 'loading'});
this.updatePermissionStatusForTypes(permissionTypesToLoad, 'loading');
}
const permissionStatus = {...this.state.permissionStatus};
try {
logger.debug('Calling navigator.mediaDevices.enumerateDevices()');
let devices = await navigator.mediaDevices.enumerateDevices();
let permissionStatus = this.state.permissionStatus;
logger.debug('Initial enumeration result', {
deviceCount: devices.length,
devices: devices.map((d) => ({
@@ -499,61 +535,99 @@ class VoiceDeviceManager {
hasLabel: !!d.label,
})),
});
const hasLabels = devices.some((device) => device.label && device.label !== '');
let usedNativeFlow = false;
if (hasLabels) {
const permissionTypesWithLabels = requestPermissionTypes.filter((type) => {
const requiredKind = type === 'audio' ? 'audioinput' : 'videoinput';
return devices.some((device) => device.kind === requiredKind && device.label !== '');
});
for (const type of permissionTypesWithLabels) {
permissionStatus[type] = 'granted';
}
if (requestPermissions && permissionTypesWithLabels.length === requestPermissionTypes.length) {
logger.debug('Devices have labels, permissions already granted');
permissionStatus = 'granted';
} else if (requestPermissions && isDesktop()) {
logger.debug('No labels detected; attempting native permission flow');
const [nativeMic, nativeCamera] = await Promise.all([
ensureNativePermission('microphone'),
ensureNativePermission('camera'),
]);
usedNativeFlow = nativeMic !== 'unsupported' || nativeCamera !== 'unsupported';
if (nativeMic === 'denied' || nativeCamera === 'denied') {
permissionStatus = 'denied';
} else if (nativeMic === 'granted' || nativeCamera === 'granted') {
permissionStatus = 'granted';
const unresolvedNativeTypes = requestPermissionTypes.filter((type) => permissionStatus[type] !== 'granted');
const nativeResults = await Promise.all(
unresolvedNativeTypes.map(async (type) => {
try {
return {
type,
result: await ensureNativePermission(type === 'audio' ? 'microphone' : 'camera'),
};
} catch (error) {
logger.warn('Native media permission request failed', {type, error});
return {type, result: 'not-determined' as const};
}
}),
);
for (const {type, result} of nativeResults) {
if (result === 'granted') permissionStatus[type] = 'granted';
if (result === 'denied') permissionStatus[type] = 'denied';
}
if (nativeResults.some(({result}) => result === 'granted')) {
try {
devices = await navigator.mediaDevices.enumerateDevices();
} catch (error) {
logger.warn('Device re-enumeration after native permission grant failed', {error});
}
}
}
if (!hasLabels && requestPermissions && (!usedNativeFlow || permissionStatus !== 'granted')) {
const isIOSPWA = Platform.isIOSWeb && Platform.isPWA;
const browserPermissionTypes = requestPermissionTypes.filter(
(type) => permissionStatus[type] !== 'granted' && permissionStatus[type] !== 'denied',
);
if (browserPermissionTypes.length > 0) {
const isIOSPWA =
browserPermissionTypes.length === 1 &&
browserPermissionTypes[0] === 'audio' &&
Platform.isIOSWeb &&
Platform.isPWA;
let skipGetUserMedia = false;
if (isIOSPWA && navigator.permissions) {
try {
const micPermission = await navigator.permissions.query({name: 'microphone' as PermissionName});
if (micPermission.state === 'granted') {
logger.debug('iOS PWA: microphone permission already granted via Permissions API, skipping getUserMedia');
permissionStatus = 'granted';
devices = await navigator.mediaDevices.enumerateDevices();
skipGetUserMedia = devices.some((d) => d.label && d.label !== '');
permissionStatus.audio = 'granted';
try {
devices = await navigator.mediaDevices.enumerateDevices();
skipGetUserMedia = devices.some((device) => device.kind === 'audioinput' && device.label !== '');
} catch (error) {
logger.warn('iOS PWA device re-enumeration after permission grant failed', {error});
}
}
} catch {}
} catch (error) {
logger.debug('iOS PWA microphone permission query failed', {error});
}
}
if (!skipGetUserMedia) {
logger.debug('No labels found, requesting permissions via getUserMedia');
try {
const stream = await navigator.mediaDevices.getUserMedia({
audio: true,
video: true,
audio: browserPermissionTypes.includes('audio'),
video: browserPermissionTypes.includes('video'),
});
logger.debug('getUserMedia succeeded, stopping tracks');
stream.getTracks().forEach((track) => {
logger.debug('Stopping track', {kind: track.kind, label: track.label});
track.stop();
});
permissionStatus = 'granted';
for (const type of browserPermissionTypes) {
permissionStatus[type] = 'granted';
}
logger.debug('Re-enumerating devices after permission grant');
devices = await navigator.mediaDevices.enumerateDevices();
logger.debug('Re-enumeration result', {
deviceCount: devices.length,
devices: devices.map((d) => ({
kind: d.kind,
hasDeviceId: d.deviceId.trim().length > 0,
label: d.label,
})),
});
try {
devices = await navigator.mediaDevices.enumerateDevices();
logger.debug('Re-enumeration result', {
deviceCount: devices.length,
devices: devices.map((d) => ({
kind: d.kind,
hasDeviceId: d.deviceId.trim().length > 0,
label: d.label,
})),
});
} catch (error) {
logger.warn('Device re-enumeration after browser permission grant failed', {error});
}
} catch (error) {
logger.debug('getUserMedia failed', {
error,
@@ -564,9 +638,13 @@ class VoiceDeviceManager {
error instanceof DOMException &&
(error.name === 'NotAllowedError' || error.name === 'PermissionDeniedError')
) {
permissionStatus = 'denied';
for (const type of browserPermissionTypes) {
permissionStatus[type] = 'denied';
}
} else {
permissionStatus = 'granted';
for (const type of browserPermissionTypes) {
permissionStatus[type] = 'idle';
}
}
}
}
@@ -582,7 +660,7 @@ class VoiceDeviceManager {
inputDevices,
outputDevices,
videoDevices,
permissionStatus: this.resolvePermissionStatus(requestPermissions, permissionStatus),
permissionStatus,
};
this.hasEnumeratedDevices = true;
logger.debug('Final device state', {
@@ -593,27 +671,16 @@ class VoiceDeviceManager {
});
this.updateState(nextState);
return this.state;
} catch (_error) {
logger.debug('enumerateDevices failed with exception', _error);
if (requestPermissions) {
this.updateState({permissionStatus: 'denied'});
} catch (error) {
logger.warn('Voice device enumeration failed', {error, requestPermissionTypes});
for (const type of requestPermissionTypes) {
if (permissionStatus[type] === 'loading') permissionStatus[type] = 'idle';
}
this.updateState({permissionStatus});
return this.state;
}
}
private resolvePermissionStatus(requestPermissions: boolean, computedStatus: PermissionStatus): PermissionStatus {
if (!requestPermissions) {
if (this.state.permissionStatus === 'denied') {
return 'denied';
}
if (this.state.permissionStatus === 'granted') {
return 'granted';
}
}
return computedStatus;
}
private updateState(partial: Partial<VoiceDeviceState>) {
this.state = {
...this.state,
@@ -624,7 +691,9 @@ class VoiceDeviceManager {
private handleDeviceChange = () => {
this.hasEnumeratedDevices = false;
void this.ensureDevices({requestPermissions: this.shouldRequestPermissions});
void this.ensureDevices({requestPermissions: false}).catch((error) => {
logger.warn('Voice device refresh after device change failed', {error});
});
};
}
@@ -0,0 +1,114 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
import {afterEach, beforeEach, describe, expect, test, vi} from 'vitest';
interface FakeMediaDevices {
enumerateDevices: () => Promise<Array<MediaDeviceInfo>>;
getUserMedia: (constraints: MediaStreamConstraints) => Promise<MediaStream>;
addEventListener: () => void;
removeEventListener: () => void;
}
const makeDevice = (kind: MediaDeviceKind, deviceId: string, label: string): MediaDeviceInfo =>
({
deviceId,
groupId: `${deviceId}-group`,
kind,
label,
toJSON: () => ({deviceId, kind, label}),
}) as MediaDeviceInfo;
const makeStream = (kinds: Array<'audio' | 'video'>): MediaStream => {
const tracks = kinds.map((kind) => ({kind, label: `${kind}-track`, stop: vi.fn()}));
return {getTracks: () => tracks} as unknown as MediaStream;
};
const getUserMediaCalls: Array<MediaStreamConstraints> = [];
let labelledDevices = false;
const fakeMediaDevices: FakeMediaDevices = {
enumerateDevices: async () =>
labelledDevices
? [makeDevice('audioinput', 'mic-1', 'Studio Mic'), makeDevice('videoinput', 'cam-1', 'Studio Camera')]
: [makeDevice('audioinput', 'mic-1', ''), makeDevice('videoinput', 'cam-1', '')],
getUserMedia: async (constraints) => {
getUserMediaCalls.push(constraints);
labelledDevices = true;
const kinds: Array<'audio' | 'video'> = [];
if (constraints.audio) kinds.push('audio');
if (constraints.video) kinds.push('video');
return makeStream(kinds);
},
addEventListener: () => {},
removeEventListener: () => {},
};
let originalNavigator: PropertyDescriptor | undefined;
let originalWindow: PropertyDescriptor | undefined;
beforeEach(() => {
getUserMediaCalls.length = 0;
labelledDevices = false;
originalNavigator = Object.getOwnPropertyDescriptor(globalThis, 'navigator');
originalWindow = Object.getOwnPropertyDescriptor(globalThis, 'window');
Object.defineProperty(globalThis, 'navigator', {
configurable: true,
value: {userAgent: 'node', mediaDevices: fakeMediaDevices},
});
Object.defineProperty(globalThis, 'window', {
configurable: true,
value: {matchMedia: () => ({matches: false})},
});
vi.resetModules();
});
afterEach(() => {
if (originalNavigator) {
Object.defineProperty(globalThis, 'navigator', originalNavigator);
} else {
Reflect.deleteProperty(globalThis, 'navigator');
}
if (originalWindow) {
Object.defineProperty(globalThis, 'window', originalWindow);
} else {
Reflect.deleteProperty(globalThis, 'window');
}
});
const loadManager = async () => {
const module = await import('@app/features/voice/utils/VoiceDeviceManager');
return module.voiceDeviceManager;
};
describe('voiceDeviceManager permission types', () => {
test('an audio-only request never asks the browser for video', async () => {
const manager = await loadManager();
const state = await manager.ensureDevices({requestPermissionTypes: ['audio'], forceRefresh: true});
expect(getUserMediaCalls).toEqual([{audio: true, video: false}]);
expect(state.permissionStatus.audio).toBe('granted');
expect(state.permissionStatus.video).toBe('idle');
});
test('a video-only request never asks the browser for audio', async () => {
const manager = await loadManager();
const state = await manager.ensureDevices({requestPermissionTypes: ['video'], forceRefresh: true});
expect(getUserMediaCalls).toEqual([{audio: false, video: true}]);
expect(state.permissionStatus.video).toBe('granted');
expect(state.permissionStatus.audio).toBe('idle');
});
test('the legacy requestPermissions boolean still asks for both types', async () => {
const manager = await loadManager();
const state = await manager.ensureDevices({requestPermissions: true, forceRefresh: true});
expect(getUserMediaCalls).toEqual([{audio: true, video: true}]);
expect(state.permissionStatus).toEqual({audio: 'granted', video: 'granted'});
});
test('a passive enumeration never prompts', async () => {
const manager = await loadManager();
const state = await manager.ensureDevices({requestPermissions: false, forceRefresh: true});
expect(getUserMediaCalls).toEqual([]);
expect(state.permissionStatus).toEqual({audio: 'idle', video: 'idle'});
expect(state.inputDevices.length).toBeGreaterThan(0);
});
});