diff --git a/biome.json b/biome.json index ca4905183..ce27b5b27 100644 --- a/biome.json +++ b/biome.json @@ -20,7 +20,7 @@ "bracketSpacing": false, "bracketSameLine": false }, - "globals": ["React"] + "globals": ["React", "__webpack_base_uri__"] }, "json": { "formatter": { diff --git a/fluxer_app/package.json b/fluxer_app/package.json index cbb5b8be5..b50202ecb 100644 --- a/fluxer_app/package.json +++ b/fluxer_app/package.json @@ -229,6 +229,8 @@ "mobx-persist-store": "catalog:", "mobx-react-lite": "catalog:", "motion": "catalog:", + "onnxruntime-common": "catalog:", + "onnxruntime-web": "catalog:", "qrcode": "catalog:", "react": "catalog:", "react-aria-components": "catalog:", @@ -265,6 +267,7 @@ "@types/react": "catalog:", "@types/react-dom": "catalog:", "@typescript/native-preview": "catalog:", + "@webgpu/types": "catalog:", "autoprefixer": "catalog:", "esbuild": "catalog:", "happy-dom": "20.12.0", diff --git a/fluxer_app/rspack.config.mjs b/fluxer_app/rspack.config.mjs index 09fe4c7f9..9ba8f0e79 100644 --- a/fluxer_app/rspack.config.mjs +++ b/fluxer_app/rspack.config.mjs @@ -377,6 +377,13 @@ export default () => { ...(workerWasmPublicPath ? {publicPath: workerWasmPublicPath} : {}), }, }, + { + test: /\.onnx$/, + type: 'asset/resource', + generator: { + filename: isProduction ? 'assets/[contenthash:16][ext]' : 'assets/[name].[hash][ext]', + }, + }, { test: /\.(png|jpg|jpeg|gif|webp|ico|woff|woff2|ttf|eot|mp3|wav|ogg|mp4|webm)$/, type: 'asset/resource', diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppMediaExecutionAdapter.tsx b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppMediaExecutionAdapter.tsx index 6edbd99e9..6b3b34cbc 100644 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppMediaExecutionAdapter.tsx +++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppMediaExecutionAdapter.tsx @@ -48,6 +48,7 @@ import { import type {VoiceStateSyncPartial} from '@app/features/voice/engine/VoiceStateSyncTypes'; import { enforceLocalMediaPublicationCap, + getLocalCameraPublications, getLocalMicrophonePublications, getPrimaryLocalMicrophonePublication, } from '@app/features/voice/engine/VoiceTrackPublicationUtils'; @@ -97,7 +98,7 @@ import LocalVoiceState from '@app/features/voice/state/LocalVoiceState'; import ParticipantVolume from '@app/features/voice/state/ParticipantVolume'; import VoiceSettings from '@app/features/voice/state/VoiceSettings'; import {buildMicrophonePublishOptions} from '@app/features/voice/utils/AudioPublishOptions'; -import {applyBackgroundProcessor} from '@app/features/voice/utils/VideoBackgroundProcessor'; +import {applyBackgroundProcessor, clearCameraVideoProcessor} from '@app/features/voice/utils/VideoBackgroundProcessor'; import { removeVoiceInputProcessor, syncVoiceInputProcessor, @@ -1133,6 +1134,13 @@ export class VoiceEngineV2AppMediaExecutionAdapter extends Store { assert.ok(participant, 'camera transition requires a local participant'); assertBoolean(enabled, 'publishCameraTransition.enabled'); await this.enforceCameraPublicationCap(participant, enabled ? 'before camera enable' : 'before camera disable'); + if (!enabled) { + this.unbindCameraLifecycle(); + const cameraTrack = getLocalCameraPublications(participant)[0]?.track as LocalVideoTrack | undefined; + if (cameraTrack != null) { + await clearCameraVideoProcessor(cameraTrack); + } + } const videoResolution = getCameraVideoPreset(VoiceSettings.getCameraResolution()); await participant.setCameraEnabled(enabled, {resolution: videoResolution, ...restOptions}); await this.enforceCameraPublicationCap(participant, enabled ? 'after camera enable' : 'after camera disable'); @@ -1228,6 +1236,8 @@ export class VoiceEngineV2AppMediaExecutionAdapter extends Store { if (!cameraTrack) { return; } + this.unbindCameraLifecycle(); + await clearCameraVideoProcessor(cameraTrack as LocalVideoTrack); await participant.unpublishTrack(cameraTrack); await this.publishCameraTransition(activeRoom, true, {deviceId: VoiceSettings.getVideoDeviceId()}); updateLocalParticipantFromRoom(activeRoom); diff --git a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppPermissionAdapter.ts b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppPermissionAdapter.ts index c55f0296c..23897cecf 100644 --- a/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppPermissionAdapter.ts +++ b/fluxer_app/src/features/voice/engine/v2/VoiceEngineV2AppPermissionAdapter.ts @@ -15,14 +15,16 @@ import { } from '@app/features/voice/engine/VoicePermissionStateMachine'; import { enforceLocalMediaPublicationCap, + getLocalCameraPublications, getLocalMicrophonePublications, getLocalScreenSharePublications, unpublishLocalMediaPublications, } from '@app/features/voice/engine/VoiceTrackPublicationUtils'; import {asVoiceTrackSource, VoiceTrackSource} from '@app/features/voice/engine/VoiceTrackSource'; +import {clearCameraVideoProcessor} from '@app/features/voice/utils/VideoBackgroundProcessor'; import {removeVoiceInputProcessor} from '@app/features/voice/utils/VoiceInputProcessor'; import {getVoiceChannelPermissions, type VoiceChannelPermissions} from '@app/features/voice/utils/VoicePermissionUtils'; -import type {LocalAudioTrack, Room} from 'livekit-client'; +import type {LocalAudioTrack, LocalVideoTrack, Room} from 'livekit-client'; export { createVoiceEngineV2AppSystemPermissionAdapter, @@ -325,11 +327,16 @@ class VoiceEngineV2AppPermissionAdapter extends Store { } break; } - case 'video': + case 'video': { await enforceLocalMediaPublicationCap(localParticipant, 'camera'); + const cameraTrack = getLocalCameraPublications(localParticipant)[0]?.track as LocalVideoTrack | undefined; + if (cameraTrack != null) { + await clearCameraVideoProcessor(cameraTrack); + } await localParticipant.setCameraEnabled(false); await enforceLocalMediaPublicationCap(localParticipant, 'camera'); break; + } case 'screenShare': await unpublishLocalMediaPublications(localParticipant, getLocalScreenSharePublications(localParticipant), { stopOnUnpublish: true, diff --git a/fluxer_app/src/features/voice/utils/CameraVideoProcessor.ts b/fluxer_app/src/features/voice/utils/CameraVideoProcessor.ts index 590d704b2..5882fcebb 100644 --- a/fluxer_app/src/features/voice/utils/CameraVideoProcessor.ts +++ b/fluxer_app/src/features/voice/utils/CameraVideoProcessor.ts @@ -1,6 +1,11 @@ // SPDX-License-Identifier: AGPL-3.0-or-later import {Logger} from '@app/features/platform/utils/AppLogger'; +import { + WebCameraEffectPipeline, + type WebCameraPipeline, +} from '@app/features/voice/utils/camera-effects/WebCameraEffectPipeline'; +import type {WebCameraEffectConfig} from '@app/features/voice/utils/camera-effects/WebCameraEffectProtocol'; import type {Track, TrackProcessor, VideoProcessorOptions} from 'livekit-client'; const logger = new Logger('CameraVideoProcessor'); @@ -32,6 +37,13 @@ interface MediaStreamTrackProcessorGlobals { export interface CameraVideoProcessorOptions { mirror?: boolean; + background?: WebCameraEffectConfig | null; +} + +export interface CameraVideoProcessorHandle extends VideoTrackProcessor { + canUpdate(options: CameraVideoProcessorOptions): boolean; + update(options: CameraVideoProcessorOptions): Promise; + setOperationalFailureHandler(handler: (error: Error) => void): void; } function isTrackProcessorConstructor(value: unknown): value is MediaStreamTrackProcessorConstructor { @@ -370,41 +382,173 @@ class MirrorVideoProcessor implements VideoTrackProcessor { }; } -class CameraVideoProcessor implements VideoTrackProcessor { +class CameraVideoProcessor implements CameraVideoProcessorHandle { name = 'camera-video-processor'; processedTrack?: MediaStreamTrack; private mirrorProcessor: MirrorVideoProcessor | null = null; + private backgroundPipeline: WebCameraPipeline | null = null; + private currentOptions: CameraVideoProcessorOptions; + private operationTail: Promise = Promise.resolve(); + private destroyPromise: Promise | null = null; + private operationalFailureHandler: ((error: Error) => void) | null = null; + private initialized = false; + private destroyed = false; - constructor(private readonly options: CameraVideoProcessorOptions) {} + constructor(options: CameraVideoProcessorOptions) { + this.currentOptions = options; + } async init(opts: VideoProcessorOptions): Promise { - await this.setup(opts); + await this.runExclusive(async () => { + if (this.initialized || this.destroyed) { + throw new Error('Camera video processor cannot be initialized in its current lifecycle state'); + } + await this.setup(opts); + this.initialized = true; + }); } async restart(opts: VideoProcessorOptions): Promise { - await this.destroy(); - await this.setup(opts); + await this.runExclusive(async () => { + if (this.destroyed) { + throw new Error('Cannot restart a destroyed camera video processor'); + } + await this.teardown(); + this.initialized = false; + await this.setup(opts); + this.initialized = true; + }); } - async destroy(): Promise { + destroy(): Promise { + if (this.destroyPromise == null) { + this.destroyPromise = this.runExclusive(async () => { + if (this.destroyed) { + return; + } + await this.teardown(); + this.initialized = false; + this.destroyed = true; + }); + } + return this.destroyPromise; + } + + canUpdate(options: CameraVideoProcessorOptions): boolean { + const currentMirror = this.currentOptions.mirror === true; + const nextMirror = options.mirror === true; + if (currentMirror !== nextMirror) { + return false; + } + const currentHasBackground = this.currentOptions.background != null; + const nextHasBackground = options.background != null; + return currentHasBackground === nextHasBackground; + } + + setOperationalFailureHandler(handler: (error: Error) => void): void { + if (this.operationalFailureHandler != null) { + throw new Error('Camera video processor operational failure handler was already assigned'); + } + this.operationalFailureHandler = handler; + } + + async update(options: CameraVideoProcessorOptions): Promise { + await this.runExclusive(async () => { + if (!this.initialized || this.destroyed) { + throw new Error('Cannot update an inactive camera video processor'); + } + if (!this.canUpdate(options)) { + throw new Error('Camera video processor update would change its output topology'); + } + const background = options.background ?? null; + if (background != null) { + const backgroundPipeline = this.backgroundPipeline; + if (backgroundPipeline == null) { + throw new Error('Camera video processor has no active background pipeline to update'); + } + await backgroundPipeline.updateConfig({background}); + } + this.currentOptions = options; + }); + } + + private runExclusive(operation: () => Promise): Promise { + const result = this.operationTail.then(operation); + this.operationTail = result.then( + () => undefined, + () => undefined, + ); + return result; + } + + private async teardown(): Promise { + const backgroundPipeline = this.backgroundPipeline; + if (backgroundPipeline) { + try { + backgroundPipeline.beginStop(); + } catch (error) { + logger.warn('Failed to signal camera background pipeline stop intent', {error}); + } + } const mirrorProcessor = this.mirrorProcessor; + this.backgroundPipeline = null; this.mirrorProcessor = null; this.processedTrack = undefined; + if (backgroundPipeline) { + try { + backgroundPipeline.stop(); + } catch (error) { + logger.warn('Failed to stop camera background pipeline', {error}); + } + } await mirrorProcessor?.destroy(); } private async setup(opts: VideoProcessorOptions): Promise { - let inputTrack = opts.track; - if (this.options.mirror) { - const mirrorProcessor = new MirrorVideoProcessor(); - this.mirrorProcessor = mirrorProcessor; - await mirrorProcessor.init({...opts, track: inputTrack}); - inputTrack = mirrorProcessor.processedTrack ?? inputTrack; + const background = this.currentOptions.background ?? null; + try { + let inputTrack = opts.track; + if (this.currentOptions.mirror) { + const mirrorProcessor = new MirrorVideoProcessor(); + this.mirrorProcessor = mirrorProcessor; + await mirrorProcessor.init({...opts, track: inputTrack}); + inputTrack = mirrorProcessor.processedTrack ?? inputTrack; + } + if (background) { + const backgroundPipeline = await WebCameraEffectPipeline.create({ + source: inputTrack, + config: {background}, + onFailure: (_pipeline, error) => { + this.handleBackgroundPipelineFailure(error); + }, + }); + this.backgroundPipeline = backgroundPipeline; + this.processedTrack = backgroundPipeline.outputTrack; + return; + } + this.processedTrack = inputTrack === opts.track ? undefined : inputTrack; + } catch (error) { + await this.teardown(); + throw error; } - this.processedTrack = inputTrack === opts.track ? undefined : inputTrack; + } + + private handleBackgroundPipelineFailure(error: unknown): void { + const normalizedError = + error instanceof Error ? error : new Error('Camera background pipeline failed', {cause: error}); + const handler = this.operationalFailureHandler; + if (handler == null) { + logger.error('Camera background pipeline failed without a recovery owner', {error: normalizedError}); + return; + } + handler(normalizedError); } } -export function createCameraVideoProcessor(options: CameraVideoProcessorOptions): VideoTrackProcessor { +export function createCameraVideoProcessor(options: CameraVideoProcessorOptions): CameraVideoProcessorHandle { return new CameraVideoProcessor(options); } + +export function isCameraVideoProcessor(processor: unknown): processor is CameraVideoProcessorHandle { + return processor instanceof CameraVideoProcessor; +} diff --git a/fluxer_app/src/features/voice/utils/VideoBackgroundProcessor.ts b/fluxer_app/src/features/voice/utils/VideoBackgroundProcessor.ts index c1b4fa4ef..6a467adfb 100644 --- a/fluxer_app/src/features/voice/utils/VideoBackgroundProcessor.ts +++ b/fluxer_app/src/features/voice/utils/VideoBackgroundProcessor.ts @@ -1,21 +1,28 @@ // SPDX-License-Identifier: AGPL-3.0-or-later import {Logger} from '@app/features/platform/utils/AppLogger'; -import VoiceSettings from '@app/features/voice/state/VoiceSettings'; +import {getBackgroundMediaObjectURL} from '@app/features/theme/utils/BackgroundImageDB'; +import VoiceSettings, { + type BackgroundImage, + BLUR_BACKGROUND_ID, + NONE_BACKGROUND_ID, +} from '@app/features/voice/state/VoiceSettings'; import { + type CameraVideoProcessorHandle, type CameraVideoProcessorOptions, createCameraVideoProcessor, + isCameraVideoProcessor, } from '@app/features/voice/utils/CameraVideoProcessor'; +import {CameraBackgroundMode} from '@app/features/voice/utils/camera-effects/CameraCaptureContract'; +import type {WebCameraEffectConfig} from '@app/features/voice/utils/camera-effects/WebCameraEffectProtocol'; +import {areVoiceBackgroundsAvailable} from '@app/features/voice/utils/VoiceBackgroundAvailability'; import type {LocalVideoTrack} from 'livekit-client'; const logger = new Logger('VideoBackgroundProcessor'); export interface BackgroundProcessorOptions { backgroundImageId?: string; - backgroundImages?: Array<{ - id: string; - createdAt: number; - }>; + backgroundImages?: Array; mirrorCamera?: boolean; } @@ -23,24 +30,120 @@ export interface AppliedBackgroundProcessor { destroy: () => Promise; } -async function clearBackgroundProcessor(track: LocalVideoTrack): Promise { +export async function clearCameraVideoProcessor(track: LocalVideoTrack): Promise { if (!track.getProcessor()) { return; } await track.stopProcessor(false); + if (track.getProcessor()) { + throw new Error('Camera video processor remained attached after it was cleared'); + } logger.info('Cleared background processor'); } +async function recoverRawCameraAfterProcessorFailure( + track: LocalVideoTrack, + processor: CameraVideoProcessorHandle, + error: Error, +): Promise { + try { + const recovered = await track.stopProcessorIfCurrent(processor, false); + if (recovered) { + logger.warn('Camera video processor failed; restored the raw camera track', {error}); + } + } catch (recoveryError) { + if (track.mediaStreamTrack.readyState === 'ended') { + logger.warn('Camera video processor failed and the camera track was already gone during recovery', { + error, + recoveryError, + }); + return; + } + logger.error('Camera video processor failed and raw camera recovery was incomplete', { + error, + recoveryError, + }); + } +} + async function applyCameraVideoProcessor( track: LocalVideoTrack, options: CameraVideoProcessorOptions, logLabel: string, -): Promise { - await clearBackgroundProcessor(track); +): Promise { + const updatedProcessor = await track.runWithTrackChangeLock(async () => { + const activeProcessor = track.getProcessor(); + if (!isCameraVideoProcessor(activeProcessor) || !activeProcessor.canUpdate(options)) { + return null; + } + await activeProcessor.update(options); + if (track.getProcessor() !== activeProcessor) { + throw new Error('Updated camera video processor is no longer attached to the active track'); + } + if (!activeProcessor.processedTrack || activeProcessor.processedTrack.readyState !== 'live') { + throw new Error('Updated camera video processor has no live output track'); + } + return activeProcessor; + }); + if (updatedProcessor != null) { + logger.info(logLabel); + return updatedProcessor; + } const processor = createCameraVideoProcessor(options); - await track.setProcessor(processor); - logger.info(logLabel); - return processor; + processor.setOperationalFailureHandler((error) => { + void recoverRawCameraAfterProcessorFailure(track, processor, error); + }); + try { + await track.setProcessor(processor); + if (track.getProcessor() !== processor) { + throw new Error('Camera video processor was not retained by the active track'); + } + if (!processor.processedTrack || processor.processedTrack.readyState !== 'live') { + throw new Error('Camera video processor produced no live output track'); + } + logger.info(logLabel); + return processor; + } catch (error) { + try { + if (track.getProcessor() === processor) { + await track.stopProcessor(false); + } + } catch (cleanupError) { + throw new AggregateError([error, cleanupError], 'Camera video processor apply and cleanup both failed'); + } + throw error; + } +} + +async function resolveBackgroundEffectConfig( + options?: BackgroundProcessorOptions, +): Promise { + const backgroundImageId = options?.backgroundImageId ?? VoiceSettings.getBackgroundImageId(); + if (backgroundImageId === NONE_BACKGROUND_ID) { + return null; + } + if (!areVoiceBackgroundsAvailable()) { + throw new Error('Camera background effects are unavailable'); + } + const blurStrength = VoiceSettings.getBackgroundBlurStrength(); + if (backgroundImageId === BLUR_BACKGROUND_ID) { + return {mode: CameraBackgroundMode.BLUR, blurStrength}; + } + const backgroundImages = options?.backgroundImages ?? VoiceSettings.getBackgroundImages(); + const hasImage = backgroundImages.some((image) => image.id === backgroundImageId); + if (!hasImage) { + throw new Error(`Custom camera background is not present in the saved media list: ${backgroundImageId}`); + } + const customMedia = await getBackgroundMediaObjectURL(backgroundImageId); + if (!customMedia) { + throw new Error(`Custom camera background media could not be resolved: ${backgroundImageId}`); + } + return { + mode: CameraBackgroundMode.CUSTOM, + blurStrength, + customMediaURL: customMedia.url, + customMediaKind: customMedia.mediaKind, + }; } export async function applyCameraMirrorProcessor( @@ -49,14 +152,14 @@ export async function applyCameraMirrorProcessor( ) { try { if (!mirrorCamera) { - await clearBackgroundProcessor(track); + await clearCameraVideoProcessor(track); logger.debug('No camera mirror processor applied'); return null; } return applyCameraVideoProcessor(track, {mirror: true}, 'Applied camera mirror'); } catch (error) { logger.warn('Failed to apply camera mirror processor', error); - return null; + throw error; } } @@ -65,16 +168,17 @@ export async function applyBackgroundProcessor( options?: BackgroundProcessorOptions, ): Promise { try { - const voiceSettings = VoiceSettings; - const mirrorCamera = options?.mirrorCamera ?? voiceSettings.getMirrorCamera(); - if (!mirrorCamera) { - await clearBackgroundProcessor(track); + const mirrorCamera = options?.mirrorCamera ?? VoiceSettings.getMirrorCamera(); + const background = await resolveBackgroundEffectConfig(options); + if (!mirrorCamera && background == null) { + await clearCameraVideoProcessor(track); logger.debug('No camera video processor applied'); return null; } - return applyCameraVideoProcessor(track, {mirror: true}, 'Applied camera mirror'); + const logLabel = background == null ? 'Applied camera mirror' : `Applied camera background (${background.mode})`; + return applyCameraVideoProcessor(track, {mirror: mirrorCamera, background}, logLabel); } catch (error) { logger.warn('Failed to apply camera video processor', error); - return null; + throw error; } } diff --git a/fluxer_app/src/features/voice/utils/camera-effects/AggregateOperations.ts b/fluxer_app/src/features/voice/utils/camera-effects/AggregateOperations.ts new file mode 100644 index 000000000..21f02adbf --- /dev/null +++ b/fluxer_app/src/features/voice/utils/camera-effects/AggregateOperations.ts @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +export interface ThrowCollectedFailuresRequest { + readonly failures: ReadonlyArray; + readonly message: string; +} + +class MissingFailureReasonError extends Error { + constructor() { + super('Operation failed without an error reason'); + this.name = 'MissingFailureReasonError'; + } +} + +function normalizeFailureReason(failure: unknown): unknown { + if (failure == null) { + return new MissingFailureReasonError(); + } + return failure; +} + +function flattenFailures(failures: ReadonlyArray): Array { + const flattened: Array = []; + for (const failure of failures) { + const normalized = normalizeFailureReason(failure); + if (normalized instanceof AggregateError && Array.isArray(normalized.errors)) { + for (const nested of normalized.errors) { + flattened.push(normalizeFailureReason(nested)); + } + continue; + } + flattened.push(normalized); + } + return flattened; +} + +export async function collectSettledFailures(operations: ReadonlyArray>): Promise> { + const outcomes = await Promise.allSettled(operations); + const failures: Array = []; + for (const outcome of outcomes) { + if (outcome.status === 'rejected') { + failures.push(normalizeFailureReason(outcome.reason)); + } + } + return flattenFailures(failures); +} + +export function throwCollectedFailures( + request: ThrowCollectedFailuresRequest & {readonly failures: readonly [unknown, ...Array]}, +): never; +export function throwCollectedFailures(request: ThrowCollectedFailuresRequest): void; +export function throwCollectedFailures(request: ThrowCollectedFailuresRequest): void { + const flattened = flattenFailures(request.failures); + if (flattened.length === 0) { + return; + } + if (flattened.length === 1) { + const failure = flattened[0]; + if (failure == null) { + throw new MissingFailureReasonError(); + } + throw failure; + } + throw new AggregateError(flattened, request.message); +} diff --git a/fluxer_app/src/features/voice/utils/camera-effects/BoundedResponse.ts b/fluxer_app/src/features/voice/utils/camera-effects/BoundedResponse.ts new file mode 100644 index 000000000..1bf8f543f --- /dev/null +++ b/fluxer_app/src/features/voice/utils/camera-effects/BoundedResponse.ts @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +export interface BoundedResponseBodyOptions { + readonly maximumBytes: number; + readonly maximumChunks: number; + readonly description: string; +} + +export interface ReadBoundedResponseBodyRequest extends BoundedResponseBodyOptions { + readonly response: Response; +} + +export interface ResponseDeadlineOptions { + readonly timeoutMilliseconds: number; + readonly description: string; + readonly signal: AbortSignal | null; + readonly operation: (signal: AbortSignal) => Promise; +} + +export interface CancellableResponseBody { + readonly body?: ReadableStream | null; +} + +export interface CancelResponseBodyAndThrowRequest { + readonly description: string; + readonly error: unknown; + readonly response: CancellableResponseBody; +} + +export const BoundedResponseBodyLimit = Object.freeze({ + BYTES: 'bytes', + CHUNKS: 'chunks', +} as const); + +export type BoundedResponseBodyLimit = (typeof BoundedResponseBodyLimit)[keyof typeof BoundedResponseBodyLimit]; + +export class BoundedResponseBodyLimitError extends Error { + readonly description: string; + readonly limit: BoundedResponseBodyLimit; + readonly maximum: number; + + constructor(description: string, limit: BoundedResponseBodyLimit, maximum: number) { + super(`${description} exceeds ${maximum.toString()} ${limit}`); + this.name = 'BoundedResponseBodyLimitError'; + this.description = description; + this.limit = limit; + this.maximum = maximum; + } +} + +class ResponseDeadlineExceededError extends Error { + constructor(description: string, timeoutMilliseconds: number) { + super(`${description} timed out after ${timeoutMilliseconds.toString()} ms`); + this.name = 'ResponseDeadlineExceededError'; + } +} + +interface BoundedResponseChunks { + readonly chunks: Array; + readonly totalBytes: number; +} + +function responseChunkForStorage(chunk: Uint8Array): Uint8Array { + const copy = new Uint8Array(chunk.byteLength); + copy.set(chunk); + return copy; +} + +async function readBoundedResponseChunks(request: ReadBoundedResponseBodyRequest): Promise { + if (request.response.body == null) { + return {chunks: [], totalBytes: 0}; + } + const reader = request.response.body.getReader(); + const chunks: Array = []; + let totalBytes = 0; + try { + for (;;) { + const result = await reader.read(); + if (result.done) { + break; + } + if (chunks.length >= request.maximumChunks) { + throw new BoundedResponseBodyLimitError( + request.description, + BoundedResponseBodyLimit.CHUNKS, + request.maximumChunks, + ); + } + totalBytes += result.value.byteLength; + if (!Number.isSafeInteger(totalBytes) || totalBytes > request.maximumBytes) { + throw new BoundedResponseBodyLimitError( + request.description, + BoundedResponseBodyLimit.BYTES, + request.maximumBytes, + ); + } + chunks.push(responseChunkForStorage(result.value)); + } + } catch (error) { + const failures: Array = [error]; + try { + await reader.cancel(error); + } catch (cancelError) { + failures.push(cancelError); + } + if (failures.length > 1) { + throw new AggregateError(failures, `${request.description} read and cancellation failed`); + } + throw error; + } finally { + try { + reader.releaseLock(); + } catch {} + } + return {chunks, totalBytes}; +} + +export async function readBoundedResponseBytes(request: ReadBoundedResponseBodyRequest): Promise { + const {chunks, totalBytes} = await readBoundedResponseChunks(request); + const bytes = new Uint8Array(totalBytes); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + return bytes; +} + +export async function readBoundedResponseArrayBuffer(request: ReadBoundedResponseBodyRequest): Promise { + const bytes = await readBoundedResponseBytes(request); + const buffer = new ArrayBuffer(bytes.byteLength); + new Uint8Array(buffer).set(bytes); + return buffer; +} + +export async function readBoundedResponseBlob(request: ReadBoundedResponseBodyRequest): Promise { + const {chunks} = await readBoundedResponseChunks(request); + const contentType = request.response.headers.get('content-type') ?? ''; + return new Blob(chunks as Array, {type: contentType}); +} + +export async function cancelResponseBodyAndThrow({ + description, + error, + response, +}: CancelResponseBodyAndThrowRequest): Promise { + if (response.body != null) { + try { + await response.body.cancel(error); + } catch (cancelError) { + throw new AggregateError([error, cancelError], `${description} rejection and cancellation failed`); + } + } + throw error; +} + +export async function runWithResponseDeadline(options: ResponseDeadlineOptions): Promise { + const upstreamSignal = options.signal; + if (upstreamSignal?.aborted) { + throw upstreamSignal.reason ?? new Error(`${options.description} was aborted upstream`); + } + const controller = new AbortController(); + const abortFromUpstream = (): void => { + controller.abort(upstreamSignal?.reason ?? new Error(`${options.description} was aborted upstream`)); + }; + if (upstreamSignal != null) { + upstreamSignal.addEventListener('abort', abortFromUpstream, {once: true}); + } + const timer = setTimeout(() => { + controller.abort(new ResponseDeadlineExceededError(options.description, options.timeoutMilliseconds)); + }, options.timeoutMilliseconds); + try { + return await options.operation(controller.signal); + } finally { + clearTimeout(timer); + if (upstreamSignal != null) { + upstreamSignal.removeEventListener('abort', abortFromUpstream); + } + } +} diff --git a/fluxer_app/src/features/voice/utils/camera-effects/CameraCaptureContract.ts b/fluxer_app/src/features/voice/utils/camera-effects/CameraCaptureContract.ts new file mode 100644 index 000000000..6ddb04e76 --- /dev/null +++ b/fluxer_app/src/features/voice/utils/camera-effects/CameraCaptureContract.ts @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +export const CameraBackgroundMode = Object.freeze({ + NONE: 'none', + BLUR: 'blur', + CUSTOM: 'custom', +} as const); + +export type CameraBackgroundMode = (typeof CameraBackgroundMode)[keyof typeof CameraBackgroundMode]; + +export const MAX_VIDEO_FRAME_RATE = 60; + +export function clampVideoFrameRate(frameRate: number): number { + if (!Number.isFinite(frameRate)) { + return MAX_VIDEO_FRAME_RATE; + } + if (frameRate <= 0) { + return MAX_VIDEO_FRAME_RATE; + } + return Math.min(Math.floor(frameRate), MAX_VIDEO_FRAME_RATE); +} diff --git a/fluxer_app/src/features/voice/utils/camera-effects/ErrorDiagnostic.ts b/fluxer_app/src/features/voice/utils/camera-effects/ErrorDiagnostic.ts new file mode 100644 index 000000000..52a9e18d2 --- /dev/null +++ b/fluxer_app/src/features/voice/utils/camera-effects/ErrorDiagnostic.ts @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +export const ErrorDiagnosticType = Object.freeze({ + AGGREGATE_ERROR: 'aggregate-error', + DOM_EXCEPTION: 'dom-exception', + ERROR: 'error', + NON_ERROR: 'non-error', +} as const); + +export type ErrorDiagnosticType = (typeof ErrorDiagnosticType)[keyof typeof ErrorDiagnosticType]; + +export interface ErrorDiagnostic { + readonly errorType: ErrorDiagnosticType; + readonly message: string; + readonly stack: string | null; +} + +const ERROR_DIAGNOSTIC_ENTRY_MAX = 16; +export const ERROR_DIAGNOSTIC_MESSAGE_MAX_LENGTH = 16_384; +export const ERROR_DIAGNOSTIC_STACK_MAX_LENGTH = 65_536; +const ERROR_DIAGNOSTIC_NAME_MAX_LENGTH = 128; +const ERROR_DIAGNOSTIC_MESSAGE_ENTRY_MAX_LENGTH = 896; +const ERROR_DIAGNOSTIC_STACK_ENTRY_MAX_LENGTH = 4_096; + +export function isErrorDiagnosticType(value: unknown): value is ErrorDiagnosticType { + if (value === ErrorDiagnosticType.AGGREGATE_ERROR) { + return true; + } + if (value === ErrorDiagnosticType.DOM_EXCEPTION) { + return true; + } + if (value === ErrorDiagnosticType.ERROR) { + return true; + } + return value === ErrorDiagnosticType.NON_ERROR; +} + +export function isErrorDiagnostic(value: object): value is ErrorDiagnostic { + const errorType = Reflect.get(value, 'errorType'); + if (!isErrorDiagnosticType(errorType)) { + return false; + } + const message = Reflect.get(value, 'message'); + if (typeof message !== 'string') { + return false; + } + if (message.length > ERROR_DIAGNOSTIC_MESSAGE_MAX_LENGTH) { + return false; + } + const stack = Reflect.get(value, 'stack'); + if (stack === null) { + return true; + } + if (typeof stack !== 'string') { + return false; + } + return stack.length <= ERROR_DIAGNOSTIC_STACK_MAX_LENGTH; +} + +export function getErrorDiagnosticType(error: unknown): ErrorDiagnosticType { + if (error instanceof AggregateError) { + return ErrorDiagnosticType.AGGREGATE_ERROR; + } + if (error instanceof DOMException) { + return ErrorDiagnosticType.DOM_EXCEPTION; + } + if (error instanceof Error) { + return ErrorDiagnosticType.ERROR; + } + return ErrorDiagnosticType.NON_ERROR; +} + +function limitDiagnosticText(value: string, maximumLength: number): string { + if (value.length <= maximumLength) { + return value; + } + return value.slice(0, maximumLength); +} + +function describeError(error: unknown): string { + if (error instanceof Error) { + const name = limitDiagnosticText(error.name, ERROR_DIAGNOSTIC_NAME_MAX_LENGTH); + const message = limitDiagnosticText(error.message, ERROR_DIAGNOSTIC_MESSAGE_ENTRY_MAX_LENGTH); + return `${name}: ${message}`; + } + try { + return limitDiagnosticText(String(error), ERROR_DIAGNOSTIC_MESSAGE_ENTRY_MAX_LENGTH); + } catch { + return 'Unprintable non-error failure'; + } +} + +function collectDiagnosticErrors(error: unknown): ReadonlyArray { + const collected: Array = [error]; + let index = 0; + while (index < collected.length) { + if (collected.length >= ERROR_DIAGNOSTIC_ENTRY_MAX) { + break; + } + const current = collected[index]; + index += 1; + if (current instanceof Error) { + const cause = current.cause; + if (cause !== undefined) { + collected.push(cause); + } + } + if (!(current instanceof AggregateError)) { + continue; + } + if (!Array.isArray(current.errors)) { + continue; + } + for (const nested of current.errors) { + if (collected.length >= ERROR_DIAGNOSTIC_ENTRY_MAX) { + break; + } + collected.push(nested); + } + } + return collected; +} + +function hasErrorStack(error: unknown): error is Error { + if (!(error instanceof Error)) { + return false; + } + return typeof error.stack === 'string'; +} + +export function getErrorDiagnostic(error: unknown): ErrorDiagnostic { + const errors = collectDiagnosticErrors(error); + const message = errors + .map((current, index) => (index === 0 ? describeError(current) : `Cause ${index}: ${describeError(current)}`)) + .join('\n'); + const stack = errors + .filter(hasErrorStack) + .map((current, index) => { + const entry = limitDiagnosticText(current.stack ?? '', ERROR_DIAGNOSTIC_STACK_ENTRY_MAX_LENGTH); + return index === 0 ? entry : `Caused by:\n${entry}`; + }) + .join('\n'); + return { + errorType: getErrorDiagnosticType(error), + message: limitDiagnosticText(message, ERROR_DIAGNOSTIC_MESSAGE_MAX_LENGTH), + stack: stack.length === 0 ? null : limitDiagnosticText(stack, ERROR_DIAGNOSTIC_STACK_MAX_LENGTH), + }; +} diff --git a/fluxer_app/src/features/voice/utils/camera-effects/MediaStreamTrackProcessorPolyfill.ts b/fluxer_app/src/features/voice/utils/camera-effects/MediaStreamTrackProcessorPolyfill.ts new file mode 100644 index 000000000..6d8cacb92 --- /dev/null +++ b/fluxer_app/src/features/voice/utils/camera-effects/MediaStreamTrackProcessorPolyfill.ts @@ -0,0 +1,172 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {Logger} from '@app/features/platform/utils/AppLogger'; + +export interface TrackProcessorReadable { + readonly readable: ReadableStream; +} + +type MediaStreamTrackProcessorInstance = { + readonly readable: ReadableStream; +}; +type MediaStreamTrackProcessorConstructor = new (options: { + track: MediaStreamTrack; +}) => MediaStreamTrackProcessorInstance; + +const logger = new Logger('MediaStreamTrackProcessorPolyfill'); + +export function mediaStreamTrackProcessorSupported(): boolean { + return 'MediaStreamTrackProcessor' in globalThis; +} + +export function videoFrameCaptureSupported(): boolean { + if (!('VideoFrame' in globalThis)) { + return false; + } + if (!('HTMLVideoElement' in globalThis)) { + return false; + } + return typeof HTMLVideoElement.prototype.requestVideoFrameCallback === 'function'; +} + +function getMediaStreamTrackProcessorConstructor(): MediaStreamTrackProcessorConstructor | null { + const candidate: unknown = Reflect.get(globalThis, 'MediaStreamTrackProcessor'); + if (typeof candidate !== 'function') { + return null; + } + return candidate as MediaStreamTrackProcessorConstructor; +} + +export function createTrackProcessor(track: MediaStreamTrack): TrackProcessorReadable { + const Processor = getMediaStreamTrackProcessorConstructor(); + if (Processor != null) { + return new Processor({track}) as unknown as TrackProcessorReadable; + } + return {readable: createPolyfilledVideoReadable(track) as ReadableStream}; +} + +function createPolyfilledVideoReadable(track: MediaStreamTrack): ReadableStream { + return new PolyfilledVideoTrackProcessor(track).createReadable(); +} + +class PolyfilledVideoTrackProcessor { + private readonly video: HTMLVideoElement; + private stopped = false; + private callbackHandle: number | null = null; + private streamController: ReadableStreamDefaultController | null = null; + + constructor(private readonly track: MediaStreamTrack) { + this.video = document.createElement('video'); + this.video.muted = true; + this.video.autoplay = true; + this.video.playsInline = true; + this.video.srcObject = new MediaStream([track]); + } + + createReadable(): ReadableStream { + return new ReadableStream({ + start: (controller): void => { + this.start(controller); + }, + cancel: (): void => { + this.stop(); + }, + }); + } + + private readonly handleTrackEnded = (): void => { + try { + this.streamController?.close(); + } catch (error) { + logger.warn('Polyfilled video track processor close failed', {error}); + } + this.stop(); + }; + + private stop(): void { + if (this.stopped) { + return; + } + this.stopped = true; + if (this.callbackHandle != null) { + try { + this.video.cancelVideoFrameCallback(this.callbackHandle); + } catch {} + this.callbackHandle = null; + } + try { + this.track.removeEventListener('ended', this.handleTrackEnded); + } catch {} + try { + this.video.srcObject = null; + } catch {} + } + + private reportFailure(error: unknown): void { + try { + this.streamController?.error(error); + } catch {} + this.stop(); + } + + private start(controller: ReadableStreamDefaultController): void { + this.streamController = controller; + this.track.addEventListener('ended', this.handleTrackEnded, {once: true}); + this.video + .play() + .then(() => { + this.scheduleNextFrame(); + }) + .catch((error) => { + if (this.stopped) { + return; + } + this.reportFailure(error); + }); + } + + private scheduleNextFrame(): void { + if (this.stopped) { + return; + } + this.callbackHandle = this.video.requestVideoFrameCallback(this.handleFrame); + } + + private readonly handleFrame: VideoFrameRequestCallback = (_callbackTimeMs, metadata): void => { + if (this.stopped) { + return; + } + if (this.track.readyState === 'ended') { + this.handleTrackEnded(); + return; + } + const controller = this.streamController; + if (controller == null) { + this.reportFailure(new Error('Polyfilled video capture started without a stream controller')); + return; + } + if (controller.desiredSize != null && controller.desiredSize <= 0) { + this.scheduleNextFrame(); + return; + } + this.captureFrame(controller, metadata); + }; + + private captureFrame( + controller: ReadableStreamDefaultController, + metadata: VideoFrameCallbackMetadata, + ): void { + const timestamp = Math.round(metadata.mediaTime * 1_000_000); + let frame: VideoFrame | null = null; + try { + frame = new VideoFrame(this.video, {timestamp}); + controller.enqueue(frame); + frame = null; + } catch (error) { + frame?.close(); + this.reportFailure(error); + return; + } + this.scheduleNextFrame(); + } +} diff --git a/fluxer_app/src/features/voice/utils/camera-effects/WebCameraBackgroundSupport.ts b/fluxer_app/src/features/voice/utils/camera-effects/WebCameraBackgroundSupport.ts new file mode 100644 index 000000000..290f1ea07 --- /dev/null +++ b/fluxer_app/src/features/voice/utils/camera-effects/WebCameraBackgroundSupport.ts @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import { + mediaStreamTrackProcessorSupported, + videoFrameCaptureSupported, +} from '@app/features/voice/utils/camera-effects/MediaStreamTrackProcessorPolyfill'; + +export interface WebCameraSegmentationRuntimeCapability { + readonly available: boolean; + readonly reason: string; +} + +class WebCameraSegmentationCapabilityOwner { + private capability: WebCameraSegmentationRuntimeCapability | null = null; + + get(): WebCameraSegmentationRuntimeCapability { + if (this.capability == null) { + this.capability = detectWebCameraSegmentationCapability(); + } + return this.capability; + } +} + +const webCameraSegmentationCapabilityOwner = new WebCameraSegmentationCapabilityOwner(); + +function transferableReadableStreamSupported(): boolean { + if (typeof structuredClone !== 'function' || !('ReadableStream' in globalThis)) { + return false; + } + const stream = new ReadableStream({ + start(controller): void { + controller.close(); + }, + }); + try { + const transferred = structuredClone(stream, {transfer: [stream]}); + return transferred instanceof ReadableStream; + } catch { + return false; + } +} + +function offscreenCanvasFallbackSupported(): boolean { + if (!('OffscreenCanvas' in globalThis)) { + return false; + } + try { + return new OffscreenCanvas(1, 1).getContext('2d') != null; + } catch { + return false; + } +} + +export function detectWebCameraSegmentationCapability(): WebCameraSegmentationRuntimeCapability { + if (!('window' in globalThis)) { + return {available: false, reason: 'Camera effects require a window and document'}; + } + if (!('document' in globalThis)) { + return {available: false, reason: 'Camera effects require a window and document'}; + } + if (!('Worker' in globalThis)) { + return {available: false, reason: 'Camera effects require module workers and VideoFrame'}; + } + if (!('VideoFrame' in globalThis)) { + return {available: false, reason: 'Camera effects require module workers and VideoFrame'}; + } + if (!('HTMLCanvasElement' in globalThis)) { + return {available: false, reason: 'Camera effects require transferable captured canvas output'}; + } + if (typeof HTMLCanvasElement.prototype.captureStream !== 'function') { + return {available: false, reason: 'Camera effects require transferable captured canvas output'}; + } + if (typeof HTMLCanvasElement.prototype.transferControlToOffscreen !== 'function') { + return {available: false, reason: 'Camera effects require transferable captured canvas output'}; + } + if (!offscreenCanvasFallbackSupported()) { + return {available: false, reason: 'Camera effects require an OffscreenCanvas 2D fallback'}; + } + if (!mediaStreamTrackProcessorSupported() && !videoFrameCaptureSupported()) { + return {available: false, reason: 'Camera effects require bounded VideoFrame capture'}; + } + if (!transferableReadableStreamSupported()) { + return {available: false, reason: 'Camera effects require transferable readable streams'}; + } + return {available: true, reason: 'Worker camera effect pipeline is available'}; +} + +export function webCameraSegmentationRuntimeAvailable(): boolean { + return webCameraSegmentationCapabilityOwner.get().available; +} diff --git a/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectCanvasRenderer.ts b/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectCanvasRenderer.ts new file mode 100644 index 000000000..9693b3cdd --- /dev/null +++ b/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectCanvasRenderer.ts @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import { + collectSettledFailures, + throwCollectedFailures, +} from '@app/features/voice/utils/camera-effects/AggregateOperations'; +import {CameraBackgroundMode} from '@app/features/voice/utils/camera-effects/CameraCaptureContract'; +import type {WebCameraEffectCustomFrameSource} from '@app/features/voice/utils/camera-effects/WebCameraEffectCustomImage'; +import { + cameraEffectBlurPixels, + validateCameraEffectFrameDimensions, + WEB_CAMERA_EFFECT_SEGMENTATION_MIN_INTERVAL_MS, + WebCameraEffectBackend, + type WebCameraPipelineConfig, +} from '@app/features/voice/utils/camera-effects/WebCameraEffectProtocol'; +import type {WebCameraEffectRenderer} from '@app/features/voice/utils/camera-effects/WebCameraEffectRenderer'; +import {WebCameraEffectWebGLMaskRefiner} from '@app/features/voice/utils/camera-effects/WebCameraEffectWebGLMaskRefiner'; +import {SEG_INPUT_EDGE, WebSelfieSegmenter} from '@app/features/voice/utils/camera-effects/WebSelfieSegmenter'; + +function twoDContext( + canvas: OffscreenCanvas, + options: CanvasRenderingContext2DSettings, +): OffscreenCanvasRenderingContext2D { + const context = canvas.getContext('2d', options); + if (context == null) { + throw new Error('OffscreenCanvas 2D context is unavailable for camera effects'); + } + return context; +} + +function resetContext(context: OffscreenCanvasRenderingContext2D): void { + context.setTransform(1, 0, 0, 1, 0, 0); + context.globalCompositeOperation = 'source-over'; + context.filter = 'none'; + context.globalAlpha = 1; +} + +async function disposeWebSelfieSegmenter(segmenter: WebSelfieSegmenter | null): Promise { + if (segmenter == null) { + return; + } + await segmenter.dispose(); +} + +export class WebCameraEffectCanvasRenderer implements WebCameraEffectRenderer { + private readonly outputCanvas: OffscreenCanvas; + private readonly outputContext: OffscreenCanvasRenderingContext2D; + private readonly segmentationCanvas = new OffscreenCanvas(SEG_INPUT_EDGE, SEG_INPUT_EDGE); + private readonly segmentationContext = twoDContext(this.segmentationCanvas, { + alpha: false, + willReadFrequently: true, + }); + private readonly maskCanvas = new OffscreenCanvas(SEG_INPUT_EDGE, SEG_INPUT_EDGE); + private readonly maskContext = twoDContext(this.maskCanvas, {alpha: true}); + private readonly maskImage = this.maskContext.createImageData(SEG_INPUT_EDGE, SEG_INPUT_EDGE); + private readonly foregroundCanvas = new OffscreenCanvas(1, 1); + private readonly foregroundContext = twoDContext(this.foregroundCanvas, {alpha: true}); + private readonly maskRefiner = WebCameraEffectWebGLMaskRefiner.create(); + private config: WebCameraPipelineConfig = {background: null}; + private segmenter: WebSelfieSegmenter | null = null; + private customFrameSource: WebCameraEffectCustomFrameSource | null = null; + private width = 0; + private height = 0; + private lastSegmentationAt = Number.NEGATIVE_INFINITY; + private maskRevision = 0; + private maskReady = false; + private disposed = false; + + get backend(): WebCameraEffectBackend { + if (this.segmenter == null) { + return WebCameraEffectBackend.CANVAS_WORKER; + } + return WebCameraEffectBackend.WASM_WORKER; + } + + private constructor(canvas: OffscreenCanvas) { + this.outputCanvas = canvas; + this.outputContext = twoDContext(canvas, {alpha: false, desynchronized: true}); + this.outputContext.imageSmoothingEnabled = true; + this.outputContext.imageSmoothingQuality = 'high'; + this.segmentationContext.imageSmoothingEnabled = true; + this.segmentationContext.imageSmoothingQuality = 'medium'; + this.foregroundContext.imageSmoothingEnabled = true; + this.foregroundContext.imageSmoothingQuality = 'high'; + } + + static async create( + canvas: OffscreenCanvas, + config: WebCameraPipelineConfig, + customFrameSource: WebCameraEffectCustomFrameSource | null, + ): Promise { + const renderer = new WebCameraEffectCanvasRenderer(canvas); + try { + await renderer.configure(config, customFrameSource); + await renderer.warmup(); + return renderer; + } catch (error) { + const cleanupFailures = await collectSettledFailures([renderer.dispose()]); + throwCollectedFailures({ + failures: [error, ...cleanupFailures], + message: 'Canvas camera effect initialization failed', + }); + } + } + + async configure( + config: WebCameraPipelineConfig, + customFrameSource: WebCameraEffectCustomFrameSource | null, + ): Promise { + if (this.disposed) { + throw new Error('Cannot configure a disposed camera effect renderer'); + } + const customBackground = config.background?.mode === CameraBackgroundMode.CUSTOM; + if (customBackground !== (customFrameSource != null)) { + throw new Error('Camera effect custom frame source does not match its configuration'); + } + if ( + config.background != null && + config.background.mode === CameraBackgroundMode.BLUR && + typeof this.outputContext.filter !== 'string' + ) { + throw new Error('OffscreenCanvas blur filters are unavailable'); + } + let createdSegmenter: WebSelfieSegmenter | null = null; + try { + if (config.background != null && this.segmenter == null) { + createdSegmenter = await WebSelfieSegmenter.create(); + } + } catch (error) { + const cleanupFailures = await collectSettledFailures([disposeWebSelfieSegmenter(createdSegmenter)]); + throwCollectedFailures({ + failures: [error, ...cleanupFailures], + message: 'Canvas camera effect configuration failed during cleanup', + }); + } + if (this.segmenter == null) { + this.segmenter = createdSegmenter; + } + const backgroundLifecycleChanged = (this.config.background == null) !== (config.background == null); + if (backgroundLifecycleChanged) { + this.segmenter?.reset(); + this.lastSegmentationAt = Number.NEGATIVE_INFINITY; + this.maskReady = false; + } + this.customFrameSource = customFrameSource; + this.config = config; + } + + async render(frame: VideoFrame, now: number): Promise { + if (this.disposed) { + throw new Error('Cannot render with a disposed camera effect renderer'); + } + const width = frame.displayWidth; + const height = frame.displayHeight; + this.ensureSize(width, height); + const source = frame; + const background = this.config.background; + resetContext(this.outputContext); + if (background == null) { + this.outputContext.globalCompositeOperation = 'copy'; + this.outputContext.drawImage(source, 0, 0, width, height); + this.outputContext.globalCompositeOperation = 'source-over'; + return; + } + await this.maybeSegment(source, now); + this.drawBackground(source, now); + if (!this.maskReady) { + return; + } + resetContext(this.foregroundContext); + this.foregroundContext.globalCompositeOperation = 'copy'; + this.foregroundContext.drawImage(source, 0, 0, width, height); + this.foregroundContext.globalCompositeOperation = 'destination-in'; + const refinedMask = this.maskRefiner?.refine(frame, this.maskCanvas, this.maskRevision, width, height) ?? null; + if (refinedMask == null) { + this.foregroundContext.drawImage(this.maskCanvas, 0, 0, SEG_INPUT_EDGE, SEG_INPUT_EDGE, 0, 0, width, height); + } else { + this.foregroundContext.drawImage(refinedMask, 0, 0, width, height); + } + this.foregroundContext.globalCompositeOperation = 'source-over'; + this.outputContext.drawImage(this.foregroundCanvas, 0, 0); + } + + async dispose(): Promise { + if (this.disposed) { + return; + } + this.disposed = true; + const segmenter = this.segmenter; + this.customFrameSource = null; + this.segmenter = null; + const failures = await collectSettledFailures([ + disposeWebSelfieSegmenter(segmenter), + Promise.resolve().then(() => this.maskRefiner?.dispose()), + ]); + throwCollectedFailures({failures, message: 'Canvas camera effect teardown failed'}); + } + + private ensureSize(width: number, height: number): void { + validateCameraEffectFrameDimensions(width, height); + if (this.width === width && this.height === height) { + return; + } + this.width = width; + this.height = height; + this.foregroundCanvas.width = width; + this.foregroundCanvas.height = height; + this.outputCanvas.width = width; + this.outputCanvas.height = height; + this.foregroundContext.imageSmoothingEnabled = true; + this.foregroundContext.imageSmoothingQuality = 'high'; + this.outputContext.imageSmoothingEnabled = true; + this.outputContext.imageSmoothingQuality = 'high'; + this.segmenter?.reset(); + this.lastSegmentationAt = Number.NEGATIVE_INFINITY; + this.maskReady = false; + this.maskRevision += 1; + } + + private async warmup(): Promise { + const width = this.outputCanvas.width; + const height = this.outputCanvas.height; + validateCameraEffectFrameDimensions(width, height); + const probeCanvas = new OffscreenCanvas(width, height); + const probeContext = probeCanvas.getContext('2d'); + if (probeContext == null) { + throw new Error('Canvas camera effect warm-up requires OffscreenCanvas 2D'); + } + probeContext.fillStyle = '#000'; + probeContext.fillRect(0, 0, width, height); + const frame = new VideoFrame(probeCanvas, {timestamp: 0}); + try { + await this.render(frame, 0); + } finally { + frame.close(); + } + if (this.segmenter != null) { + this.segmenter.reset(); + } + this.lastSegmentationAt = Number.NEGATIVE_INFINITY; + this.maskReady = false; + } + + private async maybeSegment(source: CanvasImageSource, now: number): Promise { + if (now - this.lastSegmentationAt < WEB_CAMERA_EFFECT_SEGMENTATION_MIN_INTERVAL_MS) { + return; + } + const segmenter = this.segmenter; + if (segmenter == null) { + throw new Error('Camera background rendering requires an initialized segmenter'); + } + this.lastSegmentationAt = now; + resetContext(this.segmentationContext); + this.segmentationContext.drawImage(source, 0, 0, SEG_INPUT_EDGE, SEG_INPUT_EDGE); + const input = this.segmentationContext.getImageData(0, 0, SEG_INPUT_EDGE, SEG_INPUT_EDGE); + await segmenter.segmentIntoMask(input.data, this.maskImage.data); + this.maskContext.putImageData(this.maskImage, 0, 0); + this.maskRevision += 1; + this.maskReady = true; + } + + private drawBackground(source: CanvasImageSource, now: number): void { + const background = this.config.background; + if (background == null) { + throw new Error('Camera background renderer has no configured background'); + } + this.outputContext.globalCompositeOperation = 'copy'; + if (background.mode === CameraBackgroundMode.CUSTOM) { + const customFrameSource = this.customFrameSource; + if (customFrameSource == null) { + throw new Error('Custom camera background frame source is unavailable'); + } + const lease = customFrameSource.acquireFrame(now); + try { + const frame = lease.frame; + const scale = Math.max(this.width / frame.width, this.height / frame.height); + const drawWidth = frame.width * scale; + const drawHeight = frame.height * scale; + this.outputContext.drawImage( + frame.image, + (this.width - drawWidth) / 2, + (this.height - drawHeight) / 2, + drawWidth, + drawHeight, + ); + } finally { + lease.release(); + } + this.outputContext.globalCompositeOperation = 'source-over'; + return; + } + this.outputContext.filter = `blur(${cameraEffectBlurPixels(background.blurStrength)}px)`; + this.outputContext.drawImage(source, 0, 0, this.width, this.height); + this.outputContext.filter = 'none'; + this.outputContext.globalCompositeOperation = 'source-over'; + } +} diff --git a/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectCustomFrameSource.ts b/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectCustomFrameSource.ts new file mode 100644 index 000000000..4aafbe6d3 --- /dev/null +++ b/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectCustomFrameSource.ts @@ -0,0 +1,726 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {runWithResponseDeadline} from '@app/features/voice/utils/camera-effects/BoundedResponse'; +import {validateCameraEffectFrameDimensions} from '@app/features/voice/utils/camera-effects/WebCameraEffectProtocol'; + +const MAX_ANIMATED_IMAGE_FRAMES = 300; +const MAX_ANIMATED_IMAGE_AGGREGATE_PIXELS = 32_000_000; +const MAX_ANIMATED_IMAGE_DURATION_MICROSECONDS = 60_000_000; +const MIN_ANIMATED_FRAME_DURATION_MICROSECONDS = 20_000; +const MAX_ANIMATED_FRAME_DURATION_MICROSECONDS = 10_000_000; +const MAX_ANIMATED_FRAME_SEARCH_STEPS = 10; +const VIDEO_FIRST_FRAME_TIMEOUT_MS = 8_000; +const VIDEO_LEASE_RELEASE_TIMEOUT_MS = 1_000; + +export const WebCameraEffectCustomFrameSourceKind = Object.freeze({ + STATIC: 'static', + ANIMATED: 'animated', + VIDEO: 'video', +} as const); + +export type WebCameraEffectCustomFrameSourceKind = + (typeof WebCameraEffectCustomFrameSourceKind)[keyof typeof WebCameraEffectCustomFrameSourceKind]; + +export type WebCameraEffectCustomFrameImage = ImageBitmap | VideoFrame; + +export interface WebCameraEffectCustomFrame { + readonly image: WebCameraEffectCustomFrameImage; + readonly width: number; + readonly height: number; + readonly index: number; +} + +export interface WebCameraEffectCustomFrameLease { + readonly frame: WebCameraEffectCustomFrame; + release(): void; +} + +export interface WebCameraEffectCustomFrameSource { + readonly kind: WebCameraEffectCustomFrameSourceKind; + readonly width: number; + readonly height: number; + readonly frameCount: number | null; + acquireFrame(nowMilliseconds: number): WebCameraEffectCustomFrameLease; + dispose(): Promise; +} + +interface MutableWebCameraEffectCustomFrame { + image: WebCameraEffectCustomFrameImage; + width: number; + height: number; + index: number; +} + +class ReusableWebCameraEffectCustomFrameLease implements WebCameraEffectCustomFrameLease { + private value: WebCameraEffectCustomFrame | null = null; + + constructor(private readonly onRelease: () => void) {} + + get frame(): WebCameraEffectCustomFrame { + if (this.value == null) { + throw new Error('Custom camera background frame lease is not active'); + } + return this.value; + } + + get active(): boolean { + return this.value != null; + } + + acquire(frame: WebCameraEffectCustomFrame): WebCameraEffectCustomFrameLease { + if (this.value != null) { + throw new Error('Custom camera background frame source already has an active lease'); + } + this.value = frame; + return this; + } + + release(): void { + if (this.value == null) { + throw new Error('Custom camera background frame lease was released more than once'); + } + this.value = null; + this.onRelease(); + } +} + +class StaticWebCameraEffectCustomFrameSource implements WebCameraEffectCustomFrameSource { + readonly kind = WebCameraEffectCustomFrameSourceKind.STATIC; + readonly width: number; + readonly height: number; + readonly frameCount = 1; + private readonly frame: WebCameraEffectCustomFrame; + private readonly lease = new ReusableWebCameraEffectCustomFrameLease(() => {}); + private disposed = false; + + constructor(image: WebCameraEffectCustomFrameImage) { + const dimensions = customFrameImageDimensions(image); + this.width = dimensions.width; + this.height = dimensions.height; + this.frame = {image, width: this.width, height: this.height, index: 0}; + } + + acquireFrame(nowMilliseconds: number): WebCameraEffectCustomFrameLease { + requireFrameSourceTime(nowMilliseconds); + this.requireActive(); + return this.lease.acquire(this.frame); + } + + async dispose(): Promise { + if (this.disposed) return; + if (this.lease.active) { + throw new Error('Cannot dispose a custom camera background while its frame is leased'); + } + this.disposed = true; + this.frame.image.close(); + } + + private requireActive(): void { + if (this.disposed) { + throw new Error('Cannot read a disposed custom camera background frame source'); + } + } +} + +class AnimatedWebCameraEffectCustomFrameSource implements WebCameraEffectCustomFrameSource { + readonly kind = WebCameraEffectCustomFrameSourceKind.ANIMATED; + readonly width: number; + readonly height: number; + readonly frameCount: number; + private readonly frames: ReadonlyArray; + private readonly frameEndMicroseconds: ReadonlyArray; + private readonly totalDurationMicroseconds: number; + private readonly lease = new ReusableWebCameraEffectCustomFrameLease(() => {}); + private epochMilliseconds: number | null = null; + private disposed = false; + + constructor(frames: ReadonlyArray, frameEndMicroseconds: ReadonlyArray) { + const firstFrame = frames[0]; + if (firstFrame == null) { + throw new Error('Animated custom camera background requires at least one decoded frame'); + } + this.width = firstFrame.displayWidth; + this.height = firstFrame.displayHeight; + this.frameCount = frames.length; + this.frames = frames.map((image, index) => ({image, width: this.width, height: this.height, index})); + this.frameEndMicroseconds = frameEndMicroseconds; + this.totalDurationMicroseconds = frameEndMicroseconds[frameEndMicroseconds.length - 1] ?? 0; + if (this.totalDurationMicroseconds <= 0) { + throw new Error('Animated custom camera background has no positive duration'); + } + } + + acquireFrame(nowMilliseconds: number): WebCameraEffectCustomFrameLease { + requireFrameSourceTime(nowMilliseconds); + this.requireActive(); + return this.lease.acquire(this.requireFrame(this.frameIndexAt(nowMilliseconds))); + } + + async dispose(): Promise { + if (this.disposed) return; + if (this.lease.active) { + throw new Error('Cannot dispose a custom camera background while its frame is leased'); + } + this.disposed = true; + const failures = closeFrameImages(this.frames.map((frame) => frame.image)); + throwCleanupFailures(failures, 'Animated custom camera background teardown failed'); + } + + private frameIndexAt(nowMilliseconds: number): number { + if (nowMilliseconds === 0) return 0; + if (this.epochMilliseconds == null) { + this.epochMilliseconds = nowMilliseconds; + return 0; + } + if (nowMilliseconds < this.epochMilliseconds) { + throw new Error('Custom camera background frame time moved backwards'); + } + const elapsedMicroseconds = Math.floor((nowMilliseconds - this.epochMilliseconds) * 1000); + return this.findFrameIndex(elapsedMicroseconds % this.totalDurationMicroseconds); + } + + private findFrameIndex(positionMicroseconds: number): number { + let low = 0; + let high = this.frameEndMicroseconds.length - 1; + for (let step = 0; step < MAX_ANIMATED_FRAME_SEARCH_STEPS; step += 1) { + if (low >= high) return low; + const middle = low + Math.floor((high - low) / 2); + if ((this.frameEndMicroseconds[middle] ?? 0) > positionMicroseconds) high = middle; + else low = middle + 1; + } + throw new Error('Animated custom camera background frame search exceeded its bound'); + } + + private requireFrame(index: number): WebCameraEffectCustomFrame { + const frame = this.frames[index]; + if (frame == null) { + throw new Error('Animated custom camera background selected an unavailable frame'); + } + return frame; + } + + private requireActive(): void { + if (this.disposed) { + throw new Error('Cannot read a disposed custom camera background frame source'); + } + } +} + +class VideoWebCameraEffectCustomFrameSource implements WebCameraEffectCustomFrameSource { + readonly kind = WebCameraEffectCustomFrameSourceKind.VIDEO; + readonly frameCount = null; + private readonly reader: ReadableStreamDefaultReader; + private readonly lease = new ReusableWebCameraEffectCustomFrameLease(() => this.releaseFrameLease()); + private readonly frame: MutableWebCameraEffectCustomFrame = { + image: null as unknown as VideoFrame, + width: 0, + height: 0, + index: 0, + }; + private readonly firstFramePromise: Promise; + private resolveFirstFrame: (() => void) | null = null; + private rejectFirstFrame: ((error: unknown) => void) | null = null; + private currentFrame: VideoFrame | null = null; + private pendingFrame: VideoFrame | null = null; + private currentFrameIndex = 0; + private pendingFrameIndex = 0; + private failure: unknown = null; + private failed = false; + private nextFrameIndex = 0; + private stopping = false; + private disposed = false; + private leaseReleaseResolve: (() => void) | null = null; + private leaseReleaseTimedOut = false; + private readonly pumpPromise: Promise; + private disposePromise: Promise | null = null; + + constructor(readable: ReadableStream) { + this.reader = readable.getReader(); + this.firstFramePromise = new Promise((resolve, reject) => { + this.resolveFirstFrame = resolve; + this.rejectFirstFrame = reject; + }); + this.pumpPromise = this.pump(); + } + + get width(): number { + return this.currentFrame?.displayWidth ?? 0; + } + + get height(): number { + return this.currentFrame?.displayHeight ?? 0; + } + + async waitForFirstFrame(signal: AbortSignal): Promise { + await waitForAbortablePromise(this.firstFramePromise, signal); + } + + acquireFrame(nowMilliseconds: number): WebCameraEffectCustomFrameLease { + requireFrameSourceTime(nowMilliseconds); + this.requireActive(); + this.promotePendingFrame(); + const currentFrame = this.currentFrame; + if (currentFrame == null) { + throw new Error('Custom camera background video has no current frame'); + } + this.frame.image = currentFrame; + this.frame.width = currentFrame.displayWidth; + this.frame.height = currentFrame.displayHeight; + this.frame.index = this.currentFrameIndex; + return this.lease.acquire(this.frame); + } + + dispose(): Promise { + if (this.disposePromise == null) { + this.disposePromise = this.disposeOwned(); + } + return this.disposePromise; + } + + private async pump(): Promise { + try { + while (!this.stopping) { + const result = await this.reader.read(); + if (result.done) { + if (!this.stopping) throw new Error('Custom camera background video frame stream ended'); + break; + } + if (this.stopping) { + result.value.close(); + break; + } + this.acceptFrame(result.value); + } + } catch (error) { + if (!this.stopping) this.recordFailure(error); + } + } + + private acceptFrame(frame: VideoFrame): void { + try { + validateCameraEffectFrameDimensions(frame.displayWidth, frame.displayHeight); + if (this.currentFrame != null) this.requireStableDimensions(frame); + } catch (error) { + frame.close(); + throw error; + } + const frameIndex = this.takeFrameIndex(); + const currentFrame = this.currentFrame; + if (currentFrame == null) { + this.currentFrame = frame; + this.currentFrameIndex = frameIndex; + this.resolveFirstFrame?.(); + this.clearFirstFrameSettlers(); + return; + } + if (this.lease.active) { + const replaced = this.pendingFrame; + this.pendingFrame = frame; + this.pendingFrameIndex = frameIndex; + replaced?.close(); + return; + } + this.currentFrame = frame; + this.currentFrameIndex = frameIndex; + currentFrame.close(); + } + + private requireStableDimensions(frame: VideoFrame): void { + const currentFrame = this.currentFrame; + if (currentFrame == null) { + throw new Error('Custom camera background video has no current frame for dimension validation'); + } + if (frame.displayWidth !== currentFrame.displayWidth) { + throw new Error('Custom camera background video width changed'); + } + if (frame.displayHeight !== currentFrame.displayHeight) { + throw new Error('Custom camera background video height changed'); + } + } + + private takeFrameIndex(): number { + const frameIndex = this.nextFrameIndex; + this.nextFrameIndex = (this.nextFrameIndex + 1) % Number.MAX_SAFE_INTEGER; + return frameIndex; + } + + private promotePendingFrame(): void { + const pendingFrame = this.pendingFrame; + if (this.lease.active || pendingFrame == null) return; + const replaced = this.currentFrame; + this.currentFrame = pendingFrame; + this.currentFrameIndex = this.pendingFrameIndex; + this.pendingFrame = null; + replaced?.close(); + } + + private releaseFrameLease(): void { + let failure: unknown; + try { + this.promotePendingFrame(); + } catch (error) { + failure = error; + } + this.leaseReleaseResolve?.(); + this.leaseReleaseResolve = null; + if (this.stopping && this.leaseReleaseTimedOut) { + const currentFrame = this.currentFrame; + this.currentFrame = null; + try { + currentFrame?.close(); + } catch (error) { + if (failure !== undefined) { + throw new AggregateError([failure, error], 'Custom camera background video frame release failed'); + } + throw error; + } + } + if (failure !== undefined) throw failure; + } + + private recordFailure(error: unknown): void { + if (this.failed) return; + this.failed = true; + this.failure = error ?? new Error('Custom camera background video frame stream failed without a reason'); + this.rejectFirstFrame?.(this.failure); + this.clearFirstFrameSettlers(); + } + + private clearFirstFrameSettlers(): void { + this.resolveFirstFrame = null; + this.rejectFirstFrame = null; + } + + private requireActive(): void { + if (this.disposed || this.stopping) { + throw new Error('Cannot read a disposed custom camera background video source'); + } + if (this.failed) { + throw this.failure; + } + } + + private async disposeOwned(): Promise { + if (this.disposed) return; + this.stopping = true; + this.rejectFirstFrame?.(new Error('Custom camera background video source was disposed before its first frame')); + this.clearFirstFrameSettlers(); + const cancellation = Promise.resolve().then(() => this.reader.cancel()); + const failures = await settledFailures([cancellation, this.pumpPromise]); + try { + this.reader.releaseLock(); + } catch (error) { + failures.push(error); + } + this.closeOwnedFrame('pendingFrame', failures); + if (this.lease.active) { + try { + await this.waitForLeaseRelease(); + } catch (error) { + this.leaseReleaseTimedOut = true; + failures.push(error); + } + } + if (!this.lease.active) this.closeOwnedFrame('currentFrame', failures); + this.disposed = true; + throwCleanupFailures(failures, 'Custom camera background video teardown failed'); + } + + private closeOwnedFrame(key: 'currentFrame' | 'pendingFrame', failures: Array): void { + const frame = this[key]; + this[key] = null; + if (frame == null) return; + try { + frame.close(); + } catch (error) { + failures.push(error); + } + } + + private waitForLeaseRelease(): Promise { + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + this.leaseReleaseResolve = null; + reject(new Error('Custom camera background video frame lease exceeded its release deadline')); + }, VIDEO_LEASE_RELEASE_TIMEOUT_MS); + this.leaseReleaseResolve = () => { + clearTimeout(timeout); + resolve(); + }; + }); + } +} + +function customFrameImageDimensions(image: WebCameraEffectCustomFrameImage): {width: number; height: number} { + if ('displayWidth' in image) { + return {width: image.displayWidth, height: image.displayHeight}; + } + return {width: image.width, height: image.height}; +} + +function requireFrameSourceTime(nowMilliseconds: number): void { + if (!Number.isFinite(nowMilliseconds) || nowMilliseconds < 0) { + throw new Error('Custom camera background frame time must be finite and non-negative'); + } +} + +function closeFrameImages(images: ReadonlyArray): Array { + const failures: Array = []; + for (const image of images) { + try { + image.close(); + } catch (error) { + failures.push(error); + } + } + return failures; +} + +function throwCleanupFailures(failures: ReadonlyArray, message: string): void { + if (failures.length === 0) return; + if (failures.length === 1) throw failures[0]; + throw new AggregateError(failures, message); +} + +function throwWithCleanupFailures(error: unknown, failures: ReadonlyArray, message: string): never { + if (failures.length === 0) throw error; + throw new AggregateError([error, ...failures], message); +} + +async function settledFailures(promises: ReadonlyArray>): Promise> { + const outcomes = await Promise.allSettled(promises); + const failures: Array = []; + for (const outcome of outcomes) { + if (outcome.status === 'rejected') failures.push(outcome.reason); + } + return failures; +} + +function waitForAbortablePromise(promise: Promise, signal: AbortSignal): Promise { + if (signal.aborted) return Promise.reject(signal.reason); + return new Promise((resolve, reject) => { + const handleAbort = (): void => reject(signal.reason); + signal.addEventListener('abort', handleAbort, {once: true}); + promise.then( + (value) => { + signal.removeEventListener('abort', handleAbort); + resolve(value); + }, + (error: unknown) => { + signal.removeEventListener('abort', handleAbort); + reject(error); + }, + ); + }); +} + +function throwIfAborted(signal: AbortSignal): void { + if (signal.aborted) throw signal.reason ?? new Error('Custom camera background operation was aborted'); +} + +function requireAnimatedFrameDuration(duration: number | null): number { + if (duration == null || !Number.isSafeInteger(duration) || duration < 0) { + throw new Error('Animated custom camera background has an invalid frame duration'); + } + const normalizedDuration = Math.max(duration, MIN_ANIMATED_FRAME_DURATION_MICROSECONDS); + if (normalizedDuration > MAX_ANIMATED_FRAME_DURATION_MICROSECONDS) { + throw new Error('Animated custom camera background frame duration exceeds the supported maximum'); + } + return normalizedDuration; +} + +function validateAnimatedImageBudget(width: number, height: number, frameCount: number): void { + validateCameraEffectFrameDimensions(width, height); + const aggregatePixels = width * height * frameCount; + if (!Number.isSafeInteger(aggregatePixels)) { + throw new Error('Animated custom camera background decoded pixel budget is invalid'); + } + if (aggregatePixels > MAX_ANIMATED_IMAGE_AGGREGATE_PIXELS) { + throw new Error('Animated custom camera background exceeds the aggregate decoded pixel budget'); + } +} + +function validateDecodedFrameDimensions(frame: VideoFrame, width: number, height: number): void { + validateCameraEffectFrameDimensions(frame.displayWidth, frame.displayHeight); + if (frame.displayWidth !== width || frame.displayHeight !== height) { + throw new Error('Animated custom camera background frame dimensions changed during decoding'); + } +} + +async function decodeFrames( + decoder: ImageDecoder, + frameCount: number, + animated: boolean, + signal: AbortSignal, +): Promise { + if (!animated && frameCount !== 1) { + throw new Error('Static custom camera background unexpectedly contains multiple frames'); + } + const frames: Array = []; + const frameEndMicroseconds: Array = []; + let width = 0; + let height = 0; + let totalDurationMicroseconds = 0; + let decodedFrame: VideoFrame | null = null; + try { + for (let frameIndex = 0; frameIndex < frameCount; frameIndex += 1) { + throwIfAborted(signal); + const result = await decoder.decode({frameIndex, completeFramesOnly: true}); + decodedFrame = result.image; + throwIfAborted(signal); + if (!result.complete) { + throw new Error('Animated custom camera background produced an incomplete decoded frame'); + } + if (frameIndex === 0) { + width = decodedFrame.displayWidth; + height = decodedFrame.displayHeight; + validateAnimatedImageBudget(width, height, frameCount); + } + validateDecodedFrameDimensions(decodedFrame, width, height); + if (animated) { + totalDurationMicroseconds += requireAnimatedFrameDuration(decodedFrame.duration); + if (totalDurationMicroseconds > MAX_ANIMATED_IMAGE_DURATION_MICROSECONDS) { + throw new Error('Animated custom camera background exceeds the supported duration'); + } + frameEndMicroseconds.push(totalDurationMicroseconds); + } + frames.push(decodedFrame); + decodedFrame = null; + } + } catch (error) { + const failures = closeFrameImages(decodedFrame == null ? frames : [decodedFrame, ...frames]); + throwWithCleanupFailures(error, failures, 'Animated custom camera background decoding failed during cleanup'); + } + const firstFrame = frames[0]; + if (firstFrame == null) { + throw new Error('Animated custom camera background decoding produced no frames'); + } + if (!animated) return new StaticWebCameraEffectCustomFrameSource(firstFrame); + try { + return new AnimatedWebCameraEffectCustomFrameSource(frames, frameEndMicroseconds); + } catch (error) { + const failures = closeFrameImages(frames); + throwWithCleanupFailures(error, failures, 'Animated custom camera background setup failed during cleanup'); + } +} + +type CameraImageDecoderConstructor = { + new (init: ImageDecoderInit): ImageDecoder; + isTypeSupported(type: string): Promise; +}; + +function getCameraImageDecoderConstructor(): CameraImageDecoderConstructor | null { + const candidate: unknown = Reflect.get(globalThis, 'ImageDecoder'); + if (typeof candidate !== 'function') return null; + if (typeof Reflect.get(candidate, 'isTypeSupported') !== 'function') return null; + return candidate as CameraImageDecoderConstructor; +} + +async function decodeImageDecoderSource( + blob: Blob, + mediaType: string, + signal: AbortSignal, +): Promise { + const ImageDecoderConstructor = getCameraImageDecoderConstructor(); + if (ImageDecoderConstructor == null) { + throw new Error('Animated custom camera backgrounds require ImageDecoder'); + } + if (!(await ImageDecoderConstructor.isTypeSupported(mediaType))) { + throw new Error(`Animated custom camera background type is unsupported: ${mediaType}`); + } + throwIfAborted(signal); + const decoder = new ImageDecoderConstructor({data: blob.stream(), type: mediaType, preferAnimation: true}); + let decoderClosed = false; + let source: WebCameraEffectCustomFrameSource | null = null; + const closeDecoder = (): void => { + if (decoderClosed) return; + decoderClosed = true; + decoder.close(); + }; + const handleAbort = (): void => closeDecoder(); + signal.addEventListener('abort', handleAbort, {once: true}); + try { + await Promise.all([decoder.tracks.ready, decoder.completed]); + throwIfAborted(signal); + const selectedTrack = decoder.tracks.selectedTrack; + if (selectedTrack == null) { + throw new Error('Animated custom camera background has no selected image track'); + } + const frameCount = selectedTrack.frameCount; + if (!Number.isSafeInteger(frameCount) || frameCount <= 0) { + throw new Error('Animated custom camera background has an invalid frame count'); + } + if (frameCount > MAX_ANIMATED_IMAGE_FRAMES) { + throw new Error('Animated custom camera background exceeds the supported frame count'); + } + const animated = mediaType === 'image/gif' || selectedTrack.animated; + source = await decodeFrames(decoder, frameCount, animated, signal); + closeDecoder(); + return source; + } catch (error) { + const failures: Array = []; + try { + closeDecoder(); + } catch (cleanupError) { + failures.push(cleanupError); + } + if (source != null) { + try { + await source.dispose(); + } catch (cleanupError) { + failures.push(cleanupError); + } + } + throwWithCleanupFailures(error, failures, 'Animated custom camera background initialization failed during cleanup'); + } finally { + signal.removeEventListener('abort', handleAbort); + } +} + +async function decodeStaticSource(blob: Blob, signal: AbortSignal): Promise { + const image = await createImageBitmap(blob); + try { + throwIfAborted(signal); + validateCameraEffectFrameDimensions(image.width, image.height); + return new StaticWebCameraEffectCustomFrameSource(image); + } catch (error) { + const failures = closeFrameImages([image]); + throwWithCleanupFailures(error, failures, 'Custom camera background validation failed during cleanup'); + } +} + +export function createWebCameraEffectImageFrameSource( + blob: Blob, + mediaType: string, + signal: AbortSignal, +): Promise { + if (mediaType === 'image/gif' || mediaType === 'image/webp') { + return decodeImageDecoderSource(blob, mediaType, signal); + } + return decodeStaticSource(blob, signal); +} + +export async function createWebCameraEffectVideoFrameSource( + readable: ReadableStream, +): Promise { + const source = new VideoWebCameraEffectCustomFrameSource(readable); + try { + await runWithResponseDeadline({ + timeoutMilliseconds: VIDEO_FIRST_FRAME_TIMEOUT_MS, + description: 'Custom camera background video first frame', + signal: null, + operation: (signal) => source.waitForFirstFrame(signal), + }); + return source; + } catch (error) { + try { + await source.dispose(); + } catch (cleanupError) { + throw new AggregateError( + [error, cleanupError], + 'Custom camera background video initialization failed during cleanup', + ); + } + throw error; + } +} diff --git a/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectCustomImage.ts b/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectCustomImage.ts new file mode 100644 index 000000000..c17557a87 --- /dev/null +++ b/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectCustomImage.ts @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import { + cancelResponseBodyAndThrow, + readBoundedResponseBlob, + runWithResponseDeadline, +} from '@app/features/voice/utils/camera-effects/BoundedResponse'; +import {createWebCameraEffectImageFrameSource} from '@app/features/voice/utils/camera-effects/WebCameraEffectCustomFrameSource'; + +export type { + WebCameraEffectCustomFrame, + WebCameraEffectCustomFrameImage, + WebCameraEffectCustomFrameLease, + WebCameraEffectCustomFrameSource, +} from '@app/features/voice/utils/camera-effects/WebCameraEffectCustomFrameSource'; +export { + createWebCameraEffectVideoFrameSource, + WebCameraEffectCustomFrameSourceKind, +} from '@app/features/voice/utils/camera-effects/WebCameraEffectCustomFrameSource'; + +import type {WebCameraEffectCustomFrameSource} from '@app/features/voice/utils/camera-effects/WebCameraEffectCustomFrameSource'; + +const MAX_CUSTOM_MEDIA_BYTES = 10 * 1024 * 1024; +const MAX_CUSTOM_MEDIA_URL_LENGTH = 16 * 1024; +const MAX_CUSTOM_MEDIA_RESPONSE_CHUNKS = 4096; +export const WEB_CAMERA_EFFECT_CUSTOM_MEDIA_OPERATION_TIMEOUT_MS = 8_000; + +const SUPPORTED_IMAGE_MEDIA_TYPES = new Set(['image/gif', 'image/jpeg', 'image/png', 'image/webp']); + +export function requireWebCameraEffectCustomMediaURL(URL: string): void { + if (URL.length === 0 || URL.length > MAX_CUSTOM_MEDIA_URL_LENGTH) { + throw new Error('Custom camera background URL has an invalid length'); + } +} + +function throwIfCustomMediaOperationAborted(signal: AbortSignal): void { + if (signal.aborted) { + throw signal.reason ?? new Error('Custom camera background operation was aborted'); + } +} + +export async function readWebCameraEffectCustomMediaBlob(URL: string, signal: AbortSignal): Promise { + requireWebCameraEffectCustomMediaURL(URL); + throwIfCustomMediaOperationAborted(signal); + let response: Response; + try { + response = await fetch(URL, { + credentials: 'omit', + redirect: 'error', + referrerPolicy: 'no-referrer', + signal, + }); + } catch { + if (signal.aborted) throw signal.reason; + throw new Error('Custom camera background request failed'); + } + if (!response.ok) { + await cancelResponseBodyAndThrow({ + response, + error: new Error(`Custom camera background request failed with status ${response.status}`), + description: 'Custom camera background response', + }); + } + const blob = await readBoundedResponseBlob({ + response, + maximumBytes: MAX_CUSTOM_MEDIA_BYTES, + maximumChunks: MAX_CUSTOM_MEDIA_RESPONSE_CHUNKS, + description: 'Custom camera background response', + }); + throwIfCustomMediaOperationAborted(signal); + if (blob.size === 0) { + throw new Error('Custom camera background response is empty'); + } + return blob; +} + +function normalizedMediaType(value: string): string { + return value.split(';', 1)[0]?.trim().toLowerCase() ?? ''; +} + +function bytesEqualAt(bytes: Uint8Array, offset: number, expected: ReadonlyArray): boolean { + if (bytes.byteLength < offset + expected.length) return false; + for (let index = 0; index < expected.length; index += 1) { + if (bytes[offset + index] !== expected[index]) return false; + } + return true; +} + +function sniffImageMediaType(bytes: Uint8Array): string | null { + if (bytesEqualAt(bytes, 0, [0x47, 0x49, 0x46, 0x38, 0x37, 0x61])) return 'image/gif'; + if (bytesEqualAt(bytes, 0, [0x47, 0x49, 0x46, 0x38, 0x39, 0x61])) return 'image/gif'; + if (bytesEqualAt(bytes, 0, [0xff, 0xd8, 0xff])) return 'image/jpeg'; + if (bytesEqualAt(bytes, 0, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])) return 'image/png'; + if (bytesEqualAt(bytes, 0, [0x52, 0x49, 0x46, 0x46]) && bytesEqualAt(bytes, 8, [0x57, 0x45, 0x42, 0x50])) { + return 'image/webp'; + } + return null; +} + +export async function resolveWebCameraEffectCustomImageMediaType(blob: Blob, signal: AbortSignal): Promise { + const header = new Uint8Array(await blob.slice(0, 12).arrayBuffer()); + throwIfCustomMediaOperationAborted(signal); + const detectedMediaType = sniffImageMediaType(header); + if (detectedMediaType == null) { + throw new Error('Custom camera background is not a supported image format'); + } + const declaredMediaType = normalizedMediaType(blob.type); + if (SUPPORTED_IMAGE_MEDIA_TYPES.has(declaredMediaType) && declaredMediaType !== detectedMediaType) { + throw new Error('Custom camera background media type does not match its encoded data'); + } + return detectedMediaType; +} + +export async function loadWebCameraEffectCustomFrameSource(URL: string): Promise { + requireWebCameraEffectCustomMediaURL(URL); + return runWithResponseDeadline({ + timeoutMilliseconds: WEB_CAMERA_EFFECT_CUSTOM_MEDIA_OPERATION_TIMEOUT_MS, + description: 'Custom camera background initialization', + signal: null, + operation: async (signal) => { + const blob = await readWebCameraEffectCustomMediaBlob(URL, signal); + const mediaType = await resolveWebCameraEffectCustomImageMediaType(blob, signal); + return createWebCameraEffectImageFrameSource(blob, mediaType, signal); + }, + }); +} diff --git a/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectMask.ts b/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectMask.ts new file mode 100644 index 000000000..c3f46f785 --- /dev/null +++ b/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectMask.ts @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +export const WEB_CAMERA_EFFECT_MASK_VOID_MAX = 0.3; +export const WEB_CAMERA_EFFECT_MASK_CORE_MIN = 0.7; +export const WEB_CAMERA_EFFECT_MASK_BAND_WIDTH = 0.4; +export const WEB_CAMERA_EFFECT_MASK_SPECKLE_NEIGHBOUR_MAX = 0.28; +export const WEB_CAMERA_EFFECT_MASK_HOLE_NEIGHBOUR_MIN = 0.72; +export const WEB_CAMERA_EFFECT_MASK_CORE_GROW_MIN = 0.45; +export const WEB_CAMERA_EFFECT_MASK_TEMPORAL_KEEP_STILL = 0.55; +export const WEB_CAMERA_EFFECT_MASK_TEMPORAL_MOTION_LOW = 0.1; +export const WEB_CAMERA_EFFECT_MASK_TEMPORAL_MOTION_HIGH = 0.35; +export const WEB_CAMERA_EFFECT_MASK_BAND_EPSILON = 0.004; +export const WEB_CAMERA_EFFECT_MASK_EDGE_SOFTNESS = 0.25; +export const WEB_CAMERA_EFFECT_MASK_GUIDE_SPATIAL_FALLOFF = 1.5; +export const WEB_CAMERA_EFFECT_MASK_GUIDE_RANGE_FALLOFF = 18; + +export function shapeWebCameraEffectMaskAlpha(value: number): number { + const normalized = Math.max( + 0, + Math.min(1, (value - WEB_CAMERA_EFFECT_MASK_VOID_MAX) / WEB_CAMERA_EFFECT_MASK_BAND_WIDTH), + ); + return normalized * normalized * (3 - 2 * normalized); +} diff --git a/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectPipeline.ts b/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectPipeline.ts new file mode 100644 index 000000000..48079fc3b --- /dev/null +++ b/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectPipeline.ts @@ -0,0 +1,940 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {Logger} from '@app/features/platform/utils/AppLogger'; +import { + collectSettledFailures, + throwCollectedFailures, +} from '@app/features/voice/utils/camera-effects/AggregateOperations'; +import { + CameraBackgroundMode, + clampVideoFrameRate, +} from '@app/features/voice/utils/camera-effects/CameraCaptureContract'; +import { + type ErrorDiagnostic, + type ErrorDiagnosticType, + getErrorDiagnosticType, + isErrorDiagnostic, + isErrorDiagnosticType, +} from '@app/features/voice/utils/camera-effects/ErrorDiagnostic'; +import {createTrackProcessor} from '@app/features/voice/utils/camera-effects/MediaStreamTrackProcessorPolyfill'; +import {detectWebCameraSegmentationCapability} from '@app/features/voice/utils/camera-effects/WebCameraBackgroundSupport'; +import { + isWebCameraEffectShutdownReason, + validateCameraEffectFrameDimensions, + WEB_CAMERA_EFFECT_UPDATE_REQUEST_ID_MAX, + WebCameraEffectBackend, + WebCameraEffectCommandKind, + WebCameraEffectCustomMediaKind, + WebCameraEffectEventKind, + WebCameraEffectShutdownReason, + type WebCameraEffectStartCommand, + type WebCameraEffectUpdateCommand, + type WebCameraEffectUpdateFailedEvent, + type WebCameraEffectWorkerEvent, + type WebCameraPipelineConfig, +} from '@app/features/voice/utils/camera-effects/WebCameraEffectProtocol'; +import { + createWebCameraEffectVideoFrameProducer, + type WebCameraEffectVideoFrameProducer, +} from '@app/features/voice/utils/camera-effects/WebCameraEffectVideoFrameSource'; + +const VIDEO_TRACK_KIND = 'video'; + +class InvalidCameraEffectReadyEventError extends Error { + constructor() { + super('Camera effect worker emitted an invalid ready event'); + this.name = 'InvalidCameraEffectReadyEventError'; + } +} + +const logger = new Logger('WebCameraEffectPipeline'); + +const DEFAULT_OUTPUT_FRAME_RATE = 30; +const INITIALIZATION_TIMEOUT_MS = 30_000; +const SHUTDOWN_TIMEOUT_MS = 2_000; +const UPDATE_TIMEOUT_MS = 12_000; + +const WebGPUProbeState = Object.freeze({ + UNKNOWN: 'unknown', + PROBING: 'probing', + ENABLED: 'enabled', + DISABLED: 'disabled', +} as const); + +type WebGPUProbeState = (typeof WebGPUProbeState)[keyof typeof WebGPUProbeState]; + +let webGPUProbeState: WebGPUProbeState = WebGPUProbeState.UNKNOWN; + +interface CapturedSurface { + readonly offscreen: OffscreenCanvas; + readonly track: MediaStreamTrack; +} + +interface InitializationResult { + readonly backend: WebCameraEffectBackend; + readonly fallbackErrorType: ErrorDiagnosticType | null; +} + +interface PendingUpdate { + readonly requestId: number; + readonly config: WebCameraPipelineConfig; + readonly candidateVideoProducer: WebCameraEffectVideoFrameProducer | null; + readonly ownsCandidateVideoProducer: boolean; + readonly resolve: () => void; + readonly reject: (error: Error) => void; + readonly timeout: number; +} + +export interface WebCameraPipeline { + readonly outputTrack: MediaStreamTrack; + readonly backend: WebCameraEffectBackend; + assertActive(): void; + updateConfig(config: WebCameraPipelineConfig): Promise; + beginStop(): void; + stop(): void; +} + +export interface WebCameraEffectPipelineCreateRequest { + readonly source: MediaStreamTrack; + readonly config: WebCameraPipelineConfig; + readonly onFailure: (pipeline: WebCameraPipeline, error: Error) => void; +} + +function parseCameraEffectWorkerEvent(value: unknown): WebCameraEffectWorkerEvent { + if (value == null || typeof value !== 'object') { + throw new Error('Camera effect worker emitted a non-object event'); + } + const record = value as Record; + if (record.kind === WebCameraEffectEventKind.READY) { + if ( + record.backend !== WebCameraEffectBackend.WEB_GPU && + record.backend !== WebCameraEffectBackend.WASM_WORKER && + record.backend !== WebCameraEffectBackend.CANVAS_WORKER + ) { + throw new InvalidCameraEffectReadyEventError(); + } + if (!('fallbackErrorType' in record)) { + throw new InvalidCameraEffectReadyEventError(); + } + const fallbackErrorType = record.fallbackErrorType; + let resolvedFallbackErrorType: ErrorDiagnosticType | null = null; + if (!Object.is(fallbackErrorType, null)) { + if (!isErrorDiagnosticType(fallbackErrorType)) { + throw new InvalidCameraEffectReadyEventError(); + } + resolvedFallbackErrorType = fallbackErrorType; + } + return { + kind: WebCameraEffectEventKind.READY, + backend: record.backend, + fallbackErrorType: resolvedFallbackErrorType, + }; + } + if (record.kind === WebCameraEffectEventKind.UPDATED) { + if (!isValidUpdateRequestId(record.requestId)) { + throw new Error('Camera effect worker emitted an invalid update acknowledgement'); + } + return {kind: WebCameraEffectEventKind.UPDATED, requestId: record.requestId}; + } + if (record.kind === WebCameraEffectEventKind.UPDATE_FAILED) { + if (!isValidUpdateRequestId(record.requestId) || !isErrorDiagnostic(record)) { + throw new Error('Camera effect worker emitted an invalid update rejection'); + } + return { + kind: WebCameraEffectEventKind.UPDATE_FAILED, + requestId: record.requestId, + errorType: record.errorType, + message: record.message, + stack: record.stack, + }; + } + if (record.kind === WebCameraEffectEventKind.FAILED) { + if (!isErrorDiagnostic(record)) { + throw new Error('Camera effect worker emitted an invalid failure event'); + } + return { + kind: WebCameraEffectEventKind.FAILED, + errorType: record.errorType, + message: record.message, + stack: record.stack, + }; + } + if (record.kind === WebCameraEffectEventKind.STOPPED) { + const reason = record.reason; + if (!isWebCameraEffectShutdownReason(reason)) { + throw new Error('Camera effect worker emitted a stop event without a valid reason'); + } + return { + kind: WebCameraEffectEventKind.STOPPED, + reason, + diagnostic: parseOptionalErrorDiagnostic(record.diagnostic), + }; + } + throw new Error('Camera effect worker emitted an unknown event'); +} + +function parseOptionalErrorDiagnostic(value: unknown): ErrorDiagnostic | null { + if (value == null) { + return null; + } + if (!isErrorDiagnostic(value)) { + throw new Error('Camera effect worker emitted an invalid stop diagnostic'); + } + return {errorType: value.errorType, message: value.message, stack: value.stack}; +} + +function isValidUpdateRequestId(value: unknown): value is number { + if (!Number.isSafeInteger(value)) { + return false; + } + if ((value as number) <= 0) { + return false; + } + return (value as number) <= WEB_CAMERA_EFFECT_UPDATE_REQUEST_ID_MAX; +} + +function createWorkerReportedError(event: ErrorDiagnostic, context: string): Error { + const error = new Error(`${context}: ${event.message}`); + if (event.stack != null) { + error.stack = `${error.stack ?? error.message}\nWorker stack:\n${event.stack}`; + } + return error; +} + +function throwAfterTrackCleanup(error: unknown, tracks: ReadonlyArray, message: string): never { + const failures: Array = [error]; + for (const track of tracks) { + try { + track.stop(); + } catch (cleanupError) { + failures.push(cleanupError); + } + } + throwCollectedFailures({failures, message: message}); + throw error; +} + +function createCapturedSurface(frameRate: number, width: number, height: number): CapturedSurface { + const canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + const stream = canvas.captureStream(frameRate); + const track = stream.getVideoTracks()[0]; + if (track == null) { + throw new Error('Camera effect canvas capture produced no video track'); + } + try { + return {offscreen: canvas.transferControlToOffscreen(), track}; + } catch (error) { + throwAfterTrackCleanup(error, [track], 'Camera effect surface creation failed during cleanup'); + } +} + +function initialSurfaceDimension(value: number | null): number { + if (value == null) { + return 1; + } + if (!Number.isSafeInteger(value)) { + return 1; + } + if (value <= 0) { + return 1; + } + return value; +} + +function hasVideoBackground(config: WebCameraPipelineConfig): boolean { + const background = config.background; + if (background == null || background.mode !== CameraBackgroundMode.CUSTOM) { + return false; + } + return background.customMediaKind === WebCameraEffectCustomMediaKind.VIDEO; +} + +function hasSameVideoBackground(current: WebCameraPipelineConfig, next: WebCameraPipelineConfig): boolean { + if (!hasVideoBackground(current) || !hasVideoBackground(next)) { + return false; + } + const currentBackground = current.background; + const nextBackground = next.background; + if (currentBackground?.mode !== CameraBackgroundMode.CUSTOM) { + return false; + } + if (nextBackground?.mode !== CameraBackgroundMode.CUSTOM) { + return false; + } + return currentBackground.customMediaURL === nextBackground.customMediaURL; +} + +async function createVideoFrameProducer( + config: WebCameraPipelineConfig, +): Promise { + const background = config.background; + if (background == null || background.mode !== CameraBackgroundMode.CUSTOM) { + return null; + } + if (background.customMediaKind !== WebCameraEffectCustomMediaKind.VIDEO) { + return null; + } + return createWebCameraEffectVideoFrameProducer(background.customMediaURL); +} + +function selectWebGPUPreference(config: WebCameraPipelineConfig): boolean { + if (config.background == null) { + return false; + } + if (webGPUProbeState === WebGPUProbeState.UNKNOWN) { + webGPUProbeState = WebGPUProbeState.PROBING; + return true; + } + return webGPUProbeState === WebGPUProbeState.ENABLED; +} + +function recordWebGPUSelection(preferred: boolean, backend: WebCameraEffectBackend): void { + if (!preferred) { + return; + } + if (backend === WebCameraEffectBackend.WEB_GPU) { + webGPUProbeState = WebGPUProbeState.ENABLED; + return; + } + webGPUProbeState = WebGPUProbeState.DISABLED; +} + +function releaseWebGPUProbe(preferred: boolean): void { + if (preferred && webGPUProbeState === WebGPUProbeState.PROBING) { + webGPUProbeState = WebGPUProbeState.UNKNOWN; + } +} + +export class WebCameraEffectPipeline implements WebCameraPipeline { + private readonly worker: Worker; + private readonly gpuTrack: MediaStreamTrack; + private readonly fallbackTrack: MediaStreamTrack; + private readonly onFailure: (pipeline: WebCameraPipeline, error: Error) => void; + private readonly preferWebGPU: boolean; + private readonly initializationPromise: Promise; + private readonly resolveInitialization: (result: InitializationResult) => void; + private readonly rejectInitialization: (error: Error) => void; + private initializationTimeout: number | null; + private shutdownTimeout: number | null = null; + private selectedTrack: MediaStreamTrack | null = null; + private selectedBackend: WebCameraEffectBackend | null = null; + private operationalFailure: Error | null = null; + private currentConfig: WebCameraPipelineConfig; + private currentVideoProducer: WebCameraEffectVideoFrameProducer | null = null; + private pendingUpdate: PendingUpdate | null = null; + private nextUpdateRequestId = 1; + private updatePreparationActive = false; + private stopped = false; + private stopIntent = false; + private initialized = false; + + private constructor( + worker: Worker, + GPUTrack: MediaStreamTrack, + fallbackTrack: MediaStreamTrack, + onFailure: (pipeline: WebCameraPipeline, error: Error) => void, + config: WebCameraPipelineConfig, + preferWebGPU: boolean, + ) { + this.worker = worker; + this.gpuTrack = GPUTrack; + this.fallbackTrack = fallbackTrack; + this.onFailure = onFailure; + this.currentConfig = config; + this.preferWebGPU = preferWebGPU; + let capturedResolveInitialization: ((result: InitializationResult) => void) | null = null; + let capturedRejectInitialization: ((error: Error) => void) | null = null; + this.initializationPromise = new Promise((resolve, reject) => { + capturedResolveInitialization = resolve; + capturedRejectInitialization = reject; + }); + const resolveInitialization = capturedResolveInitialization; + if (resolveInitialization == null) { + throw new Error('camera_effect_initialization_resolve_not_captured'); + } + const rejectInitialization = capturedRejectInitialization; + if (rejectInitialization == null) { + throw new Error('camera_effect_initialization_reject_not_captured'); + } + this.resolveInitialization = resolveInitialization; + this.rejectInitialization = rejectInitialization; + this.initializationTimeout = window.setTimeout(() => { + this.failInitialization(new Error('Camera effect worker initialization timed out')); + }, INITIALIZATION_TIMEOUT_MS); + worker.addEventListener('message', this.handleWorkerMessage); + worker.addEventListener('error', this.handleWorkerError); + worker.addEventListener('messageerror', this.handleWorkerMessageError); + } + + get outputTrack(): MediaStreamTrack { + if (this.selectedTrack == null) { + throw new Error('Camera effect output track requested before initialization'); + } + this.assertActive(); + return this.selectedTrack; + } + + get backend(): WebCameraEffectBackend { + if (this.selectedBackend == null) { + throw new Error('Camera effect backend requested before initialization'); + } + this.assertActive(); + return this.selectedBackend; + } + + static async create({ + source, + config, + onFailure, + }: WebCameraEffectPipelineCreateRequest): Promise { + if (source.kind !== VIDEO_TRACK_KIND) { + throw new Error('Camera effect pipeline requires a video track'); + } + const capability = detectWebCameraSegmentationCapability(); + if (!capability.available) { + throw new Error(capability.reason); + } + const settings = source.getSettings(); + let configuredFrameRate = settings.frameRate; + if (configuredFrameRate == null) { + configuredFrameRate = DEFAULT_OUTPUT_FRAME_RATE; + } + const frameRate = clampVideoFrameRate(configuredFrameRate); + let configuredWidth: number | null = null; + if (settings.width != null) { + configuredWidth = settings.width; + } + let configuredHeight: number | null = null; + if (settings.height != null) { + configuredHeight = settings.height; + } + const width = initialSurfaceDimension(configuredWidth); + const height = initialSurfaceDimension(configuredHeight); + validateCameraEffectFrameDimensions(width, height); + const GPUSurface = createCapturedSurface(frameRate, width, height); + let fallbackSurface: CapturedSurface; + try { + fallbackSurface = createCapturedSurface(frameRate, width, height); + } catch (error) { + throwAfterTrackCleanup( + error, + [GPUSurface.track], + 'Camera effect fallback surface creation failed during cleanup', + ); + } + let worker: Worker; + try { + worker = new Worker(new URL('./WebCameraEffectWorker.ts', import.meta.url), {type: 'module'}); + } catch (error) { + throwAfterTrackCleanup( + error, + [GPUSurface.track, fallbackSurface.track], + 'Camera effect worker creation failed during cleanup', + ); + } + const preferWebGPU = selectWebGPUPreference(config); + const pipeline = new WebCameraEffectPipeline( + worker, + GPUSurface.track, + fallbackSurface.track, + onFailure, + config, + preferWebGPU, + ); + let readable: ReadableStream | null = null; + try { + pipeline.currentVideoProducer = await createVideoFrameProducer(config); + readable = createTrackProcessor(source).readable; + const customBackgroundFrames = pipeline.currentVideoProducer?.readable ?? null; + const command: WebCameraEffectStartCommand = { + kind: WebCameraEffectCommandKind.START, + readable, + customBackgroundFrames, + gpuCanvas: GPUSurface.offscreen, + fallbackCanvas: fallbackSurface.offscreen, + config, + preferWebGPU, + }; + const transfer: Array = [readable, GPUSurface.offscreen, fallbackSurface.offscreen]; + if (customBackgroundFrames != null) { + transfer.push(customBackgroundFrames); + } + worker.postMessage(command, transfer); + } catch (error) { + const primaryError = new Error('Camera effect pipeline could not prepare or transfer its input streams', { + cause: error, + }); + let failures: ReadonlyArray = []; + if (readable != null) { + failures = await collectSettledFailures([readable.cancel(primaryError)]); + } + let initializationError: Error = primaryError; + if (failures.length > 0) { + initializationError = new AggregateError( + [primaryError, ...failures], + 'Camera effect pipeline initialization cleanup failed', + ); + } + pipeline.failInitialization(initializationError); + } + const initialization = await pipeline.initializationPromise; + if (initialization.fallbackErrorType != null) { + logger.debug('voice.camera_effect_backend.fallback_selected', { + errorType: initialization.fallbackErrorType, + }); + } else { + logger.info('voice.camera_effect_worker.initialized', {backend: initialization.backend}); + } + return pipeline; + } + + assertActive(): void { + if (this.operationalFailure != null) { + throw this.operationalFailure; + } + if (this.stopped || !this.initialized) { + throw new Error('Camera effect pipeline is inactive'); + } + } + + async updateConfig(config: WebCameraPipelineConfig): Promise { + this.assertActive(); + if (this.updatePreparationActive || this.pendingUpdate != null) { + throw new Error('Camera effect configuration update is already in progress'); + } + this.updatePreparationActive = true; + let candidateVideoProducer: WebCameraEffectVideoFrameProducer | null = null; + let ownsCandidateVideoProducer = false; + try { + if (hasSameVideoBackground(this.currentConfig, config)) { + candidateVideoProducer = this.currentVideoProducer; + } else { + candidateVideoProducer = await createVideoFrameProducer(config); + ownsCandidateVideoProducer = candidateVideoProducer != null; + } + this.assertActive(); + } catch (error) { + if (ownsCandidateVideoProducer) { + candidateVideoProducer?.stop(); + } + this.updatePreparationActive = false; + throw error; + } + const requestId = this.takeUpdateRequestId(); + let resolveUpdate: (() => void) | null = null; + let rejectUpdate: ((error: Error) => void) | null = null; + const completion = new Promise((resolve, reject) => { + resolveUpdate = resolve; + rejectUpdate = reject; + }); + if (resolveUpdate == null || rejectUpdate == null) { + if (ownsCandidateVideoProducer) { + candidateVideoProducer?.stop(); + } + this.updatePreparationActive = false; + throw new Error('Camera effect update promise handlers were not captured'); + } + const timeout = window.setTimeout(() => { + const timeoutError = new Error('Camera effect configuration update timed out'); + this.rejectPendingUpdate(timeoutError); + this.stopAfterWorkerFailure(timeoutError); + }, UPDATE_TIMEOUT_MS); + this.pendingUpdate = { + requestId, + config, + candidateVideoProducer, + ownsCandidateVideoProducer, + resolve: resolveUpdate, + reject: rejectUpdate, + timeout, + }; + this.updatePreparationActive = false; + const customBackgroundFrames = ownsCandidateVideoProducer ? (candidateVideoProducer?.readable ?? null) : null; + const command: WebCameraEffectUpdateCommand = { + kind: WebCameraEffectCommandKind.UPDATE, + requestId, + config, + customBackgroundFrames, + }; + try { + const transfer: Array = []; + if (customBackgroundFrames != null) { + transfer.push(customBackgroundFrames); + } + this.worker.postMessage(command, transfer); + } catch (error) { + const updateError = new Error('Camera effect worker could not receive its updated configuration', { + cause: error, + }); + this.rejectPendingUpdate(updateError); + this.stopAfterWorkerFailure(updateError); + } + return completion; + } + + private takeUpdateRequestId(): number { + const requestId = this.nextUpdateRequestId; + this.nextUpdateRequestId += 1; + if (this.nextUpdateRequestId > WEB_CAMERA_EFFECT_UPDATE_REQUEST_ID_MAX) { + this.nextUpdateRequestId = 1; + } + return requestId; + } + + private completeUpdate(requestId: number): void { + const pending = this.pendingUpdate; + if (pending == null || pending.requestId !== requestId) { + this.stopAfterWorkerFailure(new Error('Camera effect worker acknowledged an unknown update')); + return; + } + window.clearTimeout(pending.timeout); + this.pendingUpdate = null; + const previousVideoProducer = this.currentVideoProducer; + this.currentConfig = pending.config; + this.currentVideoProducer = pending.candidateVideoProducer; + pending.resolve(); + if (previousVideoProducer !== pending.candidateVideoProducer) { + try { + previousVideoProducer?.stop(); + } catch (error) { + const cleanupError = new Error('Camera effect update committed but previous video source cleanup failed', { + cause: error, + }); + this.stopAfterWorkerFailure(cleanupError); + return; + } + } + } + + private rejectUpdate(event: WebCameraEffectUpdateFailedEvent): void { + const pending = this.pendingUpdate; + if (pending == null || pending.requestId !== event.requestId) { + this.stopAfterWorkerFailure(new Error('Camera effect worker rejected an unknown update')); + return; + } + this.rejectPendingUpdate(createWorkerReportedError(event, 'Camera effect configuration update failed')); + } + + private rejectPendingUpdate(error: Error): void { + const pending = this.pendingUpdate; + if (pending == null) { + return; + } + window.clearTimeout(pending.timeout); + this.pendingUpdate = null; + let rejection = error; + if (pending.ownsCandidateVideoProducer) { + try { + pending.candidateVideoProducer?.stop(); + } catch (cleanupError) { + rejection = new AggregateError([error, cleanupError], 'Camera effect update rejection cleanup failed'); + } + } + pending.reject(rejection); + } + + beginStop(): void { + if (this.stopIntent) { + return; + } + this.stopIntent = true; + const failures: Array = []; + const operations = [ + () => { + if (this.selectedTrack != null) { + this.selectedTrack.removeEventListener('ended', this.handleOutputTrackEnded); + } + }, + () => this.worker.postMessage({kind: WebCameraEffectCommandKind.STOP}), + ]; + for (const operation of operations) { + try { + operation(); + } catch (error) { + failures.push(error); + } + } + throwCollectedFailures({failures, message: 'Camera effect pipeline stop intent failed'}); + } + + stop(): void { + if (this.stopped) { + return; + } + if (!this.initialized) { + this.failInitialization(new Error('Camera effect pipeline stopped during initialization')); + return; + } + this.stopped = true; + const failures: Array = []; + const operations = [ + () => this.beginStop(), + () => this.rejectPendingUpdate(new Error('Camera effect pipeline stopped during configuration update')), + () => { + this.currentVideoProducer?.stop(); + this.currentVideoProducer = null; + }, + () => this.gpuTrack.stop(), + () => this.fallbackTrack.stop(), + () => { + this.shutdownTimeout = window.setTimeout(() => { + try { + this.terminateWorker(); + } catch (error) { + logger.error('voice.camera_effect_worker_termination.failed', { + errorType: getErrorDiagnosticType(error), + }); + } + }, SHUTDOWN_TIMEOUT_MS); + }, + ]; + for (const operation of operations) { + try { + operation(); + } catch (error) { + failures.push(error); + } + } + throwCollectedFailures({failures, message: 'Camera effect pipeline shutdown failed'}); + } + + private readonly handleWorkerMessage = (event: MessageEvent): void => { + let message: WebCameraEffectWorkerEvent; + try { + message = parseCameraEffectWorkerEvent(event.data); + } catch (error) { + const eventError = new Error('Camera effect worker emitted an invalid event', {cause: error}); + if (!this.initialized) { + this.failInitialization(eventError); + return; + } + logger.error('voice.camera_effect_worker_event_invalid.rejected', {errorType: getErrorDiagnosticType(error)}); + this.stopAfterWorkerFailure(eventError); + return; + } + if (message.kind === WebCameraEffectEventKind.READY) { + this.completeInitialization(message); + return; + } + if (message.kind === WebCameraEffectEventKind.UPDATED) { + this.completeUpdate(message.requestId); + return; + } + if (message.kind === WebCameraEffectEventKind.UPDATE_FAILED) { + this.rejectUpdate(message); + return; + } + if (message.kind === WebCameraEffectEventKind.FAILED) { + const error = createWorkerReportedError(message, 'Camera effect worker failed'); + if (!this.initialized) { + this.failInitialization(error); + return; + } + if (this.stopped) { + return; + } + this.rejectPendingUpdate(error); + if (this.stopIntent) { + logger.debug('voice.camera_effect_worker_failed_after_stop_intent.ignored', { + errorType: message.errorType, + }); + this.stop(); + return; + } + logger.error('voice.camera_effect_worker.failed', { + errorType: message.errorType, + message: message.message, + stack: message.stack, + }); + this.stopAfterWorkerFailure(error); + return; + } + if (!this.initialized) { + this.failInitialization(new Error('Camera effect worker stopped during initialization')); + return; + } + if (message.reason === WebCameraEffectShutdownReason.CLEANUP_FAILED) { + logger.warn('voice.camera_effect_worker_cleanup.incomplete', { + errorType: message.diagnostic?.errorType ?? null, + message: message.diagnostic?.message ?? null, + stack: message.diagnostic?.stack ?? null, + }); + } else if (message.reason !== WebCameraEffectShutdownReason.OWNER_STOP && !this.stopIntent && !this.stopped) { + const error = new Error('Camera effect worker stopped unexpectedly'); + logger.error('voice.camera_effect_worker_stopped_unexpectedly.failed', {reason: message.reason}); + this.stopAfterWorkerFailure(error); + return; + } + try { + this.terminateWorker(); + } catch (error) { + logger.error('voice.camera_effect_worker_termination.failed', { + errorType: getErrorDiagnosticType(error), + }); + } + }; + + private readonly handleWorkerError = (event: ErrorEvent): void => { + const location = event.filename ? ` (${event.filename}:${event.lineno}:${event.colno})` : ''; + const detail = event.message ? `: ${event.message}${location}` : location; + const error = new Error(`Camera effect worker raised an error${detail}`, { + cause: event.error ?? undefined, + }); + if (!this.initialized) { + this.failInitialization(error); + return; + } + if (!this.stopped) { + logger.error('voice.camera_effect_worker.failed', { + errorType: getErrorDiagnosticType(event.error), + message: event.message, + filename: event.filename, + lineno: event.lineno, + colno: event.colno, + }); + this.stopAfterWorkerFailure(error); + } + }; + + private readonly handleWorkerMessageError = (): void => { + const error = new Error('Camera effect worker message could not be deserialized'); + if (!this.initialized) { + this.failInitialization(error); + return; + } + if (!this.stopped) { + logger.error('voice.camera_effect_worker_message.failed'); + this.stopAfterWorkerFailure(error); + } + }; + + private readonly handleOutputTrackEnded = (): void => { + if (!this.initialized || this.stopped) { + return; + } + this.stopAfterWorkerFailure(new Error('Camera effect output track ended unexpectedly')); + }; + + private completeInitialization(result: InitializationResult): void { + if (this.initialized) { + logger.error('voice.camera_effect_worker_sent_more_than_one_ready_event.failed'); + this.stopAfterWorkerFailure(new Error('Camera effect worker sent more than one ready event')); + return; + } + if (this.stopped) { + return; + } + recordWebGPUSelection(this.preferWebGPU, result.backend); + let selectedTrack = this.fallbackTrack; + let unusedTrack = this.gpuTrack; + if (result.backend === WebCameraEffectBackend.WEB_GPU) { + selectedTrack = this.gpuTrack; + unusedTrack = this.fallbackTrack; + } + try { + unusedTrack.stop(); + selectedTrack.addEventListener('ended', this.handleOutputTrackEnded, {once: true}); + } catch (error) { + this.failInitialization( + new Error('Camera effect pipeline could not release its unused output track', {cause: error}), + ); + return; + } + this.initialized = true; + this.selectedBackend = result.backend; + this.selectedTrack = selectedTrack; + this.clearInitializationTimeout(); + this.resolveInitialization(result); + } + + private stopAfterWorkerFailure(error: Error): void { + if (this.operationalFailure != null || this.stopped) { + return; + } + this.operationalFailure = error; + const failures: Array = []; + try { + this.onFailure(this, error); + } catch (callbackError) { + failures.push(callbackError); + } + try { + this.stop(); + } catch (cleanupError) { + failures.push(cleanupError); + } + if (failures.length > 0) { + const failure = new AggregateError( + [error, ...failures], + 'Camera effect worker failure notification or cleanup failed', + ); + logger.error('voice.camera_effect_worker_failure_cleanup.failed', { + errorType: getErrorDiagnosticType(failure), + }); + } + } + + private failInitialization(error: Error): void { + if (this.initialized || this.stopped) { + return; + } + this.stopped = true; + this.clearInitializationTimeout(); + releaseWebGPUProbe(this.preferWebGPU); + const failures: Array = [error]; + const cleanup = [ + () => { + this.currentVideoProducer?.stop(); + this.currentVideoProducer = null; + }, + () => this.gpuTrack.stop(), + () => this.fallbackTrack.stop(), + () => this.terminateWorker(), + ]; + for (const operation of cleanup) { + try { + operation(); + } catch (cleanupError) { + failures.push(cleanupError); + } + } + if (failures.length === 1) { + this.rejectInitialization(error); + return; + } + this.rejectInitialization( + new AggregateError(failures, 'Camera effect pipeline initialization failed during cleanup'), + ); + } + + private clearInitializationTimeout(): void { + if (this.initializationTimeout != null) { + window.clearTimeout(this.initializationTimeout); + this.initializationTimeout = null; + } + } + + private terminateWorker(): void { + if (this.shutdownTimeout != null) { + window.clearTimeout(this.shutdownTimeout); + this.shutdownTimeout = null; + } + const failures: Array = []; + const cleanup = [ + () => this.worker.removeEventListener('message', this.handleWorkerMessage), + () => this.worker.removeEventListener('error', this.handleWorkerError), + () => this.worker.removeEventListener('messageerror', this.handleWorkerMessageError), + () => this.worker.terminate(), + ]; + for (const operation of cleanup) { + try { + operation(); + } catch (error) { + failures.push(error); + } + } + throwCollectedFailures({failures, message: 'Camera effect worker termination failed'}); + } +} diff --git a/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectProtocol.ts b/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectProtocol.ts new file mode 100644 index 000000000..6ff4c77a2 --- /dev/null +++ b/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectProtocol.ts @@ -0,0 +1,327 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {CameraBackgroundMode} from '@app/features/voice/utils/camera-effects/CameraCaptureContract'; +import type {ErrorDiagnostic, ErrorDiagnosticType} from '@app/features/voice/utils/camera-effects/ErrorDiagnostic'; + +class InvalidCameraEffectFrameDimensionsError extends Error { + constructor() { + super('Camera effect frame dimensions must be positive safe integers'); + this.name = 'InvalidCameraEffectFrameDimensionsError'; + } +} + +class CameraEffectFrameDimensionsExceededError extends Error { + constructor() { + super('Camera effect frame dimensions exceed the bounded working set'); + this.name = 'CameraEffectFrameDimensionsExceededError'; + } +} + +export interface WebCameraEffectBlurConfig { + readonly mode: typeof CameraBackgroundMode.BLUR; + readonly blurStrength: number; +} + +export interface WebCameraEffectCustomConfig { + readonly mode: typeof CameraBackgroundMode.CUSTOM; + readonly blurStrength: number; + readonly customMediaURL: string; + readonly customMediaKind: WebCameraEffectCustomMediaKind; +} + +export type WebCameraEffectConfig = WebCameraEffectBlurConfig | WebCameraEffectCustomConfig; + +export interface WebCameraPipelineConfig { + readonly background: WebCameraEffectConfig | null; +} + +export const WEB_CAMERA_EFFECT_SEGMENTATION_MIN_INTERVAL_MS = 50; + +export const WEB_CAMERA_EFFECT_STOP_GRACE_MS = 250; + +const MIN_BLUR_PIXELS = 4; +const MAX_BLUR_PIXELS = 20; +const WEB_CAMERA_EFFECT_BLUR_STRENGTH_MIN = 0; +const WEB_CAMERA_EFFECT_BLUR_STRENGTH_MAX = 100; +const MAX_FRAME_EDGE = 4096; +const MAX_FRAME_PIXELS = 3840 * 2160; + +export function requireCameraEffectBlurStrength(strength: number): number { + if (!Number.isFinite(strength)) { + throw new Error('Camera effect blur strength must be finite'); + } + if (strength < WEB_CAMERA_EFFECT_BLUR_STRENGTH_MIN) { + throw new Error('Camera effect blur strength is below the supported range'); + } + if (strength > WEB_CAMERA_EFFECT_BLUR_STRENGTH_MAX) { + throw new Error('Camera effect blur strength exceeds the supported range'); + } + return strength; +} + +export function cameraEffectBlurPixels(strength: number): number { + const validatedStrength = requireCameraEffectBlurStrength(strength); + const range = MAX_BLUR_PIXELS - MIN_BLUR_PIXELS; + return Math.round(MIN_BLUR_PIXELS + (validatedStrength / WEB_CAMERA_EFFECT_BLUR_STRENGTH_MAX) * range); +} + +export function validateCameraEffectFrameDimensions(width: number, height: number): void { + if (!Number.isSafeInteger(width)) { + throw new InvalidCameraEffectFrameDimensionsError(); + } + if (width <= 0) { + throw new InvalidCameraEffectFrameDimensionsError(); + } + if (!Number.isSafeInteger(height)) { + throw new InvalidCameraEffectFrameDimensionsError(); + } + if (height <= 0) { + throw new InvalidCameraEffectFrameDimensionsError(); + } + if (width > MAX_FRAME_EDGE) { + throw new CameraEffectFrameDimensionsExceededError(); + } + if (height > MAX_FRAME_EDGE) { + throw new CameraEffectFrameDimensionsExceededError(); + } + if (width * height > MAX_FRAME_PIXELS) { + throw new CameraEffectFrameDimensionsExceededError(); + } +} + +export const WebCameraEffectBackend = Object.freeze({ + WEB_GPU: 'webgpu', + WASM_WORKER: 'wasm-worker', + CANVAS_WORKER: 'canvas-worker', +} as const); + +export type WebCameraEffectBackend = (typeof WebCameraEffectBackend)[keyof typeof WebCameraEffectBackend]; + +export const WebCameraEffectCommandKind = Object.freeze({ + START: 'start', + UPDATE: 'update', + STOP: 'stop', +} as const); + +export type WebCameraEffectCommandKind = (typeof WebCameraEffectCommandKind)[keyof typeof WebCameraEffectCommandKind]; + +export const WebCameraEffectEventKind = Object.freeze({ + READY: 'ready', + UPDATED: 'updated', + UPDATE_FAILED: 'update_failed', + FAILED: 'failed', + STOPPED: 'stopped', +} as const); + +export type WebCameraEffectEventKind = (typeof WebCameraEffectEventKind)[keyof typeof WebCameraEffectEventKind]; + +export const WebCameraEffectShutdownReason = Object.freeze({ + OWNER_STOP: 'owner-stop', + INPUT_ENDED: 'input-ended', + OPERATION_FAILED: 'operation-failed', + CLEANUP_FAILED: 'cleanup-failed', +} as const); + +export type WebCameraEffectShutdownReason = + (typeof WebCameraEffectShutdownReason)[keyof typeof WebCameraEffectShutdownReason]; + +export function isWebCameraEffectShutdownReason(value: unknown): value is WebCameraEffectShutdownReason { + if (value === WebCameraEffectShutdownReason.OWNER_STOP) { + return true; + } + if (value === WebCameraEffectShutdownReason.INPUT_ENDED) { + return true; + } + if (value === WebCameraEffectShutdownReason.OPERATION_FAILED) { + return true; + } + return value === WebCameraEffectShutdownReason.CLEANUP_FAILED; +} + +export const WebCameraEffectCustomMediaKind = Object.freeze({ + STATIC: 'static', + ANIMATED: 'animated', + VIDEO: 'video', +} as const); + +export type WebCameraEffectCustomMediaKind = + (typeof WebCameraEffectCustomMediaKind)[keyof typeof WebCameraEffectCustomMediaKind]; + +export function isWebCameraEffectCustomMediaKind(value: unknown): value is WebCameraEffectCustomMediaKind { + if (value === WebCameraEffectCustomMediaKind.STATIC) { + return true; + } + if (value === WebCameraEffectCustomMediaKind.ANIMATED) { + return true; + } + return value === WebCameraEffectCustomMediaKind.VIDEO; +} + +export const WEB_CAMERA_EFFECT_UPDATE_REQUEST_ID_MAX = 2_147_483_647; + +function isWebCameraPipelineConfig(value: unknown): value is WebCameraPipelineConfig { + if (value == null || typeof value !== 'object') { + return false; + } + const background = Reflect.get(value, 'background'); + if (background === null) { + return true; + } + if (background == null || typeof background !== 'object') { + return false; + } + const mode = Reflect.get(background, 'mode'); + const blurStrength = Reflect.get(background, 'blurStrength'); + if (typeof blurStrength !== 'number') { + return false; + } + try { + requireCameraEffectBlurStrength(blurStrength); + } catch { + return false; + } + if (mode === CameraBackgroundMode.BLUR) { + return true; + } + if (mode !== CameraBackgroundMode.CUSTOM) { + return false; + } + const customMediaURL = Reflect.get(background, 'customMediaURL'); + if (typeof customMediaURL !== 'string' || customMediaURL.length === 0) { + return false; + } + return isWebCameraEffectCustomMediaKind(Reflect.get(background, 'customMediaKind')); +} + +function hasExpectedCustomBackgroundFrames( + config: WebCameraPipelineConfig, + frames: unknown, + requireVideoFrames: boolean, +): frames is ReadableStream | null { + const background = config.background; + if (background == null || background.mode !== CameraBackgroundMode.CUSTOM) { + return frames === null; + } + if (background.customMediaKind !== WebCameraEffectCustomMediaKind.VIDEO) { + return frames === null; + } + if (!requireVideoFrames && frames === null) { + return true; + } + return frames instanceof ReadableStream; +} + +function isValidStartCommand(value: object): value is WebCameraEffectStartCommand { + if (typeof Reflect.get(value, 'preferWebGPU') !== 'boolean') { + return false; + } + const config = Reflect.get(value, 'config'); + if (!isWebCameraPipelineConfig(config)) { + return false; + } + if (!(Reflect.get(value, 'readable') instanceof ReadableStream)) { + return false; + } + if (!(Reflect.get(value, 'gpuCanvas') instanceof OffscreenCanvas)) { + return false; + } + if (!(Reflect.get(value, 'fallbackCanvas') instanceof OffscreenCanvas)) { + return false; + } + return hasExpectedCustomBackgroundFrames(config, Reflect.get(value, 'customBackgroundFrames'), true); +} + +function isValidUpdateRequestId(value: unknown): value is number { + if (!Number.isSafeInteger(value)) { + return false; + } + if ((value as number) <= 0) { + return false; + } + return (value as number) <= WEB_CAMERA_EFFECT_UPDATE_REQUEST_ID_MAX; +} + +function isValidUpdateCommand(value: object): value is WebCameraEffectUpdateCommand { + if (!isValidUpdateRequestId(Reflect.get(value, 'requestId'))) { + return false; + } + const config = Reflect.get(value, 'config'); + if (!isWebCameraPipelineConfig(config)) { + return false; + } + return hasExpectedCustomBackgroundFrames(config, Reflect.get(value, 'customBackgroundFrames'), false); +} + +export interface WebCameraEffectStartCommand { + readonly kind: typeof WebCameraEffectCommandKind.START; + readonly readable: ReadableStream; + readonly customBackgroundFrames: ReadableStream | null; + readonly gpuCanvas: OffscreenCanvas; + readonly fallbackCanvas: OffscreenCanvas; + readonly config: WebCameraPipelineConfig; + readonly preferWebGPU: boolean; +} + +export interface WebCameraEffectUpdateCommand { + readonly kind: typeof WebCameraEffectCommandKind.UPDATE; + readonly requestId: number; + readonly config: WebCameraPipelineConfig; + readonly customBackgroundFrames: ReadableStream | null; +} + +export interface WebCameraEffectStopCommand { + readonly kind: typeof WebCameraEffectCommandKind.STOP; +} + +export type WebCameraEffectWorkerCommand = + | WebCameraEffectStartCommand + | WebCameraEffectUpdateCommand + | WebCameraEffectStopCommand; + +export const WebCameraEffectCommandPolicy = Object.freeze({ + isValid(value: object): value is WebCameraEffectWorkerCommand { + switch (Reflect.get(value, 'kind')) { + case WebCameraEffectCommandKind.START: + return isValidStartCommand(value); + case WebCameraEffectCommandKind.UPDATE: + return isValidUpdateCommand(value); + case WebCameraEffectCommandKind.STOP: + return true; + default: + return false; + } + }, +}); + +export interface WebCameraEffectReadyEvent { + readonly kind: typeof WebCameraEffectEventKind.READY; + readonly backend: WebCameraEffectBackend; + readonly fallbackErrorType: ErrorDiagnosticType | null; +} + +export interface WebCameraEffectUpdatedEvent { + readonly kind: typeof WebCameraEffectEventKind.UPDATED; + readonly requestId: number; +} + +export interface WebCameraEffectUpdateFailedEvent extends ErrorDiagnostic { + readonly kind: typeof WebCameraEffectEventKind.UPDATE_FAILED; + readonly requestId: number; +} + +export interface WebCameraEffectFailedEvent extends ErrorDiagnostic { + readonly kind: typeof WebCameraEffectEventKind.FAILED; +} + +export interface WebCameraEffectStoppedEvent { + readonly kind: typeof WebCameraEffectEventKind.STOPPED; + readonly reason: WebCameraEffectShutdownReason; + readonly diagnostic: ErrorDiagnostic | null; +} + +export type WebCameraEffectWorkerEvent = + | WebCameraEffectReadyEvent + | WebCameraEffectUpdatedEvent + | WebCameraEffectUpdateFailedEvent + | WebCameraEffectFailedEvent + | WebCameraEffectStoppedEvent; diff --git a/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectRenderer.ts b/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectRenderer.ts new file mode 100644 index 000000000..929714d4d --- /dev/null +++ b/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectRenderer.ts @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import type {WebCameraEffectCustomFrameSource} from '@app/features/voice/utils/camera-effects/WebCameraEffectCustomImage'; +import type { + WebCameraEffectBackend, + WebCameraPipelineConfig, +} from '@app/features/voice/utils/camera-effects/WebCameraEffectProtocol'; + +export interface WebCameraEffectRenderer { + readonly backend: WebCameraEffectBackend; + configure(config: WebCameraPipelineConfig, customFrameSource: WebCameraEffectCustomFrameSource | null): Promise; + render(frame: VideoFrame, now: number): Promise; + dispose(): Promise; +} diff --git a/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectSegmentationOwner.ts b/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectSegmentationOwner.ts new file mode 100644 index 000000000..71fee2531 --- /dev/null +++ b/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectSegmentationOwner.ts @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import invariant from 'tiny-invariant'; + +const WEB_CAMERA_EFFECT_OUTPUT_FRAMES_BEHIND_SEGMENTATION_MAX = 2; + +interface MutableWebCameraEffectSegmentationPhysicalOperation { + readonly lifecycle: number; + admittedOutputFrames: number; + settlement: Promise; +} + +export class WebCameraEffectSegmentationOwner { + private lifecycle = 0; + private physicalOperation: MutableWebCameraEffectSegmentationPhysicalOperation | null = null; + private deferredFailure: unknown | null = null; + + canStartPhysicalOperation(): boolean { + return this.physicalOperation == null; + } + + advanceLifecycle(): void { + const nextLifecycle = this.lifecycle + 1; + invariant(Number.isSafeInteger(nextLifecycle), 'WebGPU camera segmentation lifecycle must stay safe'); + this.lifecycle = nextLifecycle; + } + + startPhysicalOperation(operation: Promise, publishCurrentCompletion: () => void): void { + invariant(this.physicalOperation == null, 'WebGPU camera segmentation physical operation overlapped'); + invariant(this.deferredFailure == null, 'WebGPU camera segmentation failure must be observed before restart'); + const physicalOperation: MutableWebCameraEffectSegmentationPhysicalOperation = { + lifecycle: this.lifecycle, + admittedOutputFrames: 0, + settlement: Promise.resolve(), + }; + this.physicalOperation = physicalOperation; + physicalOperation.settlement = operation.then( + () => this.completeSuccessfulPhysicalOperation(physicalOperation, publishCurrentCompletion), + (error: unknown) => this.completeFailedPhysicalOperation(physicalOperation, error), + ); + } + + admitOutputFrame(): Promise | null { + const physicalOperation = this.physicalOperation; + if (physicalOperation == null) { + return null; + } + if (physicalOperation.admittedOutputFrames < WEB_CAMERA_EFFECT_OUTPUT_FRAMES_BEHIND_SEGMENTATION_MAX) { + physicalOperation.admittedOutputFrames += 1; + return null; + } + return this.waitForOutputFrameAdmission(physicalOperation); + } + + private async waitForOutputFrameAdmission( + physicalOperation: MutableWebCameraEffectSegmentationPhysicalOperation, + ): Promise { + await physicalOperation.settlement; + this.requireNoDeferredFailure(); + } + + async settlePhysicalOperation(): Promise { + const physicalOperation = this.physicalOperation; + if (physicalOperation != null) { + await physicalOperation.settlement; + } + this.requireNoDeferredFailure(); + } + + async settleForDisposal(): Promise> { + const failures: Array = []; + const physicalOperation = this.physicalOperation; + if (physicalOperation != null) { + try { + await physicalOperation.settlement; + } catch (error) { + failures.push(error); + } + } + const failure = this.deferredFailure; + this.deferredFailure = null; + if (failure != null) { + failures.push(failure); + } + return failures; + } + + requireNoDeferredFailure(): void { + const failure = this.deferredFailure; + if (failure == null) { + return; + } + this.deferredFailure = null; + throw failure; + } + + private completeSuccessfulPhysicalOperation( + operation: MutableWebCameraEffectSegmentationPhysicalOperation, + publishCurrentCompletion: () => void, + ): void { + invariant(this.physicalOperation === operation, 'WebGPU camera segmentation physical ownership changed'); + this.physicalOperation = null; + if (operation.lifecycle !== this.lifecycle) { + return; + } + try { + publishCurrentCompletion(); + } catch (error) { + this.recordDeferredFailure(error); + } + } + + private completeFailedPhysicalOperation( + operation: MutableWebCameraEffectSegmentationPhysicalOperation, + failure: unknown, + ): void { + invariant(this.physicalOperation === operation, 'WebGPU camera segmentation physical ownership changed'); + this.physicalOperation = null; + this.recordDeferredFailure(failure); + } + + private recordDeferredFailure(failure: unknown): void { + let normalizedFailure = failure; + if (normalizedFailure == null) { + normalizedFailure = new Error('WebGPU camera segmentation failed without an error value'); + } + invariant(this.deferredFailure == null, 'WebGPU camera segmentation owner retained multiple failures'); + this.deferredFailure = normalizedFailure; + } +} diff --git a/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectVideoFrameSource.ts b/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectVideoFrameSource.ts new file mode 100644 index 000000000..dde4355c9 --- /dev/null +++ b/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectVideoFrameSource.ts @@ -0,0 +1,428 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {runWithResponseDeadline} from '@app/features/voice/utils/camera-effects/BoundedResponse'; +import { + readWebCameraEffectCustomMediaBlob, + requireWebCameraEffectCustomMediaURL, + WEB_CAMERA_EFFECT_CUSTOM_MEDIA_OPERATION_TIMEOUT_MS, +} from '@app/features/voice/utils/camera-effects/WebCameraEffectCustomImage'; +import {validateCameraEffectFrameDimensions} from '@app/features/voice/utils/camera-effects/WebCameraEffectProtocol'; + +const MAX_CUSTOM_VIDEO_DURATION_SECONDS = 60; +const SUPPORTED_VIDEO_MEDIA_TYPES = new Set(['video/mp4', 'video/webm']); +const VIDEO_HEADER_SNIFF_BYTES = 128; +const MP4_FILE_TYPE_BOX_OFFSET = 4; +const MP4_FILE_TYPE_BOX: ReadonlyArray = [0x66, 0x74, 0x79, 0x70]; +const EBML_MAGIC: ReadonlyArray = [0x1a, 0x45, 0xdf, 0xa3]; +const EBML_DOC_TYPE_ELEMENT_ID: ReadonlyArray = [0x42, 0x82]; +const EBML_VINT_SINGLE_BYTE_MARKER = 0x80; +const EBML_VINT_SINGLE_BYTE_VALUE_MASK = 0x7f; +const MAX_EBML_DOC_TYPE_LENGTH = 16; +const WEBM_DOC_TYPE = 'webm'; + +export interface WebCameraEffectVideoFrameProducer { + readonly readable: ReadableStream; + readonly width: number; + readonly height: number; + stop(): void; +} + +class WebCameraEffectVideoFrameProducerOwner implements WebCameraEffectVideoFrameProducer { + readonly readable: ReadableStream; + private readonly video: HTMLVideoElement; + private readonly objectURL: string; + private controller: ReadableStreamDefaultController | null = null; + private pendingFrame: VideoFrame | null = null; + private pullResolve: (() => void) | null = null; + private callbackHandle: number | null = null; + private started = false; + private stopped = false; + private videoWidth = 0; + private videoHeight = 0; + + constructor(blob: Blob) { + this.video = document.createElement('video'); + this.video.autoplay = true; + this.video.crossOrigin = 'anonymous'; + this.video.loop = true; + this.video.muted = true; + this.video.playsInline = true; + this.video.preload = 'auto'; + this.readable = new ReadableStream( + { + start: (controller): void => { + this.controller = controller; + }, + pull: (controller) => this.pullFrame(controller), + cancel: () => this.cancelFromStream(), + }, + {highWaterMark: 0}, + ); + this.objectURL = URL.createObjectURL(blob); + } + + get width(): number { + return this.videoWidth; + } + + get height(): number { + return this.videoHeight; + } + + async start(signal: AbortSignal): Promise { + if (this.started) { + throw new Error('Custom camera background video producer was started more than once'); + } + this.started = true; + if (typeof this.video.requestVideoFrameCallback !== 'function') { + throw new Error('Custom camera background video requires requestVideoFrameCallback'); + } + if (!('VideoFrame' in globalThis)) { + throw new Error('Custom camera background video requires VideoFrame'); + } + this.video.src = this.objectURL; + this.video.load(); + await waitForVideoMetadata(this.video, signal); + this.validateMetadata(); + this.video.addEventListener('error', this.handlePlaybackError); + this.video.addEventListener('ended', this.handleUnexpectedEnd); + await waitForAbortablePromise(this.video.play(), signal); + throwIfAborted(signal); + this.scheduleNextFrame(); + } + + stop(): void { + const failures = this.releaseOwnedResources(true); + throwCleanupFailures(failures, 'Custom camera background video producer teardown failed'); + } + + private pullFrame(controller: ReadableStreamDefaultController): void | Promise { + if (this.stopped) return; + if (this.pendingFrame != null) { + const frame = this.pendingFrame; + this.pendingFrame = null; + controller.enqueue(frame); + return; + } + if (this.pullResolve != null) { + throw new Error('Custom camera background video stream has more than one pending pull'); + } + return new Promise((resolve) => { + this.pullResolve = resolve; + }); + } + + private cancelFromStream(): void { + const failures = this.releaseOwnedResources(false); + throwCleanupFailures(failures, 'Custom camera background video stream cancellation failed'); + } + + private validateMetadata(): void { + validateCameraEffectFrameDimensions(this.video.videoWidth, this.video.videoHeight); + if (!Number.isFinite(this.video.duration) || this.video.duration <= 0) { + throw new Error('Custom camera background video has an invalid duration'); + } + if (this.video.duration > MAX_CUSTOM_VIDEO_DURATION_SECONDS) { + throw new Error('Custom camera background video exceeds the supported duration'); + } + this.videoWidth = this.video.videoWidth; + this.videoHeight = this.video.videoHeight; + } + + private scheduleNextFrame(): void { + if (this.stopped) return; + try { + this.callbackHandle = this.video.requestVideoFrameCallback(this.handleVideoFrame); + } catch (error) { + this.fail(error); + } + } + + private readonly handleVideoFrame: VideoFrameRequestCallback = (_now, metadata): void => { + this.callbackHandle = null; + if (this.stopped) return; + let frame: VideoFrame | null = null; + try { + if (!Number.isFinite(metadata.mediaTime) || metadata.mediaTime < 0) { + throw new Error('Custom camera background video produced an invalid frame timestamp'); + } + const timestamp = Math.round(metadata.mediaTime * 1_000_000); + if (!Number.isSafeInteger(timestamp)) { + throw new Error('Custom camera background video frame timestamp exceeds the safe range'); + } + frame = new VideoFrame(this.video, {timestamp}); + validateCameraEffectFrameDimensions(frame.displayWidth, frame.displayHeight); + this.publishFrame(frame); + frame = null; + } catch (error) { + frame?.close(); + this.fail(error); + return; + } + this.scheduleNextFrame(); + }; + + private publishFrame(frame: VideoFrame): void { + const controller = this.controller; + if (controller == null) { + throw new Error('Custom camera background video stream has no controller'); + } + if (this.pullResolve != null) { + const resolve = this.pullResolve; + this.pullResolve = null; + controller.enqueue(frame); + resolve(); + return; + } + const replaced = this.pendingFrame; + this.pendingFrame = frame; + replaced?.close(); + } + + private readonly handlePlaybackError = (): void => { + this.fail(videoElementError(this.video)); + }; + + private readonly handleUnexpectedEnd = (): void => { + this.fail(new Error('Custom camera background video playback ended unexpectedly')); + }; + + private fail(error: unknown): void { + if (this.stopped) return; + const failures = this.releaseOwnedResources(false); + const reportedError = + failures.length === 0 + ? error + : new AggregateError([error, ...failures], 'Custom camera background video failed during cleanup'); + this.controller?.error(reportedError); + } + + private releaseOwnedResources(closeStream: boolean): Array { + if (this.stopped) return []; + this.stopped = true; + const failures: Array = []; + this.cancelFrameCallback(failures); + this.closePendingFrame(failures); + this.resolvePendingPull(); + this.releaseVideoElement(failures); + if (closeStream) { + try { + this.controller?.close(); + } catch (error) { + failures.push(error); + } + } + return failures; + } + + private cancelFrameCallback(failures: Array): void { + if (this.callbackHandle == null) return; + try { + this.video.cancelVideoFrameCallback(this.callbackHandle); + } catch (error) { + failures.push(error); + } + this.callbackHandle = null; + } + + private closePendingFrame(failures: Array): void { + const pendingFrame = this.pendingFrame; + this.pendingFrame = null; + if (pendingFrame == null) return; + try { + pendingFrame.close(); + } catch (error) { + failures.push(error); + } + } + + private resolvePendingPull(): void { + const resolve = this.pullResolve; + this.pullResolve = null; + resolve?.(); + } + + private releaseVideoElement(failures: Array): void { + this.video.removeEventListener('error', this.handlePlaybackError); + this.video.removeEventListener('ended', this.handleUnexpectedEnd); + try { + this.video.pause(); + this.video.removeAttribute('src'); + this.video.load(); + } catch (error) { + failures.push(error); + } + try { + URL.revokeObjectURL(this.objectURL); + } catch (error) { + failures.push(error); + } + } +} + +function normalizedMediaType(value: string): string { + return value.split(';', 1)[0]?.trim().toLowerCase() ?? ''; +} + +function bytesEqualAt(bytes: Uint8Array, offset: number, expected: ReadonlyArray): boolean { + if (bytes.byteLength < offset + expected.length) return false; + for (let index = 0; index < expected.length; index += 1) { + if (bytes[offset + index] !== expected[index]) return false; + } + return true; +} + +function isEBMLDocTypeByte(byte: number): boolean { + if (byte >= 0x30 && byte <= 0x39) return true; + if (byte >= 0x41 && byte <= 0x5a) return true; + if (byte >= 0x61 && byte <= 0x7a) return true; + return byte === 0x2d || byte === 0x5f; +} + +function readEBMLDocTypeValue(bytes: Uint8Array, start: number, length: number): string | null { + let docType = ''; + for (let index = 0; index < length; index += 1) { + const byte = bytes[start + index] ?? 0; + if (byte === 0) break; + if (!isEBMLDocTypeByte(byte)) return null; + docType += String.fromCharCode(byte); + } + if (docType.length === 0) return null; + return docType; +} + +function readEBMLDocType(bytes: Uint8Array): string | null { + const searchLimit = bytes.byteLength - EBML_DOC_TYPE_ELEMENT_ID.length; + for (let offset = EBML_MAGIC.length; offset < searchLimit; offset += 1) { + if (!bytesEqualAt(bytes, offset, EBML_DOC_TYPE_ELEMENT_ID)) continue; + const sizeOffset = offset + EBML_DOC_TYPE_ELEMENT_ID.length; + const sizeByte = bytes[sizeOffset] ?? 0; + if ((sizeByte & EBML_VINT_SINGLE_BYTE_MARKER) === 0) continue; + const length = sizeByte & EBML_VINT_SINGLE_BYTE_VALUE_MASK; + if (length === 0 || length > MAX_EBML_DOC_TYPE_LENGTH) continue; + const valueOffset = sizeOffset + 1; + if (valueOffset + length > bytes.byteLength) return null; + return readEBMLDocTypeValue(bytes, valueOffset, length); + } + return null; +} + +function sniffVideoMediaType(bytes: Uint8Array): string | null { + if (bytesEqualAt(bytes, MP4_FILE_TYPE_BOX_OFFSET, MP4_FILE_TYPE_BOX)) return 'video/mp4'; + if (!bytesEqualAt(bytes, 0, EBML_MAGIC)) return null; + const docType = readEBMLDocType(bytes); + if (docType == null) { + throw new Error('Custom camera background video has an unreadable EBML document type'); + } + if (docType !== WEBM_DOC_TYPE) { + throw new Error(`Custom camera background video document type is unsupported: ${docType}`); + } + return 'video/webm'; +} + +async function normalizeCustomVideoBlob(blob: Blob, signal: AbortSignal): Promise { + const header = new Uint8Array(await blob.slice(0, VIDEO_HEADER_SNIFF_BYTES).arrayBuffer()); + throwIfAborted(signal); + const detectedMediaType = sniffVideoMediaType(header); + if (detectedMediaType == null) { + throw new Error('Custom camera background is not a supported video format'); + } + const declaredMediaType = normalizedMediaType(blob.type); + if (SUPPORTED_VIDEO_MEDIA_TYPES.has(declaredMediaType) && declaredMediaType !== detectedMediaType) { + throw new Error('Custom camera background video type does not match its encoded data'); + } + if (declaredMediaType === detectedMediaType) return blob; + return new Blob([blob], {type: detectedMediaType}); +} + +function videoElementError(video: HTMLVideoElement): Error { + const code = video.error?.code ?? 0; + const message = video.error?.message.trim() ?? ''; + if (message.length > 0) return new Error(`Custom camera background video failed (${code}): ${message}`); + return new Error(`Custom camera background video failed with media error ${code}`); +} + +function waitForVideoMetadata(video: HTMLVideoElement, signal: AbortSignal): Promise { + if (signal.aborted) return Promise.reject(signal.reason); + if (video.readyState >= 1 && video.videoWidth > 0 && video.videoHeight > 0) return Promise.resolve(); + return new Promise((resolve, reject) => { + const cleanup = (): void => { + video.removeEventListener('loadedmetadata', handleMetadata); + video.removeEventListener('error', handleError); + signal.removeEventListener('abort', handleAbort); + }; + const handleMetadata = (): void => { + cleanup(); + resolve(); + }; + const handleError = (): void => { + cleanup(); + reject(videoElementError(video)); + }; + const handleAbort = (): void => { + cleanup(); + reject(signal.reason); + }; + video.addEventListener('loadedmetadata', handleMetadata, {once: true}); + video.addEventListener('error', handleError, {once: true}); + signal.addEventListener('abort', handleAbort, {once: true}); + }); +} + +function waitForAbortablePromise(promise: Promise, signal: AbortSignal): Promise { + if (signal.aborted) return Promise.reject(signal.reason); + return new Promise((resolve, reject) => { + const handleAbort = (): void => reject(signal.reason); + signal.addEventListener('abort', handleAbort, {once: true}); + promise.then( + (value) => { + signal.removeEventListener('abort', handleAbort); + resolve(value); + }, + (error: unknown) => { + signal.removeEventListener('abort', handleAbort); + reject(error); + }, + ); + }); +} + +function throwIfAborted(signal: AbortSignal): void { + if (signal.aborted) throw signal.reason ?? new Error('Custom camera background video operation was aborted'); +} + +function throwCleanupFailures(failures: ReadonlyArray, message: string): void { + if (failures.length === 0) return; + if (failures.length === 1) throw failures[0]; + throw new AggregateError(failures, message); +} + +export async function createWebCameraEffectVideoFrameProducer( + mediaURL: string, +): Promise { + requireWebCameraEffectCustomMediaURL(mediaURL); + return runWithResponseDeadline({ + timeoutMilliseconds: WEB_CAMERA_EFFECT_CUSTOM_MEDIA_OPERATION_TIMEOUT_MS, + description: 'Custom camera background video initialization', + signal: null, + operation: async (signal) => { + const blob = await readWebCameraEffectCustomMediaBlob(mediaURL, signal); + const normalizedBlob = await normalizeCustomVideoBlob(blob, signal); + const producer = new WebCameraEffectVideoFrameProducerOwner(normalizedBlob); + try { + await producer.start(signal); + return producer; + } catch (error) { + try { + producer.stop(); + } catch (cleanupError) { + throw new AggregateError( + [error, cleanupError], + 'Custom camera background video initialization failed during cleanup', + ); + } + throw error; + } + }, + }); +} diff --git a/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectWebGLMaskRefiner.ts b/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectWebGLMaskRefiner.ts new file mode 100644 index 000000000..7db4c69b8 --- /dev/null +++ b/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectWebGLMaskRefiner.ts @@ -0,0 +1,265 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import { + WEB_CAMERA_EFFECT_MASK_BAND_EPSILON, + WEB_CAMERA_EFFECT_MASK_EDGE_SOFTNESS, + WEB_CAMERA_EFFECT_MASK_GUIDE_RANGE_FALLOFF, + WEB_CAMERA_EFFECT_MASK_GUIDE_SPATIAL_FALLOFF, +} from '@app/features/voice/utils/camera-effects/WebCameraEffectMask'; +import {SEG_INPUT_EDGE} from '@app/features/voice/utils/camera-effects/WebSelfieSegmenter'; + +const VERTEX_SHADER_SOURCE = `#version 300 es +precision highp float; + +out vec2 textureCoordinates; + +void main() { + vec2 positions[3] = vec2[3]( + vec2(-1.0, -1.0), + vec2(3.0, -1.0), + vec2(-1.0, 3.0) + ); + vec2 position = positions[gl_VertexID]; + gl_Position = vec4(position, 0.0, 1.0); + textureCoordinates = vec2((position.x + 1.0) * 0.5, (1.0 - position.y) * 0.5); +} +`; + +const FRAGMENT_SHADER_SOURCE = `#version 300 es +precision highp float; + +uniform sampler2D sourceTexture; +uniform sampler2D maskTexture; +in vec2 textureCoordinates; +out vec4 outputColour; + +float luminance(vec3 colour) { + return dot(colour, vec3(0.2126, 0.7152, 0.0722)); +} + +float refinedMask(vec2 coordinates, float guide) { + vec2 maskSize = vec2(${SEG_INPUT_EDGE}.0); + vec2 maskPosition = coordinates * maskSize - vec2(0.5); + vec2 base = round(maskPosition); + float maskTotal = 0.0; + float weightTotal = 0.0; + for (int y = -1; y <= 1; y += 1) { + for (int x = -1; x <= 1; x += 1) { + vec2 offset = vec2(float(x), float(y)); + vec2 sampleCoordinates = clamp((base + offset + vec2(0.5)) / maskSize, vec2(0.0), vec2(1.0)); + float sampleGuide = luminance(textureLod(sourceTexture, sampleCoordinates, 0.0).rgb); + vec2 sampleDistance = maskPosition - (base + offset); + float spatialWeight = exp(-float(${WEB_CAMERA_EFFECT_MASK_GUIDE_SPATIAL_FALLOFF}) * dot(sampleDistance, sampleDistance)); + float rangeWeight = exp(-float(${WEB_CAMERA_EFFECT_MASK_GUIDE_RANGE_FALLOFF}) * abs(guide - sampleGuide)); + float weight = spatialWeight * rangeWeight; + maskTotal += textureLod(maskTexture, sampleCoordinates, 0.0).a * weight; + weightTotal += weight; + } + } + return maskTotal / max(weightTotal, 0.0001); +} + +void main() { + float guide = luminance(texture(sourceTexture, textureCoordinates).rgb); + float coarse = textureLod(maskTexture, textureCoordinates, 0.0).a; + float alpha = step(0.5, coarse); + if (coarse > ${WEB_CAMERA_EFFECT_MASK_BAND_EPSILON} && coarse < 1.0 - ${WEB_CAMERA_EFFECT_MASK_BAND_EPSILON}) { + float refined = refinedMask(textureCoordinates, guide); + float curve = clamp((refined - 0.5) / (2.0 * ${WEB_CAMERA_EFFECT_MASK_EDGE_SOFTNESS}) + 0.5, 0.0, 1.0); + alpha = curve * curve * (3.0 - 2.0 * curve); + } + outputColour = vec4(0.0, 0.0, 0.0, alpha); +} +`; + +function createShader(gl: WebGL2RenderingContext, type: number, source: string): WebGLShader { + const shader = gl.createShader(type); + if (shader == null) { + throw new Error('Camera mask refinement could not allocate a WebGL shader'); + } + gl.shaderSource(shader, source); + gl.compileShader(shader); + if (gl.getShaderParameter(shader, gl.COMPILE_STATUS) === true) { + return shader; + } + const diagnostic = gl.getShaderInfoLog(shader) ?? 'no shader diagnostic was provided'; + gl.deleteShader(shader); + throw new Error(`Camera mask refinement WebGL shader compilation failed: ${diagnostic}`); +} + +function createProgram(gl: WebGL2RenderingContext): WebGLProgram { + const vertexShader = createShader(gl, gl.VERTEX_SHADER, VERTEX_SHADER_SOURCE); + let fragmentShader: WebGLShader | null = null; + let program: WebGLProgram | null = null; + try { + fragmentShader = createShader(gl, gl.FRAGMENT_SHADER, FRAGMENT_SHADER_SOURCE); + program = gl.createProgram(); + if (program == null) { + throw new Error('Camera mask refinement could not allocate a WebGL program'); + } + gl.attachShader(program, vertexShader); + gl.attachShader(program, fragmentShader); + gl.linkProgram(program); + if (gl.getProgramParameter(program, gl.LINK_STATUS) !== true) { + const diagnostic = gl.getProgramInfoLog(program) ?? 'no program diagnostic was provided'; + throw new Error(`Camera mask refinement WebGL program linking failed: ${diagnostic}`); + } + return program; + } catch (error) { + if (program != null) { + gl.deleteProgram(program); + } + throw error; + } finally { + gl.deleteShader(vertexShader); + if (fragmentShader != null) { + gl.deleteShader(fragmentShader); + } + } +} + +function requireUniform(gl: WebGL2RenderingContext, program: WebGLProgram, name: string): WebGLUniformLocation { + const location = gl.getUniformLocation(program, name); + if (location == null) { + throw new Error(`Camera mask refinement WebGL uniform is unavailable: ${name}`); + } + return location; +} + +function createTexture(gl: WebGL2RenderingContext, unit: number): WebGLTexture { + const texture = gl.createTexture(); + if (texture == null) { + throw new Error('Camera mask refinement could not allocate a WebGL texture'); + } + gl.activeTexture(gl.TEXTURE0 + unit); + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + return texture; +} + +export class WebCameraEffectWebGLMaskRefiner { + private width = 0; + private height = 0; + private uploadedMaskRevision = -1; + private disposed = false; + + private constructor( + private readonly canvas: OffscreenCanvas, + private readonly gl: WebGL2RenderingContext, + private readonly program: WebGLProgram, + private readonly vertexArray: WebGLVertexArrayObject, + private readonly sourceTexture: WebGLTexture, + private readonly maskTexture: WebGLTexture, + ) {} + + static create(): WebCameraEffectWebGLMaskRefiner | null { + const canvas = new OffscreenCanvas(1, 1); + const gl = canvas.getContext('webgl2', { + alpha: true, + antialias: false, + depth: false, + premultipliedAlpha: false, + preserveDrawingBuffer: true, + stencil: false, + }); + if (gl == null) { + return null; + } + let program: WebGLProgram | null = null; + let vertexArray: WebGLVertexArrayObject | null = null; + let sourceTexture: WebGLTexture | null = null; + let maskTexture: WebGLTexture | null = null; + try { + program = createProgram(gl); + vertexArray = gl.createVertexArray(); + if (vertexArray == null) { + throw new Error('Camera mask refinement could not allocate a WebGL vertex array'); + } + sourceTexture = createTexture(gl, 0); + maskTexture = createTexture(gl, 1); + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, SEG_INPUT_EDGE, SEG_INPUT_EDGE, 0, gl.RGBA, gl.UNSIGNED_BYTE, null); + gl.useProgram(program); + gl.uniform1i(requireUniform(gl, program, 'sourceTexture'), 0); + gl.uniform1i(requireUniform(gl, program, 'maskTexture'), 1); + gl.bindVertexArray(vertexArray); + gl.disable(gl.BLEND); + gl.disable(gl.CULL_FACE); + gl.disable(gl.DEPTH_TEST); + gl.disable(gl.DITHER); + return new WebCameraEffectWebGLMaskRefiner(canvas, gl, program, vertexArray, sourceTexture, maskTexture); + } catch (error) { + if (sourceTexture != null) gl.deleteTexture(sourceTexture); + if (maskTexture != null) gl.deleteTexture(maskTexture); + if (vertexArray != null) gl.deleteVertexArray(vertexArray); + if (program != null) gl.deleteProgram(program); + gl.getExtension('WEBGL_lose_context')?.loseContext(); + throw error; + } + } + + refine( + source: VideoFrame, + mask: OffscreenCanvas, + maskRevision: number, + width: number, + height: number, + ): OffscreenCanvas | null { + if (this.disposed) { + throw new Error('Cannot refine a camera mask with a disposed WebGL owner'); + } + if (this.gl.isContextLost()) { + return null; + } + if (this.width !== width || this.height !== height) { + this.width = width; + this.height = height; + this.canvas.width = width; + this.canvas.height = height; + this.gl.viewport(0, 0, width, height); + this.gl.activeTexture(this.gl.TEXTURE0); + this.gl.bindTexture(this.gl.TEXTURE_2D, this.sourceTexture); + this.gl.texImage2D( + this.gl.TEXTURE_2D, + 0, + this.gl.RGBA, + width, + height, + 0, + this.gl.RGBA, + this.gl.UNSIGNED_BYTE, + null, + ); + } + this.gl.pixelStorei(this.gl.UNPACK_FLIP_Y_WEBGL, 0); + this.gl.pixelStorei(this.gl.UNPACK_PREMULTIPLY_ALPHA_WEBGL, 0); + this.gl.activeTexture(this.gl.TEXTURE0); + this.gl.bindTexture(this.gl.TEXTURE_2D, this.sourceTexture); + this.gl.texSubImage2D(this.gl.TEXTURE_2D, 0, 0, 0, this.gl.RGBA, this.gl.UNSIGNED_BYTE, source); + if (this.uploadedMaskRevision !== maskRevision) { + this.gl.activeTexture(this.gl.TEXTURE1); + this.gl.bindTexture(this.gl.TEXTURE_2D, this.maskTexture); + this.gl.texSubImage2D(this.gl.TEXTURE_2D, 0, 0, 0, this.gl.RGBA, this.gl.UNSIGNED_BYTE, mask); + this.uploadedMaskRevision = maskRevision; + } + this.gl.useProgram(this.program); + this.gl.bindVertexArray(this.vertexArray); + this.gl.drawArrays(this.gl.TRIANGLES, 0, 3); + this.gl.flush(); + return this.canvas; + } + + dispose(): void { + if (this.disposed) { + return; + } + this.disposed = true; + this.gl.deleteTexture(this.sourceTexture); + this.gl.deleteTexture(this.maskTexture); + this.gl.deleteVertexArray(this.vertexArray); + this.gl.deleteProgram(this.program); + this.gl.getExtension('WEBGL_lose_context')?.loseContext(); + } +} diff --git a/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectWebGPUConstants.ts b/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectWebGPUConstants.ts new file mode 100644 index 000000000..eeed3edf5 --- /dev/null +++ b/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectWebGPUConstants.ts @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +export const WEB_GPU_SHADER_STAGE_FRAGMENT: GPUShaderStageFlags = 2; +export const WEB_GPU_SHADER_STAGE_COMPUTE: GPUShaderStageFlags = 4; + +export const WEB_GPU_BUFFER_USAGE_COPY_DST: GPUBufferUsageFlags = 8; +export const WEB_GPU_BUFFER_USAGE_UNIFORM: GPUBufferUsageFlags = 64; +export const WEB_GPU_BUFFER_USAGE_STORAGE: GPUBufferUsageFlags = 128; + +export const WEB_GPU_TEXTURE_USAGE_COPY_DST: GPUTextureUsageFlags = 2; +export const WEB_GPU_TEXTURE_USAGE_TEXTURE_BINDING: GPUTextureUsageFlags = 4; +export const WEB_GPU_TEXTURE_USAGE_STORAGE_BINDING: GPUTextureUsageFlags = 8; +export const WEB_GPU_TEXTURE_USAGE_RENDER_ATTACHMENT: GPUTextureUsageFlags = 16; diff --git a/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectWebGPUPipelines.ts b/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectWebGPUPipelines.ts new file mode 100644 index 000000000..720ef4725 --- /dev/null +++ b/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectWebGPUPipelines.ts @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import { + WEB_GPU_SHADER_STAGE_COMPUTE, + WEB_GPU_SHADER_STAGE_FRAGMENT, +} from '@app/features/voice/utils/camera-effects/WebCameraEffectWebGPUConstants'; +import { + CAMERA_BLUR_SHADER, + CAMERA_COMPOSITE_SHADER, + CAMERA_COPY_SHADER, + CAMERA_COVER_SHADER, + CAMERA_MASK_SHADER, + CAMERA_PREPROCESS_SHADER, + CAMERA_SOURCE_SHADER, +} from '@app/features/voice/utils/camera-effects/WebCameraEffectWebGPUShaders'; + +export interface WebCameraEffectWebGPUPipelines { + readonly sourceLayout: GPUBindGroupLayout; + readonly preprocessLayout: GPUBindGroupLayout; + readonly blurLayout: GPUBindGroupLayout; + readonly copyLayout: GPUBindGroupLayout; + readonly coverLayout: GPUBindGroupLayout; + readonly compositeLayout: GPUBindGroupLayout; + readonly maskLayout: GPUBindGroupLayout; + readonly source: GPURenderPipeline; + readonly preprocess: GPURenderPipeline; + readonly blur: GPURenderPipeline; + readonly copy: GPURenderPipeline; + readonly cover: GPURenderPipeline; + readonly composite: GPURenderPipeline; + readonly mask: GPUComputePipeline; +} + +interface WebCameraEffectRenderPipelineCreation { + readonly device: GPUDevice; + readonly label: string; + readonly code: string; + readonly layout: GPUBindGroupLayout; + readonly format: GPUTextureFormat; +} + +function renderPipeline({ + device, + label, + code, + layout, + format, +}: WebCameraEffectRenderPipelineCreation): GPURenderPipeline { + const module = device.createShaderModule({label, code}); + return device.createRenderPipeline({ + label, + layout: device.createPipelineLayout({bindGroupLayouts: [layout]}), + vertex: {module, entryPoint: 'vertexMain'}, + fragment: {module, entryPoint: 'fragmentMain', targets: [{format}]}, + primitive: {topology: 'triangle-list'}, + }); +} + +export function createWebCameraEffectWebGPUPipelines( + device: GPUDevice, + canvasFormat: GPUTextureFormat, +): WebCameraEffectWebGPUPipelines { + const sourceLayout = device.createBindGroupLayout({ + label: 'camera-source-layout', + entries: [ + {binding: 0, visibility: WEB_GPU_SHADER_STAGE_FRAGMENT, externalTexture: {}}, + {binding: 1, visibility: WEB_GPU_SHADER_STAGE_FRAGMENT, sampler: {type: 'filtering'}}, + ], + }); + const preprocessLayout = device.createBindGroupLayout({ + label: 'camera-preprocess-layout', + entries: [ + {binding: 0, visibility: WEB_GPU_SHADER_STAGE_FRAGMENT, externalTexture: {}}, + {binding: 1, visibility: WEB_GPU_SHADER_STAGE_FRAGMENT, sampler: {type: 'filtering'}}, + {binding: 2, visibility: WEB_GPU_SHADER_STAGE_FRAGMENT, buffer: {type: 'storage'}}, + ], + }); + const blurLayout = device.createBindGroupLayout({ + label: 'camera-blur-layout', + entries: [ + {binding: 0, visibility: WEB_GPU_SHADER_STAGE_FRAGMENT, texture: {sampleType: 'float'}}, + {binding: 1, visibility: WEB_GPU_SHADER_STAGE_FRAGMENT, sampler: {type: 'filtering'}}, + {binding: 2, visibility: WEB_GPU_SHADER_STAGE_FRAGMENT, buffer: {type: 'uniform'}}, + ], + }); + const copyLayout = device.createBindGroupLayout({ + label: 'camera-copy-layout', + entries: [ + {binding: 0, visibility: WEB_GPU_SHADER_STAGE_FRAGMENT, texture: {sampleType: 'float'}}, + {binding: 1, visibility: WEB_GPU_SHADER_STAGE_FRAGMENT, sampler: {type: 'filtering'}}, + ], + }); + const coverLayout = device.createBindGroupLayout({ + label: 'camera-cover-layout', + entries: [ + {binding: 0, visibility: WEB_GPU_SHADER_STAGE_FRAGMENT, texture: {sampleType: 'float'}}, + {binding: 1, visibility: WEB_GPU_SHADER_STAGE_FRAGMENT, sampler: {type: 'filtering'}}, + {binding: 2, visibility: WEB_GPU_SHADER_STAGE_FRAGMENT, buffer: {type: 'uniform'}}, + ], + }); + const compositeLayout = device.createBindGroupLayout({ + label: 'camera-composite-layout', + entries: [ + {binding: 0, visibility: WEB_GPU_SHADER_STAGE_FRAGMENT, texture: {sampleType: 'float'}}, + {binding: 1, visibility: WEB_GPU_SHADER_STAGE_FRAGMENT, texture: {sampleType: 'float'}}, + {binding: 2, visibility: WEB_GPU_SHADER_STAGE_FRAGMENT, texture: {sampleType: 'float'}}, + {binding: 3, visibility: WEB_GPU_SHADER_STAGE_FRAGMENT, sampler: {type: 'filtering'}}, + ], + }); + const maskLayout = device.createBindGroupLayout({ + label: 'camera-mask-layout', + entries: [ + {binding: 0, visibility: WEB_GPU_SHADER_STAGE_COMPUTE, buffer: {type: 'read-only-storage'}}, + {binding: 1, visibility: WEB_GPU_SHADER_STAGE_COMPUTE, buffer: {type: 'storage'}}, + { + binding: 2, + visibility: WEB_GPU_SHADER_STAGE_COMPUTE, + storageTexture: {access: 'write-only', format: 'rgba8unorm'}, + }, + {binding: 3, visibility: WEB_GPU_SHADER_STAGE_COMPUTE, buffer: {type: 'uniform'}}, + ], + }); + const maskModule = device.createShaderModule({label: 'camera-mask', code: CAMERA_MASK_SHADER}); + return { + sourceLayout, + preprocessLayout, + blurLayout, + copyLayout, + coverLayout, + compositeLayout, + maskLayout, + source: renderPipeline({ + device, + label: 'camera-source', + code: CAMERA_SOURCE_SHADER, + layout: sourceLayout, + format: 'rgba8unorm', + }), + preprocess: renderPipeline({ + device, + label: 'camera-preprocess', + code: CAMERA_PREPROCESS_SHADER, + layout: preprocessLayout, + format: 'rgba8unorm', + }), + blur: renderPipeline({ + device, + label: 'camera-blur', + code: CAMERA_BLUR_SHADER, + layout: blurLayout, + format: 'rgba8unorm', + }), + copy: renderPipeline({ + device, + label: 'camera-copy', + code: CAMERA_COPY_SHADER, + layout: copyLayout, + format: canvasFormat, + }), + cover: renderPipeline({ + device, + label: 'camera-cover', + code: CAMERA_COVER_SHADER, + layout: coverLayout, + format: 'rgba8unorm', + }), + composite: renderPipeline({ + device, + label: 'camera-composite', + code: CAMERA_COMPOSITE_SHADER, + layout: compositeLayout, + format: canvasFormat, + }), + mask: device.createComputePipeline({ + label: 'camera-mask', + layout: device.createPipelineLayout({bindGroupLayouts: [maskLayout]}), + compute: {module: maskModule, entryPoint: 'computeMain'}, + }), + }; +} diff --git a/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectWebGPURenderer.ts b/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectWebGPURenderer.ts new file mode 100644 index 000000000..1ee4a4c89 --- /dev/null +++ b/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectWebGPURenderer.ts @@ -0,0 +1,973 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import { + collectSettledFailures, + throwCollectedFailures, +} from '@app/features/voice/utils/camera-effects/AggregateOperations'; +import {CameraBackgroundMode} from '@app/features/voice/utils/camera-effects/CameraCaptureContract'; +import type { + WebCameraEffectCustomFrame, + WebCameraEffectCustomFrameSource, +} from '@app/features/voice/utils/camera-effects/WebCameraEffectCustomImage'; +import { + cameraEffectBlurPixels, + requireCameraEffectBlurStrength, + validateCameraEffectFrameDimensions, + WEB_CAMERA_EFFECT_SEGMENTATION_MIN_INTERVAL_MS, + WebCameraEffectBackend, + type WebCameraPipelineConfig, +} from '@app/features/voice/utils/camera-effects/WebCameraEffectProtocol'; +import type {WebCameraEffectRenderer} from '@app/features/voice/utils/camera-effects/WebCameraEffectRenderer'; +import {WebCameraEffectSegmentationOwner} from '@app/features/voice/utils/camera-effects/WebCameraEffectSegmentationOwner'; +import { + WEB_GPU_BUFFER_USAGE_COPY_DST, + WEB_GPU_BUFFER_USAGE_STORAGE, + WEB_GPU_BUFFER_USAGE_UNIFORM, + WEB_GPU_TEXTURE_USAGE_COPY_DST, + WEB_GPU_TEXTURE_USAGE_RENDER_ATTACHMENT, + WEB_GPU_TEXTURE_USAGE_STORAGE_BINDING, + WEB_GPU_TEXTURE_USAGE_TEXTURE_BINDING, +} from '@app/features/voice/utils/camera-effects/WebCameraEffectWebGPUConstants'; +import { + createWebCameraEffectWebGPUPipelines, + type WebCameraEffectWebGPUPipelines, +} from '@app/features/voice/utils/camera-effects/WebCameraEffectWebGPUPipelines'; +import { + loadWebSelfieRuntime, + MissingSegmentationAlphasOutputError, + SEG_INPUT_EDGE, + SEG_INPUT_NAME, + SEG_INPUT_PIXELS, + SEG_OUTPUT_NAME, + type WebSelfieOrtModule, +} from '@app/features/voice/utils/camera-effects/WebSelfieSegmenter'; +import type * as OrtNamespace from 'onnxruntime-web/webgpu'; +import invariant from 'tiny-invariant'; + +class WebGPUCameraFrameResourcesUnavailableError extends Error { + constructor() { + super('WebGPU camera frame resources are unavailable'); + this.name = 'WebGPUCameraFrameResourcesUnavailableError'; + } +} + +class MissingSegmentationGPUFloatMaskError extends Error { + constructor() { + super('Segmentation model did not produce the required GPU-resident float mask'); + this.name = 'MissingSegmentationGPUFloatMaskError'; + } +} + +class MissingWebGPUCameraFrameTexturesError extends Error { + constructor() { + super('Cannot build WebGPU camera bind groups without frame textures'); + this.name = 'MissingWebGPUCameraFrameTexturesError'; + } +} + +const INPUT_BUFFER_BYTES = 3 * SEG_INPUT_PIXELS * Float32Array.BYTES_PER_ELEMENT; +const MASK_BUFFER_BYTES = SEG_INPUT_PIXELS * Float32Array.BYTES_PER_ELEMENT; + +interface CustomTexture { + readonly texture: GPUTexture; + readonly width: number; + readonly height: number; + frameIndex: number; +} + +interface FrameBindGroups { + readonly sourceCopy: GPUBindGroup; + readonly backgroundCopy: GPUBindGroup; + readonly horizontalBlur: GPUBindGroup; + readonly verticalBlur: GPUBindGroup; + readonly composite: GPUBindGroup; +} + +interface FrameConfigurationResources { + readonly horizontalBlurParamsBuffer: GPUBuffer; + readonly verticalBlurParamsBuffer: GPUBuffer; + readonly coverParamsBuffer: GPUBuffer; + readonly frameBindGroups: FrameBindGroups; + readonly customCoverBindGroup: GPUBindGroup | null; +} + +interface CustomBackgroundCoverScale { + readonly scaleX: number; + readonly scaleY: number; +} + +interface WebCameraEffectWebGPUInitialization { + readonly canvas: OffscreenCanvas; + readonly context: GPUCanvasContext; + readonly canvasFormat: GPUTextureFormat; + readonly device: GPUDevice; + readonly ort: WebSelfieOrtModule; + readonly session: OrtNamespace.InferenceSession; +} + +function errorMessage(error: unknown): string { + if (error instanceof Error) { + return error.message; + } + return String(error); +} + +function resolveValidationDetail(validationError: GPUError | null): string { + if (validationError == null) { + return ''; + } + return `; validation failed: ${validationError.message}`; +} + +async function popValidationErrorScope( + device: GPUDevice, + validationScopeActive: boolean, + validationFailures: Array, +): Promise { + if (!validationScopeActive) { + return null; + } + try { + return await device.popErrorScope(); + } catch (validationFailure) { + validationFailures.push(validationFailure); + return null; + } +} + +function resolveCustomBackgroundCoverScale(imageAspect: number, canvasAspect: number): CustomBackgroundCoverScale { + if (imageAspect > canvasAspect) { + return { + scaleX: canvasAspect / imageAspect, + scaleY: 1, + }; + } + return { + scaleX: 1, + scaleY: imageAspect / canvasAspect, + }; +} + +function GPUBooleanFlag(value: boolean): number { + if (value) { + return 1; + } + return 0; +} + +function destroyGPUTexture(texture: GPUTexture | null): void { + if (texture == null) { + return; + } + texture.destroy(); +} + +function destroyGPUBuffer(buffer: GPUBuffer | null): void { + if (buffer == null) { + return; + } + buffer.destroy(); +} + +function destroyCustomTexture(customTexture: CustomTexture | null): void { + if (customTexture == null) { + return; + } + customTexture.texture.destroy(); +} + +function collectInferenceOutputDisposalFailures( + outputs: Readonly>, +): ReadonlyArray { + const failures: Array = []; + for (const output of Object.values(outputs)) { + try { + output.dispose(); + } catch (error) { + failures.push(error); + } + } + return failures; +} + +async function collectWebGPUInitializationCleanupFailures( + renderer: WebCameraEffectWebGPURenderer | null, + session: OrtNamespace.InferenceSession | null, + device: GPUDevice, +): Promise> { + if (renderer != null) { + return collectSettledFailures([renderer.dispose()]); + } + return collectSettledFailures([ + Promise.resolve().then(() => { + if (session != null) { + session.release(); + } + }), + Promise.resolve().then(() => device.destroy()), + ]); +} + +function requireGPUMask(output: OrtNamespace.Tensor): OrtNamespace.Tensor { + let elements = 1; + for (const dimension of output.dims) { + elements *= dimension; + } + if (output.type !== 'float32') { + throw new MissingSegmentationGPUFloatMaskError(); + } + if (output.location !== 'gpu-buffer') { + throw new MissingSegmentationGPUFloatMaskError(); + } + if (elements !== SEG_INPUT_PIXELS) { + throw new MissingSegmentationGPUFloatMaskError(); + } + return output; +} + +function beginRenderPass(encoder: GPUCommandEncoder, target: GPUTextureView): GPURenderPassEncoder { + return encoder.beginRenderPass({ + colorAttachments: [ + { + view: target, + clearValue: {r: 0, g: 0, b: 0, a: 1}, + loadOp: 'clear', + storeOp: 'store', + }, + ], + }); +} + +function drawRenderPass(pass: GPURenderPassEncoder, pipeline: GPURenderPipeline, bindGroup: GPUBindGroup): void { + pass.setPipeline(pipeline); + pass.setBindGroup(0, bindGroup); + pass.draw(3); + pass.end(); +} + +export class WebCameraEffectWebGPURenderer implements WebCameraEffectRenderer { + readonly backend = WebCameraEffectBackend.WEB_GPU; + private readonly canvas: OffscreenCanvas; + private readonly canvasContext: GPUCanvasContext; + private readonly canvasFormat: GPUTextureFormat; + private readonly device: GPUDevice; + private readonly session: OrtNamespace.InferenceSession; + private readonly pipelines: WebCameraEffectWebGPUPipelines; + private readonly sampler: GPUSampler; + private readonly inputBuffer: GPUBuffer; + private readonly inputTensor: OrtNamespace.Tensor; + private readonly smoothedMaskBuffer: GPUBuffer; + private horizontalBlurParamsBuffer: GPUBuffer; + private verticalBlurParamsBuffer: GPUBuffer; + private coverParamsBuffer: GPUBuffer; + private readonly maskParamsBuffer: GPUBuffer; + private readonly horizontalBlurParams = new Float32Array(4); + private readonly verticalBlurParams = new Float32Array(4); + private readonly coverParams = new Float32Array(4); + private readonly maskParams = new ArrayBuffer(16); + private readonly maskParamsView = new DataView(this.maskParams); + private readonly segmentationOwner = new WebCameraEffectSegmentationOwner(); + private readonly preprocessTarget: GPUTexture; + private readonly preprocessTargetView: GPUTextureView; + private readonly maskTexture: GPUTexture; + private readonly maskTextureView: GPUTextureView; + private config: WebCameraPipelineConfig = {background: null}; + private customFrameSource: WebCameraEffectCustomFrameSource | null = null; + private customTexture: CustomTexture | null = null; + private sourceTexture: GPUTexture | null = null; + private sourceTextureView: GPUTextureView | null = null; + private blurTexture: GPUTexture | null = null; + private blurTextureView: GPUTextureView | null = null; + private backgroundTexture: GPUTexture | null = null; + private backgroundTextureView: GPUTextureView | null = null; + private frameBindGroups: FrameBindGroups | null = null; + private customCoverBindGroup: GPUBindGroup | null = null; + private width = 0; + private height = 0; + private customBackgroundDirty = false; + private lastSegmentationAt = Number.NEGATIVE_INFINITY; + private maskPrimed = false; + private maskReady = false; + private disposed = false; + private disposePromise: Promise | null = null; + private deviceLostReason: string | null = null; + + private constructor({canvas, context, canvasFormat, device, ort, session}: WebCameraEffectWebGPUInitialization) { + this.canvas = canvas; + this.canvasContext = context; + this.canvasFormat = canvasFormat; + this.device = device; + this.session = session; + this.pipelines = createWebCameraEffectWebGPUPipelines(device, canvasFormat); + this.sampler = device.createSampler({ + label: 'camera-linear-sampler', + addressModeU: 'clamp-to-edge', + addressModeV: 'clamp-to-edge', + magFilter: 'linear', + minFilter: 'linear', + }); + this.inputBuffer = device.createBuffer({ + label: 'camera-segmentation-input', + size: INPUT_BUFFER_BYTES, + usage: WEB_GPU_BUFFER_USAGE_STORAGE | WEB_GPU_BUFFER_USAGE_COPY_DST, + }); + this.inputTensor = ort.Tensor.fromGpuBuffer(this.inputBuffer, { + dataType: 'float32', + dims: [1, 3, SEG_INPUT_EDGE, SEG_INPUT_EDGE], + }); + this.smoothedMaskBuffer = device.createBuffer({ + label: 'camera-smoothed-mask', + size: MASK_BUFFER_BYTES, + usage: WEB_GPU_BUFFER_USAGE_STORAGE | WEB_GPU_BUFFER_USAGE_COPY_DST, + }); + this.horizontalBlurParamsBuffer = this.uniformBuffer('camera-horizontal-blur-params'); + this.verticalBlurParamsBuffer = this.uniformBuffer('camera-vertical-blur-params'); + this.coverParamsBuffer = this.uniformBuffer('camera-cover-params'); + this.maskParamsBuffer = this.uniformBuffer('camera-mask-params'); + this.preprocessTarget = device.createTexture({ + label: 'camera-preprocess-target', + size: [SEG_INPUT_EDGE, SEG_INPUT_EDGE], + format: 'rgba8unorm', + usage: WEB_GPU_TEXTURE_USAGE_RENDER_ATTACHMENT, + }); + this.maskTexture = device.createTexture({ + label: 'camera-mask', + size: [SEG_INPUT_EDGE, SEG_INPUT_EDGE], + format: 'rgba8unorm', + usage: WEB_GPU_TEXTURE_USAGE_STORAGE_BINDING | WEB_GPU_TEXTURE_USAGE_TEXTURE_BINDING, + }); + this.preprocessTargetView = this.preprocessTarget.createView(); + this.maskTextureView = this.maskTexture.createView(); + void device.lost + .then((info) => { + if (!this.disposed) { + this.deviceLostReason = `${info.reason}: ${info.message}`; + } + }) + .catch(() => {}); + } + + static async create( + canvas: OffscreenCanvas, + config: WebCameraPipelineConfig, + customFrameSource: WebCameraEffectCustomFrameSource | null, + ): Promise { + const GPU = navigator.gpu; + if (GPU == null) { + throw new Error('WebGPU is unavailable'); + } + const adapter = await GPU.requestAdapter({powerPreference: 'high-performance'}); + if (adapter == null) { + throw new Error('WebGPU did not provide an adapter'); + } + const device = await adapter.requestDevice(); + const context = canvas.getContext('webgpu') as GPUCanvasContext | null; + if (context == null) { + device.destroy(); + throw new Error('OffscreenCanvas WebGPU context is unavailable'); + } + const format = GPU.getPreferredCanvasFormat(); + context.configure({device, format, alphaMode: 'opaque'}); + let session: OrtNamespace.InferenceSession | null = null; + let renderer: WebCameraEffectWebGPURenderer | null = null; + let validationScopeActive = true; + device.pushErrorScope('validation'); + try { + const {ort, modelBytes} = await loadWebSelfieRuntime(); + session = await ort.InferenceSession.create(modelBytes, { + executionProviders: [{name: 'webgpu', device, preferredLayout: 'NCHW'}], + graphOptimizationLevel: 'all', + logSeverityLevel: 3, + preferredOutputLocation: {[SEG_OUTPUT_NAME]: 'gpu-buffer'}, + }); + renderer = new WebCameraEffectWebGPURenderer({ + canvas, + context, + canvasFormat: format, + device, + ort, + session, + }); + await renderer.configure(config, customFrameSource); + await renderer.warmup(); + const validationError = await device.popErrorScope(); + validationScopeActive = false; + if (validationError != null) { + throw new Error(`WebGPU validation failed: ${validationError.message}`); + } + return renderer; + } catch (error) { + const validationFailures: Array = []; + const validationError = await popValidationErrorScope(device, validationScopeActive, validationFailures); + const validationDetail = resolveValidationDetail(validationError); + const initializationError = new Error( + `WebGPU camera effect initialization failed: ${errorMessage(error)}${validationDetail}`, + { + cause: error, + }, + ); + const cleanupFailures = await collectWebGPUInitializationCleanupFailures(renderer, session, device); + throwCollectedFailures({ + failures: [initializationError, ...validationFailures, ...cleanupFailures], + message: 'WebGPU camera effect initialization and cleanup failed', + }); + } + } + + async configure( + config: WebCameraPipelineConfig, + customFrameSource: WebCameraEffectCustomFrameSource | null, + ): Promise { + this.requireActive(); + if (config.background != null) { + requireCameraEffectBlurStrength(config.background.blurStrength); + } + const customBackground = config.background?.mode === CameraBackgroundMode.CUSTOM; + if (customBackground !== (customFrameSource != null)) { + throw new Error('Camera effect custom frame source does not match its configuration'); + } + let nextCustomTexture = this.customTexture; + if (this.customFrameSource !== customFrameSource) { + nextCustomTexture = await this.createCustomTexture(customFrameSource); + } + let nextFrameConfiguration: FrameConfigurationResources | null = null; + try { + if (this.width > 0 && this.height > 0) { + nextFrameConfiguration = await this.createFrameConfigurationResources(config, nextCustomTexture); + } + const backgroundLifecycleChanged = (this.config.background == null) !== (config.background == null); + if (backgroundLifecycleChanged) { + this.segmentationOwner.advanceLifecycle(); + } + } catch (error) { + const cleanupFailures = await collectSettledFailures([ + Promise.resolve().then(() => { + if (nextCustomTexture !== this.customTexture) { + destroyCustomTexture(nextCustomTexture); + } + }), + ]); + throwCollectedFailures({ + failures: [error, ...cleanupFailures], + message: 'WebGPU camera effect configuration failed during cleanup', + }); + } + const backgroundLifecycleChanged = (this.config.background == null) !== (config.background == null); + const previousCustomTexture = this.customTexture; + const previousHorizontalBlurParamsBuffer = this.horizontalBlurParamsBuffer; + const previousVerticalBlurParamsBuffer = this.verticalBlurParamsBuffer; + const previousCoverParamsBuffer = this.coverParamsBuffer; + this.customTexture = nextCustomTexture; + this.customFrameSource = customFrameSource; + this.config = config; + if (nextFrameConfiguration != null) { + this.horizontalBlurParamsBuffer = nextFrameConfiguration.horizontalBlurParamsBuffer; + this.verticalBlurParamsBuffer = nextFrameConfiguration.verticalBlurParamsBuffer; + this.coverParamsBuffer = nextFrameConfiguration.coverParamsBuffer; + this.frameBindGroups = nextFrameConfiguration.frameBindGroups; + this.customCoverBindGroup = nextFrameConfiguration.customCoverBindGroup; + } + if (backgroundLifecycleChanged) { + this.lastSegmentationAt = Number.NEGATIVE_INFINITY; + this.maskPrimed = false; + this.maskReady = false; + } + this.customBackgroundDirty = nextCustomTexture != null; + if (previousCustomTexture !== nextCustomTexture) { + destroyCustomTexture(previousCustomTexture); + } + if (nextFrameConfiguration != null) { + destroyGPUBuffer(previousHorizontalBlurParamsBuffer); + destroyGPUBuffer(previousVerticalBlurParamsBuffer); + destroyGPUBuffer(previousCoverParamsBuffer); + } + } + + private async createFrameConfigurationResources( + config: WebCameraPipelineConfig, + customTexture: CustomTexture | null, + ): Promise { + let horizontalBlurParamsBuffer: GPUBuffer | null = null; + let verticalBlurParamsBuffer: GPUBuffer | null = null; + let coverParamsBuffer: GPUBuffer | null = null; + try { + horizontalBlurParamsBuffer = this.uniformBuffer('camera-horizontal-blur-params'); + verticalBlurParamsBuffer = this.uniformBuffer('camera-vertical-blur-params'); + coverParamsBuffer = this.uniformBuffer('camera-cover-params'); + this.updateFrameParams( + config, + customTexture, + horizontalBlurParamsBuffer, + verticalBlurParamsBuffer, + coverParamsBuffer, + ); + return { + horizontalBlurParamsBuffer, + verticalBlurParamsBuffer, + coverParamsBuffer, + frameBindGroups: this.createFrameBindGroups(horizontalBlurParamsBuffer, verticalBlurParamsBuffer), + customCoverBindGroup: this.createCustomCoverBindGroup(customTexture, coverParamsBuffer), + }; + } catch (error) { + const cleanupFailures = await collectSettledFailures([ + Promise.resolve().then(() => destroyGPUBuffer(horizontalBlurParamsBuffer)), + Promise.resolve().then(() => destroyGPUBuffer(verticalBlurParamsBuffer)), + Promise.resolve().then(() => destroyGPUBuffer(coverParamsBuffer)), + ]); + throwCollectedFailures({ + failures: [error, ...cleanupFailures], + message: 'WebGPU camera effect frame configuration failed during cleanup', + }); + } + } + + private async createCustomTexture( + customFrameSource: WebCameraEffectCustomFrameSource | null, + ): Promise { + if (customFrameSource == null) { + return null; + } + const lease = customFrameSource.acquireFrame(0); + try { + return await this.loadCustomTexture(lease.frame); + } finally { + lease.release(); + } + } + + async render(frame: VideoFrame, now: number): Promise { + this.requireActive(); + const outputFrameAdmission = this.segmentationOwner.admitOutputFrame(); + if (outputFrameAdmission != null) { + await outputFrameAdmission; + this.requireActive(); + } + this.ensureSize(frame.displayWidth, frame.displayHeight); + this.refreshCustomTexture(now); + const externalTexture = this.device.importExternalTexture({source: frame}); + const sourceBindGroup = this.device.createBindGroup({ + label: 'camera-source-frame', + layout: this.pipelines.sourceLayout, + entries: [ + {binding: 0, resource: externalTexture}, + {binding: 1, resource: this.sampler}, + ], + }); + const sourceView = this.sourceTextureView; + const backgroundView = this.backgroundTextureView; + const frameBindGroups = this.frameBindGroups; + if (sourceView == null) { + throw new WebGPUCameraFrameResourcesUnavailableError(); + } + if (backgroundView == null) { + throw new WebGPUCameraFrameResourcesUnavailableError(); + } + if (frameBindGroups == null) { + throw new WebGPUCameraFrameResourcesUnavailableError(); + } + const encoder = this.device.createCommandEncoder({label: 'camera-frame'}); + drawRenderPass(beginRenderPass(encoder, sourceView), this.pipelines.source, sourceBindGroup); + const background = this.config.background; + if (background != null && background.mode === CameraBackgroundMode.BLUR) { + const blurView = this.blurTextureView; + if (blurView == null) { + throw new Error('WebGPU camera blur texture is unavailable'); + } + drawRenderPass(beginRenderPass(encoder, blurView), this.pipelines.blur, frameBindGroups.horizontalBlur); + drawRenderPass(beginRenderPass(encoder, backgroundView), this.pipelines.blur, frameBindGroups.verticalBlur); + } else if (background != null && background.mode === CameraBackgroundMode.CUSTOM && this.customBackgroundDirty) { + const coverBindGroup = this.customCoverBindGroup; + if (coverBindGroup == null) { + throw new Error('WebGPU custom camera background resources are unavailable'); + } + drawRenderPass(beginRenderPass(encoder, backgroundView), this.pipelines.cover, coverBindGroup); + this.customBackgroundDirty = false; + } + const outputView = this.canvasContext.getCurrentTexture().createView(); + if (background == null) { + drawRenderPass(beginRenderPass(encoder, outputView), this.pipelines.copy, frameBindGroups.sourceCopy); + } else if (!this.maskReady) { + drawRenderPass(beginRenderPass(encoder, outputView), this.pipelines.copy, frameBindGroups.backgroundCopy); + } else { + drawRenderPass(beginRenderPass(encoder, outputView), this.pipelines.composite, frameBindGroups.composite); + } + this.device.queue.submit([encoder.finish()]); + if (this.config.background != null) { + this.maybeStartSegmentation(frame, now); + } + } + + dispose(): Promise { + if (this.disposePromise == null) { + this.disposePromise = this.disposeOwned(); + } + return this.disposePromise; + } + + private async disposeOwned(): Promise { + invariant(!this.disposed, 'WebGPU camera effect disposal must have one owner'); + this.disposed = true; + this.segmentationOwner.advanceLifecycle(); + let segmentationFailures: ReadonlyArray; + try { + segmentationFailures = await this.segmentationOwner.settleForDisposal(); + } catch (error) { + segmentationFailures = [error]; + } + const inputFailures = await collectSettledFailures([Promise.resolve().then(() => this.inputTensor.dispose())]); + const sessionFailures = await collectSettledFailures([Promise.resolve().then(() => this.session.release())]); + const resourceFailures = await collectSettledFailures([ + Promise.resolve().then(() => destroyGPUTexture(this.sourceTexture)), + Promise.resolve().then(() => destroyGPUTexture(this.blurTexture)), + Promise.resolve().then(() => destroyGPUTexture(this.backgroundTexture)), + Promise.resolve().then(() => destroyCustomTexture(this.customTexture)), + Promise.resolve().then(() => this.preprocessTarget.destroy()), + Promise.resolve().then(() => this.maskTexture.destroy()), + Promise.resolve().then(() => this.inputBuffer.destroy()), + Promise.resolve().then(() => this.smoothedMaskBuffer.destroy()), + Promise.resolve().then(() => this.horizontalBlurParamsBuffer.destroy()), + Promise.resolve().then(() => this.verticalBlurParamsBuffer.destroy()), + Promise.resolve().then(() => this.coverParamsBuffer.destroy()), + Promise.resolve().then(() => this.maskParamsBuffer.destroy()), + ]); + const deviceFailures = await collectSettledFailures([Promise.resolve().then(() => this.device.destroy())]); + throwCollectedFailures({ + failures: [...segmentationFailures, ...inputFailures, ...sessionFailures, ...resourceFailures, ...deviceFailures], + message: 'WebGPU camera effect teardown failed', + }); + } + + private uniformBuffer(label: string): GPUBuffer { + return this.device.createBuffer({ + label, + size: 16, + usage: WEB_GPU_BUFFER_USAGE_UNIFORM | WEB_GPU_BUFFER_USAGE_COPY_DST, + }); + } + + private async warmup(): Promise { + const width = this.canvas.width; + const height = this.canvas.height; + validateCameraEffectFrameDimensions(width, height); + const probeCanvas = new OffscreenCanvas(width, height); + const probeContext = probeCanvas.getContext('2d'); + if (probeContext == null) { + throw new Error('WebGPU camera effect warm-up requires OffscreenCanvas 2D'); + } + probeContext.fillStyle = '#000'; + probeContext.fillRect(0, 0, width, height); + const frame = new VideoFrame(probeCanvas, {timestamp: 0}); + try { + await this.render(frame, 0); + await this.segmentationOwner.settlePhysicalOperation(); + } finally { + frame.close(); + } + this.device.queue.writeBuffer(this.smoothedMaskBuffer, 0, new Uint8Array(MASK_BUFFER_BYTES)); + this.lastSegmentationAt = Number.NEGATIVE_INFINITY; + this.maskPrimed = false; + this.maskReady = false; + } + + private requireActive(): void { + if (this.disposed) { + throw new Error('Cannot use a disposed WebGPU camera effect renderer'); + } + this.segmentationOwner.requireNoDeferredFailure(); + if (this.deviceLostReason != null) { + throw new Error(`WebGPU camera effect device was lost: ${this.deviceLostReason}`); + } + } + + private ensureSize(width: number, height: number): void { + validateCameraEffectFrameDimensions(width, height); + if (this.width === width && this.height === height) { + return; + } + this.width = width; + this.height = height; + this.canvas.width = width; + this.canvas.height = height; + this.canvasContext.configure({device: this.device, format: this.canvasFormat, alphaMode: 'opaque'}); + destroyGPUTexture(this.sourceTexture); + destroyGPUTexture(this.blurTexture); + destroyGPUTexture(this.backgroundTexture); + this.sourceTexture = this.frameTexture('camera-source'); + this.blurTexture = this.frameTexture('camera-blur'); + this.backgroundTexture = this.frameTexture('camera-background'); + this.sourceTextureView = this.sourceTexture.createView(); + this.blurTextureView = this.blurTexture.createView(); + this.backgroundTextureView = this.backgroundTexture.createView(); + this.updateFrameParams( + this.config, + this.customTexture, + this.horizontalBlurParamsBuffer, + this.verticalBlurParamsBuffer, + this.coverParamsBuffer, + ); + this.frameBindGroups = this.createFrameBindGroups(this.horizontalBlurParamsBuffer, this.verticalBlurParamsBuffer); + this.customCoverBindGroup = this.createCustomCoverBindGroup(this.customTexture, this.coverParamsBuffer); + this.customBackgroundDirty = this.customTexture != null; + this.device.queue.writeBuffer(this.smoothedMaskBuffer, 0, new Uint8Array(MASK_BUFFER_BYTES)); + this.lastSegmentationAt = Number.NEGATIVE_INFINITY; + this.maskPrimed = false; + this.maskReady = false; + } + + private frameTexture(label: string): GPUTexture { + return this.device.createTexture({ + label, + size: [this.width, this.height], + format: 'rgba8unorm', + usage: WEB_GPU_TEXTURE_USAGE_RENDER_ATTACHMENT | WEB_GPU_TEXTURE_USAGE_TEXTURE_BINDING, + }); + } + + private updateFrameParams( + config: WebCameraPipelineConfig, + customTexture: CustomTexture | null, + horizontalBlurParamsBuffer: GPUBuffer, + verticalBlurParamsBuffer: GPUBuffer, + coverParamsBuffer: GPUBuffer, + ): void { + let radius = 0; + const background = config.background; + if (background != null) { + radius = cameraEffectBlurPixels(background.blurStrength); + } + this.horizontalBlurParams[0] = 1 / this.width; + this.horizontalBlurParams[1] = 0; + this.horizontalBlurParams[2] = radius; + this.horizontalBlurParams[3] = 0; + this.device.queue.writeBuffer(horizontalBlurParamsBuffer, 0, this.horizontalBlurParams); + this.verticalBlurParams[0] = 0; + this.verticalBlurParams[1] = 1 / this.height; + this.verticalBlurParams[2] = radius; + this.verticalBlurParams[3] = 0; + this.device.queue.writeBuffer(verticalBlurParamsBuffer, 0, this.verticalBlurParams); + if (customTexture == null) { + return; + } + const imageAspect = customTexture.width / customTexture.height; + const canvasAspect = this.width / this.height; + const {scaleX, scaleY} = resolveCustomBackgroundCoverScale(imageAspect, canvasAspect); + this.coverParams[0] = scaleX; + this.coverParams[1] = scaleY; + this.coverParams[2] = (1 - scaleX) / 2; + this.coverParams[3] = (1 - scaleY) / 2; + this.device.queue.writeBuffer(coverParamsBuffer, 0, this.coverParams); + } + + private createFrameBindGroups( + horizontalBlurParamsBuffer: GPUBuffer, + verticalBlurParamsBuffer: GPUBuffer, + ): FrameBindGroups { + const sourceView = this.sourceTextureView; + const blurView = this.blurTextureView; + const backgroundView = this.backgroundTextureView; + if (sourceView == null) { + throw new MissingWebGPUCameraFrameTexturesError(); + } + if (blurView == null) { + throw new MissingWebGPUCameraFrameTexturesError(); + } + if (backgroundView == null) { + throw new MissingWebGPUCameraFrameTexturesError(); + } + return { + sourceCopy: this.device.createBindGroup({ + label: 'camera-source-copy', + layout: this.pipelines.copyLayout, + entries: [ + {binding: 0, resource: sourceView}, + {binding: 1, resource: this.sampler}, + ], + }), + backgroundCopy: this.device.createBindGroup({ + label: 'camera-background-copy', + layout: this.pipelines.copyLayout, + entries: [ + {binding: 0, resource: backgroundView}, + {binding: 1, resource: this.sampler}, + ], + }), + horizontalBlur: this.device.createBindGroup({ + label: 'camera-horizontal-blur', + layout: this.pipelines.blurLayout, + entries: [ + {binding: 0, resource: sourceView}, + {binding: 1, resource: this.sampler}, + {binding: 2, resource: {buffer: horizontalBlurParamsBuffer}}, + ], + }), + verticalBlur: this.device.createBindGroup({ + label: 'camera-vertical-blur', + layout: this.pipelines.blurLayout, + entries: [ + {binding: 0, resource: blurView}, + {binding: 1, resource: this.sampler}, + {binding: 2, resource: {buffer: verticalBlurParamsBuffer}}, + ], + }), + composite: this.device.createBindGroup({ + label: 'camera-composite', + layout: this.pipelines.compositeLayout, + entries: [ + {binding: 0, resource: sourceView}, + {binding: 1, resource: backgroundView}, + {binding: 2, resource: this.maskTextureView}, + {binding: 3, resource: this.sampler}, + ], + }), + }; + } + + private createCustomCoverBindGroup( + customTexture: CustomTexture | null, + coverParamsBuffer: GPUBuffer, + ): GPUBindGroup | null { + if (customTexture == null) { + return null; + } + return this.device.createBindGroup({ + label: 'camera-custom-cover', + layout: this.pipelines.coverLayout, + entries: [ + {binding: 0, resource: customTexture.texture.createView()}, + {binding: 1, resource: this.sampler}, + {binding: 2, resource: {buffer: coverParamsBuffer}}, + ], + }); + } + + private async loadCustomTexture(frame: WebCameraEffectCustomFrame): Promise { + let texture: GPUTexture | null = null; + try { + texture = this.device.createTexture({ + label: 'camera-custom-background', + size: [frame.width, frame.height], + format: 'rgba8unorm', + usage: WEB_GPU_TEXTURE_USAGE_COPY_DST | WEB_GPU_TEXTURE_USAGE_TEXTURE_BINDING, + }); + this.device.queue.copyExternalImageToTexture( + {source: frame.image}, + {texture}, + {width: frame.width, height: frame.height}, + ); + } catch (error) { + const cleanupFailures = await collectSettledFailures([Promise.resolve().then(() => destroyGPUTexture(texture))]); + throwCollectedFailures({ + failures: [error, ...cleanupFailures], + message: 'WebGPU custom camera background upload failed during cleanup', + }); + } + if (texture == null) { + throw new Error('WebGPU custom camera background upload produced no texture'); + } + return {texture, width: frame.width, height: frame.height, frameIndex: frame.index}; + } + + private refreshCustomTexture(now: number): void { + const customFrameSource = this.customFrameSource; + const customTexture = this.customTexture; + if (customFrameSource == null || customTexture == null) { + return; + } + const lease = customFrameSource.acquireFrame(now); + try { + const frame = lease.frame; + if (customTexture.frameIndex === frame.index) { + return; + } + if (customTexture.width !== frame.width || customTexture.height !== frame.height) { + throw new Error('Custom camera background frame dimensions changed after initialization'); + } + this.device.queue.copyExternalImageToTexture( + {source: frame.image}, + {texture: customTexture.texture}, + {width: frame.width, height: frame.height}, + ); + customTexture.frameIndex = frame.index; + this.customBackgroundDirty = true; + } finally { + lease.release(); + } + } + + private maybeStartSegmentation(frame: VideoFrame, now: number): void { + if (!this.segmentationOwner.canStartPhysicalOperation()) { + return; + } + if (now - this.lastSegmentationAt < WEB_CAMERA_EFFECT_SEGMENTATION_MIN_INTERVAL_MS) { + return; + } + this.lastSegmentationAt = now; + let operation: Promise; + try { + this.submitSegmentationPreprocess(frame); + operation = this.executeSegmentation(); + } catch (error) { + operation = Promise.reject(error); + } + this.segmentationOwner.startPhysicalOperation(operation, () => { + this.maskPrimed = true; + this.maskReady = true; + }); + } + + private submitSegmentationPreprocess(frame: VideoFrame): void { + const externalTexture = this.device.importExternalTexture({source: frame}); + const preprocessBindGroup = this.device.createBindGroup({ + label: 'camera-preprocess-frame', + layout: this.pipelines.preprocessLayout, + entries: [ + {binding: 0, resource: externalTexture}, + {binding: 1, resource: this.sampler}, + {binding: 2, resource: {buffer: this.inputBuffer}}, + ], + }); + const preprocessEncoder = this.device.createCommandEncoder({label: 'camera-preprocess'}); + drawRenderPass( + beginRenderPass(preprocessEncoder, this.preprocessTargetView), + this.pipelines.preprocess, + preprocessBindGroup, + ); + this.device.queue.submit([preprocessEncoder.finish()]); + } + + private async executeSegmentation(): Promise { + const outputs = await this.session.run({[SEG_INPUT_NAME]: this.inputTensor}); + const failures: Array = []; + try { + const inferenceOutput = outputs[SEG_OUTPUT_NAME]; + if (inferenceOutput == null) { + throw new MissingSegmentationAlphasOutputError(); + } + const output = requireGPUMask(inferenceOutput); + this.maskParamsView.setUint32(0, GPUBooleanFlag(this.maskPrimed), true); + this.device.queue.writeBuffer(this.maskParamsBuffer, 0, this.maskParams); + const maskBindGroup = this.device.createBindGroup({ + label: 'camera-mask-frame', + layout: this.pipelines.maskLayout, + entries: [ + {binding: 0, resource: {buffer: output.gpuBuffer}}, + {binding: 1, resource: {buffer: this.smoothedMaskBuffer}}, + {binding: 2, resource: this.maskTextureView}, + {binding: 3, resource: {buffer: this.maskParamsBuffer}}, + ], + }); + const maskEncoder = this.device.createCommandEncoder({label: 'camera-mask'}); + const pass = maskEncoder.beginComputePass(); + pass.setPipeline(this.pipelines.mask); + pass.setBindGroup(0, maskBindGroup); + pass.dispatchWorkgroups(SEG_INPUT_EDGE / 8, SEG_INPUT_EDGE / 8); + pass.end(); + this.device.queue.submit([maskEncoder.finish()]); + await this.device.queue.onSubmittedWorkDone(); + } catch (error) { + failures.push(error); + } + failures.push(...collectInferenceOutputDisposalFailures(outputs)); + throwCollectedFailures({failures, message: 'WebGPU camera segmentation failed'}); + } +} diff --git a/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectWebGPUShaders.ts b/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectWebGPUShaders.ts new file mode 100644 index 000000000..a5c08a813 --- /dev/null +++ b/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectWebGPUShaders.ts @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import { + WEB_CAMERA_EFFECT_MASK_BAND_EPSILON, + WEB_CAMERA_EFFECT_MASK_BAND_WIDTH, + WEB_CAMERA_EFFECT_MASK_CORE_GROW_MIN, + WEB_CAMERA_EFFECT_MASK_CORE_MIN, + WEB_CAMERA_EFFECT_MASK_EDGE_SOFTNESS, + WEB_CAMERA_EFFECT_MASK_GUIDE_RANGE_FALLOFF, + WEB_CAMERA_EFFECT_MASK_GUIDE_SPATIAL_FALLOFF, + WEB_CAMERA_EFFECT_MASK_HOLE_NEIGHBOUR_MIN, + WEB_CAMERA_EFFECT_MASK_SPECKLE_NEIGHBOUR_MAX, + WEB_CAMERA_EFFECT_MASK_TEMPORAL_KEEP_STILL, + WEB_CAMERA_EFFECT_MASK_TEMPORAL_MOTION_HIGH, + WEB_CAMERA_EFFECT_MASK_TEMPORAL_MOTION_LOW, + WEB_CAMERA_EFFECT_MASK_VOID_MAX, +} from '@app/features/voice/utils/camera-effects/WebCameraEffectMask'; + +const FULLSCREEN_VERTEX = ` +struct VertexOutput { + @builtin(position) position: vec4f, + @location(0) uv: vec2f, +} + +@vertex +fn vertexMain(@builtin(vertex_index) vertexIndex: u32) -> VertexOutput { + let positions = array( + vec2f(-1.0, -1.0), + vec2f(3.0, -1.0), + vec2f(-1.0, 3.0), + ); + let position = positions[vertexIndex]; + var output: VertexOutput; + output.position = vec4f(position, 0.0, 1.0); + output.uv = vec2f((position.x + 1.0) * 0.5, (1.0 - position.y) * 0.5); + return output; +} +`; + +export const CAMERA_SOURCE_SHADER = ` +${FULLSCREEN_VERTEX} + +@group(0) @binding(0) var source: texture_external; +@group(0) @binding(1) var sourceSampler: sampler; + +@fragment +fn fragmentMain(input: VertexOutput) -> @location(0) vec4f { + return textureSampleBaseClampToEdge(source, sourceSampler, input.uv); +} +`; + +export const CAMERA_PREPROCESS_SHADER = ` +${FULLSCREEN_VERTEX} + +@group(0) @binding(0) var source: texture_external; +@group(0) @binding(1) var sourceSampler: sampler; +@group(0) @binding(2) var tensor: array; + +@fragment +fn fragmentMain(input: VertexOutput) -> @location(0) vec4f { + let tapOffset = 0.25 / 256.0; + var colourTotal = textureSampleBaseClampToEdge(source, sourceSampler, input.uv + vec2f(-tapOffset, -tapOffset)); + colourTotal += textureSampleBaseClampToEdge(source, sourceSampler, input.uv + vec2f(tapOffset, -tapOffset)); + colourTotal += textureSampleBaseClampToEdge(source, sourceSampler, input.uv + vec2f(-tapOffset, tapOffset)); + colourTotal += textureSampleBaseClampToEdge(source, sourceSampler, input.uv + vec2f(tapOffset, tapOffset)); + let colour = colourTotal * 0.25; + let position = vec2u(input.position.xy); + let index = position.y * 256u + position.x; + tensor[index] = colour.r; + tensor[65536u + index] = colour.g; + tensor[131072u + index] = colour.b; + return vec4f(0.0); +} +`; + +export const CAMERA_BLUR_SHADER = ` +${FULLSCREEN_VERTEX} + +struct BlurParams { + direction: vec2f, + radius: f32, + padding: f32, +} + +@group(0) @binding(0) var source: texture_2d; +@group(0) @binding(1) var sourceSampler: sampler; +@group(0) @binding(2) var params: BlurParams; + +@fragment +fn fragmentMain(input: VertexOutput) -> @location(0) vec4f { + let scale = max(1.0, params.radius * 0.25); + let nearOffset = params.direction * 1.3846153846 * scale; + let farOffset = params.direction * 3.2307692308 * scale; + var colour = textureSample(source, sourceSampler, input.uv) * 0.2270270270; + colour += textureSample(source, sourceSampler, input.uv + nearOffset) * 0.3162162162; + colour += textureSample(source, sourceSampler, input.uv - nearOffset) * 0.3162162162; + colour += textureSample(source, sourceSampler, input.uv + farOffset) * 0.0702702703; + colour += textureSample(source, sourceSampler, input.uv - farOffset) * 0.0702702703; + return colour; +} +`; + +export const CAMERA_COPY_SHADER = ` +${FULLSCREEN_VERTEX} + +@group(0) @binding(0) var source: texture_2d; +@group(0) @binding(1) var sourceSampler: sampler; + +@fragment +fn fragmentMain(input: VertexOutput) -> @location(0) vec4f { + return textureSample(source, sourceSampler, input.uv); +} +`; + +export const CAMERA_COVER_SHADER = ` +${FULLSCREEN_VERTEX} + +struct CoverParams { + scale: vec2f, + offset: vec2f, +} + +@group(0) @binding(0) var source: texture_2d; +@group(0) @binding(1) var sourceSampler: sampler; +@group(0) @binding(2) var params: CoverParams; + +@fragment +fn fragmentMain(input: VertexOutput) -> @location(0) vec4f { + return textureSample(source, sourceSampler, input.uv * params.scale + params.offset); +} +`; + +export const CAMERA_COMPOSITE_SHADER = ` +${FULLSCREEN_VERTEX} + +@group(0) @binding(0) var foreground: texture_2d; +@group(0) @binding(1) var background: texture_2d; +@group(0) @binding(2) var mask: texture_2d; +@group(0) @binding(3) var linearSampler: sampler; + +fn luminance(colour: vec3f) -> f32 { + return dot(colour, vec3f(0.2126, 0.7152, 0.0722)); +} + +fn refinedMask(uv: vec2f, guide: f32) -> f32 { + let maskSize = vec2f(256.0); + let maskPosition = uv * maskSize - vec2f(0.5); + let base = round(maskPosition); + var maskTotal = 0.0; + var weightTotal = 0.0; + for (var y: i32 = -1; y <= 1; y += 1) { + for (var x: i32 = -1; x <= 1; x += 1) { + let offset = vec2f(f32(x), f32(y)); + let sampleUv = clamp((base + offset + vec2f(0.5)) / maskSize, vec2f(0.0), vec2f(1.0)); + let sampleGuide = luminance(textureSampleLevel(foreground, linearSampler, sampleUv, 0.0).rgb); + let sampleDistance = maskPosition - (base + offset); + let spatialWeight = exp(-f32(${WEB_CAMERA_EFFECT_MASK_GUIDE_SPATIAL_FALLOFF}) * dot(sampleDistance, sampleDistance)); + let rangeWeight = exp(-f32(${WEB_CAMERA_EFFECT_MASK_GUIDE_RANGE_FALLOFF}) * abs(guide - sampleGuide)); + let weight = spatialWeight * rangeWeight; + maskTotal += textureSampleLevel(mask, linearSampler, sampleUv, 0.0).r * weight; + weightTotal += weight; + } + } + return maskTotal / max(weightTotal, 0.0001); +} + +@fragment +fn fragmentMain(input: VertexOutput) -> @location(0) vec4f { + let foregroundColour = textureSample(foreground, linearSampler, input.uv); + let backgroundColour = textureSample(background, linearSampler, input.uv); + let coarse = textureSampleLevel(mask, linearSampler, input.uv, 0.0).r; + var alpha = step(0.5, coarse); + if (coarse > ${WEB_CAMERA_EFFECT_MASK_BAND_EPSILON} && coarse < 1.0 - ${WEB_CAMERA_EFFECT_MASK_BAND_EPSILON}) { + let refined = refinedMask(input.uv, luminance(foregroundColour.rgb)); + let curve = clamp((refined - 0.5) / (2.0 * ${WEB_CAMERA_EFFECT_MASK_EDGE_SOFTNESS}) + 0.5, 0.0, 1.0); + alpha = curve * curve * (3.0 - 2.0 * curve); + } + return vec4f(mix(backgroundColour.rgb, foregroundColour.rgb, alpha), 1.0); +} +`; + +export const CAMERA_MASK_SHADER = ` +struct MaskParams { + primed: u32, + padding2: f32, + padding0: u32, + padding1: u32, +} + +@group(0) @binding(0) var inferenceMask: array; +@group(0) @binding(1) var smoothedMask: array; +@group(0) @binding(2) var outputMask: texture_storage_2d; +@group(0) @binding(3) var params: MaskParams; + +fn inferenceMaskAt(x: i32, y: i32) -> f32 { + let clampedX = u32(clamp(x, 0, 255)); + let clampedY = u32(clamp(y, 0, 255)); + return clamp(inferenceMask[clampedY * 256u + clampedX], 0.0, 1.0); +} + +@compute @workgroup_size(8, 8) +fn computeMain(@builtin(global_invocation_id) id: vec3u) { + if (id.x >= 256u || id.y >= 256u) { + return; + } + let index = id.y * 256u + id.x; + let texelX = i32(id.x); + let texelY = i32(id.y); + let centre = clamp(inferenceMask[index], 0.0, 1.0); + var neighbourhoodSum = 0.0; + var maxNeighbour = 0.0; + for (var offsetY: i32 = -1; offsetY <= 1; offsetY += 1) { + for (var offsetX: i32 = -1; offsetX <= 1; offsetX += 1) { + let neighbourValue = inferenceMaskAt(texelX + offsetX, texelY + offsetY); + neighbourhoodSum += neighbourValue; + if (offsetX != 0 || offsetY != 0) { + maxNeighbour = max(maxNeighbour, neighbourValue); + } + } + } + let neighbourhoodMean = neighbourhoodSum / 9.0; + var clean = centre; + if (centre >= ${WEB_CAMERA_EFFECT_MASK_CORE_MIN} && neighbourhoodMean < ${WEB_CAMERA_EFFECT_MASK_SPECKLE_NEIGHBOUR_MAX}) { + clean = 0.0; + } + if (centre <= ${WEB_CAMERA_EFFECT_MASK_VOID_MAX} && neighbourhoodMean > ${WEB_CAMERA_EFFECT_MASK_HOLE_NEIGHBOUR_MIN}) { + clean = 1.0; + } + if (centre >= ${WEB_CAMERA_EFFECT_MASK_CORE_GROW_MIN} && maxNeighbour >= ${WEB_CAMERA_EFFECT_MASK_CORE_MIN}) { + clean = 1.0; + } + let normalized = clamp((clean - ${WEB_CAMERA_EFFECT_MASK_VOID_MAX}) / ${WEB_CAMERA_EFFECT_MASK_BAND_WIDTH}, 0.0, 1.0); + let shaped = normalized * normalized * (3.0 - 2.0 * normalized); + let previous = smoothedMask[index]; + let delta = abs(shaped - previous); + let rawMotion = clamp((delta - ${WEB_CAMERA_EFFECT_MASK_TEMPORAL_MOTION_LOW}) / (${WEB_CAMERA_EFFECT_MASK_TEMPORAL_MOTION_HIGH} - ${WEB_CAMERA_EFFECT_MASK_TEMPORAL_MOTION_LOW}), 0.0, 1.0); + let motion = rawMotion * rawMotion * (3.0 - 2.0 * rawMotion); + let keep = select(0.0, ${WEB_CAMERA_EFFECT_MASK_TEMPORAL_KEEP_STILL} * (1.0 - motion), params.primed != 0u); + var next = keep * previous + (1.0 - keep) * shaped; + if (clean >= ${WEB_CAMERA_EFFECT_MASK_CORE_MIN}) { + next = 1.0; + } + if (clean <= ${WEB_CAMERA_EFFECT_MASK_VOID_MAX}) { + next = 0.0; + } + smoothedMask[index] = next; + textureStore(outputMask, vec2u(id.xy), vec4f(next, 0.0, 0.0, 1.0)); +} +`; diff --git a/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectWorker.ts b/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectWorker.ts new file mode 100644 index 000000000..e5250f605 --- /dev/null +++ b/fluxer_app/src/features/voice/utils/camera-effects/WebCameraEffectWorker.ts @@ -0,0 +1,611 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {CameraBackgroundMode} from '@app/features/voice/utils/camera-effects/CameraCaptureContract'; +import { + type ErrorDiagnosticType, + getErrorDiagnostic, + getErrorDiagnosticType, +} from '@app/features/voice/utils/camera-effects/ErrorDiagnostic'; +import {WebCameraEffectCanvasRenderer} from '@app/features/voice/utils/camera-effects/WebCameraEffectCanvasRenderer'; +import { + createWebCameraEffectVideoFrameSource, + loadWebCameraEffectCustomFrameSource, + type WebCameraEffectCustomFrameSource, +} from '@app/features/voice/utils/camera-effects/WebCameraEffectCustomImage'; +import { + WEB_CAMERA_EFFECT_STOP_GRACE_MS, + WebCameraEffectCommandKind, + WebCameraEffectCommandPolicy, + WebCameraEffectCustomMediaKind, + WebCameraEffectEventKind, + WebCameraEffectShutdownReason, + type WebCameraEffectStartCommand, + type WebCameraEffectUpdateCommand, + type WebCameraEffectWorkerEvent, + type WebCameraPipelineConfig, +} from '@app/features/voice/utils/camera-effects/WebCameraEffectProtocol'; +import type {WebCameraEffectRenderer} from '@app/features/voice/utils/camera-effects/WebCameraEffectRenderer'; +import {WebCameraEffectWebGPURenderer} from '@app/features/voice/utils/camera-effects/WebCameraEffectWebGPURenderer'; + +type CameraEffectWorkerScope = DedicatedWorkerGlobalScope & {close(): void}; + +function requireCameraEffectWorkerScope(value: unknown): asserts value is CameraEffectWorkerScope { + if (typeof value !== 'object' || value == null) { + throw new Error('Camera effect worker global scope is unavailable'); + } + if (!('postMessage' in value) || typeof value.postMessage !== 'function') { + throw new Error('Camera effect worker global scope cannot post messages'); + } + if (!('addEventListener' in value) || typeof value.addEventListener !== 'function') { + throw new Error('Camera effect worker global scope cannot receive messages'); + } + if (!('close' in value) || typeof value.close !== 'function') { + throw new Error('Camera effect worker global scope cannot be closed'); + } +} + +requireCameraEffectWorkerScope(self); +const workerScope = self; +__webpack_base_uri__ = workerScope.location.href; +const OPERATION_TIMEOUT_MS = 10_000; +const OPERATION_QUEUE_MAX = 8; +const DISPOSAL_TIMEOUT_MS = 1_500; + +const WebCameraEffectLifecycle = Object.freeze({ + RUNNING: 'running', + DRAINING: 'draining', + STOPPING: 'stopping', + CLOSED: 'closed', +} as const); + +type WebCameraEffectLifecycle = (typeof WebCameraEffectLifecycle)[keyof typeof WebCameraEffectLifecycle]; + +function postEvent(event: WebCameraEffectWorkerEvent): void { + workerScope.postMessage(event); +} + +function postFailure(error: unknown): void { + postEvent({kind: WebCameraEffectEventKind.FAILED, ...getErrorDiagnostic(error)}); +} + +function rejectCommand(error: unknown): void { + try { + postFailure(error); + } finally { + workerScope.close(); + } +} + +class CameraEffectInputStreamEndedError extends Error { + constructor() { + super('Camera effect input stream ended'); + this.name = 'CameraEffectInputStreamEndedError'; + } +} + +class CameraEffectDisposalTimeoutError extends Error { + constructor() { + super('Camera effect worker cleanup exceeded its deadline'); + this.name = 'CameraEffectDisposalTimeoutError'; + } +} + +interface RendererSelection { + readonly renderer: WebCameraEffectRenderer; + readonly fallbackErrorType: ErrorDiagnosticType | null; +} + +function hasSameCustomBackground(current: WebCameraPipelineConfig, next: WebCameraPipelineConfig): boolean { + const currentBackground = current.background; + const nextBackground = next.background; + if (currentBackground == null || nextBackground == null) { + return false; + } + if (currentBackground.mode !== CameraBackgroundMode.CUSTOM) { + return false; + } + if (nextBackground.mode !== CameraBackgroundMode.CUSTOM) { + return false; + } + if (currentBackground.customMediaURL !== nextBackground.customMediaURL) { + return false; + } + return currentBackground.customMediaKind === nextBackground.customMediaKind; +} + +async function createCustomFrameSource( + config: WebCameraPipelineConfig, + videoFrames: ReadableStream | null, +): Promise { + const background = config.background; + if (background == null || background.mode !== CameraBackgroundMode.CUSTOM) { + return null; + } + let source: WebCameraEffectCustomFrameSource; + if (background.customMediaKind === WebCameraEffectCustomMediaKind.VIDEO) { + if (videoFrames == null) { + throw new Error('Video camera background update requires a transferred frame stream'); + } + source = await createWebCameraEffectVideoFrameSource(videoFrames); + } else { + source = await loadWebCameraEffectCustomFrameSource(background.customMediaURL); + } + const decodedKindMatches = source.kind === background.customMediaKind; + const singleFrameAnimation = + background.customMediaKind === WebCameraEffectCustomMediaKind.ANIMATED && + source.kind === WebCameraEffectCustomMediaKind.STATIC; + if (decodedKindMatches || singleFrameAnimation) { + return source; + } + let cleanupError: unknown; + try { + await source.dispose(); + } catch (error) { + cleanupError = error; + } + const mismatchError = new Error('Custom camera background media kind does not match its decoded content'); + if (cleanupError !== undefined) { + throw new AggregateError([mismatchError, cleanupError], 'Custom camera background rejection cleanup failed'); + } + throw mismatchError; +} + +async function selectRenderer( + command: WebCameraEffectStartCommand, + customFrameSource: WebCameraEffectCustomFrameSource | null, +): Promise { + if (command.config.background == null) { + return { + renderer: await WebCameraEffectCanvasRenderer.create(command.fallbackCanvas, command.config, customFrameSource), + fallbackErrorType: null, + }; + } + if (!command.preferWebGPU) { + return { + renderer: await WebCameraEffectCanvasRenderer.create(command.fallbackCanvas, command.config, customFrameSource), + fallbackErrorType: null, + }; + } + try { + return { + renderer: await WebCameraEffectWebGPURenderer.create(command.gpuCanvas, command.config, customFrameSource), + fallbackErrorType: null, + }; + } catch (webGPUError) { + try { + return { + renderer: await WebCameraEffectCanvasRenderer.create(command.fallbackCanvas, command.config, customFrameSource), + fallbackErrorType: getErrorDiagnosticType(webGPUError), + }; + } catch (WASMError) { + throw new AggregateError([webGPUError, WASMError], 'Every camera effect renderer failed'); + } + } +} + +class WebCameraEffectWorkerController { + private readonly reader: ReadableStreamDefaultReader; + private operationTail: Promise = Promise.resolve(); + private updatePreparationPromise: Promise = Promise.resolve(); + private disposePromise: Promise | null = null; + private pendingOperations = 0; + private updateInProgress = false; + private lifecycle: WebCameraEffectLifecycle = WebCameraEffectLifecycle.RUNNING; + private terminalReason: WebCameraEffectShutdownReason | null = null; + private terminalDiagnosticError: unknown = null; + private resolveOwnerStopIntent: (() => void) | null = null; + private readonly ownerStopIntent: Promise; + + constructor( + private readonly renderer: WebCameraEffectRenderer, + readable: ReadableStream, + private config: WebCameraPipelineConfig, + private customFrameSource: WebCameraEffectCustomFrameSource | null, + ) { + this.reader = readable.getReader(); + let capturedResolveOwnerStopIntent: (() => void) | null = null; + this.ownerStopIntent = new Promise((resolve) => { + capturedResolveOwnerStopIntent = resolve; + }); + const resolveOwnerStopIntent = capturedResolveOwnerStopIntent; + if (resolveOwnerStopIntent == null) { + throw new Error('Camera effect worker stop intent resolver was not captured'); + } + this.resolveOwnerStopIntent = resolveOwnerStopIntent; + } + + private isShuttingDown(): boolean { + return this.lifecycle !== WebCameraEffectLifecycle.RUNNING; + } + + run(): void { + this.pump().catch((error) => { + this.fail(error); + }); + } + + update(command: WebCameraEffectUpdateCommand): void { + if (this.isShuttingDown()) { + return; + } + if (this.updateInProgress) { + postEvent({ + kind: WebCameraEffectEventKind.UPDATE_FAILED, + requestId: command.requestId, + ...getErrorDiagnostic(new Error('Camera effect worker already has an update in progress')), + }); + return; + } + this.updateInProgress = true; + const preparation = this.prepareUpdate(command).finally(() => { + this.updateInProgress = false; + }); + this.updatePreparationPromise = preparation; + preparation.catch((error) => { + this.fail(error); + }); + } + + private async prepareUpdate(command: WebCameraEffectUpdateCommand): Promise { + let nextCustomFrameSource = this.customFrameSource; + let ownsCandidate = false; + try { + if (!hasSameCustomBackground(this.config, command.config)) { + nextCustomFrameSource = await createCustomFrameSource(command.config, command.customBackgroundFrames); + ownsCandidate = nextCustomFrameSource != null; + } + } catch (error) { + const diagnosticError = await this.disposeRejectedCandidate(error, nextCustomFrameSource, ownsCandidate); + this.postUpdateFailure(command.requestId, diagnosticError); + return; + } + if (this.isShuttingDown()) { + await this.disposeStoppedCandidate(nextCustomFrameSource, ownsCandidate); + return; + } + let configurationError: unknown; + let configurationFailed = false; + let skipped = false; + try { + await this.enqueue(async () => { + if (this.isShuttingDown()) { + skipped = true; + return; + } + try { + await this.renderer.configure(command.config, nextCustomFrameSource); + } catch (error) { + configurationError = error; + configurationFailed = true; + } + }, false); + } catch (error) { + const diagnosticError = await this.disposeRejectedCandidate(error, nextCustomFrameSource, ownsCandidate); + this.postUpdateFailure(command.requestId, diagnosticError); + return; + } + if (skipped) { + await this.disposeStoppedCandidate(nextCustomFrameSource, ownsCandidate); + return; + } + if (configurationFailed) { + const diagnosticError = await this.disposeRejectedCandidate( + configurationError, + nextCustomFrameSource, + ownsCandidate, + ); + this.postUpdateFailure(command.requestId, diagnosticError); + return; + } + const previousCustomFrameSource = this.customFrameSource; + this.config = command.config; + this.customFrameSource = nextCustomFrameSource; + if (!this.isShuttingDown()) { + postEvent({kind: WebCameraEffectEventKind.UPDATED, requestId: command.requestId}); + } + if (previousCustomFrameSource !== nextCustomFrameSource) { + try { + await previousCustomFrameSource?.dispose(); + } catch (error) { + throw new Error('Camera effect update committed but previous custom source cleanup failed', {cause: error}); + } + } + } + + private postUpdateFailure(requestId: number, error: unknown): void { + if (this.isShuttingDown()) { + return; + } + postEvent({ + kind: WebCameraEffectEventKind.UPDATE_FAILED, + requestId, + ...getErrorDiagnostic(error), + }); + } + + private async disposeStoppedCandidate( + candidate: WebCameraEffectCustomFrameSource | null, + ownsCandidate: boolean, + ): Promise { + if (!ownsCandidate || candidate == null) { + return; + } + await candidate.dispose(); + } + + private async disposeRejectedCandidate( + primaryError: unknown, + candidate: WebCameraEffectCustomFrameSource | null, + ownsCandidate: boolean, + ): Promise { + if (!ownsCandidate || candidate == null) { + return primaryError; + } + try { + await candidate.dispose(); + return primaryError; + } catch (cleanupError) { + return new AggregateError([primaryError, cleanupError], 'Camera effect update and candidate cleanup both failed'); + } + } + + stop(): void { + if (this.lifecycle === WebCameraEffectLifecycle.STOPPING || this.lifecycle === WebCameraEffectLifecycle.CLOSED) { + this.resolveOwnerStopIntent?.(); + return; + } + if (this.terminalReason == null) { + this.terminalReason = WebCameraEffectShutdownReason.OWNER_STOP; + } + this.lifecycle = WebCameraEffectLifecycle.STOPPING; + this.resolveOwnerStopIntent?.(); + this.dispose().catch(() => { + workerScope.close(); + }); + } + + private async pump(): Promise { + while (this.lifecycle === WebCameraEffectLifecycle.RUNNING) { + const {value, done} = await this.reader.read(); + if (done || value == null) { + await this.settleStreamEnd(); + break; + } + try { + await this.enqueue(() => this.renderer.render(value, performance.now())); + } finally { + value.close(); + } + } + await this.dispose(); + } + + private async settleStreamEnd(): Promise { + if (this.isShuttingDown()) { + return; + } + this.lifecycle = WebCameraEffectLifecycle.DRAINING; + if (await this.awaitOwnerStopIntent()) { + return; + } + throw new CameraEffectInputStreamEndedError(); + } + + private awaitOwnerStopIntent(): Promise { + return new Promise((resolve) => { + const timeout = setTimeout(() => { + resolve(false); + }, WEB_CAMERA_EFFECT_STOP_GRACE_MS); + void this.ownerStopIntent.then(() => { + clearTimeout(timeout); + resolve(true); + }); + }); + } + + private enqueue(operation: () => Promise, enforceDeadline = true): Promise { + if (this.pendingOperations >= OPERATION_QUEUE_MAX) { + return Promise.reject(new Error('Camera effect worker operation queue is full')); + } + this.pendingOperations += 1; + const result = this.operationTail.then(() => this.executeOperation(operation, enforceDeadline)); + const trackedResult = result.finally(() => { + this.pendingOperations -= 1; + }); + this.operationTail = trackedResult.catch((error) => { + this.fail(error); + }); + return trackedResult; + } + + private async executeOperation(operation: () => Promise, enforceDeadline: boolean): Promise { + if (!enforceDeadline) { + await operation(); + return; + } + const timeout = setTimeout(() => { + this.fail(new Error('Camera effect worker operation exceeded its deadline')); + }, OPERATION_TIMEOUT_MS); + try { + await operation(); + } finally { + clearTimeout(timeout); + } + } + + private fail(error: unknown): void { + if (this.terminalReason != null) { + return; + } + this.terminalReason = + error instanceof CameraEffectInputStreamEndedError + ? WebCameraEffectShutdownReason.INPUT_ENDED + : WebCameraEffectShutdownReason.OPERATION_FAILED; + try { + postFailure(error); + } finally { + this.stop(); + } + } + + private dispose(): Promise { + if (this.disposePromise == null) { + const updateSettlement = this.updatePreparationPromise.then( + () => [] as Array, + (error: unknown) => [error], + ); + const resourceDisposal = updateSettlement.then(async (preparationFailures) => { + await this.operationTail; + const outcomes = await Promise.allSettled([ + this.renderer.dispose(), + Promise.resolve().then(() => this.customFrameSource?.dispose()), + ]); + const failures = [ + ...preparationFailures, + ...outcomes.flatMap((outcome) => (outcome.status === 'rejected' ? [outcome.reason] : [])), + ]; + if (failures.length > 0) { + throw this.resolveDisposalError(failures); + } + }); + this.disposePromise = Promise.allSettled([this.reader.cancel(), this.withDisposalDeadline(resourceDisposal)]) + .then((outcomes) => this.reportDisposalFailures(outcomes)) + .finally(() => { + this.lifecycle = WebCameraEffectLifecycle.CLOSED; + try { + postEvent({ + kind: WebCameraEffectEventKind.STOPPED, + reason: this.terminalReason ?? WebCameraEffectShutdownReason.OWNER_STOP, + diagnostic: + this.terminalDiagnosticError == null ? null : getErrorDiagnostic(this.terminalDiagnosticError), + }); + } finally { + workerScope.close(); + } + }); + } + return this.disposePromise; + } + + private withDisposalDeadline(disposal: Promise): Promise { + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + reject(new CameraEffectDisposalTimeoutError()); + }, DISPOSAL_TIMEOUT_MS); + disposal.then( + () => { + clearTimeout(timeout); + resolve(); + }, + (error: unknown) => { + clearTimeout(timeout); + reject(error); + }, + ); + }); + } + + private reportDisposalFailures(outcomes: ReadonlyArray>): void { + const failures = outcomes + .filter((outcome): outcome is PromiseRejectedResult => outcome.status === 'rejected') + .map((outcome) => outcome.reason); + if (failures.length === 0) { + return; + } + const error = this.resolveDisposalError(failures); + if (this.terminalReason === WebCameraEffectShutdownReason.OWNER_STOP) { + this.terminalReason = WebCameraEffectShutdownReason.CLEANUP_FAILED; + this.terminalDiagnosticError = error; + return; + } + postFailure(error); + } + + private resolveDisposalError(failures: ReadonlyArray): unknown { + if (failures.length === 1) { + return failures[0]; + } + return new AggregateError(failures, 'Camera effect worker cleanup failed'); + } +} + +class WebCameraEffectWorkerEntry { + private controller: WebCameraEffectWorkerController | null = null; + private starting = false; + + install(): void { + workerScope.addEventListener('message', (event: MessageEvent) => { + this.handle(event.data); + }); + } + + handle(data: unknown): void { + if (data == null || typeof data !== 'object') { + rejectCommand(new Error('Camera effect worker received a non-object command')); + return; + } + if (!WebCameraEffectCommandPolicy.isValid(data)) { + rejectCommand(new Error('Camera effect worker received an unknown command')); + return; + } + if (data.kind === WebCameraEffectCommandKind.START) { + this.start(data).catch((error) => { + rejectCommand(error); + }); + return; + } + if (data.kind === WebCameraEffectCommandKind.UPDATE) { + if (this.controller == null) { + rejectCommand(new Error('Camera effect worker received update before start')); + return; + } + this.controller.update(data); + return; + } + if (data.kind === WebCameraEffectCommandKind.STOP) { + if (this.controller == null) { + rejectCommand(new Error('Camera effect worker received stop before start')); + return; + } + this.controller.stop(); + } + } + + private async start(command: WebCameraEffectStartCommand): Promise { + if (this.starting || this.controller != null) { + throw new Error('Camera effect worker received more than one start command'); + } + this.starting = true; + const customFrameSource = await createCustomFrameSource(command.config, command.customBackgroundFrames); + let selection: RendererSelection; + try { + selection = await selectRenderer(command, customFrameSource); + } catch (error) { + let cleanupError: unknown; + try { + await customFrameSource?.dispose(); + } catch (caughtCleanupError) { + cleanupError = caughtCleanupError; + } + if (cleanupError !== undefined) { + throw new AggregateError([error, cleanupError], 'Camera effect renderer selection cleanup failed'); + } + throw error; + } + const nextController = new WebCameraEffectWorkerController( + selection.renderer, + command.readable, + command.config, + customFrameSource, + ); + this.controller = nextController; + postEvent({ + kind: WebCameraEffectEventKind.READY, + backend: selection.renderer.backend, + fallbackErrorType: selection.fallbackErrorType, + }); + nextController.run(); + } +} + +new WebCameraEffectWorkerEntry().install(); diff --git a/fluxer_app/src/features/voice/utils/camera-effects/WebSelfieSegmenter.ts b/fluxer_app/src/features/voice/utils/camera-effects/WebSelfieSegmenter.ts new file mode 100644 index 000000000..08a1c4646 --- /dev/null +++ b/fluxer_app/src/features/voice/utils/camera-effects/WebSelfieSegmenter.ts @@ -0,0 +1,359 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +import {resolveWorkerAssetUrl} from '@app/features/platform/utils/WorkerAssetUrl'; +import { + collectSettledFailures, + throwCollectedFailures, +} from '@app/features/voice/utils/camera-effects/AggregateOperations'; +import { + cancelResponseBodyAndThrow, + readBoundedResponseArrayBuffer, + runWithResponseDeadline, +} from '@app/features/voice/utils/camera-effects/BoundedResponse'; +import { + shapeWebCameraEffectMaskAlpha, + WEB_CAMERA_EFFECT_MASK_CORE_GROW_MIN, + WEB_CAMERA_EFFECT_MASK_CORE_MIN, + WEB_CAMERA_EFFECT_MASK_HOLE_NEIGHBOUR_MIN, + WEB_CAMERA_EFFECT_MASK_SPECKLE_NEIGHBOUR_MAX, + WEB_CAMERA_EFFECT_MASK_TEMPORAL_KEEP_STILL, + WEB_CAMERA_EFFECT_MASK_TEMPORAL_MOTION_HIGH, + WEB_CAMERA_EFFECT_MASK_TEMPORAL_MOTION_LOW, + WEB_CAMERA_EFFECT_MASK_VOID_MAX, +} from '@app/features/voice/utils/camera-effects/WebCameraEffectMask'; +import ortWasmUrl from 'onnxruntime-web/ort-wasm-simd-threaded.asyncify.wasm'; +import type * as OrtNamespace from 'onnxruntime-web/webgpu'; +import modelAssetUrl from './models/selfie_segmentation_256x256.onnx'; + +export class MissingSegmentationAlphasOutputError extends Error { + constructor() { + super('Segmentation model produced no alphas output'); + this.name = 'MissingSegmentationAlphasOutputError'; + } +} + +export const SEG_INPUT_EDGE = 256; +export const SEG_INPUT_NAME = 'pixel_values'; +export const SEG_OUTPUT_NAME = 'alphas'; +export const SEG_INPUT_PIXELS = SEG_INPUT_EDGE * SEG_INPUT_EDGE; + +const MODEL_RESPONSE_MAX_BYTES = 4 * 1024 * 1024; +const MODEL_RESPONSE_MAX_CHUNKS = 2048; +const MODEL_REQUEST_TIMEOUT_MS = 30_000; + +export type WebSelfieOrtModule = typeof OrtNamespace; + +export interface WebSelfieRuntime { + readonly ort: WebSelfieOrtModule; + readonly modelBytes: Uint8Array; +} + +class WebSelfieRuntimeAssetOwner { + private ortModule: Promise | null = null; + private modelBytes: Promise | null = null; + + loadOrt(): Promise { + if (this.ortModule == null) { + const loading = import('onnxruntime-web/webgpu').then((ort) => this.configureOrt(ort)); + const cached = loading.catch((error) => { + this.clearFailedOrtLoad(cached); + throw error; + }); + this.ortModule = cached; + } + return this.ortModule; + } + + private configureOrt(ort: WebSelfieOrtModule): WebSelfieOrtModule { + ort.env.wasm.wasmPaths = {wasm: resolveWorkerAssetUrl(ortWasmUrl)}; + ort.env.wasm.proxy = false; + ort.env.wasm.numThreads = resolveOrtThreadCount(); + ort.env.logLevel = 'error'; + return ort; + } + + private clearFailedOrtLoad(cached: Promise): void { + if (this.ortModule === cached) { + this.ortModule = null; + } + } + + loadModel(): Promise { + if (this.modelBytes == null) { + const loading = runWithResponseDeadline({ + timeoutMilliseconds: MODEL_REQUEST_TIMEOUT_MS, + description: 'Web selfie segmentation model request', + signal: null, + operation: async (signal) => { + let response: Response; + try { + response = await fetch(resolveWorkerAssetUrl(modelAssetUrl), { + credentials: 'omit', + redirect: 'error', + referrerPolicy: 'no-referrer', + signal, + }); + } catch { + if (signal.aborted) throw signal.reason; + throw new Error('Web selfie segmentation model request failed'); + } + if (!response.ok) { + await cancelResponseBodyAndThrow({ + response, + error: new Error(`Segmentation model request failed with status ${response.status}`), + description: 'Web selfie segmentation model response', + }); + } + return readBoundedResponseArrayBuffer({ + response, + maximumBytes: MODEL_RESPONSE_MAX_BYTES, + maximumChunks: MODEL_RESPONSE_MAX_CHUNKS, + description: 'Web selfie segmentation model response', + }); + }, + }).then((bytes) => new Uint8Array(bytes)); + const cached = loading.catch((error) => { + this.clearFailedModelLoad(cached); + throw error; + }); + this.modelBytes = cached; + } + return this.modelBytes; + } + + private clearFailedModelLoad(cached: Promise): void { + if (this.modelBytes === cached) { + this.modelBytes = null; + } + } +} + +const webSelfieRuntimeAssetOwner = new WebSelfieRuntimeAssetOwner(); + +async function loadWebSelfieOrt(): Promise { + return webSelfieRuntimeAssetOwner.loadOrt(); +} + +async function loadWebSelfieModel(): Promise { + return webSelfieRuntimeAssetOwner.loadModel(); +} + +function resolveOrtThreadCount(): number { + if (!globalThis.crossOriginIsolated) { + return 1; + } + if (!('navigator' in globalThis)) { + return 1; + } + const hardwareConcurrency = navigator.hardwareConcurrency; + if (!Number.isSafeInteger(hardwareConcurrency)) { + return 1; + } + if (hardwareConcurrency < 1) { + return 1; + } + return Math.min(4, hardwareConcurrency); +} + +export async function loadWebSelfieRuntime(): Promise { + const [ortOutcome, modelBytesOutcome] = await Promise.allSettled([loadWebSelfieOrt(), loadWebSelfieModel()]); + const failures: Array = []; + if (ortOutcome.status === 'rejected') failures.push(ortOutcome.reason); + if (modelBytesOutcome.status === 'rejected') failures.push(modelBytesOutcome.reason); + throwCollectedFailures({failures, message: 'Web selfie runtime loading failed'}); + if (ortOutcome.status !== 'fulfilled' || modelBytesOutcome.status !== 'fulfilled') { + throw new Error('Web selfie runtime loading produced no result'); + } + return {ort: ortOutcome.value, modelBytes: modelBytesOutcome.value}; +} + +function requireFloatMask(output: OrtNamespace.Tensor): Float32Array { + const data = output.data; + if (!(data instanceof Float32Array) || data.length !== SEG_INPUT_PIXELS) { + throw new Error('Segmentation model produced an unexpected CPU output'); + } + return data; +} + +function collectInferenceOutputDisposalFailures( + outputs: Readonly>, +): ReadonlyArray { + const failures: Array = []; + for (const output of Object.values(outputs)) { + try { + output.dispose(); + } catch (error) { + failures.push(error); + } + } + return failures; +} + +export class WebSelfieSegmenter { + private readonly session: OrtNamespace.InferenceSession; + private readonly inputChw = new Float32Array(3 * SEG_INPUT_PIXELS); + private readonly inputTensor: OrtNamespace.Tensor; + private readonly previous = new Float32Array(SEG_INPUT_PIXELS); + private primed = false; + private disposed = false; + + private constructor(ort: WebSelfieOrtModule, session: OrtNamespace.InferenceSession) { + this.session = session; + this.inputTensor = new ort.Tensor('float32', this.inputChw, [1, 3, SEG_INPUT_EDGE, SEG_INPUT_EDGE]); + } + + static async create(): Promise { + const {ort, modelBytes} = await loadWebSelfieRuntime(); + const session = await ort.InferenceSession.create(modelBytes, { + executionProviders: ['wasm'], + graphOptimizationLevel: 'all', + logSeverityLevel: 3, + }); + const segmenter = new WebSelfieSegmenter(ort, session); + try { + await segmenter.warmup(); + return segmenter; + } catch (error) { + const cleanupFailures = await collectSettledFailures([segmenter.dispose()]); + throwCollectedFailures({ + failures: [error, ...cleanupFailures], + message: 'Web selfie segmenter initialization failed', + }); + } + } + + private async warmup(): Promise { + const outputs = await this.session.run({[SEG_INPUT_NAME]: this.inputTensor}); + const failures: Array = []; + try { + const output = outputs[SEG_OUTPUT_NAME]; + if (output == null) { + throw new MissingSegmentationAlphasOutputError(); + } + requireFloatMask(output); + } catch (error) { + failures.push(error); + } + failures.push(...collectInferenceOutputDisposalFailures(outputs)); + throwCollectedFailures({failures, message: 'Web selfie segmenter warm-up failed'}); + } + + private writeShapedMask(alphas: Float32Array, maskRGBA: Uint8ClampedArray): void { + const lastTexel = SEG_INPUT_EDGE - 1; + for (let y = 0; y < SEG_INPUT_EDGE; y += 1) { + for (let x = 0; x < SEG_INPUT_EDGE; x += 1) { + const index = y * SEG_INPUT_EDGE + x; + const centre = Math.max(0, Math.min(1, alphas[index])); + let neighbourhoodSum = 0; + let maxNeighbour = 0; + for (let offsetY = -1; offsetY <= 1; offsetY += 1) { + const sampleY = Math.max(0, Math.min(lastTexel, y + offsetY)); + for (let offsetX = -1; offsetX <= 1; offsetX += 1) { + const sampleX = Math.max(0, Math.min(lastTexel, x + offsetX)); + const sample = Math.max(0, Math.min(1, alphas[sampleY * SEG_INPUT_EDGE + sampleX])); + neighbourhoodSum += sample; + if (offsetX !== 0 || offsetY !== 0) { + maxNeighbour = Math.max(maxNeighbour, sample); + } + } + } + const neighbourhoodMean = neighbourhoodSum / 9; + let clean = centre; + if ( + centre >= WEB_CAMERA_EFFECT_MASK_CORE_MIN && + neighbourhoodMean < WEB_CAMERA_EFFECT_MASK_SPECKLE_NEIGHBOUR_MAX + ) { + clean = 0; + } + if ( + centre <= WEB_CAMERA_EFFECT_MASK_VOID_MAX && + neighbourhoodMean > WEB_CAMERA_EFFECT_MASK_HOLE_NEIGHBOUR_MIN + ) { + clean = 1; + } + if (centre >= WEB_CAMERA_EFFECT_MASK_CORE_GROW_MIN && maxNeighbour >= WEB_CAMERA_EFFECT_MASK_CORE_MIN) { + clean = 1; + } + const shaped = shapeWebCameraEffectMaskAlpha(clean); + const previous = this.previous[index]; + const delta = Math.abs(shaped - previous); + const rawMotion = Math.max( + 0, + Math.min( + 1, + (delta - WEB_CAMERA_EFFECT_MASK_TEMPORAL_MOTION_LOW) / + (WEB_CAMERA_EFFECT_MASK_TEMPORAL_MOTION_HIGH - WEB_CAMERA_EFFECT_MASK_TEMPORAL_MOTION_LOW), + ), + ); + const motion = rawMotion * rawMotion * (3 - 2 * rawMotion); + const keep = this.primed ? WEB_CAMERA_EFFECT_MASK_TEMPORAL_KEEP_STILL * (1 - motion) : 0; + let next = keep * previous + (1 - keep) * shaped; + if (clean >= WEB_CAMERA_EFFECT_MASK_CORE_MIN) { + next = 1; + } + if (clean <= WEB_CAMERA_EFFECT_MASK_VOID_MAX) { + next = 0; + } + this.previous[index] = next; + const RGBAIndex = index * 4; + maskRGBA[RGBAIndex] = 0; + maskRGBA[RGBAIndex + 1] = 0; + maskRGBA[RGBAIndex + 2] = 0; + maskRGBA[RGBAIndex + 3] = Math.round(next * 255); + } + } + } + + async segmentIntoMask(RGBA: Uint8ClampedArray, maskRGBA: Uint8ClampedArray): Promise { + if (this.disposed) { + throw new Error('Cannot segment with a disposed web selfie segmenter'); + } + if (RGBA.length !== SEG_INPUT_PIXELS * 4) { + throw new Error('Segmentation input must be one 256 by 256 RGBA frame'); + } + if (maskRGBA.length !== SEG_INPUT_PIXELS * 4) { + throw new Error('Segmentation mask must hold RGBA for every model output sample'); + } + const green = SEG_INPUT_PIXELS; + const blue = SEG_INPUT_PIXELS * 2; + for (let pixel = 0; pixel < SEG_INPUT_PIXELS; pixel += 1) { + const source = pixel * 4; + this.inputChw[pixel] = RGBA[source] / 255; + this.inputChw[green + pixel] = RGBA[source + 1] / 255; + this.inputChw[blue + pixel] = RGBA[source + 2] / 255; + } + const outputs = await this.session.run({[SEG_INPUT_NAME]: this.inputTensor}); + const failures: Array = []; + try { + const output = outputs[SEG_OUTPUT_NAME]; + if (output == null) { + throw new MissingSegmentationAlphasOutputError(); + } + const alphas = requireFloatMask(output); + this.writeShapedMask(alphas, maskRGBA); + this.primed = true; + } catch (error) { + failures.push(error); + } + failures.push(...collectInferenceOutputDisposalFailures(outputs)); + throwCollectedFailures({failures, message: 'Web selfie segmentation failed'}); + } + + reset(): void { + this.previous.fill(0); + this.primed = false; + } + + async dispose(): Promise { + if (this.disposed) { + return; + } + this.disposed = true; + const inputFailures = await collectSettledFailures([Promise.resolve().then(() => this.inputTensor.dispose())]); + const sessionFailures = await collectSettledFailures([Promise.resolve().then(() => this.session.release())]); + throwCollectedFailures({ + failures: [...inputFailures, ...sessionFailures], + message: 'Web selfie segmenter teardown failed', + }); + } +} diff --git a/fluxer_app/src/features/voice/utils/camera-effects/models/NOTICE.txt b/fluxer_app/src/features/voice/utils/camera-effects/models/NOTICE.txt new file mode 100644 index 000000000..a4dca7d71 --- /dev/null +++ b/fluxer_app/src/features/voice/utils/camera-effects/models/NOTICE.txt @@ -0,0 +1,13 @@ +selfie_segmentation_256x256.onnx + +Source: https://huggingface.co/onnx-community/mediapipe_selfie_segmentation (onnx/model.onnx) +Upstream: Google MediaPipe Selfie Segmentation. +License: Apache License 2.0. +SHA-256: 3241ac4ad8aa35bdaf33946776db29f7c283a413aa0b0dacb9483594b4531aad + +Tensor interface: + input "pixel_values": float32 [batch, 3, 256, 256], NCHW, RGB, values rescaled to [0, 1] (x / 255), no mean/std normalization. + output "alphas": float32 [batch, 1, 256, 256], single-channel foreground probability mask in [0, 1]. + +A copy of the Apache License 2.0 is available at https://www.apache.org/licenses/LICENSE-2.0. +This model is redistributed unmodified under the terms of that license. diff --git a/fluxer_app/src/features/voice/utils/camera-effects/models/selfie_segmentation_256x256.d.onnx.ts b/fluxer_app/src/features/voice/utils/camera-effects/models/selfie_segmentation_256x256.d.onnx.ts new file mode 100644 index 000000000..611112b6c --- /dev/null +++ b/fluxer_app/src/features/voice/utils/camera-effects/models/selfie_segmentation_256x256.d.onnx.ts @@ -0,0 +1,4 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +declare const url: string; +export default url; diff --git a/fluxer_app/src/features/voice/utils/camera-effects/models/selfie_segmentation_256x256.onnx b/fluxer_app/src/features/voice/utils/camera-effects/models/selfie_segmentation_256x256.onnx new file mode 100644 index 000000000..f926ab523 Binary files /dev/null and b/fluxer_app/src/features/voice/utils/camera-effects/models/selfie_segmentation_256x256.onnx differ diff --git a/fluxer_app/src/types/assets.d.ts b/fluxer_app/src/types/assets.d.ts index 965e5e82e..c5ea98d39 100644 --- a/fluxer_app/src/types/assets.d.ts +++ b/fluxer_app/src/types/assets.d.ts @@ -58,3 +58,8 @@ declare module '@pkgs/libfluxcore/libfluxcore_bg.wasm' { const url: string; export default url; } + +declare module '*.onnx' { + const url: string; + export default url; +} diff --git a/fluxer_app/src/types/ort-wasm-asset.d.ts b/fluxer_app/src/types/ort-wasm-asset.d.ts new file mode 100644 index 000000000..611112b6c --- /dev/null +++ b/fluxer_app/src/types/ort-wasm-asset.d.ts @@ -0,0 +1,4 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later + +declare const url: string; +export default url; diff --git a/fluxer_app/src/types/rspack-runtime.d.ts b/fluxer_app/src/types/rspack-runtime.d.ts new file mode 100644 index 000000000..354b2409a --- /dev/null +++ b/fluxer_app/src/types/rspack-runtime.d.ts @@ -0,0 +1 @@ +declare let __webpack_base_uri__: string; diff --git a/fluxer_app/src/types/webgpu.d.ts b/fluxer_app/src/types/webgpu.d.ts new file mode 100644 index 000000000..52b3dd415 --- /dev/null +++ b/fluxer_app/src/types/webgpu.d.ts @@ -0,0 +1,2 @@ +// SPDX-License-Identifier: AGPL-3.0-or-later +/// diff --git a/fluxer_app/tsconfig.json b/fluxer_app/tsconfig.json index c0bd1048d..8ddf765c0 100644 --- a/fluxer_app/tsconfig.json +++ b/fluxer_app/tsconfig.json @@ -15,6 +15,7 @@ "emitDecoratorMetadata": true, "paths": { "@app/*": ["./src/*"], + "onnxruntime-web/ort-wasm-simd-threaded.asyncify.wasm": ["./src/types/ort-wasm-asset.d.ts"], "@fluxer/*": ["../packages/*", "../packages/*/src/index.ts"], "@app_scripts/*": ["./scripts/*"], "@pkgs/*": ["./pkgs/*"], diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index dd62d5c5f..5a6fb9d35 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -144,6 +144,9 @@ catalogs: '@vvo/tzdb': specifier: 6.198.0 version: 6.198.0 + '@webgpu/types': + specifier: 0.1.72 + version: 0.1.72 archiver: specifier: 7.0.1 version: 7.0.1 @@ -264,6 +267,12 @@ catalogs: nodemailer: specifier: 9.0.6 version: 9.0.6 + onnxruntime-common: + specifier: 1.27.0 + version: 1.27.0 + onnxruntime-web: + specifier: 1.27.0 + version: 1.27.0 pg: specifier: 8.16.3 version: 8.16.3 @@ -1526,6 +1535,12 @@ importers: motion: specifier: 'catalog:' version: 12.34.3(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + onnxruntime-common: + specifier: 'catalog:' + version: 1.27.0 + onnxruntime-web: + specifier: 'catalog:' + version: 1.27.0 qrcode: specifier: 'catalog:' version: 1.5.4 @@ -1629,6 +1644,9 @@ importers: '@typescript/native-preview': specifier: 'catalog:' version: 7.0.0-dev.20260224.1 + '@webgpu/types': + specifier: 'catalog:' + version: 0.1.72 autoprefixer: specifier: 'catalog:' version: 10.4.24(postcss@8.5.26) @@ -5676,6 +5694,33 @@ packages: '@preact/signals-core@1.14.4': resolution: {integrity: sha512-HNB6HYeYKhQbJ1aKl+YRjrS4+QWHLKX6qKoUsfS/m0vqzsVaEBiZiaKbG/e+NKk2ch5ALQr/ihWaMHxiCuuWHA==} + '@protobufjs/aspromise@1.1.2': + resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==} + + '@protobufjs/base64@1.1.2': + resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==} + + '@protobufjs/codegen@2.0.5': + resolution: {integrity: sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==} + + '@protobufjs/eventemitter@1.1.1': + resolution: {integrity: sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==} + + '@protobufjs/fetch@1.1.1': + resolution: {integrity: sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==} + + '@protobufjs/float@1.0.2': + resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==} + + '@protobufjs/path@1.1.2': + resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==} + + '@protobufjs/pool@1.1.0': + resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==} + + '@protobufjs/utf8@1.1.2': + resolution: {integrity: sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==} + '@radix-ui/primitive@1.1.3': resolution: {integrity: sha512-JTF99U/6XIjCBo0wqkU5sK10glYe27MRRsfwoiq5zzOEZLHU3A3KCMa5X/azekYRCJ0HlwI0crAXS/5dEHTzDg==} @@ -7193,6 +7238,9 @@ packages: '@vvo/tzdb@6.198.0': resolution: {integrity: sha512-bNRWBhWYl0edVgyX6AYbhoCM2tk2lXJjGCyO2VDc2xn6Dw8dLd7WGj2DDXkVOkmOIQTNjEAcxrEpIzz5pWVwFg==} + '@webgpu/types@0.1.72': + resolution: {integrity: sha512-0cF7RFM2edNoiIS1ODJp0/Gzv4/xSXhwoR0YCza+OWpJWtn4wmo9DvK91aLlH9+uUnwIriP7ZiC3WitmyhuzBw==} + '@xmldom/xmldom@0.8.13': resolution: {integrity: sha512-KRYzxepc14G/CEpEGc3Yn+JKaAeT63smlDr+vjB8jRfgTBBI9wRj/nkQEO+ucV8p8I9bfKLWp37uHgFrbntPvw==} engines: {node: '>=10.0.0'} @@ -8412,6 +8460,9 @@ packages: resolution: {integrity: sha512-uSisMYERbaB9bkA9M4/4dnqyktaEkf1kMHNKq/7DHyxVeWqHQ2mBmVqm5u6/FVHwF3iCNalKcg82Zfl+tffWoA==} engines: {node: ^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0} + guid-typescript@1.0.9: + resolution: {integrity: sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==} + gzip-size@6.0.0: resolution: {integrity: sha512-ax7ZYomf6jqPTQ4+XCpUGyXKHk5WweS+e05MBO4/y3WJ5RkmPXNKvX+bx1behVILVwr6JSQvZAku021CHPXG3Q==} engines: {node: '>=10'} @@ -8948,6 +8999,9 @@ packages: long@5.2.5: resolution: {integrity: sha512-e0r9YBBgNCq1D1o5Dp8FMH0N5hsFtXDBiVa0qoJPHpakvZkmDKPRoGffZJII/XsHvj9An9blm+cRJ01yQqU+Dw==} + long@5.3.2: + resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==} + lower-case@2.0.2: resolution: {integrity: sha512-7fm3l3NAF9WfN6W3JOmf5drwpVqX78JtoGJ3A6W0a6ZnldM41w2fV5D490psKFTpMds8TJse/eHLFFsNHHjHgg==} @@ -9302,6 +9356,12 @@ packages: resolution: {integrity: sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==} engines: {node: '>=6'} + onnxruntime-common@1.27.0: + resolution: {integrity: sha512-3KxL5wIVqa8Ex08jxSzncm9CMgw8CjOFyOQ7SxvG9o0cVLlhTNKXyIQuTbtX4tGPJEf73OER2xrjt4HJSBL4ow==} + + onnxruntime-web@1.27.0: + resolution: {integrity: sha512-ogDLsqIozHZwifPuN37OproAo0byX6t43/bP8GzeZWBWD6MOGExswFAx3up4NS/vvWBOg2u2PXomDt3rMmdQSg==} + open@10.2.0: resolution: {integrity: sha512-YgBpdJHPyQ2UE5x+hlSXcnejzAvD0b22U2OuAP+8OnlJT+PjWPxtgmGqKKc+RgTM63U9gN0YzrYc71R2WT/hTA==} engines: {node: '>=18'} @@ -9467,6 +9527,9 @@ packages: resolution: {integrity: sha512-emEcLuomt2j03vxD54giVB4SxTjnsqkU692xZOZXHDVoYyypEm+b3jpiTcc+Cf+myooc+/Ly0z01jqeNHVgJGw==} engines: {node: '>=16.0.0'} + platform@1.3.6: + resolution: {integrity: sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==} + plist@3.1.0: resolution: {integrity: sha512-uysumyrvkUX0rX/dEVqt8gC3sTBzd4zoWfLeS29nb53imdaXVvLINYXTI2GNqzaMuvacNx4uJQ8+b3zXR0pkgQ==} engines: {node: '>=10.4.0'} @@ -9744,6 +9807,10 @@ packages: proper-lockfile@4.1.2: resolution: {integrity: sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==} + protobufjs@7.6.6: + resolution: {integrity: sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==} + engines: {node: '>=12.0.0'} + proxy-addr@2.0.7: resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} engines: {node: '>= 0.10'} @@ -14457,6 +14524,26 @@ snapshots: '@preact/signals-core@1.14.4': {} + '@protobufjs/aspromise@1.1.2': {} + + '@protobufjs/base64@1.1.2': {} + + '@protobufjs/codegen@2.0.5': {} + + '@protobufjs/eventemitter@1.1.1': {} + + '@protobufjs/fetch@1.1.1': + dependencies: + '@protobufjs/aspromise': 1.1.2 + + '@protobufjs/float@1.0.2': {} + + '@protobufjs/path@1.1.2': {} + + '@protobufjs/pool@1.1.0': {} + + '@protobufjs/utf8@1.1.2': {} + '@radix-ui/primitive@1.1.3': {} '@radix-ui/react-checkbox@1.3.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': @@ -16393,6 +16480,8 @@ snapshots: '@vvo/tzdb@6.198.0': {} + '@webgpu/types@0.1.72': {} + '@xmldom/xmldom@0.8.13': {} '@xmldom/xmldom@0.9.10': {} @@ -17770,6 +17859,8 @@ snapshots: graphql@16.13.0: {} + guid-typescript@1.0.9: {} + gzip-size@6.0.0: dependencies: duplexer: 0.1.2 @@ -18273,6 +18364,8 @@ snapshots: long@5.2.5: {} + long@5.3.2: {} + lower-case@2.0.2: dependencies: tslib: 2.8.1 @@ -18607,6 +18700,17 @@ snapshots: dependencies: mimic-fn: 2.1.0 + onnxruntime-common@1.27.0: {} + + onnxruntime-web@1.27.0: + dependencies: + flatbuffers: 25.9.23 + guid-typescript: 1.0.9 + long: 5.2.5 + onnxruntime-common: 1.27.0 + platform: 1.3.6 + protobufjs: 7.6.6 + open@10.2.0: dependencies: default-browser: 5.5.0 @@ -18808,6 +18912,8 @@ snapshots: pvutils: 1.1.5 tslib: 2.8.1 + platform@1.3.6: {} + plist@3.1.0: dependencies: '@xmldom/xmldom': 0.8.13 @@ -19140,6 +19246,20 @@ snapshots: retry: 0.12.0 signal-exit: 3.0.7 + protobufjs@7.6.6: + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/base64': 1.1.2 + '@protobufjs/codegen': 2.0.5 + '@protobufjs/eventemitter': 1.1.1 + '@protobufjs/fetch': 1.1.1 + '@protobufjs/float': 1.0.2 + '@protobufjs/path': 1.1.2 + '@protobufjs/pool': 1.1.0 + '@protobufjs/utf8': 1.1.2 + '@types/node': 24.10.9 + long: 5.3.2 + proxy-addr@2.0.7: dependencies: forwarded: 0.2.0 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 51d5e93b6..ef7c9b9a6 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -77,6 +77,7 @@ catalog: '@types/react': 19.2.14 '@types/react-dom': 19.2.3 '@types/validator': 13.15.10 + '@webgpu/types': 0.1.72 '@typescript/native-preview': 7.0.0-dev.20260224.1 '@vitest/coverage-v8': 4.1.11 '@vvo/tzdb': 6.198.0 @@ -121,6 +122,8 @@ catalog: msw: 2.12.10 nats: 2.29.3 nodemailer: 9.0.6 + onnxruntime-common: 1.27.0 + onnxruntime-web: 1.27.0 pg: 8.16.3 pino: 10.3.1 pino-pretty: 13.1.3